diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a895ef..e8b0f4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features - Aurora PostgreSQL CloudWatch with slow query auditing example - Google Cloud SQL for MySQL module +- Google Cloud SQL for PostgreSQL module ### Bug Fixes - Modified Server Host Name of AWS RDS MS SQL SERVER Dsfhub assets diff --git a/DSF_VERSION_COMPATABILITY.md b/DSF_VERSION_COMPATABILITY.md index 05570b0..e2e5aca 100644 --- a/DSF_VERSION_COMPATABILITY.md +++ b/DSF_VERSION_COMPATABILITY.md @@ -107,5 +107,9 @@ The following table lists the DSF versions that each module is tested and mainta onboard-gcp-mysql 4.17+ + + onboard-gcp-postgresql + 4.17+ + \ No newline at end of file diff --git a/examples/onboard-gcp-mysql/README.md b/examples/onboard-gcp-mysql/README.md index ed9e377..2c30fd4 100644 --- a/examples/onboard-gcp-mysql/README.md +++ b/examples/onboard-gcp-mysql/README.md @@ -45,4 +45,4 @@ No inputs. ## Outputs No outputs. - \ No newline at end of file + diff --git a/examples/onboard-gcp-postgresql/README.md b/examples/onboard-gcp-postgresql/README.md new file mode 100644 index 0000000..d7a27b9 --- /dev/null +++ b/examples/onboard-gcp-postgresql/README.md @@ -0,0 +1,60 @@ +# Onboard Google Cloud SQL for PostgreSQL example +This example includes additional prerequisites that will need to be completed to fully utilize the module. More details can be found in the [onboarding documentation](https://docs.imperva.com/bundle/onboarding-databases-to-sonar-reference-guide/page/Cloud-SQL-for-PostgreSQL-Onboarding-Steps_48367600.html). + +This example creates 'dsfhub' and 'google' resources. More information regarding authentication to each can be found in the relevant provider documentation: +- [dsfhub](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs) +- [google](https://registry.terraform.io/providers/hashicorp/google/latest/docs) + +## Prerequisites +### Service Account +A Google Service Account will need to be created with permissions to read from PubSub subscriptions. This can be done via the ``google-service-account-dsf`` module. Depending on the authentication mechanism chosen, the service account will either need to be attached to a GCP Compute Engine instance or the service account's credentials file will need to be copied to your Agentless Gateway. + +### Google PubSub Subscription +A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF will need to be created in advance. This prerequisite is handled by the ``onboard-gcp-pubsub`` module. + +### Database Configuration +Part of the onboarding process involves connecting to your Google PostgreSQL instance and running SQL commands to create an extension. This module includes an example for how to connect to the instance from your local machine and create this. + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gcp-postgresql-1](#module\_gcp-postgresql-1) | ../../modules/onboard-gcp-postgresql | n/a | +| [gcp-postgresql-2](#module\_gcp-postgresql-2) | ../../modules/onboard-gcp-postgresql | n/a | +| [gcp-postgresql-3](#module\_gcp-postgresql-3) | ../../modules/onboard-gcp-postgresql | n/a | +| [gcp-pubsub-1](#module\_gcp-pubsub-1) | ../../modules/onboard-gcp-pubsub | n/a | +| [gcp-pubsub-2](#module\_gcp-pubsub-2) | ../../modules/onboard-gcp-pubsub | n/a | +| [gcp-pubsub-3](#module\_gcp-pubsub-3) | ../../modules/onboard-gcp-pubsub | n/a | +| [service-account](#module\_service-account) | ../../modules/google-service-account-dsf | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_sql_user.gcp-postgresql-admin-user-1](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [google_sql_user.gcp-postgresql-admin-user-2](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [google_sql_user.gcp-postgresql-admin-user-3](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [terraform_data.configure_database_1](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.configure_database_2](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.configure_database_3](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +No inputs. + +## Outputs + +No outputs. + \ No newline at end of file diff --git a/examples/onboard-gcp-postgresql/configure_database.sh b/examples/onboard-gcp-postgresql/configure_database.sh new file mode 100755 index 0000000..ff713bc --- /dev/null +++ b/examples/onboard-gcp-postgresql/configure_database.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Configures Google Postgresql database for auditing by connecting to the +# database using 'psql' +# +# Connection to the database uses PG environment variables +# See postgres documentation for more information: +# https://www.postgresql.org/docs/current/libpq-envars.html +################################################################################ + +# Settings +current_directory=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +sql_file="${current_directory}/configure_database.sql" + +# Functions +function is_pkg_installed { + local pkg="$1" + if ! command -v "${pkg}" &> /dev/null + then + echo "Package '${pkg}' is not installed." + echo "Install on MacOS: brew install libpq" + echo "Install on Ubuntu: apt-get install -y libpq-dev" + echo "Install on CentOS: yum install -y libpq" + echo "Exiting..." + exit 1 + else + return 0 + fi +} + +################################################################################ +is_pkg_installed "psql" +if [ ! -r "${sql_file}" ]; then + echo "Unable to read ${sql_file}" + echo "Exiting..." + exit 1 +else + psql --file="${sql_file}" +fi diff --git a/examples/onboard-gcp-postgresql/configure_database.sql b/examples/onboard-gcp-postgresql/configure_database.sql new file mode 100644 index 0000000..829e3e1 --- /dev/null +++ b/examples/onboard-gcp-postgresql/configure_database.sql @@ -0,0 +1,11 @@ +DO +$$ +BEGIN + RAISE NOTICE 'Creating audit extension "pgaudit".'; + IF EXISTS (SELECT FROM pg_catalog.pg_extension WHERE extname = 'pgaudit') THEN + RAISE NOTICE 'Audit extension "pgaudit" already exists. Skipping.'; + ELSE + CREATE EXTENSION pgaudit; + END IF; +END +$$; diff --git a/examples/onboard-gcp-postgresql/main.tf b/examples/onboard-gcp-postgresql/main.tf new file mode 100644 index 0000000..e4dfadc --- /dev/null +++ b/examples/onboard-gcp-postgresql/main.tf @@ -0,0 +1,360 @@ +locals { + admin_email = "test@example.com" + gateway_id = "a1b2c3d4-e5f6-g8h9-wxyz-123456790" + pubsub_auth_mechanism = "default" + + gcp_postgresql_instance_authorized_networks = [ + { + name = "local" + value = "127.0.0.1" + } + ] + gcp_project_id = "my-gcp-project" + gcp_service_account_name = "dsf-service-account" + + excluded_traffic_filter = [ + { + name = "exclude-cloudsqladmin-traffic" + filter = "textPayload:\"user=cloudsqladmin\"" + } + ] + postgresql_admin = "admin" + postgresql_admin_password = "Abcd1234" +} + +################################################################################ +# Providers +################################################################################ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +provider "google" { + # Authenticated via "gcloud" CLI + project = local.gcp_project_id +} + +provider "dsfhub" {} + +################################################################################ +# Prerequisites +# 1. A service account with permissions to read from the PubSub subscription +# 2. A Google sink router, PubSub topic and subscription (handled below) +# 3. A method to create 'pgaudit' extension on the postgresql instance. (handled +# below) +################################################################################ +module "service-account" { + source = "../../modules/google-service-account-dsf" + + account_id = local.gcp_service_account_name + auth_mechanism = local.pubsub_auth_mechanism + description = "PostgreSQL audit pull service account" + project = local.gcp_project_id + project_roles = [ + "roles/pubsub.subscriber", + "roles/pubsub.viewer" + ] +} + +################################################################################ +# GCP PostgreSQL 16 +################################################################################ +locals { + gcp_postgresql_1_instance_name = "tf-postgresql-16" +} + +module "gcp-pubsub-1" { + source = "../../modules/onboard-gcp-pubsub" + + gcp_pubsub_admin_email = local.admin_email + gcp_pubsub_audit_type = "POSTGRESQL" + gcp_pubsub_auth_mechanism = local.pubsub_auth_mechanism + gcp_pubsub_gateway_id = local.gateway_id + + project = local.gcp_project_id + + pubsub_subscription_name = "${local.gcp_postgresql_1_instance_name}-sub" + + pubsub_topic_name = "${local.gcp_postgresql_1_instance_name}-topic" + + sink_router_description = "PostgreSQL 16 sink" + sink_router_exclusions = local.excluded_traffic_filter + sink_router_filter = < +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [dsfhub](#provider\_dsfhub) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [dsfhub_data_source.this](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs/resources/data_source) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [admin\_email](#input\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [asset\_display\_name](#input\_asset\_display\_name) | User-friendly name of the asset, defined by user | `string` | n/a | yes | +| [asset\_id](#input\_asset\_id) | Unique identifier for the PostgreSQL instance in the form '{project-id}:{instance-region}:{instance-name}'. | `string` | n/a | yes | +| [audit\_pull\_enabled](#input\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | +| [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [logs\_destination\_asset\_id](#input\_logs\_destination\_asset\_id) | The asset\_id of the GCP PUSUB asset that this asset is sending its audit logs to. | `string` | `null` | no | +| [parent\_asset\_id](#input\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this data source is located. | `string` | `null` | no | +| [server\_host\_name](#input\_server\_host\_name) | Hostname (or IP if host is unknown) of the GCP PostgreSQL instance | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | IP address (or hostname if IP is unknown) of the GCP PostgreSQL instance | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [this](#output\_this) | GCP POSTGRESQL asset. | + \ No newline at end of file diff --git a/modules/dsfhub-gcp-postgresql/main.tf b/modules/dsfhub-gcp-postgresql/main.tf new file mode 100644 index 0000000..c1a8804 --- /dev/null +++ b/modules/dsfhub-gcp-postgresql/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + dsfhub = { + source = "imperva/dsfhub" + } + } +} + +resource "dsfhub_data_source" "this" { + server_type = "GCP POSTGRESQL" + + admin_email = var.admin_email + asset_display_name = var.asset_display_name + asset_id = var.asset_id + audit_pull_enabled = var.audit_pull_enabled + gateway_id = var.gateway_id + logs_destination_asset_id = var.logs_destination_asset_id + parent_asset_id = var.parent_asset_id + server_host_name = var.server_host_name + server_ip = var.server_ip + server_port = "5432" +} diff --git a/modules/dsfhub-gcp-postgresql/outputs.tf b/modules/dsfhub-gcp-postgresql/outputs.tf new file mode 100644 index 0000000..18b138c --- /dev/null +++ b/modules/dsfhub-gcp-postgresql/outputs.tf @@ -0,0 +1,4 @@ +output "this" { + description = "GCP POSTGRESQL asset." + value = dsfhub_data_source.this +} diff --git a/modules/dsfhub-gcp-postgresql/variables.tf b/modules/dsfhub-gcp-postgresql/variables.tf new file mode 100644 index 0000000..21db9de --- /dev/null +++ b/modules/dsfhub-gcp-postgresql/variables.tf @@ -0,0 +1,47 @@ +variable "admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "asset_display_name" { + description = "User-friendly name of the asset, defined by user" + type = string +} + +variable "asset_id" { + description = "Unique identifier for the PostgreSQL instance in the form '{project-id}:{instance-region}:{instance-name}'." + type = string +} + +variable "audit_pull_enabled" { + description = "If true, sonargateway will collect the audit logs for this system if it can." + type = bool + default = false +} + +variable "gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "logs_destination_asset_id" { + description = "The asset_id of the GCP PUSUB asset that this asset is sending its audit logs to." + type = string + default = null +} + +variable "parent_asset_id" { + description = "The asset_id of the GCP asset representing the GCP account where this data source is located." + type = string + default = null +} + +variable "server_host_name" { + description = "Hostname (or IP if host is unknown) of the GCP PostgreSQL instance" + type = string +} + +variable "server_ip" { + description = "IP address (or hostname if IP is unknown) of the GCP PostgreSQL instance" + type = string +} diff --git a/modules/onboard-gcp-mysql/README.md b/modules/onboard-gcp-mysql/README.md index c5848fb..684762c 100644 --- a/modules/onboard-gcp-mysql/README.md +++ b/modules/onboard-gcp-mysql/README.md @@ -53,4 +53,4 @@ No resources. |------|-------------| | [gcp-mysql-asset](#output\_gcp-mysql-asset) | GCP MYSQL asset | | [gcp-mysql-instance](#output\_gcp-mysql-instance) | Google MySQL database instance | - \ No newline at end of file + diff --git a/modules/onboard-gcp-postgresql/README.md b/modules/onboard-gcp-postgresql/README.md new file mode 100644 index 0000000..38349f7 --- /dev/null +++ b/modules/onboard-gcp-postgresql/README.md @@ -0,0 +1,57 @@ +# onboard-gcp-postgresql + +Onboard Cloud SQL for PostgreSQL to DSF Hub. + +## Notes +There are three prerequisites for using this module: +1. A Google Service Account with permissions to read from PubSub subscriptions. +2. A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF Hub. +3. A method to create the 'pgaudit' extension on the postgres instance. + +See the corresponding example for more details. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gcp-postgresql-asset](#module\_gcp-postgresql-asset) | ../dsfhub-gcp-postgresql | n/a | +| [gcp-postgresql-instance](#module\_gcp-postgresql-instance) | ../google-sql-database-instance | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [gcp\_postgresql\_admin\_email](#input\_gcp\_postgresql\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes | +| [gcp\_postgresql\_audit\_pull\_enabled](#input\_gcp\_postgresql\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no | +| [gcp\_postgresql\_gateway\_id](#input\_gcp\_postgresql\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes | +| [gcp\_postgresql\_logs\_destination\_asset\_id](#input\_gcp\_postgresql\_logs\_destination\_asset\_id) | The asset\_id of the GCP PUSUB asset that this asset is sending its audit logs to. | `string` | `null` | no | +| [gcp\_postgresql\_parent\_asset\_id](#input\_gcp\_postgresql\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this data source is located. | `string` | `null` | no | +| [instance\_authorized\_networks](#input\_instance\_authorized\_networks) | A list of authorized network blocks as defined below.

authorized\_network:
- expiration\_time: (Optional) The RFC 3339 formatted date time string indicating when this whitelist expires.
- name: (Optional) A name for this whitelist entry.
- value: A CIDR notation IPv4 or IPv6 address that is allowed to access this instance. | 
list(
    object(
      {
        expiration_time = optional(string)
        name            = optional(string)
        value           = string
      }
    )
  )
| n/a | yes | +| [instance\_database\_flags](#input\_instance\_database\_flags) | List of database flags to assign to the instance. | 
list(
    object(
      {
        name  = string
        value = string
      }
    )
  )
| 
[
  {
    "name": "cloudsql.enable_pgaudit",
    "value": "on"
  },
  {
    "name": "log_error_verbosity",
    "value": "verbose"
  },
  {
    "name": "log_connections",
    "value": "on"
  },
  {
    "name": "log_disconnections",
    "value": "on"
  },
  {
    "name": "log_hostname",
    "value": "on"
  },
  {
    "name": "pgaudit.log",
    "value": "all"
  },
  {
    "name": "log_line_prefix",
    "value": "SONAR_AUDIT=1|TIMESTAMP=%m|APPLICATION_NAME=%a|USER=%u|DATABASE=%d|REMOTE_HOST_AND_PORT=%r|SQL_STATE=%e|SESSION_ID=%c|SESSION_START=%s|PROCESS_ID=[%p]|VIRTUAL_TRANSACTION_ID=%v|TRANSACTION_ID=%x| "
  }
]
| no | +| [instance\_database\_version](#input\_instance\_database\_version) | The PostgreSQL version to use. The full list of supported versions can be found at https://cloud.google.com/sql/docs/db-versions. | `string` | `"POSTGRES_16"` | no | +| [instance\_deletion\_protection](#input\_instance\_deletion\_protection) | Whether Terraform will be prevented from destroying the instance. When the field is set to true or unset in Terraform state, a terraform apply or terraform destroy that would delete the instance will fail. When the field is set to false, deleting the instance is allowed. | `bool` | `false` | no | +| [instance\_name](#input\_instance\_name) | The name of the instance. | `string` | n/a | yes | +| [instance\_project](#input\_instance\_project) | The ID of the project that the service account will be created in. | `string` | `null` | no | +| [instance\_region](#input\_instance\_region) | The region the instance will sit in. If a region is not provided in the resource definition, the provider region will be used instead. | `string` | `null` | no | +| [instance\_tier](#input\_instance\_tier) | The machine type to use. See [tiers](https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1beta4/tiers) for more details and supported versions | `string` | `"db-perf-optimized-N-2"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcp-postgresql-asset](#output\_gcp-postgresql-asset) | GCP POSTGRESQL asset | +| [gcp-postgresql-instance](#output\_gcp-postgresql-instance) | Google PostgreSQL database instance | + \ No newline at end of file diff --git a/modules/onboard-gcp-postgresql/main.tf b/modules/onboard-gcp-postgresql/main.tf new file mode 100644 index 0000000..94eb66a --- /dev/null +++ b/modules/onboard-gcp-postgresql/main.tf @@ -0,0 +1,28 @@ +module "gcp-postgresql-instance" { + source = "../google-sql-database-instance" + + authorized_networks = var.instance_authorized_networks + database_flags = var.instance_database_flags + database_version = var.instance_database_version + deletion_protection = var.instance_deletion_protection + name = var.instance_name + project = var.instance_project + region = var.instance_region + root_password = null + sql_server_audit_config = null + tier = var.instance_tier +} + +module "gcp-postgresql-asset" { + source = "../dsfhub-gcp-postgresql" + + admin_email = var.gcp_postgresql_admin_email + asset_display_name = module.gcp-postgresql-instance.this.name + asset_id = "${module.gcp-postgresql-instance.this.project}:${module.gcp-postgresql-instance.this.region}:${module.gcp-postgresql-instance.this.name}" + audit_pull_enabled = var.gcp_postgresql_audit_pull_enabled + gateway_id = var.gcp_postgresql_gateway_id + logs_destination_asset_id = var.gcp_postgresql_logs_destination_asset_id + parent_asset_id = var.gcp_postgresql_parent_asset_id + server_host_name = module.gcp-postgresql-instance.this.ip_address.0.ip_address + server_ip = module.gcp-postgresql-instance.this.ip_address.0.ip_address +} diff --git a/modules/onboard-gcp-postgresql/outputs.tf b/modules/onboard-gcp-postgresql/outputs.tf new file mode 100644 index 0000000..7791c86 --- /dev/null +++ b/modules/onboard-gcp-postgresql/outputs.tf @@ -0,0 +1,9 @@ +output "gcp-postgresql-instance" { + description = "Google PostgreSQL database instance" + value = module.gcp-postgresql-instance.this +} + +output "gcp-postgresql-asset" { + description = "GCP POSTGRESQL asset" + value = module.gcp-postgresql-asset.this +} diff --git a/modules/onboard-gcp-postgresql/variables.tf b/modules/onboard-gcp-postgresql/variables.tf new file mode 100644 index 0000000..c334773 --- /dev/null +++ b/modules/onboard-gcp-postgresql/variables.tf @@ -0,0 +1,130 @@ +variable "gcp_postgresql_admin_email" { + description = "The email address to notify about the asset." + type = string +} + +variable "gcp_postgresql_audit_pull_enabled" { + description = "If true, sonargateway will collect the audit logs for this system if it can." + type = bool + default = false +} + +variable "gcp_postgresql_gateway_id" { + description = "Unique identifier (UID) attached to the jSonar machine controlling the asset" + type = string +} + +variable "gcp_postgresql_logs_destination_asset_id" { + description = "The asset_id of the GCP PUSUB asset that this asset is sending its audit logs to." + type = string + default = null +} + +variable "gcp_postgresql_parent_asset_id" { + description = "The asset_id of the GCP asset representing the GCP account where this data source is located." + type = string + default = null +} + +variable "instance_authorized_networks" { + description = <