Add BigQuery module

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 1.0.8 (TBD)
+
+### Features
+- GCP account asset module
+- Google Cloud BigQuery module
+
 ## 1.0.7 (2024-10-03)
 
 ### Features

DSF_VERSION_COMPATABILITY.md

@@ -99,5 +99,9 @@ The following table lists the DSF versions that each module is tested and mainta
       <td>onboard-azure-sql-managed-instance</td>
       <td>4.17+</td>
    </tr>
+   <tr>
+      <td>onboard-gcp-bigquery</td>
+      <td>4.17+</td>
+   </tr>
 
 </table>

examples/onboard-gcp-bigquery/README.md

@@ -0,0 +1,47 @@
+# Onboard Google Cloud BigQuery example
+This example includes additional prerequisites that will need to be completed to fully utilize the module. More details can be found in the [onboarding documentation](hhttps://docs.imperva.com/bundle/onboarding-databases-to-sonar-reference-guide/page/BigQuery-Onboarding-Steps_48367536.html).
+
+This example creates 'dsfhub' and 'google' resources. More information regarding authentication to each can be found in the relevant provider documentation:
+- [dsfhub](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs)
+- [google](https://registry.terraform.io/providers/hashicorp/google/latest/docs)
+
+## Prerequisites
+### Service Account
+A Google Service Account will need to be created with permissions to read from PubSub subscriptions. This can be done via the ``google-service-account-dsf`` module. Depending on the authentication mechanism chosen, the service account will either need to be attached to a GCP Compute Engine instance or the service account's credentials file will need to be copied to your Agentless Gateway.
+
+### Google PubSub Subscription
+A Google logging sink, PubSub topic, and PubSub subscription in addition to a GCP PUBSUB asset in DSF will need to be created in advance. This prerequisite is handled by the ``onboard-gcp-pubsub`` module.
+
+
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+No providers.
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_gcp-bigquery-1"></a> [gcp-bigquery-1](#module\_gcp-bigquery-1) | ../../modules/dsfhub-gcp-bigquery | n/a |
+| <a name="module_gcp-pubsub"></a> [gcp-pubsub](#module\_gcp-pubsub) | ../../modules/onboard-gcp-pubsub | n/a |
+| <a name="module_service-account"></a> [service-account](#module\_service-account) | ../../modules/google-service-account-dsf | n/a |
+
+## Resources
+
+No resources.
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_dsfhub_host"></a> [dsfhub\_host](#input\_dsfhub\_host) | n/a | `any` | n/a | yes |
+| <a name="input_dsfhub_token"></a> [dsfhub\_token](#input\_dsfhub\_token) | n/a | `any` | n/a | yes |
+
+## Outputs
+
+No outputs.
+<!-- END_TF_DOCS -->

examples/onboard-gcp-bigquery/main.tf

@@ -0,0 +1,91 @@
+locals {
+  gcp_project_id           = "my-gcp-project"
+  pubsub_subscription_name = "tf-bigquery-sub"
+  pubsub_topic_name        = "tf-bigquery-topic"
+  service_account_name     = "dsf-service-account"
+  sink_router_name         = "tf-bigquery-sink"
+
+
+  admin_email    = "test@example.com"
+  auth_mechanism = "default"
+  gateway_id     = "a1b2c3d4-e5f6-g8h9-wxyz-123456790"
+}
+
+################################################################################
+# Providers
+################################################################################
+terraform {
+  required_providers {
+    dsfhub = {
+      source = "imperva/dsfhub"
+    }
+  }
+}
+
+provider "google" {
+  # Authenticated via "gcloud" CLI
+  project = local.gcp_project_id
+}
+
+variable "dsfhub_host" {}  # TF_VAR_dsfhub_host env variable
+variable "dsfhub_token" {} # TF_VAR_dsfhub_token env variable
+
+provider "dsfhub" {
+  dsfhub_host  = var.dsfhub_host
+  dsfhub_token = var.dsfhub_token
+}
+
+################################################################################
+# Prerequisites
+# 1. A service account with permissions to read from the PubSub subscription
+# 2. A Google sink router, PubSub topic and subscription
+################################################################################
+module "service-account" {
+  source = "../../modules/google-service-account-dsf"
+
+  account_id     = local.service_account_name
+  auth_mechanism = local.auth_mechanism
+  description    = "BigQuery audit pull service account"
+  project        = local.gcp_project_id
+  project_roles = [
+    "roles/pubsub.subscriber",
+    "roles/pubsub.viewer"
+  ]
+}
+
+module "gcp-pubsub" {
+  source = "../../modules/onboard-gcp-pubsub"
+
+  gcp_pubsub_admin_email    = local.admin_email
+  gcp_pubsub_audit_type     = "BIGQUERY"
+  gcp_pubsub_auth_mechanism = local.auth_mechanism
+  gcp_pubsub_gateway_id     = local.gateway_id
+
+  project = local.gcp_project_id
+
+  pubsub_subscription_name = local.pubsub_subscription_name
+
+  pubsub_topic_name = local.pubsub_topic_name
+
+  sink_router_description = "BigQuery sink"
+  sink_router_exclusions  = null
+  sink_router_filter      = <<EOF
+    resource.type="bigquery_resource"
+  EOF
+  sink_router_name        = local.sink_router_name
+}
+
+################################################################################
+# GCP BigQuery
+################################################################################
+module "gcp-bigquery-1" {
+  source = "../../modules/dsfhub-gcp-bigquery"
+
+  admin_email               = local.admin_email
+  asset_display_name        = "projects/${local.gcp_project_id}/bigquery"
+  asset_id                  = "projects/${local.gcp_project_id}/bigquery"
+  audit_pull_enabled        = true
+  gateway_id                = local.gateway_id
+  logs_destination_asset_id = module.gcp-pubsub.gcp-pubsub-asset.asset_id
+  pubsub_subscription       = module.gcp-pubsub.gcp-pubsub-asset.asset_id
+}

modules/dsfhub-gcp-bigquery/README.md

@@ -0,0 +1,40 @@
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_dsfhub"></a> [dsfhub](#provider\_dsfhub) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [dsfhub_data_source.this](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs/resources/data_source) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_admin_email"></a> [admin\_email](#input\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes |
+| <a name="input_asset_display_name"></a> [asset\_display\_name](#input\_asset\_display\_name) | User-friendly name of the asset, defined by user | `string` | n/a | yes |
+| <a name="input_asset_id"></a> [asset\_id](#input\_asset\_id) | Unique identifier for the BigQuery service in the form 'projects/{{project}}/bigquery'. | `string` | n/a | yes |
+| <a name="input_audit_pull_enabled"></a> [audit\_pull\_enabled](#input\_audit\_pull\_enabled) | If true, sonargateway will collect the audit logs for this system if it can. | `bool` | `false` | no |
+| <a name="input_gateway_id"></a> [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes |
+| <a name="input_logs_destination_asset_id"></a> [logs\_destination\_asset\_id](#input\_logs\_destination\_asset\_id) | The asset\_id of the GCP PUSUB asset that this asset is sending its audit logs to. | `string` | `null` | no |
+| <a name="input_parent_asset_id"></a> [parent\_asset\_id](#input\_parent\_asset\_id) | The asset\_id of the GCP asset representing the GCP account where this data source is located. | `string` | `null` | no |
+| <a name="input_pubsub_subscription"></a> [pubsub\_subscription](#input\_pubsub\_subscription) | ID of the Google PubSub Subscription containing the BigQuery audit logs in the form 'projects/{{project}}/subscriptions/{{name}}'. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_this"></a> [this](#output\_this) | GCP BIGQUERY asset. |
+<!-- END_TF_DOCS -->

modules/dsfhub-gcp-bigquery/main.tf

@@ -0,0 +1,23 @@
+terraform {
+  required_providers {
+    dsfhub = {
+      source = "imperva/dsfhub"
+    }
+  }
+}
+
+resource "dsfhub_data_source" "this" {
+  server_type = "GCP BIGQUERY"
+
+  admin_email               = var.admin_email
+  asset_display_name        = var.asset_display_name
+  asset_id                  = var.asset_id
+  audit_pull_enabled        = var.audit_pull_enabled
+  gateway_id                = var.gateway_id
+  logs_destination_asset_id = var.logs_destination_asset_id
+  parent_asset_id           = var.parent_asset_id
+  pubsub_subscription       = var.pubsub_subscription
+  server_host_name          = "bigquery.googleapis.com"
+  server_ip                 = "bigquery.googleapis.com"
+  server_port               = "443"
+}

modules/dsfhub-gcp-bigquery/outputs.tf

@@ -0,0 +1,4 @@
+output "this" {
+  description = "GCP BIGQUERY asset."
+  value       = dsfhub_data_source.this
+}

modules/dsfhub-gcp-bigquery/variables.tf

@@ -0,0 +1,46 @@
+variable "admin_email" {
+  description = "The email address to notify about the asset."
+  type        = string
+}
+
+variable "asset_display_name" {
+  description = "User-friendly name of the asset, defined by user"
+  type        = string
+}
+
+variable "asset_id" {
+  description = "Unique identifier for the BigQuery service in the form 'projects/{{project}}/bigquery'."
+  type        = string
+}
+
+variable "audit_pull_enabled" {
+  description = "If true, sonargateway will collect the audit logs for this system if it can."
+  type        = bool
+  default     = false
+}
+
+variable "gateway_id" {
+  description = "Unique identifier (UID) attached to the jSonar machine controlling the asset"
+  type        = string
+}
+
+variable "logs_destination_asset_id" {
+  description = "The asset_id of the GCP PUSUB asset that this asset is sending its audit logs to."
+  type        = string
+  default     = null
+}
+
+variable "parent_asset_id" {
+  description = "The asset_id of the GCP asset representing the GCP account where this data source is located."
+  type        = string
+  default     = null
+}
+
+variable "pubsub_subscription" {
+  description = "ID of the Google PubSub Subscription containing the BigQuery audit logs in the form 'projects/{{project}}/subscriptions/{{name}}'."
+  type        = string
+  validation {
+    condition     = can(regex("projects/.+/subscriptions/.+", var.pubsub_subscription))
+    error_message = "Invalid pubsub subscription ID. Must be in the form 'projects/{{project}}/subscriptions/{{name}}'."
+  }
+}

modules/dsfhub-gcp-cloud-account/README.md

@@ -0,0 +1,39 @@
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_dsfhub"></a> [dsfhub](#provider\_dsfhub) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [dsfhub_cloud_account.this](https://registry.terraform.io/providers/imperva/dsfhub/latest/docs/resources/cloud_account) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_admin_email"></a> [admin\_email](#input\_admin\_email) | The email address to notify about the asset. | `string` | n/a | yes |
+| <a name="input_asset_display_name"></a> [asset\_display\_name](#input\_asset\_display\_name) | User-friendly name of the asset, defined by user | `string` | n/a | yes |
+| <a name="input_asset_id"></a> [asset\_id](#input\_asset\_id) | Unique identifier of the GCP account in the form '<service account email>:<default project ID>' (e.g. my-service-account-name@my-project-id.iam.gserviceaccount.com:default-project-id-for-this-asset). | `string` | n/a | yes |
+| <a name="input_auth_mechanism"></a> [auth\_mechanism](#input\_auth\_mechanism) | Specifies the auth mechanism used by the connection. Supported values: default, service\_account. | `string` | `"default"` | no |
+| <a name="input_gateway_id"></a> [gateway\_id](#input\_gateway\_id) | Unique identifier (UID) attached to the jSonar machine controlling the asset | `string` | n/a | yes |
+| <a name="input_key_file"></a> [key\_file](#input\_key\_file) | Path to JSON file with credentials info (service account's key) residing on your Agentless Gateway. File must be accessible by the sonarw OS user. Required when auth\_mechanism is set to 'service\_account'. | `string` | `null` | no |
+| <a name="input_reason"></a> [reason](#input\_reason) | Used to differentiate connections that belong to the same asset | `string` | `"default"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_this"></a> [this](#output\_this) | GCP cloud account asset. |
+<!-- END_TF_DOCS -->

modules/dsfhub-gcp-cloud-account/main.tf

@@ -0,0 +1,22 @@
+terraform {
+  required_providers {
+    dsfhub = {
+      source = "imperva/dsfhub"
+    }
+  }
+}
+
+resource "dsfhub_cloud_account" "this" {
+  server_type = "GCP"
+
+  admin_email        = var.admin_email
+  asset_display_name = var.asset_display_name
+  asset_id           = var.asset_id
+  gateway_id         = var.gateway_id
+
+  asset_connection {
+    auth_mechanism = var.auth_mechanism
+    reason         = var.reason
+    key_file       = var.auth_mechanism == "service_account" ? var.key_file : null
+  }
+}

modules/dsfhub-gcp-cloud-account/outputs.tf

@@ -0,0 +1,4 @@
+output "this" {
+  description = "GCP cloud account asset."
+  value       = dsfhub_cloud_account.this
+}

modules/dsfhub-gcp-cloud-account/variables.tf

@@ -0,0 +1,41 @@
+variable "admin_email" {
+  description = "The email address to notify about the asset."
+  type        = string
+}
+
+variable "asset_display_name" {
+  description = "User-friendly name of the asset, defined by user"
+  type        = string
+}
+
+variable "asset_id" {
+  description = "Unique identifier of the GCP account in the form '<service account email>:<default project ID>' (e.g. my-service-account-name@my-project-id.iam.gserviceaccount.com:default-project-id-for-this-asset)."
+  type        = string
+}
+
+variable "auth_mechanism" {
+  description = "Specifies the auth mechanism used by the connection. Supported values: default, service_account."
+  type        = string
+  default     = "default"
+  validation {
+    condition     = contains(["default", "service_account"], var.auth_mechanism)
+    error_message = "Invalid authentication mechanism. Supported values: default, service_account."
+  }
+}
+
+variable "gateway_id" {
+  description = "Unique identifier (UID) attached to the jSonar machine controlling the asset"
+  type        = string
+}
+
+variable "key_file" {
+  description = "Path to JSON file with credentials info (service account's key) residing on your Agentless Gateway. File must be accessible by the sonarw OS user. Required when auth_mechanism is set to 'service_account'."
+  type        = string
+  default     = null
+}
+
+variable "reason" {
+  description = "Used to differentiate connections that belong to the same asset"
+  type        = string
+  default     = "default"
+}