From e7598e268a61f2acb30c02bd322b66b3815f6ef5 Mon Sep 17 00:00:00 2001 From: akocbek <106765658+akocbek@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:16:59 +0000 Subject: [PATCH] feat: Root module updates:
- `existing_kms_instance_guid` is no longer a supported input. The code will now parse the GUID from the KMS key CRN
- added new input `use_same_kms_key_for_backups` to give more control over KMS key usage
- `kms_encryption_enabled` has been renamed to `use_ibm_owned_encryption_key`
- `fscloud` submodule updates:
- added new inputs `use_default_backup_encryption_key` and `use_same_kms_key_for_backups`
DA updates
- Removed the input `existing_backup_kms_instance_crn`. If you want to use a different KMS key for backups, you can use the `existing_backup_kms_key_crn` input to use an existing key. The DA only supports creating an new key that will be used for both data and backups encryption.
- The `skip_iam_authorization_policy` input has been renamed to `skip_es_kms_auth_policy`
If passing a value for `ibmcloud_kms_api_key`, and creating a KMS auth policy, that policy will now be scoped to the exact KMS key. (#351) --- .catalog-onboard-pipeline.yaml | 6 + README.md | 14 +- cra-config.yaml | 9 +- examples/fscloud/main.tf | 19 +- examples/fscloud/variables.tf | 5 - ibm_catalog.json | 232 +++++++++++++++- main.tf | 70 +++-- modules/fscloud/README.md | 5 +- modules/fscloud/main.tf | 1 - modules/fscloud/outputs.tf | 2 +- modules/fscloud/variables.tf | 7 +- reference-architecture/da-enterprise.svg | 4 + solutions/enterprise/DA-schemas-topics-cbr.md | 98 +++++++ solutions/enterprise/DA-types.md | 93 +++++++ solutions/enterprise/README.md | 15 ++ .../catalogValidationValues.json.template | 7 + solutions/enterprise/main.tf | 164 +++++++++++ solutions/enterprise/outputs.tf | 64 +++++ solutions/enterprise/provider.tf | 12 + solutions/enterprise/variables.tf | 254 ++++++++++++++++++ solutions/enterprise/version.tf | 13 + solutions/quickstart/main.tf | 4 +- solutions/quickstart/provider.tf | 1 + solutions/quickstart/variables.tf | 20 +- tests/pr_test.go | 48 +++- variables.tf | 8 +- 26 files changed, 1110 insertions(+), 65 deletions(-) create mode 100644 reference-architecture/da-enterprise.svg create mode 100644 solutions/enterprise/DA-schemas-topics-cbr.md create mode 100644 solutions/enterprise/DA-types.md create mode 100644 solutions/enterprise/README.md create mode 100644 solutions/enterprise/catalogValidationValues.json.template create mode 100644 solutions/enterprise/main.tf create mode 100644 solutions/enterprise/outputs.tf create mode 100644 solutions/enterprise/provider.tf create mode 100644 solutions/enterprise/variables.tf create mode 100644 solutions/enterprise/version.tf diff --git a/.catalog-onboard-pipeline.yaml b/.catalog-onboard-pipeline.yaml index 720cb2f5..d283bb63 100644 --- a/.catalog-onboard-pipeline.yaml +++ b/.catalog-onboard-pipeline.yaml @@ -12,3 +12,9 @@ offerings: scc: instance_id: 1c7d5f78-9262-44c3-b779-b28fe4d88c37 region: us-south + - name: enterprise + mark_ready: true + install_type: fullstack + scc: + instance_id: 1c7d5f78-9262-44c3-b779-b28fe4d88c37 + region: us-south diff --git a/README.md b/README.md index e2e43173..be729f5b 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,10 @@ unless real values don't help users know what to change. ```hcl module "event_streams" { source = "terraform-ibm-modules/event-streams/ibm" - version = "latest" # Replace "latest" with a release version to lock into a specific release - resource_group = "event-streams-rg" - plan = "standard" - topics = [ + version = "X.Y.Z" # Replace "X.Y.Z" with a release version to lock into a specific release + resource_group_id = "xxXXxxXXxXxXXXXxxXxxxXXXXxXXXXX" # Replace with the actual ID of resource group to use + plan = "standard" + topics = [ { name = "topic-1" partitions = 1 @@ -63,7 +63,7 @@ module "event_streams" { } } ] - schema_id = [{ + schemas = [{ schema_id = "my-es-schema_1" schema = { type = "string" @@ -115,6 +115,7 @@ You need the following permissions to run this module. |------|--------|---------| | [cbr\_rule](#module\_cbr\_rule) | terraform-ibm-modules/cbr/ibm//modules/cbr-rule-module | 1.29.0 | | [es\_guid\_crn\_parser](#module\_es\_guid\_crn\_parser) | terraform-ibm-modules/common-utilities/ibm//modules/crn-parser | 1.1.0 | +| [kms\_key\_crn\_parser](#module\_kms\_key\_crn\_parser) | terraform-ibm-modules/common-utilities/ibm//modules/crn-parser | 1.1.0 | ### Resources @@ -143,7 +144,6 @@ You need the following permissions to run this module. | [create\_timeout](#input\_create\_timeout) | The timeout value for creating an Event Streams instance. Specify `3h` for an Enterprise plan instance. Add 1 h for each level of non-default throughput. Add 30 min for each level of non-default storage size. | `string` | `"3h"` | no | | [delete\_timeout](#input\_delete\_timeout) | The timeout value for deleting an Event Streams instance. | `string` | `"15m"` | no | | [es\_name](#input\_es\_name) | The name to give the Event Streams instance created by this module. | `string` | n/a | yes | -| [existing\_kms\_instance\_guid](#input\_existing\_kms\_instance\_guid) | The GUID of the Hyper Protect Crypto Services or Key Protect instance in which the key specified in var.kms\_key\_crn is coming from. Required only if var.kms\_encryption\_enabled is set to true, var.skip\_kms\_iam\_authorization\_policy is set to false, and you pass a value for var.kms\_key\_crn. | `string` | `null` | no | | [kms\_encryption\_enabled](#input\_kms\_encryption\_enabled) | Set this to true to control the encryption keys used to encrypt the data that you store in IBM Cloud® Databases. If set to false, the data is encrypted by using randomly generated keys. For more info on Key Protect integration, see https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-key-protect. For more info on HPCS integration, see https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs | `bool` | `false` | no | | [kms\_key\_crn](#input\_kms\_key\_crn) | The root key CRN of the key management service (Key Protect or Hyper Protect Crypto Services) to use to encrypt the payload data. [Learn more](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-managing_encryption) about integrating Hyper Protect Crypto Services with Event Streams. | `string` | `null` | no | | [metrics](#input\_metrics) | Enhanced metrics to activate, as list of strings. Only allowed for enterprise plans. Allowed values: 'topic', 'partition', 'consumers'. | `list(string)` | `[]` | no | @@ -158,7 +158,7 @@ You need the following permissions to run this module. | [service\_credential\_names](#input\_service\_credential\_names) | The mapping of names and roles for service credentials that you want to create for the Event streams. | `map(string)` | `{}` | no | | [service\_endpoints](#input\_service\_endpoints) | The type of service endpoints. Possible values: 'public', 'private', 'public-and-private'. | `string` | `"public"` | no | | [skip\_es\_s2s\_iam\_authorization\_policy](#input\_skip\_es\_s2s\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that will allow all Event Streams instances in the given resource group access to read from the mirror source instance. This policy is required when creating a mirroring instance, and will only be created if a value is passed in the mirroring input. | `bool` | `false` | no | -| [skip\_kms\_iam\_authorization\_policy](#input\_skip\_kms\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the `existing_kms_instance_guid` variable. In addition, no policy is created if var.kms\_encryption\_enabled is set to false. | `bool` | `false` | no | +| [skip\_kms\_iam\_authorization\_policy](#input\_skip\_kms\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the `kms_key_crn` variable. In addition, no policy is created if var.kms\_encryption\_enabled is set to false. | `bool` | `false` | no | | [storage\_size](#input\_storage\_size) | Storage size of the Event Streams in GB. Applies only to Enterprise plan instances. Possible values: `2048`, `4096`, `6144`, `8192`, `10240`, `12288`. Storage capacity cannot be reduced after the instance is created. When the `throughput` input variable is set to `300`, storage size starts at 4096. When `throughput` is `450`, storage size starts starts at `6144`. | `number` | `"2048"` | no | | [tags](#input\_tags) | The list of tags associated with the Event Steams instance. | `list(string)` | `[]` | no | | [throughput](#input\_throughput) | Throughput capacity in MB per second. Applies only to Enterprise plan instances. Possible values: `150`, `300`, `450`. | `number` | `"150"` | no | diff --git a/cra-config.yaml b/cra-config.yaml index a0c3a536..749bcd5e 100644 --- a/cra-config.yaml +++ b/cra-config.yaml @@ -1,9 +1,12 @@ # More info about this file at https://github.com/terraform-ibm-modules/common-pipeline-assets/blob/main/.github/workflows/terraform-test-pipeline.md#cra-config-yaml version: "v1" CRA_TARGETS: - - CRA_TARGET: "examples/complete" # Target directory for CRA scan. If not provided, the CRA Scan will not be run. + - CRA_TARGET: "solutions/enterprise" # Target directory for CRA scan. If not provided, the CRA Scan will not be run. CRA_IGNORE_RULES_FILE: "cra-tf-validate-ignore-rules.json" # CRA Ignore file to use. If not provided, it checks the repo root directory for `cra-tf-validate-ignore-rules.json` PROFILE_ID: "fe96bd4d-9b37-40f2-b39f-a62760e326a3" # SCC profile ID (currently set to 'IBM Cloud Framework for Financial Services' '1.7.0' profile). CRA_ENVIRONMENT_VARIABLES: - TF_VAR_existing_kms_instance_guid: "e6dce284-e80f-46e1-a3c1-830f7adff7a9" - TF_VAR_kms_key_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9:key:76170fae-4e0c-48c3-8ebe-326059ebb533" + TF_VAR_existing_kms_instance_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9::" + TF_VAR_resource_group_name: "test-event-s-cra" + TF_VAR_provider_visibility: "public" + TF_VAR_use_existing_resource_group: false + TF_VAR_kms_endpoint_type: "public" diff --git a/examples/fscloud/main.tf b/examples/fscloud/main.tf index f93f8418..cfa117c0 100644 --- a/examples/fscloud/main.tf +++ b/examples/fscloud/main.tf @@ -69,16 +69,15 @@ module "cbr_zone_schematics" { # ############################################################################# module "event_streams" { - source = "../../modules/fscloud" - resource_group_id = module.resource_group.resource_group_id - es_name = "${var.prefix}-es-fs" - kms_key_crn = var.kms_key_crn - schemas = var.schemas - tags = var.resource_tags - topics = var.topics - existing_kms_instance_guid = var.existing_kms_instance_guid - metrics = ["topic", "partition", "consumers"] - mirroring_topic_patterns = ["topic-1", "topic-2"] + source = "../../modules/fscloud" + resource_group_id = module.resource_group.resource_group_id + es_name = "${var.prefix}-es-fs" + kms_key_crn = var.kms_key_crn + schemas = var.schemas + tags = var.resource_tags + topics = var.topics + metrics = ["topic", "partition", "consumers"] + mirroring_topic_patterns = ["topic-1", "topic-2"] mirroring = { source_crn = var.event_streams_source_crn # Required for mirroring source_alias = "source-alias" # Required for mirroring diff --git a/examples/fscloud/variables.tf b/examples/fscloud/variables.tf index 4ec0733b..2f7d2e50 100644 --- a/examples/fscloud/variables.tf +++ b/examples/fscloud/variables.tf @@ -54,11 +54,6 @@ variable "topics" { default = [] } -variable "existing_kms_instance_guid" { - description = "The GUID of the Hyper Protect Crypto service in which the key specified in var.kms_key_crn is coming from" - type = string -} - variable "kms_key_crn" { type = string description = "The root key CRN of a Hyper Protect Crypto Service (HPCS) that you want to use for disk encryption. See https://cloud.ibm.com/docs/cloud-databases?topic=cloud-databases-hpcs&interface=ui for more information on integrating HPCS with Event Streams instance." diff --git a/ibm_catalog.json b/ibm_catalog.json index e57c7737..e42fab3c 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -46,7 +46,25 @@ "key": "ibmcloud_api_key" }, { - "key": "prefix" + "key": "provider_visibility", + "options": [ + { + "displayname": "private", + "value": "private" + }, + { + "displayname": "public", + "value": "public" + }, + { + "displayname": "public-and-private", + "value": "public-and-private" + } + ] + }, + { + "key": "prefix", + "required": true }, { "key": "use_existing_resource_group" @@ -128,6 +146,9 @@ }, { "key": "topics" + }, + { + "key": "service_credential_names" } ], "iam_permissions": [ @@ -167,6 +188,215 @@ } ] } + }, + { + "label": "enterprise", + "name": "enterprise", + "install_type": "fullstack", + "working_directory": "solutions/enterprise", + "configuration": [ + { + "key": "ibmcloud_api_key" + }, + { + "key": "provider_visibility", + "options": [ + { + "displayname": "private", + "value": "private" + }, + { + "displayname": "public", + "value": "public" + }, + { + "displayname": "public-and-private", + "value": "public-and-private" + } + ] + }, + { + "key": "prefix", + "required": true + }, + { + "key": "use_existing_resource_group" + }, + { + "key": "resource_group_name" + }, + { + "key": "event_streams_name" + }, + { + "key": "region", + "required": true, + "options": [ + { + "displayname": "Dallas (us-south)", + "value": "us-south" + }, + { + "displayname": "Frankfurt (eu-de)", + "value": "eu-de" + }, + { + "displayname": "London (eu-gb)", + "value": "eu-gb" + }, + { + "displayname": "Madrid (eu-es)", + "value": "eu-es" + }, + { + "displayname": "Osaka (jp-osa)", + "value": "jp-osa" + }, + { + "displayname": "Sydney (au-syd)", + "value": "au-syd" + }, + { + "displayname": "Tokyo (jp-tok)", + "value": "jp-tok" + }, + { + "displayname": "Osaka (jp-osa)", + "value": "jp-osa" + }, + { + "displayname": "Sao Paulo (br-sao)", + "value": "br-sao" + }, + { + "displayname": "Toronto (ca-tor)", + "value": "ca-tor" + }, + { + "displayname": "Washington (us-east)", + "value": "us-east" + } + ] + }, + { + "key": "resource_tags" + }, + { + "key": "access_tags" + }, + { + "key": "schemas" + }, + { + "key": "schema_global_rule" + }, + { + "key": "topics" + }, + { + "key": "skip_event_streams_s2s_iam_auth_policy" + }, + { + "key": "cbr_rules" + }, + { + "key": "service_credential_names" + }, + { + "key": "quotas" + }, + { + "key": "metrics" + }, + { + "key": "mirroring_topic_patterns" + }, + { + "key": "mirroring" + }, + { + "key": "event_streams_key_name" + }, + { + "key": "event_streams_key_ring_name" + }, + { + "key": "existing_kms_instance_crn" + }, + { + "key": "ibmcloud_kms_api_key" + }, + { + "key": "kms_endpoint_type" + }, + { + "key": "skip_event_streams_kms_auth_policy" + }, + { + "key": "existing_kms_key_crn" + } + ], + "iam_permissions": [ + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::serviceRole:Manager" + ], + "service_name": "messagehub" + }, + { + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" + ], + "service_name": "messagehub" + } + ], + "architecture": { + "descriptions": "This architecture creates a Financial Services compliant instance of IBM Event Streams for IBM Cloud in enterprise plan. It also supports the creation of topics and schemas in the Event Streams instance.", + "features": [ + { + "title": "Creates an Event Streams instance", + "description": "Creates and configures the IBM Cloud Event Streams instance." + }, + { + "title": "Creates topics in the Event Streams instance", + "description": "Creates topics in IBM Cloud Event Streams instance." + }, + { + "title": "Creates schemas in the Event Streams instance", + "description": "Creates schemas in IBM Cloud Event Streams instance." + }, + { + "title": "Supports mirroring", + "description": "Supports mirroring which enables messages in one Event Streams service instance to be continuously copied to a second instance." + }, + { + "title": "Suports quotas.", + "description": "Supports quotas to control the resources, such as network bandwidth, that a service can consume.." + }, + { + "title": "CBR", + "description": "Create context-based restriction (CBR) rules for Event Streams instance." + }, + { + "title": "Encrypt messages with key management services", + "description": "Supports key management encryption (BYOK and KYOK)." + }, + { + "title": "Managing users and roles", + "description": "Manage IAM service credentials for Event Streams." + } + ], + "diagrams": [ + { + "diagram": { + "caption": "Financial Services compliant instance of IBM Event Streams for IBM Cloud.", + "url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-event-streams/main/reference-architecture/da-enterprise.svg", + "type": "image/svg+xml" + }, + "description": "This architecture creates a Financial Services compliant instance of IBM Event Streams for IBM Cloud." + } + ] + } } ] } diff --git a/main.tf b/main.tf index 471702d5..2313cfa5 100644 --- a/main.tf +++ b/main.tf @@ -19,7 +19,7 @@ locals { # tflint-ignore: terraform_unused_declarations validate_kms_vars = var.kms_encryption_enabled && var.kms_key_crn == null ? tobool("When setting var.kms_encryption_enabled to true, a value must be passed for var.kms_key_crn and/or var.backup_encryption_key_crn.") : true # tflint-ignore: terraform_unused_declarations - validate_auth_policy = var.kms_encryption_enabled && var.skip_kms_iam_authorization_policy == false && var.existing_kms_instance_guid == null ? tobool("When var.skip_kms_iam_authorization_policy is set to false, and var.kms_encryption_enabled to true, a value must be passed for var.existing_kms_instance_guid in order to create the auth policy.") : true + validate_auth_policy = var.kms_encryption_enabled && var.skip_kms_iam_authorization_policy == false && var.kms_key_crn == null ? tobool("When var.skip_kms_iam_authorization_policy is set to false, and var.kms_encryption_enabled to true, a value must be passed for var.kms_key_crn in order to create the auth policy.") : true # tflint-ignore: terraform_unused_declarations validate_throughput_lite_standard = ((var.plan == "lite" || var.plan == "standard") && var.throughput != 150) ? tobool("Throughput value cannot be changed in lite and standard plan. Default value is 150.") : true # tflint-ignore: terraform_unused_declarations @@ -30,13 +30,11 @@ locals { validate_mirroring_topics = var.mirroring == null && var.mirroring_topic_patterns != null ? tobool("When passing values for var.mirroring_topic_patterns, values must also be passed for var.mirroring.") : true # tflint-ignore: terraform_unused_declarations validate_mirroring_config = var.mirroring != null && var.mirroring_topic_patterns == null ? tobool("When passing values for var.mirroring, values must also be passed for var.mirroring_topic_patterns.") : true - - # Determine what KMS service is being used for database encryption - kms_service = var.kms_key_crn != null ? ( - can(regex(".*kms.*", var.kms_key_crn)) ? "kms" : ( - can(regex(".*hs-crypto.*", var.kms_key_crn)) ? "hs-crypto" : null - ) - ) : null + parsed_kms_key_crn = var.kms_key_crn != null ? split(":", var.kms_key_crn) : [] + kms_service = length(local.parsed_kms_key_crn) > 0 ? local.parsed_kms_key_crn[4] : null + kms_scope = length(local.parsed_kms_key_crn) > 0 ? local.parsed_kms_key_crn[6] : null + kms_account_id = length(local.parsed_kms_key_crn) > 0 ? split("/", local.kms_scope)[1] : null + kms_key_id = length(local.parsed_kms_key_crn) > 0 ? local.parsed_kms_key_crn[9] : null } # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 @@ -78,6 +76,20 @@ resource "ibm_resource_instance" "es_instance" { ) } +######################################################################################################################## +# Parse KMS info from given CRNs +######################################################################################################################## + +module "kms_key_crn_parser" { + count = var.kms_encryption_enabled == true ? 1 : 0 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.kms_key_crn +} + +locals { + kms_instance_guid = var.kms_key_crn != null ? module.kms_key_crn_parser[0].service_instance : null +} ############################################################################## # SCHEMA AND COMPATIBILITY RULE ############################################################################## @@ -135,13 +147,41 @@ resource "ibm_event_streams_quota" "eventstreams_quotas" { # Create IAM Authorization Policies to allow messagehub to access kms for the encryption key resource "ibm_iam_authorization_policy" "kms_policy" { - count = var.kms_encryption_enabled == false || var.skip_kms_iam_authorization_policy ? 0 : 1 - source_service_name = "messagehub" - source_resource_group_id = var.resource_group_id - target_service_name = local.kms_service - target_resource_instance_id = var.existing_kms_instance_guid - roles = ["Reader"] - description = "Allow all Event Streams instances in the resource group ${var.resource_group_id} to read from the ${local.kms_service} instance GUID ${var.existing_kms_instance_guid}" + count = var.kms_encryption_enabled == false || var.skip_kms_iam_authorization_policy ? 0 : 1 + source_service_name = "messagehub" + source_resource_group_id = var.resource_group_id + roles = ["Reader"] + description = "Allow the Event Streams instances in the resource group ${var.resource_group_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance ${local.kms_instance_guid}" + resource_attributes { + name = "serviceName" + operator = "stringEquals" + value = local.kms_service + } + resource_attributes { + name = "accountId" + operator = "stringEquals" + value = local.kms_account_id + } + resource_attributes { + name = "serviceInstance" + operator = "stringEquals" + value = local.kms_instance_guid + } + resource_attributes { + name = "resourceType" + operator = "stringEquals" + value = "key" + } + resource_attributes { + name = "resource" + operator = "stringEquals" + value = local.kms_key_id + } + # Scope of policy now includes the key, so ensure to create new policy before + # destroying old one to prevent any disruption to every day services. + lifecycle { + create_before_destroy = true + } } # workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 diff --git a/modules/fscloud/README.md b/modules/fscloud/README.md index 4208639c..a94f1887 100644 --- a/modules/fscloud/README.md +++ b/modules/fscloud/README.md @@ -29,7 +29,6 @@ No resources. | [access\_tags](#input\_access\_tags) | The list of access tags associated with the Event Steams instance. | `list(string)` | `[]` | no | | [cbr\_rules](#input\_cbr\_rules) | The list of context-based restriction rules to create. | 
list(object({
    description = string
    account_id  = string
    rule_contexts = list(object({
      attributes = optional(list(object({
        name  = string
        value = string
    }))) }))
    enforcement_mode = string
  }))
| `[]` | no | | [es\_name](#input\_es\_name) | The name of the Event Streams instance. | `string` | n/a | yes | -| [existing\_kms\_instance\_guid](#input\_existing\_kms\_instance\_guid) | The GUID of the Hyper Protect Crypto service in which the key specified in var.kms\_key\_crn is coming from | `string` | n/a | yes | | [kms\_key\_crn](#input\_kms\_key\_crn) | The root key CRN of the key management service (Key Protect or Hyper Protect Crypto Services) to use to encrypt the payload data. | `string` | n/a | yes | | [metrics](#input\_metrics) | Enhanced metrics to activate, as list of strings. Allowed values: 'topic', 'partition', 'consumers'. | `list(string)` | `[]` | no | | [mirroring](#input\_mirroring) | Event Streams mirroring configuration. Required only if creating mirroring instance. For more information on mirroring, see https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-mirroring. | 
object({
    source_crn   = string
    source_alias = string
    target_alias = string
    options = optional(object({
      topic_name_transform = object({
        type = string
        rename = optional(object({
          add_prefix    = optional(string)
          add_suffix    = optional(string)
          remove_prefix = optional(string)
          remove_suffix = optional(string)
        }))
      })
      group_id_transform = object({
        type = string
        rename = optional(object({
          add_prefix    = optional(string)
          add_suffix    = optional(string)
          remove_prefix = optional(string)
          remove_suffix = optional(string)
        }))
      })
    }))
  })
| `null` | no | @@ -41,7 +40,7 @@ No resources. | [schemas](#input\_schemas) | The list of schema objects. Include the `schema_id` and the `type` and `name` of the schema in the `schema` object. | 
list(object(
    {
      schema_id = string
      schema = object({
        type = string
        name = string
        fields = optional(list(object({
          name = string
          type = string
        })))
      })
    }
  ))
| `[]` | no | | [service\_credential\_names](#input\_service\_credential\_names) | The mapping of names and roles for service credentials that you want to create for the Event streams. | `map(string)` | `{}` | no | | [skip\_es\_s2s\_iam\_authorization\_policy](#input\_skip\_es\_s2s\_iam\_authorization\_policy) | Set to true to skip the creation of an Event Streams s2s IAM authorization policy to provision an Event Streams mirroring instance. This is required to read from the source cluster. This policy is required when creating mirroring instance. | `bool` | `false` | no | -| [skip\_kms\_iam\_authorization\_policy](#input\_skip\_kms\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the existing\_kms\_instance\_guid variable. In addition, no policy is created if var.kms\_encryption\_enabled is set to false. | `bool` | `false` | no | +| [skip\_kms\_iam\_authorization\_policy](#input\_skip\_kms\_iam\_authorization\_policy) | Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the kms\_key\_crn variable. In addition, no policy is created if var.kms\_encryption\_enabled is set to false. | `bool` | `false` | no | | [tags](#input\_tags) | The list of tags associated with the Event Steams instance. | `list(string)` | `[]` | no | | [topics](#input\_topics) | The list of topics to apply to resources. Only one topic is allowed for Lite plan instances. | 
list(object(
    {
      name       = string
      partitions = number
      config     = map(string)
    }
  ))
| `[]` | no | @@ -51,7 +50,7 @@ No resources. |------|-------------| | [crn](#output\_crn) | Event Streams instance crn | | [guid](#output\_guid) | Event Streams instance guid | -| [id](#output\_id) | Event Streams instance crn | +| [id](#output\_id) | Event Streams instance id | | [kafka\_broker\_version](#output\_kafka\_broker\_version) | The Kafka version | | [kafka\_brokers\_sasl](#output\_kafka\_brokers\_sasl) | (Array of Strings) Kafka brokers use for interacting with Kafka native API | | [kafka\_http\_url](#output\_kafka\_http\_url) | The API endpoint to interact with Event Streams REST API | diff --git a/modules/fscloud/main.tf b/modules/fscloud/main.tf index 75e89032..862caf74 100644 --- a/modules/fscloud/main.tf +++ b/modules/fscloud/main.tf @@ -5,7 +5,6 @@ module "event_streams" { plan = "enterprise-3nodes-2tb" region = var.region kms_key_crn = var.kms_key_crn - existing_kms_instance_guid = var.existing_kms_instance_guid skip_kms_iam_authorization_policy = var.skip_kms_iam_authorization_policy skip_es_s2s_iam_authorization_policy = var.skip_es_s2s_iam_authorization_policy schemas = var.schemas diff --git a/modules/fscloud/outputs.tf b/modules/fscloud/outputs.tf index 034776af..0367f115 100644 --- a/modules/fscloud/outputs.tf +++ b/modules/fscloud/outputs.tf @@ -8,7 +8,7 @@ output "crn" { } output "id" { - description = "Event Streams instance crn" + description = "Event Streams instance id" value = module.event_streams.id } diff --git a/modules/fscloud/variables.tf b/modules/fscloud/variables.tf index 22bbe815..9f8f1727 100644 --- a/modules/fscloud/variables.tf +++ b/modules/fscloud/variables.tf @@ -66,7 +66,7 @@ variable "topics" { variable "skip_kms_iam_authorization_policy" { type = bool - description = "Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the existing_kms_instance_guid variable. In addition, no policy is created if var.kms_encryption_enabled is set to false." + description = "Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the kms_key_crn variable. In addition, no policy is created if var.kms_encryption_enabled is set to false." default = false } @@ -76,11 +76,6 @@ variable "skip_es_s2s_iam_authorization_policy" { default = false } -variable "existing_kms_instance_guid" { - description = "The GUID of the Hyper Protect Crypto service in which the key specified in var.kms_key_crn is coming from" - type = string -} - variable "kms_key_crn" { type = string description = "The root key CRN of the key management service (Key Protect or Hyper Protect Crypto Services) to use to encrypt the payload data." diff --git a/reference-architecture/da-enterprise.svg b/reference-architecture/da-enterprise.svg new file mode 100644 index 00000000..29cddf70 --- /dev/null +++ b/reference-architecture/da-enterprise.svg @@ -0,0 +1,4 @@ + + + +
IBM Cloud
Region
Resource Group
IBM Cloud Event Stream Instance
Topics
Schemas
Key Ring
Existing KMS instance
CBR rule \ No newline at end of file diff --git a/solutions/enterprise/DA-schemas-topics-cbr.md b/solutions/enterprise/DA-schemas-topics-cbr.md new file mode 100644 index 00000000..39ee711e --- /dev/null +++ b/solutions/enterprise/DA-schemas-topics-cbr.md @@ -0,0 +1,98 @@ +# Configuring topics and schemas in Event Streams + +When you add a Event Streams deployable architecture from the IBM Cloud catalog to IBM Cloud Projects, you can configure topics, schemas and context-based restriction rules. When you edit your project configuration, select the **Configure** panel, and then click the **Optional** tab. + +To enter a custom value, use the edit action to open the "Edit Array" panel. Add the topics configurations to the array. + +## Options with topics + +- `name` (required): The name of the topic. +- `partitions` (optional): The number of partitions of the topic. The default value is `1`. +- `config` (optional): The configuration parameters of the topic. Supported configurations are: `cleanup.policy`, `retention.ms`, `retention.bytes`, `segment.bytes`, `segment.ms`, `segment.index.bytes`. + +The following example includes all the configuration options for topics. + +```hcl +[ + { + name = "my-es-topic" + partitions = 1 + config = { + "cleanup.policy" = "delete" + "retention.ms" = "86400000" + "retention.bytes" = "10485760" + "segment.bytes" = "10485760" + } + }, + { + name = "topic-2" + partitions = 1 + config = { + "cleanup.policy" = "compact,delete" + "retention.ms" = "86400000" + "retention.bytes" = "1073741824" + "segment.bytes" = "536870912" + } + } +] +``` + +## Options with schemas + +- `schema_id` (required): The unique ID to be assigned to schema. If this value is not specified, a generated UUID is assigned. +- `schema` (required): The schema in JSON format. Supported parameters are: `type`, `name` and `fields` (optional). + +The following example includes all the configuration options for schemas. + +```hcl +[ + { + schema_id = "my-es-schema_1" + schema = { + type = "string" + name = "name_1" + fields = [{ + name = "field_name" + type = "string" + }] + } + }, + { + schema_id = "my-es-schema_2" + schema = { + type = "string" + name = "name_2" + } + } +] +``` + +## Options with context-based restriction (cbr) rules + +- `description` (required): The description of the context-based restriction rule. +- `enforcement_mode` (required): The rule enforcement mode. Supported values are: `enabled`, `disabled` and `report`. +- `account_id` (required): The id of the account owning this cbr rule . +- `rule_contexts` (required): The list of contexts the rule applies to. Supported `attributes` configurations are: `name` and `value`. + +The following example includes all the configuration options for cbr rule. + +```hcl +[ + { + description = "Event streams access only from vpc" + enforcement_mode = "enabled" + account_id = "defc0df06b644a9cabc6e44f55b3880s" + rule_contexts = [{ + attributes = [ + { + "name" : "endpointType", + "value" : "private" + }, + { + name = "networkZoneId" + value = "93a51a1debe2674193217209601dde6f" # pragma: allowlist secret + }] + }] + } +] +``` diff --git a/solutions/enterprise/DA-types.md b/solutions/enterprise/DA-types.md new file mode 100644 index 00000000..1168fb4a --- /dev/null +++ b/solutions/enterprise/DA-types.md @@ -0,0 +1,93 @@ +# Configuring complex inputs in Event Streams + +Several optional input variables in the IBM Cloud Event Streams deployable architecture use complex object types. You specify these inputs when you configure you deployable architecture. + +- [Service credentials](#svc-credential-name) (`service_credential_names`) +- [Quotas](#quotas) (`quotas`) +- [Mirroring](#mirroring) (`quotas`) + +## Service credentials + +You can specify a set of IAM credentials to connect to the instance with the `service_credential_names` input variable. Include a credential name and IAM service role for each key-value pair. Each role provides a specific level of access to the instance. For more information, see [Adding and viewing credentials](https://cloud.ibm.com/docs/account?topic=account-service_credentials&interface=ui). + +- Variable name: `service_credential_names`. +- Type: A map. The key is the name of the service credential. The value is the role that is assigned to that credential. +- Default value: An empty map (`{}`). + +### Options for service_credential_names + +- Key (required): The name of the service credential. +- Value (required): The IAM service role that is assigned to the credential. The following values are valid for service credential roles: 'Manager', 'Writer', 'Reader'. For more information, see [IBM Cloud IAM roles](https://cloud.ibm.com/docs/account?topic=account-userroles). + +### Example service credentials + +```hcl +{ + "es_writer" : "Writer", + "es_reader" : "Reader", + "es_manager" : "Manager" +} +``` + +## Quotas + +You can set quotas of an Event Streams service instance. Both the default quota and user quotas may be managed. Quotas are only available on Event Streams Enterprise plan service instances. For more information, see [Event Streams Quotas](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-enabling_kafka_quotas). + +### Options for quotas + +- `entity` (required): Either default to set the default quota, or an IAM ID for a user quota. +- `producer_byte_rate` (optional): The producer quota in bytes/second. Use -1 for no quota. +- `consumer_byte_rate` (optional): The consumer quota in bytes/second. Use -1 for no quota. + +### Example quotas + +```hcl +{ + entity = "default" + producer_byte_rate = "16384" + consumer_byte_rate = "32768" +} +``` + +## Mirroring + +You can set mirroring which enables messages in one Event Streams service instance to be continuously copied to a second instance by using the `mirroring` input variable. For more information, see [Event Streams Mirroring](https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-mirroring). + +### Options for mirroring + +- `source_crn` (required): The CRN of the source from where data is mirrored. +- `source_alias` (required): The source alias (e.g. `us-south`) from where data is mirrored. +- `target_alias` (required): The target alias (e.g. `us-east`) to where data is mirrored. +- `options` (optional): Transform configuration for `topic name` and `group id`. Supported configurations are: `topic_name_transform` and `group_id_transform`. Valid values for `type` are `rename`, `none`, or `use_alias`. If `type` is set to `rename`, then `rename` object must include the following fields: `add_prefix`, `add_suffix`, `remove_prefix` and `remove_suffix`. + +### Example mirroring + +The following example includes all the configuration options for mirroring. + +```hcl +{ + source_crn = "event_streams_crn" + source_alias = "us-south" + target_alias = "us-east" + options = { + topic_name_transform = { + type = "rename" + rename = { + add_prefix = "add_prefix" + add_suffix = "add_suffix" + remove_prefix = "remove_prefix" + remove_suffix = "remove_suffix" + } + } + group_id_transform = { + type = "rename" + rename = { + add_prefix = "add_prefix" + add_suffix = "add_suffix" + remove_prefix = "remove_prefix" + remove_suffix = "remove_suffix" + } + } + } + } +``` diff --git a/solutions/enterprise/README.md b/solutions/enterprise/README.md new file mode 100644 index 00000000..4c585914 --- /dev/null +++ b/solutions/enterprise/README.md @@ -0,0 +1,15 @@ +# Enterprise Event Streams Solution + +This solution supports provisioning and configuring the following infrastructure: + +- A resource group, if one is not passed in. +- An Event Streams instance using `enterprise` plan. +- An IAM authorization between all Event Stream instances in the given resource group and the Key Protect or Hyper Protect Crypto Services instance that is passed in. +- An Event Streams instance that is encrypted with the Key Protect or Hyper Protect Crypto Services root key that is passed in. +- A context-based restriction (CBR) rule. +- Topics to apply to resources. +- Schemas to apply to resources. + +![da-enterprise](../../reference-architecture/da-enterprise.svg) + +**Important:** Because this solution contains a provider configuration and is not compatible with the `for_each`, `count`, and `depends_on` arguments, do not call this solution from one or more other modules. For more information about how resources are associated with provider configurations with multiple modules, see [Providers Within Modules](https://developer.hashicorp.com/terraform/language/modules/develop/providers). diff --git a/solutions/enterprise/catalogValidationValues.json.template b/solutions/enterprise/catalogValidationValues.json.template new file mode 100644 index 00000000..a50bd0d5 --- /dev/null +++ b/solutions/enterprise/catalogValidationValues.json.template @@ -0,0 +1,7 @@ +{ + "ibmcloud_api_key": $VALIDATION_APIKEY, + "resource_tags": $TAGS, + "prefix": $PREFIX, + "resource_group_name": $PREFIX, + "existing_kms_instance_crn": $HPCS_US_SOUTH_CRN +} diff --git a/solutions/enterprise/main.tf b/solutions/enterprise/main.tf new file mode 100644 index 00000000..a022db8c --- /dev/null +++ b/solutions/enterprise/main.tf @@ -0,0 +1,164 @@ +####################################################################################################################### +# Resource Group +####################################################################################################################### +module "resource_group" { + source = "terraform-ibm-modules/resource-group/ibm" + version = "1.1.6" + resource_group_name = var.use_existing_resource_group == false ? ((var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.resource_group_name}" : var.resource_group_name) : null + existing_resource_group_name = var.use_existing_resource_group == true ? var.resource_group_name : null +} + +####################################################################################################################### +# KMS encryption key +####################################################################################################################### + +locals { + create_new_kms_key = var.existing_kms_key_crn == null ? 1 : 0 # no need to create any KMS resources if passing an existing key + event_streams_key_name = var.prefix != null ? "${var.prefix}-${var.event_streams_key_name}" : var.event_streams_key_name + event_streams_key_ring_name = var.prefix != null ? "${var.prefix}-${var.event_streams_key_ring_name}" : var.event_streams_key_ring_name + + # tflint-ignore: terraform_unused_declarations + validate_kms = var.existing_kms_instance_crn == null && var.existing_kms_key_crn == null ? tobool("Both 'existing_kms_instance_crn' and 'existing_kms_key_crn' input variables can not be null. Set 'existing_kms_instance_crn' to create a new KMS key or 'existing_kms_key_crn' to use an existing KMS key.") : true +} + +module "kms" { + providers = { + ibm = ibm.kms + } + count = local.create_new_kms_key + source = "terraform-ibm-modules/kms-all-inclusive/ibm" + version = "4.18.1" + create_key_protect_instance = false + region = local.kms_region + existing_kms_instance_crn = var.existing_kms_instance_crn + key_ring_endpoint_type = var.kms_endpoint_type + key_endpoint_type = var.kms_endpoint_type + keys = [ + { + key_ring_name = local.event_streams_key_ring_name + existing_key_ring = false + keys = [ + { + key_name = local.event_streams_key_name + standard_key = false + rotation_interval_month = 3 + dual_auth_delete_enabled = false + force_delete = true + } + ] + } + ] +} + +######################################################################################################################## +# Parse KMS info from given CRNs +######################################################################################################################## + +module "kms_instance_crn_parser" { + count = var.existing_kms_instance_crn != null ? 1 : 0 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.existing_kms_instance_crn +} + +module "kms_key_crn_parser" { + count = var.existing_kms_key_crn != null ? 1 : 0 + source = "terraform-ibm-modules/common-utilities/ibm//modules/crn-parser" + version = "1.1.0" + crn = var.existing_kms_key_crn +} + +####################################################################################################################### +# KMS IAM Authorization Policies +# - only created if user passes a value for 'ibmcloud_kms_api_key' (used when KMS is in different account to Event Streams) +# - if no value passed for 'ibmcloud_kms_api_key', the auth policy is created by the Event Streams module +####################################################################################################################### + +# Lookup account ID +data "ibm_iam_account_settings" "iam_account_settings" { +} + +locals { + account_id = data.ibm_iam_account_settings.iam_account_settings.account_id + create_cross_account_kms_auth_policy = !var.skip_event_streams_kms_auth_policy && var.ibmcloud_kms_api_key != null + + # If KMS encryption enabled, parse details from the existing key if being passed, otherwise get it from the key that the DA creates + kms_account_id = var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].account_id : module.kms_instance_crn_parser[0].account_id + kms_service = var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].service_name : module.kms_instance_crn_parser[0].service_name + kms_instance_guid = var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].service_instance : module.kms_instance_crn_parser[0].service_instance + kms_key_crn = var.existing_kms_key_crn != null ? var.existing_kms_key_crn : module.kms[0].keys[format("%s.%s", local.event_streams_key_ring_name, local.event_streams_key_name)].crn + kms_key_id = var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].resource : module.kms[0].keys[format("%s.%s", local.event_streams_key_ring_name, local.event_streams_key_name)].key_id + kms_region = var.existing_kms_key_crn != null ? module.kms_key_crn_parser[0].region : module.kms_instance_crn_parser[0].region +} + +# Create auth policy (scoped to exact KMS key) +resource "ibm_iam_authorization_policy" "kms_policy" { + count = local.create_cross_account_kms_auth_policy ? 1 : 0 + provider = ibm.kms + source_service_account = local.account_id + source_service_name = "event_streams" + source_resource_group_id = module.resource_group.resource_group_id + roles = ["Reader"] + description = "Allow all Event Streams instances in the resource group ${module.resource_group.resource_group_id} in the account ${local.account_id} to read the ${local.kms_service} key ${local.kms_key_id} from the instance GUID ${local.kms_instance_guid}" + resource_attributes { + name = "serviceName" + operator = "stringEquals" + value = local.kms_service + } + resource_attributes { + name = "accountId" + operator = "stringEquals" + value = local.kms_account_id + } + resource_attributes { + name = "serviceInstance" + operator = "stringEquals" + value = local.kms_instance_guid + } + resource_attributes { + name = "resourceType" + operator = "stringEquals" + value = "key" + } + resource_attributes { + name = "resource" + operator = "stringEquals" + value = local.kms_key_id + } + # Scope of policy now includes the key, so ensure to create new policy before + # destroying old one to prevent any disruption to every day services. + lifecycle { + create_before_destroy = true + } +} + +# workaround for https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4478 +resource "time_sleep" "wait_for_authorization_policy" { + count = local.create_cross_account_kms_auth_policy ? 1 : 0 + depends_on = [ibm_iam_authorization_policy.kms_policy] + create_duration = "30s" +} + +####################################################################################################################### +# Event Streams Instance +####################################################################################################################### +module "event_streams" { + source = "../../modules/fscloud" + resource_group_id = module.resource_group.resource_group_id + es_name = (var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.event_streams_name}" : var.event_streams_name + kms_key_crn = local.kms_key_crn + schemas = var.schemas + region = var.region + topics = var.topics + tags = var.resource_tags + access_tags = var.access_tags + service_credential_names = var.service_credential_names + metrics = var.metrics + quotas = var.quotas + mirroring_topic_patterns = var.mirroring_topic_patterns + mirroring = var.mirroring + cbr_rules = var.cbr_rules + schema_global_rule = var.schema_global_rule + skip_kms_iam_authorization_policy = var.skip_event_streams_kms_auth_policy + skip_es_s2s_iam_authorization_policy = var.skip_event_streams_s2s_iam_auth_policy +} diff --git a/solutions/enterprise/outputs.tf b/solutions/enterprise/outputs.tf new file mode 100644 index 00000000..6a54e241 --- /dev/null +++ b/solutions/enterprise/outputs.tf @@ -0,0 +1,64 @@ +############################################################################## +# Outputs +############################################################################## + +output "resource_group_name" { + description = "Resource group name" + value = module.resource_group.resource_group_name +} + +output "resource_group_id" { + description = "Resource group ID" + value = module.resource_group.resource_group_id +} + +output "crn" { + description = "Event Streams instance crn" + value = module.event_streams.crn +} + +output "id" { + description = "Event Streams instance id" + value = module.event_streams.id +} + +output "guid" { + description = "Event Streams instance guid" + value = module.event_streams.guid +} + +output "kafka_brokers_sasl" { + description = "(Array of Strings) Kafka brokers use for interacting with Kafka native API" + value = module.event_streams.kafka_brokers_sasl +} + +output "kafka_http_url" { + description = "The API endpoint to interact with Event Streams REST API" + value = module.event_streams.kafka_http_url +} +output "kafka_broker_version" { + description = "The Kafka version" + value = module.event_streams.kafka_broker_version +} + +output "service_credentials_json" { + description = "Service credentials json map" + value = module.event_streams.service_credentials_json + sensitive = true +} + +output "service_credentials_object" { + description = "Service credentials object" + value = module.event_streams.service_credentials_object + sensitive = true +} + +output "mirroring_config_id" { + description = "The ID of the mirroring config in CRN format" + value = module.event_streams.mirroring_config_id +} + +output "mirroring_topic_patterns" { + description = "Mirroring topic patterns" + value = module.event_streams.mirroring_topic_patterns +} diff --git a/solutions/enterprise/provider.tf b/solutions/enterprise/provider.tf new file mode 100644 index 00000000..515e60c2 --- /dev/null +++ b/solutions/enterprise/provider.tf @@ -0,0 +1,12 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = var.region + visibility = var.provider_visibility +} + +provider "ibm" { + alias = "kms" + ibmcloud_api_key = var.ibmcloud_kms_api_key != null ? var.ibmcloud_kms_api_key : var.ibmcloud_api_key + region = local.kms_region + visibility = var.provider_visibility +} diff --git a/solutions/enterprise/variables.tf b/solutions/enterprise/variables.tf new file mode 100644 index 00000000..45f78850 --- /dev/null +++ b/solutions/enterprise/variables.tf @@ -0,0 +1,254 @@ +variable "ibmcloud_api_key" { + type = string + description = "The IBM Cloud API Key" + sensitive = true +} + +variable "region" { + type = string + description = "Region to provision all resources created by this example" + default = "us-south" +} + +variable "prefix" { + type = string + description = "Optional. The prefix to append to all resources that this solution creates. Prefix is ignored if it is `null` or empty string (\"\")." + default = "enterprise" +} + +variable "resource_group_name" { + type = string + description = "The name of a new or the existing resource group to provision the Event Streams instance. If a prefix input variable is passed, it is prefixed to the value in the `-value` format." +} + +variable "use_existing_resource_group" { + type = bool + description = "Whether to use an existing resource group." + default = false + nullable = false +} + +variable "event_streams_name" { + description = "The name of the Event Streams instance to create. If a prefix input variable is passed, it is prefixed to the value in the `-value` format." + type = string + default = "event-streams" +} + +variable "resource_tags" { + type = list(string) + description = "List of tags associated with the Event Steams instance" + default = [] +} + +variable "access_tags" { + type = list(string) + description = "The list of access tags associated with the Event Steams instance." + default = [] +} + +variable "schemas" { + type = list(object( + { + schema_id = string + schema = object({ + type = string + name = string + fields = optional(list(object({ + name = string + type = string + }))) + }) + } + )) + description = "The list of schema objects. Include the `schema_id`, `type` and `name` of the schema in the `schema` object. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-event-streams/tree/main/solutions/enterprise/DA-schemas-topics-cbr.md#options-with-schemas)." + default = [] +} + +variable "schema_global_rule" { + type = string + description = "Schema global compatibility rule. Allowed values are 'NONE', 'FULL', 'FULL_TRANSITIVE', 'FORWARD', 'FORWARD_TRANSITIVE', 'BACKWARD', 'BACKWARD_TRANSITIVE'." + default = null + + validation { + condition = var.schema_global_rule == null || contains(["NONE", "FULL", "FULL_TRANSITIVE", "FORWARD", "FORWARD_TRANSITIVE", "BACKWARD", "BACKWARD_TRANSITIVE"], coalesce(var.schema_global_rule, "NONE")) + error_message = "The schema_global_rule must be null or one of 'NONE', 'FULL', 'FULL_TRANSITIVE', 'FORWARD', 'FORWARD_TRANSITIVE', 'BACKWARD', 'BACKWARD_TRANSITIVE'." + } +} + +variable "topics" { + type = list(object( + { + name = string + partitions = number + config = object({}) + } + )) + description = "The list of topics to apply to resources. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-event-streams/tree/main/solutions/enterprise/DA-schemas-topics-cbr.md#options-with-topics)." + default = [] +} + +############################################################## +# Context-based restriction (CBR) +############################################################## + +variable "cbr_rules" { + type = list(object({ + description = string + account_id = string + rule_contexts = list(object({ + attributes = optional(list(object({ + name = string + value = string + }))) })) + enforcement_mode = string + })) + description = "A single context-based restriction rule to create. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-event-streams/tree/main/solutions/enterprise/DA-schemas-topics-cbr.md#options-with-cbr)." + default = [] + nullable = false + # Additional validation happens in the rule module + validation { + condition = !(length(var.cbr_rules) > 1) + error_message = "Only one context-based restriction rule is allowed." + } +} + +variable "service_credential_names" { + description = "The mapping of names and roles for service credentials that you want to create for the Event streams.[Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-event-streams/tree/main/solutions/enterprise/DA-types.md#svc-credential-name)" + type = map(string) + default = {} +} + +variable "metrics" { + type = list(string) + description = "Enhanced metrics to activate, as list of strings. Allowed values: 'topic', 'partition', 'consumers'." + default = [] + + validation { + condition = alltrue([for name in var.metrics : contains(["topic", "partition", "consumers"], name)]) + error_message = "The specified metrics are not valid. The following values are valid for metrics: 'topic', 'partition', 'consumers'." + } +} + +variable "quotas" { + type = list(object({ + entity = string + producer_byte_rate = optional(number, -1) + consumer_byte_rate = optional(number, -1) + })) + description = "Quotas to be applied to the Event Streams instance. Entity may be 'default' to apply to all users, or an IAM ServiceID for a specific user. Rates are bytes/second, with -1 meaning no quota. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-event-streams/tree/main/solutions/enterprise/DA-types.md#quotas)" + default = [] +} + +############################################################## +# Mirroring +############################################################## + +variable "mirroring_topic_patterns" { + type = list(string) + description = "The list of the topics to set in instance. Required only if creating mirroring instance." + default = null +} + +variable "mirroring" { + description = "Event Streams mirroring configuration. Required only if creating mirroring instance. For more information on mirroring, see https://github.com/terraform-ibm-modules/terraform-ibm-event-streams/tree/main/solutions/enterprise/DA-types.md#mirroring and https://cloud.ibm.com/docs/EventStreams?topic=EventStreams-mirroring." + type = object({ + source_crn = string + source_alias = string + target_alias = string + options = optional(object({ + topic_name_transform = object({ + type = string + rename = optional(object({ + add_prefix = optional(string) + add_suffix = optional(string) + remove_prefix = optional(string) + remove_suffix = optional(string) + })) + }) + group_id_transform = object({ + type = string + rename = optional(object({ + add_prefix = optional(string) + add_suffix = optional(string) + remove_prefix = optional(string) + remove_suffix = optional(string) + })) + }) + })) + }) + default = null +} + +variable "skip_event_streams_s2s_iam_auth_policy" { + type = bool + description = "Set to true to skip the creation of an Event Streams s2s IAM authorization policy to provision an Event Streams mirroring instance. This is required to read from the source cluster. This policy is required when creating mirroring instance." + default = false + nullable = false +} + +############################################################## +# Provider +############################################################## + +variable "provider_visibility" { + description = "Set the visibility value for the IBM terraform provider. Supported values are `public`, `private`, `public-and-private`. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/guides/custom-service-endpoints)." + type = string + default = "private" + + validation { + condition = contains(["public", "private", "public-and-private"], var.provider_visibility) + error_message = "Invalid visibility option. Allowed values are 'public', 'private', or 'public-and-private'." + } +} + +############################################################## +# Encryption +############################################################## + +variable "existing_kms_instance_crn" { + type = string + description = "The CRN of a Key Protect or Hyper Protect Crypto Services instance. Required only when creating a new encryption key and key ring which will be used to encrypt event streams. To use an existing key, pass values for `existing_kms_key_crn`." + default = null +} + +variable "existing_kms_key_crn" { + type = string + description = "The CRN of a Key Protect or Hyper Protect Crypto Services encryption key to encrypt your data. If no value is passed a new key will be created in the instance specified in the `existing_kms_instance_crn` input variable." + default = null +} + +variable "kms_endpoint_type" { + type = string + description = "The type of endpoint to use for communicating with the Key Protect or Hyper Protect Crypto Services instance. Possible values: `public`, `private`." + default = "private" + validation { + condition = can(regex("public|private", var.kms_endpoint_type)) + error_message = "The kms_endpoint_type value must be 'public' or 'private'." + } +} + +variable "skip_event_streams_kms_auth_policy" { + type = bool + description = "Set to true to skip the creation of IAM authorization policies that permits all Event Streams instances in the given resource group 'Reader' access to the Key Protect or Hyper Protect Crypto Services key. This policy is required in order to enable KMS encryption, so only skip creation if there is one already present in your account." + default = false + nullable = false +} + +variable "event_streams_key_ring_name" { + type = string + default = "event-streams-key-ring" + description = "The name for the key ring created for the Event Streams key. Applies only if not specifying an existing key. If a prefix input variable is specified, the prefix is added to the name in the `-` format." +} + +variable "event_streams_key_name" { + type = string + default = "event-streams-key" + description = "The name for the key created for the Event Streams key. Applies only if not specifying an existing key. If a prefix input variable is specified, the prefix is added to the name in the `-` format." +} + +variable "ibmcloud_kms_api_key" { + type = string + description = "The IBM Cloud API key that can create a root key and key ring in the key management service (KMS) instance. If not specified, the 'ibmcloud_api_key' variable is used. Specify this key if the instance in `existing_kms_instance_crn` is in an account that's different from the Event Streams instance. Leave this input empty if the same account owns both instances." + sensitive = true + default = null +} diff --git a/solutions/enterprise/version.tf b/solutions/enterprise/version.tf new file mode 100644 index 00000000..37cdf6fe --- /dev/null +++ b/solutions/enterprise/version.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.3.0" + required_providers { + ibm = { + source = "IBM-Cloud/ibm" + version = "1.71.3" + } + time = { + source = "hashicorp/time" + version = "0.12.1" + } + } +} diff --git a/solutions/quickstart/main.tf b/solutions/quickstart/main.tf index 49ffad27..e2d6ff62 100644 --- a/solutions/quickstart/main.tf +++ b/solutions/quickstart/main.tf @@ -4,7 +4,7 @@ module "resource_group" { source = "terraform-ibm-modules/resource-group/ibm" version = "1.1.6" - resource_group_name = var.use_existing_resource_group == false ? (var.prefix != null ? "${var.prefix}-${var.resource_group_name}" : var.resource_group_name) : null + resource_group_name = var.use_existing_resource_group == false ? ((var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.resource_group_name}" : var.resource_group_name) : null existing_resource_group_name = var.use_existing_resource_group == true ? var.resource_group_name : null } @@ -14,7 +14,7 @@ module "resource_group" { module "event_streams" { source = "../../" resource_group_id = module.resource_group.resource_group_id - es_name = var.prefix != null ? "${var.prefix}-${var.es_name}" : var.es_name + es_name = (var.prefix != null && var.prefix != "") ? "${var.prefix}-${var.es_name}" : var.es_name plan = var.plan region = var.region topics = var.topics diff --git a/solutions/quickstart/provider.tf b/solutions/quickstart/provider.tf index df45ef50..f69fb6da 100644 --- a/solutions/quickstart/provider.tf +++ b/solutions/quickstart/provider.tf @@ -1,4 +1,5 @@ provider "ibm" { ibmcloud_api_key = var.ibmcloud_api_key region = var.region + visibility = var.provider_visibility } diff --git a/solutions/quickstart/variables.tf b/solutions/quickstart/variables.tf index f8268763..178f62c0 100644 --- a/solutions/quickstart/variables.tf +++ b/solutions/quickstart/variables.tf @@ -6,8 +6,8 @@ variable "ibmcloud_api_key" { variable "prefix" { type = string - description = "Optional. The prefix to append to all resources that this solution creates." - default = null + description = "Optional. The prefix to append to all resources that this solution creates. Prefix is ignored if it is `null` or empty string (\"\")." + default = "dev" } variable "use_existing_resource_group" { @@ -19,7 +19,6 @@ variable "use_existing_resource_group" { variable "resource_group_name" { type = string description = "The name of a new or the existing resource group to provision the Event Streams instance. If a prefix input variable is passed, it is prefixed to the value in the `-value` format." - default = null } variable "es_name" { @@ -73,3 +72,18 @@ variable "service_credential_names" { type = map(string) default = {} } + +############################################################## +# Provider +############################################################## + +variable "provider_visibility" { + description = "Set the visibility value for the IBM terraform provider. Supported values are `public`, `private`, `public-and-private`. [Learn more](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/guides/custom-service-endpoints)." + type = string + default = "public" + + validation { + condition = contains(["public", "private", "public-and-private"], var.provider_visibility) + error_message = "Invalid visibility option. Allowed values are 'public', 'private', or 'public-and-private'." + } +} diff --git a/tests/pr_test.go b/tests/pr_test.go index d3a7d686..f72a76ad 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -14,6 +14,7 @@ import ( const completeExampleTerraformDir = "examples/complete" const quickstartSolutionTerraformDir = "solutions/quickstart" +const enterpriseSolutionTerraformDir = "solutions/enterprise" const fsCloudTerraformDir = "examples/fscloud" // Use existing group for tests @@ -76,6 +77,7 @@ func TestRunQuickstartSolution(t *testing.T) { "resource_group_name": options.ResourceGroup, "use_existing_resource_group": true, "prefix": options.Prefix, + "provider_visibility": "public", } output, err := options.RunTestConsistency() @@ -83,7 +85,52 @@ func TestRunQuickstartSolution(t *testing.T) { assert.NotNil(t, output, "Expected some output") } +func TestEnterpriseSolutionInSchematics(t *testing.T) { + t.Parallel() + + options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ + Testing: t, + Prefix: "es-ent", + BestRegionYAMLPath: regionSelectionPath, + TarIncludePatterns: []string{ + "*.tf", + enterpriseSolutionTerraformDir + "/*.tf", + "modules/fscloud/*.tf", + }, + /* + Comment out the 'ResourceGroup' input to force this tests to create a unique resource group to ensure tests do + not clash. This is due to the fact that an auth policy may already exist in this resource group since we are + re-using a permanent HPCS instance and a permanent Event Streams instance. By using a new resource group, the auth policy will not already exist + since this module scopes auth policies by resource group. + */ + //ResourceGroup: resourceGroup, + TemplateFolder: enterpriseSolutionTerraformDir, + Tags: []string{"test-schematic"}, + DeleteWorkspaceOnFail: false, + WaitJobCompleteMinutes: 180, + }) + + options.TerraformVars = []testschematic.TestSchematicTerraformVar{ + {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, + {Name: "prefix", Value: options.Prefix, DataType: "string"}, + {Name: "resource_group_name", Value: options.Prefix, DataType: "string"}, + {Name: "service_credential_names", Value: "{\"es_writer\": \"Writer\", \"es_reader\": \"Reader\"}", DataType: "map(string)"}, + {Name: "provider_visibility", Value: "private", DataType: "string"}, + {Name: "existing_kms_instance_crn", Value: permanentResources["hpcs_south_crn"], DataType: "string"}, + {Name: "access_tags", Value: permanentResources["accessTags"], DataType: "list(string)"}, + {Name: "resource_tags", Value: options.Tags, DataType: "list(string)"}, + } + + err := options.RunSchematicTest() + assert.Nil(t, err, "This should not have errored") +} + func TestFSCloudInSchematics(t *testing.T) { + // TODO: When running fscloud and enterprise test in parallel, we get the following error: + // You have an active provision that is less than 30 minutes old. Please wait until either 30 minutes have passed since your previous provision, or until the previous provision has completed. + // + // We need to run in parallel, otherwise takes too long. For now we will skip fscloud test. We asked for exemption for that error. (waiting) + t.Skip() t.Parallel() options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ @@ -111,7 +158,6 @@ func TestFSCloudInSchematics(t *testing.T) { options.TerraformVars = []testschematic.TestSchematicTerraformVar{ {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, {Name: "prefix", Value: options.Prefix, DataType: "string"}, - {Name: "existing_kms_instance_guid", Value: permanentResources["hpcs_south"].(string), DataType: "string"}, {Name: "kms_key_crn", Value: permanentResources["hpcs_south_root_key_crn"].(string), DataType: "string"}, {Name: "event_streams_source_crn", Value: permanentResources["event_streams_us_south_crn"].(string), DataType: "string"}, } diff --git a/variables.tf b/variables.tf index cae359b3..b838b21a 100644 --- a/variables.tf +++ b/variables.tf @@ -83,7 +83,7 @@ variable "service_endpoints" { variable "skip_kms_iam_authorization_policy" { type = bool - description = "Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the `existing_kms_instance_guid` variable. In addition, no policy is created if var.kms_encryption_enabled is set to false." + description = "Set to true to skip the creation of an IAM authorization policy that permits all Event Streams database instances in the resource group to read the encryption key from the KMS instance. If set to false, pass in a value for the KMS instance in the `kms_key_crn` variable. In addition, no policy is created if var.kms_encryption_enabled is set to false." default = false } @@ -153,12 +153,6 @@ variable "kms_key_crn" { } } -variable "existing_kms_instance_guid" { - description = "The GUID of the Hyper Protect Crypto Services or Key Protect instance in which the key specified in var.kms_key_crn is coming from. Required only if var.kms_encryption_enabled is set to true, var.skip_kms_iam_authorization_policy is set to false, and you pass a value for var.kms_key_crn." - type = string - default = null -} - variable "create_timeout" { type = string description = "The timeout value for creating an Event Streams instance. Specify `3h` for an Enterprise plan instance. Add 1 h for each level of non-default throughput. Add 30 min for each level of non-default storage size."