From be099503e0ded26e6aa3beeeeb2bef4e0cb53169 Mon Sep 17 00:00:00 2001 From: Ritika Patil <94649368+riragh@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:41:07 -0600 Subject: [PATCH] feat: (PSKD-709) Add Support for Google NetApp Volumes (#224) * feat: (PSKD-709) Add support for Google NetApp volumes Signed-off-by: Ritika Patil --- README.md | 2 +- docs/CONFIG-VARS.md | 21 +++++++- docs/user/TerraformGCPAuthentication.md | 6 ++- examples/sample-input-ha.tfvars | 3 +- locals.tf | 6 +++ main.tf | 18 ++++++- modules/google_netapp/main.tf | 70 +++++++++++++++++++++++++ modules/google_netapp/outputs.tf | 7 +++ modules/google_netapp/variables.tf | 54 +++++++++++++++++++ network.tf | 2 +- outputs.tf | 6 ++- variables.tf | 52 ++++++++++++++++++ vms.tf | 12 +++-- 13 files changed, 246 insertions(+), 13 deletions(-) create mode 100644 modules/google_netapp/main.tf create mode 100644 modules/google_netapp/outputs.tf create mode 100644 modules/google_netapp/variables.tf diff --git a/README.md b/README.md index 25bce1e..a4d06af 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This project contains Terraform scripts to provision Google Cloud infrastructure >- Managed Google Kubernetes Engine (GKE) cluster >- System and User GKE Node pools with required Labels and Taints >- Infrastructure to deploy SAS Viya platform CAS in SMP or MPP mode - >- Shared Storage options for SAS Viya platform - Google Filestore (ha) or NFS Server (standard) + >- Shared Storage options for SAS Viya platform - Google Filestore (ha), Google NetApp Volumes (ha) or NFS Server (standard) >- Google Cloud SQL for PostgreSQL instance, optional [Architecture Diagram](./docs/images/viya4-iac-gcp-diag.png?raw=true) diff --git a/docs/CONFIG-VARS.md b/docs/CONFIG-VARS.md index 9f80726..438cf58 100644 --- a/docs/CONFIG-VARS.md +++ b/docs/CONFIG-VARS.md @@ -17,7 +17,8 @@ Supported configuration variables are listed in the table below. All variables - [Additional Nodepools](#additional-nodepools) - [Storage](#storage) - [For `storage_type=standard` only (NFS server VM)](#for-storage_typestandard-only-nfs-server-vm) - - [For `storage_type=ha` only (Google Filestore)](#for-storage_typeha-only-google-filestore) + - [For `storage_type=ha` with Google Filestore](#for-storage_typeha-with-google-filestore) + - [For `storage_type=ha` with Google NetApp Volumes](#for-storage_typeha-with-google-netapp-volumes) - [Google Artifact Registry (GAR) and Google Container Registry (GCR)](#google-artifact-registry-gar-and-google-container-registry-gcr) - [Postgres Servers](#postgres-servers) - [Monitoring](#monitoring) @@ -69,6 +70,7 @@ You can use `default_public_access_cidrs` to set a default range for all created | misc_subnet_cidr | Address space for the the auxiliary resources (Jump VM and optionally NFS VM) subnet | string | "192.168.2.0/24" | This variable is ignored when `subnet_names` is set (aka bring your own subnet) | | filestore_subnet_cidr | Address space for Google Filestore subnet | string | "192.168.3.0/29" | Needs to be at least a /29 range. Only used when `storage_type="ha"` | | database_subnet_cidr | Address space for Google Cloud SQL Postgres subnet | string | "192.168.4.0/23" | Only used with external postgres | +| netapp_subnet_cidr | Address space for Google Cloud NetApp Volumes subnet | string | "192.168.5.0/24" | Needs to be at least a /24 range. Only used when `storage_type="ha"` and `storage_type_backend="netapp"` | ### Use Existing @@ -212,6 +214,7 @@ stateful = { | Name | Description | Type | Default | Notes | | :--- | ---: | ---: | ---: | ---: | | storage_type | Type of Storage. Valid Values: "standard", "ha" | string | "standard" | "standard" creates NFS server VM, "ha" Google Filestore instance | +| storage_type_backend | The storage backend for the chosen `storage_type`. | string | If `storage_type=standard` the default is "nfs";
If `storage_type=ha` the default is "filestore" | Valid Values: "nfs" if `storage_type=standard`; "filestore" or "netapp" if `storage_type=ha` | ### For `storage_type=standard` only (NFS server VM) @@ -221,13 +224,27 @@ stateful = { | nfs_vm_admin | OS Admin User for the NFS server VM | string | "nfsuser" | The NFS server VM is only created when storage_type="standard" | | nfs_raid_disk_size | Size in Gb for each disk of the RAID5 cluster on the NFS server VM | number | 1000 | The NFS server VM is only created when storage_type="standard" | -### For `storage_type=ha` only (Google Filestore) +### For `storage_type=ha` with Google Filestore | Name | Description | Type | Default | Notes | | :--- | ---: | ---: | ---: | ---: | | filestore_tier | The service tier for the Google Filestore Instance | string | "BASIC_HDD" | Valid Values: "BASIC_HDD", "BASIC_SSD" (previously called "STANDARD" and "PREMIUM" respectively.) | | filestore_size_in_gb | Size in GB of Filesystem in the Google Filestore Instance | number | 1024 for BASIC_HDD, 2560 for BASIC_SDD | 2560 GB is the minimum size for the BASIC_SSD tier. The BASIC_HDD tier allows a minimum size of 1024 GB. | +### For `storage_type=ha` with Google NetApp Volumes + +When `storage_type=ha` and `storage_type_backend=netapp` are specified, [Google NetApp Volumes](https://cloud.google.com/netapp/volumes/docs/discover/overview) service is created. Before using this storage option, +- Enable the Google Cloud NetApp Volumes API for your project, see how to enable [here](https://cloud.google.com/netapp/volumes/docs/get-started/configure-access/initiate-console-settings#enable_the_api). +- Grant access to NetApp Volumes operations by granting IAM roles to users. The two predefined roles are `roles/netapp.admin` and `roles/netapp.viewer`. You can assign these roles to specific users or service accounts. +- NetApp Volumes is available in several regions. For details about region availability, see [NetApp Volumes locations](https://cloud.google.com/netapp/volumes/docs/locations). + +| Name | Description | Type | Default | Notes | +| :--- | ---: | ---: | ---: | ---: | +| netapp_service_level | The service level of the storage pool. | string | "PREMIUM" | Valid Values are: PREMIUM, EXTREME, STANDARD, FLEX. | +| netapp_protocols | The target volume protocol expressed as a list. | list(string) | ["NFSV3"] | Each value may be one of: NFSV3, NFSV4, SMB. Currently, only NFSV3 is supported by SAS Viya Platform. | +| netapp_capacity_gib | Capacity of the storage pool (in GiB). Storage Pool capacity specified must be between 2048 GiB and 10485760 GiB. | string | "2048" | | +| netapp_volume_path | A unique file path for the volume. Used when creating mount targets. Needs to be unique per location.| string | | | + ## Google Artifact Registry (GAR) and Google Container Registry (GCR) | Name | Description | Type | Default | Notes | diff --git a/docs/user/TerraformGCPAuthentication.md b/docs/user/TerraformGCPAuthentication.md index 441cd12..0a71727 100644 --- a/docs/user/TerraformGCPAuthentication.md +++ b/docs/user/TerraformGCPAuthentication.md @@ -39,7 +39,9 @@ The Service Account will need the following [IAM roles](https://cloud.google.com | `roles/container.admin` | Kubernetes Engine Admin | Cluster creation | | `roles/container.clusterAdmin` | Kubernetes Engine Cluster Admin | Terraform Kubernetes Engine Module | | `roles/container.developer` | Kubernetes Engine Developer | Cluster creation | -| `roles/file.editor` | Cloud Filestore Editor | Needed for [`storage_type=="HA"`](../CONFIG-VARS.md#storage) | +| `roles/file.editor` | Cloud Filestore Editor | Needed for [`storage_type=="ha" && storage_type_backend = "filestore"`](../CONFIG-VARS.md#storage) | +| `roles/netapp.admin` | NetApp Admin | Needed for [`storage_type=="ha" && storage_type_backend = "netapp"`](../CONFIG-VARS.md#storage) | +| `roles/netapp.viewer` | NetApp Viewer | Needed for [`storage_type=="ha" && storage_type_backend = "netapp"`](../CONFIG-VARS.md#storage) | | `roles/iam.serviceAccountAdmin` | Service Account Admin | Terraform Kubernetes Engine Module | | `roles/iam.serviceAccountUser` | Service Account User | Terraform Kubernetes Engine Module | | `roles/resourcemanager.projectIamAdmin` | Project IAM Admin | Terraform Kubernetes Engine Module | @@ -59,6 +61,8 @@ gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAM gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/container.clusterAdmin gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/container.developer gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/file.editor +gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/netapp.admin +gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/netapp.viewer gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/iam.serviceAccountAdmin gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/iam.serviceAccountUser gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:${SA_NAME}@${PROJECT}.iam.gserviceaccount.com --role roles/resourcemanager.projectIamAdmin diff --git a/examples/sample-input-ha.tfvars b/examples/sample-input-ha.tfvars index 8a8d14b..8576eed 100644 --- a/examples/sample-input-ha.tfvars +++ b/examples/sample-input-ha.tfvars @@ -94,5 +94,6 @@ jump_vm_admin = "jumpuser" # Storage for Viya Compute Services # Supported storage_type values # "standard" - Custom managed NFS Server VM and disks -# "ha" - Google Filestore +# "ha" - Google Filestore or Google NetApp Volumes storage_type = "ha" +storage_type_backend = "filestore" # "filestore" is the default, use "netapp" to create Google NetApp Volumes diff --git a/locals.tf b/locals.tf index 763bb18..e184431 100644 --- a/locals.tf +++ b/locals.tf @@ -25,6 +25,12 @@ locals { : null ) + # Storage + storage_type_backend = (var.storage_type == "none" ? "none" + : var.storage_type == "standard" ? "nfs" + : var.storage_type == "ha" && var.storage_type_backend == "netapp" ? "netapp" + : var.storage_type == "ha" ? "filestore" : "none") + # Kubernetes kubeconfig_path = var.iac_tooling == "docker" ? "/workspace/${var.prefix}-gke-kubeconfig.conf" : "${var.prefix}-gke-kubeconfig.conf" diff --git a/main.tf b/main.tf index 74268a4..5b232a8 100644 --- a/main.tf +++ b/main.tf @@ -66,7 +66,7 @@ EOT resource "google_filestore_instance" "rwx" { name = "${var.prefix}-rwx-filestore" - count = var.storage_type == "ha" ? 1 : 0 + count = var.storage_type == "ha" && local.storage_type_backend == "filestore" ? 1 : 0 tier = upper(var.filestore_tier) location = local.zone labels = var.tags @@ -301,3 +301,19 @@ module "sql_proxy_sa" { project_roles = ["${var.project}=>roles/cloudsql.admin"] display_name = "IAC-managed service account for cluster ${var.prefix} and sql-proxy integration." } + +module "google_netapp" { + source = "./modules/google_netapp" + + count = var.storage_type == "ha" && local.storage_type_backend == "netapp" ? 1 : 0 + + prefix = var.prefix + region = local.region + network = module.vpc.network_name + netapp_subnet_cidr = var.netapp_subnet_cidr + service_level = var.netapp_service_level + capacity_gib = var.netapp_capacity_gib + protocols = var.netapp_protocols + volume_path = "${var.prefix}-${var.netapp_volume_path}" + allowed_clients = join(",", [local.gke_subnet_cidr, local.misc_subnet_cidr]) +} diff --git a/modules/google_netapp/main.tf b/modules/google_netapp/main.tf new file mode 100644 index 0000000..3ec9c3b --- /dev/null +++ b/modules/google_netapp/main.tf @@ -0,0 +1,70 @@ +# Copyright © 2021-2024, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Terraform Registry : https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/netapp_volume +# GitHub Repository : https://github.com/terraform-google-modules +# + +# Reserve compute address CIDR for NetApp Volumes to use +resource "google_compute_global_address" "private_ip_alloc" { + name = "${var.network}-ip-range" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + address = split("/", var.netapp_subnet_cidr)[0] + prefix_length = split("/", var.netapp_subnet_cidr)[1] + network = var.network +} + +# Create the PSA peering +resource "google_service_networking_connection" "default" { + network = var.network + service = "netapp.servicenetworking.goog" + reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] + + deletion_policy = "ABANDON" +} + +# Modify the PSA Connection to allow import/export of custom routes +resource "google_compute_network_peering_routes_config" "route_updates" { + peering = google_service_networking_connection.default.peering + network = var.network + + import_custom_routes = true + export_custom_routes = true +} + +resource "google_netapp_storage_pool" "netapp-tf-pool" { + name = "${var.prefix}-netapp-storage-pool" + location = var.region + service_level = var.service_level + capacity_gib = var.capacity_gib + network = var.network + + lifecycle { + ignore_changes = [network] + } +} + +resource "google_netapp_volume" "netapp-nfs-volume" { + location = var.region + name = "${var.prefix}-netapp-volume" + capacity_gib = var.capacity_gib # Size can be up to space available in pool + share_name = var.volume_path + storage_pool = google_netapp_storage_pool.netapp-tf-pool.name + protocols = var.protocols + unix_permissions = "0777" + export_policy { + rules { + access_type = "READ_WRITE" + allowed_clients = var.allowed_clients + has_root_access = true + nfsv3 = contains(var.protocols, "NFSV3") ? true : false + nfsv4 = contains(var.protocols, "NFSV4") ? true : false + } + } + + depends_on = [ + google_netapp_storage_pool.netapp-tf-pool, + google_service_networking_connection.default + ] +} diff --git a/modules/google_netapp/outputs.tf b/modules/google_netapp/outputs.tf new file mode 100644 index 0000000..71fc2df --- /dev/null +++ b/modules/google_netapp/outputs.tf @@ -0,0 +1,7 @@ +output "mountpath" { + value = google_netapp_volume.netapp-nfs-volume.mount_options[0].export +} + +output "export_ip" { + value = split(":", google_netapp_volume.netapp-nfs-volume.mount_options[0].export_full)[0] +} diff --git a/modules/google_netapp/variables.tf b/modules/google_netapp/variables.tf new file mode 100644 index 0000000..18ed8ba --- /dev/null +++ b/modules/google_netapp/variables.tf @@ -0,0 +1,54 @@ +# Copyright © 2021-2024, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +variable "prefix" { + description = "A prefix used in the name for all cloud resources created by this script. The prefix string must start with lowercase letter and contain only lowercase alphanumeric characters and hyphen or dash(-), but can not start or end with '-'." + type = string +} + +variable "region" { + description = "The region to create the VM in" + type = string +} + +variable "service_level" { + description = "Service level of the storage pool. Possible values are: PREMIUM, EXTREME, STANDARD, FLEX." + type = string + default = "PREMIUM" +} + +variable "protocols" { + description = "The target volume protocol expressed as a list. Allowed combinations are ['NFSV3'], ['NFSV4'], ['SMB'], ['NFSV3', 'NFSV4'], ['SMB', 'NFSV3'] and ['SMB', 'NFSV4']. Each value may be one of: NFSV3, NFSV4, SMB." + type = list(string) + default = ["NFSV3"] +} + +variable "capacity_gib" { + description = "Capacity of the storage pool (in GiB)." + type = string + default = 2048 +} + +variable "volume_path" { + description = "A unique file path for the volume. Used when creating mount targets. Needs to be unique per location." + type = string + default = "export" +} + +variable "network" { + description = "VPC network name with format: `projects/{{project}}/global/networks/{{network}}`" + type = string +} + + +variable "allowed_clients" { + description = "CIDR blocks allowed to mount nfs exports" + type = string + default = "0.0.0.0/0" +} + +variable "netapp_subnet_cidr" { + description = "Address space for Google Cloud NetApp Volumes subnet" + type = string + default = "192.168.5.0/24" +} diff --git a/network.tf b/network.tf index 575804f..939449b 100644 --- a/network.tf +++ b/network.tf @@ -72,7 +72,7 @@ resource "google_service_networking_connection" "private_vpc_connection" { # required as of hashicorp/google v5.12.0 when using google_service_networking_connection in # conjunction with CloudSQL instances in order to cleanly delete resources # https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection - deletion_policy = "ABANDON" + deletion_policy = "ABANDON" } resource "google_compute_firewall" "nfs_vm_cluster_firewall" { diff --git a/outputs.tf b/outputs.tf index aa95b10..6493a77 100644 --- a/outputs.tf +++ b/outputs.tf @@ -27,7 +27,8 @@ output "rwx_filestore_endpoint" { description = "Shared Storage private IP" value = (var.storage_type == "none" ? null - : var.storage_type == "ha" ? google_filestore_instance.rwx[0].networks[0].ip_addresses[0] : module.nfs_server[0].private_ip + : var.storage_type == "ha" && local.storage_type_backend == "filestore" ? google_filestore_instance.rwx[0].networks[0].ip_addresses[0] + : var.storage_type == "ha" && local.storage_type_backend == "netapp" ? module.google_netapp[0].export_ip : module.nfs_server[0].private_ip ) } @@ -35,7 +36,8 @@ output "rwx_filestore_path" { description = "Shared Storage mount path" value = (var.storage_type == "none" ? null - : var.storage_type == "ha" ? "/${google_filestore_instance.rwx[0].file_shares[0].name}" : "/export" + : var.storage_type == "ha" && local.storage_type_backend == "filestore" ? "/${google_filestore_instance.rwx[0].file_shares[0].name}" + : var.storage_type == "ha" && local.storage_type_backend == "netapp" ? module.google_netapp[0].mountpath : "/export" ) } diff --git a/variables.tf b/variables.tf index e50c25c..4cc74e2 100644 --- a/variables.tf +++ b/variables.tf @@ -184,6 +184,18 @@ variable "storage_type" { } } +variable "storage_type_backend" { + description = "The storage backend used for the chosen storage type. Defaults to 'nfs' for storage_type='standard'. Defaults to 'filestore for storage_type='ha'. 'filestore' and 'netapp' are valid choices for storage_type='ha'." + type = string + default = "nfs" + # If storage_type is standard, this will be set to "nfs" + + validation { + condition = contains(["nfs", "filestore", "netapp", "none"], lower(var.storage_type_backend)) + error_message = "ERROR: Supported values for `storage_type_backend` are nfs, filestore, netapp or none." + } +} + variable "minimum_initial_nodes" { description = "Number of initial nodes to aim for to overcome the Ingress quota limit of 100" type = number @@ -426,6 +438,41 @@ variable "enable_registry_access" { default = true } +## Google NetApp Volumes +variable "netapp_service_level" { + description = "Service level of the storage pool. Possible values are: PREMIUM, EXTREME, STANDARD, FLEX." + type = string + default = "PREMIUM" + + validation { + condition = var.netapp_service_level != null ? contains(["PREMIUM", "EXTREME", "STANDARD", "FLEX"], var.netapp_service_level) : null + error_message = "ERROR: netapp_service_level - Valid values include - PREMIUM, EXTREME, STANDARD, FLEX." + } +} + +variable "netapp_protocols" { + description = "The target volume protocol expressed as a list. Each value may be one of: NFSV3, NFSV4, SMB. Currently, only NFS is supported." + type = list(string) + default = ["NFSV3"] + + validation { + condition = var.netapp_protocols != null ? startswith(var.netapp_protocols[0], "NFS") : null + error_message = "ERROR: Currently, only NFS protocol is supported." + } +} + +variable "netapp_capacity_gib" { + description = "Capacity of the storage pool (in GiB). Storage Pool capacity specified must be between 2048 GiB and 10485760 GiB." + type = string + default = 2048 +} + +variable "netapp_volume_path" { + description = "A unique file path for the volume. Used when creating mount targets. Needs to be unique per location." + type = string + default = "export" +} + # GKE Monitoring variable "create_gke_monitoring_service" { description = "Enable GKE metrics from pods in the cluster to the Google Cloud Monitoring API." @@ -519,6 +566,11 @@ variable "database_subnet_cidr" { default = "192.168.4.0/23" } +variable "netapp_subnet_cidr" { + description = "Address space for Google Cloud NetApp Volumes subnet" + type = string + default = "192.168.5.0/24" +} variable "gke_network_policy" { description = "Sets up network policy to be used with GKE CNI. Network policy allows us to control the traffic flow between pods. Currently supported values are true (calico) and false (kubenet). Changing this forces a new resource to be created." diff --git a/vms.tf b/vms.tf index 3b1fcfa..01b5201 100644 --- a/vms.tf +++ b/vms.tf @@ -4,18 +4,22 @@ locals { rwx_filestore_endpoint = (var.storage_type == "none" ? "" - : var.storage_type == "ha" ? google_filestore_instance.rwx[0].networks[0].ip_addresses[0] : module.nfs_server[0].private_ip + : var.storage_type == "ha" && local.storage_type_backend == "filestore" ? google_filestore_instance.rwx[0].networks[0].ip_addresses[0] + : var.storage_type == "ha" && local.storage_type_backend == "netapp" ? module.google_netapp[0].export_ip : module.nfs_server[0].private_ip ) rwx_filestore_path = (var.storage_type == "none" ? "" - : var.storage_type == "ha" ? "/${google_filestore_instance.rwx[0].file_shares[0].name}" : "/export" + : var.storage_type == "ha" && local.storage_type_backend == "filestore" ? "/${google_filestore_instance.rwx[0].file_shares[0].name}" + : var.storage_type == "ha" && local.storage_type_backend == "netapp" ? module.google_netapp[0].mountpath : "/export" ) + protocol_version = var.storage_type == "ha" && local.storage_type_backend == "netapp" ? split("V", var.netapp_protocols[0])[1] == "4" ? "4.1" : "3" : "3" + } module "nfs_server" { source = "./modules/google_vm" project = var.project - count = var.storage_type == "standard" ? 1 : 0 + count = var.storage_type == "standard" && local.storage_type_backend == "nfs" ? 1 : 0 create_public_ip = var.create_nfs_public_ip name = "${var.prefix}-nfs-server" @@ -68,7 +72,7 @@ module "jump_server" { ["${local.rwx_filestore_endpoint}:${local.rwx_filestore_path}", var.jump_rwx_filestore_path, "nfs", - "_netdev,auto,x-systemd.automount,x-systemd.mount-timeout=10,timeo=14,x-systemd.idle-timeout=1min,relatime,hard,rsize=65536,wsize=65536,vers=3,tcp,namlen=255,retrans=2,sec=sys,local_lock=none", + "_netdev,auto,x-systemd.automount,x-systemd.mount-timeout=10,timeo=14,x-systemd.idle-timeout=1min,relatime,hard,rsize=65536,wsize=65536,vers=${local.protocol_version},tcp,namlen=255,retrans=2,sec=sys,local_lock=none", "0", "0" ])