From 3e42a6bb9909892044f25d7da5e476837537933b Mon Sep 17 00:00:00 2001 From: Manu Agrawal Date: Tue, 13 Jun 2023 09:56:15 +0530 Subject: [PATCH] Fixing workload identity merge issue (#75) There seems to be some versioning/merge issue due to which some of the changes did not get added in the previous merge for enabling workload identity. Adding the remaining changes here. Also updated some of the usecases that were not using workload identity. Tested the setup again by creating new set of resources, details of the test for reference: { "infraConfig": { "spannerInstanceConfig": { "projectId" : "span-cloud-testing", "instanceId": "machmeter-finance-ledger-fin", "dbName": "finance-ledger-fin", "configuration": "regional-us-east4", "displayName": "machmeter-finance-ledger-fin", "processingUnits": 1000, "environment": "testing" }, "gkeConfig": { "clusterName": "machmeter-test-cluster-fin", "namespace": "spanner-test-fin", "region": "us-east4", "network": "gke-network-fin", "subnetwork": "gke-subnet-fin", "ipRangePodsName": "ip-range-pods-fin", "ipRangeServicesName": "ip-range-services-fin", "google_service_account": "machmeter-test-gsa-fin", "kube_service_account": "machmeter-test-ksa-fin", "machineType": "e2-standard-4", "nodeLocations": "us-east4-b,us-east4-a,us-east4-c", "minCount": 3, "maxCount": 3, "initialNodeCount": 3 } }, "ddlConfig" : { "spannerInstanceConfig" : { "instanceId" : "machmeter-finance-ledger-fin", "dbName" : "finance-ledger-fin", "configuration": "regional-us-east4", "projectId" : "span-cloud-testing" }, "schemaFilePath" : "usecases/finance/ledger/templates/schema.sql" } } --- docs/requirements.md | 4 +- .../cloud/machmeter/model/GKEConfig.java | 33 +++--- .../machmeter/plugins/CleanupPlugin.java | 3 - .../machmeter/plugins/ExecutePlugin.java | 2 + .../machmeter/plugins/InfraSetupPlugin.java | 4 +- .../src/main/resources/terraform/main.tf | 101 +++++++++--------- .../movies-database/sample-configs/setup.json | 3 +- .../finance/ledger/sample-configs/setup.json | 3 +- machmeter/usecases/finance/profile/README.md | 3 +- .../profile/configs/profile-infra-config.json | 3 +- .../shopping/cart/sample-configs/setup.json | 3 +- 11 files changed, 87 insertions(+), 75 deletions(-) diff --git a/docs/requirements.md b/docs/requirements.md index 4b4d228..8343276 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -42,9 +42,7 @@ cd machmeter cd machmeter # Building the maven project - -# You provide the path to service accounts key. -export GOOGLE_APPLICATION_CREDENTIALS=~/service-accounts.json +mvn clean package -P assembly # Install the gcloud gke plugin gcloud components install gke-gcloud-auth-plugin diff --git a/machmeter/src/main/java/com/google/cloud/machmeter/model/GKEConfig.java b/machmeter/src/main/java/com/google/cloud/machmeter/model/GKEConfig.java index 4759deb..8aebbf5 100644 --- a/machmeter/src/main/java/com/google/cloud/machmeter/model/GKEConfig.java +++ b/machmeter/src/main/java/com/google/cloud/machmeter/model/GKEConfig.java @@ -49,10 +49,6 @@ public class GKEConfig { @Expose private String namespace; - @SerializedName(value = "service_account_json", alternate = "serviceAccountJson") - @Expose - private String serviceAccountJson; - @SerializedName(value = "machine_type", alternate = "machineType") @Expose private String machineType; @@ -73,6 +69,27 @@ public class GKEConfig { @Expose private int initialNodeCount; + @SerializedName(value = "google_service_account", alternate = "googleServiceAccount") + @Expose + private String googleServiceAccount; + + public void setGoogleServiceAccount(String googleServiceAccount) { + this.googleServiceAccount = googleServiceAccount; + } + public String getGoogleServiceAccount() { + return googleServiceAccount; + } + + @SerializedName(value = "kube_service_account", alternate = "kubeServiceAccount") + @Expose + private String kubeServiceAccount; + + public void setKubeServiceAccount(String kubeServiceAccount) { + this.kubeServiceAccount = kubeServiceAccount; + } + public String getKubeServiceAccount() { + return kubeServiceAccount; + } public String getClusterName() { return clusterName; } @@ -129,14 +146,6 @@ public void setNamespace(String namespace) { this.namespace = namespace; } - public String getServiceAccountJson() { - return serviceAccountJson; - } - - public void setServiceAccountJson(String serviceAccountJson) { - this.serviceAccountJson = serviceAccountJson; - } - public String getMachineType() { return machineType; } diff --git a/machmeter/src/main/java/com/google/cloud/machmeter/plugins/CleanupPlugin.java b/machmeter/src/main/java/com/google/cloud/machmeter/plugins/CleanupPlugin.java index 1fa1c9c..580c770 100644 --- a/machmeter/src/main/java/com/google/cloud/machmeter/plugins/CleanupPlugin.java +++ b/machmeter/src/main/java/com/google/cloud/machmeter/plugins/CleanupPlugin.java @@ -49,9 +49,6 @@ public void execute(SetupConfig config) { SpannerInstanceConfig spannerInstanceConfig = config.getInfraConfig().getSpannerInstanceConfig(); GKEConfig gkeConfig = config.getInfraConfig().getGkeConfig(); - if (gkeConfig.getServiceAccountJson().isEmpty()) { - gkeConfig.setServiceAccountJson(System.getenv("GOOGLE_APPLICATION_CREDENTIALS")); - } String terraformDestroyCMD = String.format( "terraform destroy -var=gcp_project=%s -var='spanner_config=%s' -var='gke_config=%s'", diff --git a/machmeter/src/main/java/com/google/cloud/machmeter/plugins/ExecutePlugin.java b/machmeter/src/main/java/com/google/cloud/machmeter/plugins/ExecutePlugin.java index 337d7c2..4b81cfa 100644 --- a/machmeter/src/main/java/com/google/cloud/machmeter/plugins/ExecutePlugin.java +++ b/machmeter/src/main/java/com/google/cloud/machmeter/plugins/ExecutePlugin.java @@ -76,6 +76,7 @@ public void execute(ExecuteConfig config) { try { shellExecutor.run(kubectlStop, "machmeter_output/terraform"); } catch (Exception e) { + e.printStackTrace(); throw new RuntimeException(e); } })); @@ -87,6 +88,7 @@ public void execute(ExecuteConfig config) { } catch (IOException | InterruptedException e) { e.printStackTrace(); } catch (Exception e) { + e.printStackTrace(); throw new RuntimeException(e); } } diff --git a/machmeter/src/main/java/com/google/cloud/machmeter/plugins/InfraSetupPlugin.java b/machmeter/src/main/java/com/google/cloud/machmeter/plugins/InfraSetupPlugin.java index a083b73..3ac2cd0 100644 --- a/machmeter/src/main/java/com/google/cloud/machmeter/plugins/InfraSetupPlugin.java +++ b/machmeter/src/main/java/com/google/cloud/machmeter/plugins/InfraSetupPlugin.java @@ -49,9 +49,6 @@ public void execute(SetupConfig config) { SpannerInstanceConfig spannerInstanceConfig = config.getInfraConfig().getSpannerInstanceConfig(); GKEConfig gkeConfig = config.getInfraConfig().getGkeConfig(); - if (gkeConfig.getServiceAccountJson().isEmpty()) { - gkeConfig.setServiceAccountJson(System.getenv("GOOGLE_APPLICATION_CREDENTIALS")); - } String terraformPlanCMD = String.format( "terraform plan -var=gcp_project=%s -var='spanner_config=%s' -var='gke_config=%s'", @@ -74,6 +71,7 @@ public void execute(SetupConfig config) { } catch (IOException | InterruptedException e) { e.printStackTrace(); } catch (Exception e) { + e.printStackTrace(); throw new RuntimeException(e); } } diff --git a/machmeter/src/main/resources/terraform/main.tf b/machmeter/src/main/resources/terraform/main.tf index a31ef37..35f543f 100644 --- a/machmeter/src/main/resources/terraform/main.tf +++ b/machmeter/src/main/resources/terraform/main.tf @@ -25,12 +25,13 @@ variable "gke_config" { subnetwork = string ip_range_pods_name = string ip_range_services_name = string - service_account_json = string machine_type = string node_locations = string min_count = number max_count = number initial_node_count = number + google_service_account = string + kube_service_account = string }) description = "The configuration specifications for the GKE cluster" } @@ -120,6 +121,9 @@ module "gke" { grant_registry_access = true remove_default_node_pool = false create_service_account = false + node_metadata = "GKE_METADATA" + identity_namespace = "${var.gcp_project}.svc.id.goog" + node_pools = [ { name = "default-node-pool" @@ -149,6 +153,51 @@ module "gke" { } } +resource "google_service_account" "app-service-account" { + account_id = "${var.gke_config.google_service_account}" + project = var.gcp_project +} + +resource "kubernetes_service_account" "k8s-service-account" { + automount_service_account_token = true + metadata { + name = "${var.gke_config.kube_service_account}" + namespace = var.gke_config.namespace + annotations = { + "iam.gke.io/gcp-service-account" : "${google_service_account.app-service-account.email}" + } + } +} + +data "google_iam_policy" "spanner-policy" { + binding { + role = "roles/iam.workloadIdentityUser" + members = [ + "serviceAccount:${var.gcp_project}.svc.id.goog[${var.gke_config.namespace}/${kubernetes_service_account.k8s-service-account.metadata[0].name}]" + ] + } +} + +resource "google_service_account_iam_policy" "app-service-account-iam" { + service_account_id = google_service_account.app-service-account.name + policy_data = data.google_iam_policy.spanner-policy.policy_data +} + +data "google_iam_policy" "database-reader-policy" { + binding { + role = "roles/owner" + members = [ + "serviceAccount:${google_service_account.app-service-account.email}" + ] + } +} + +resource "google_spanner_database_iam_policy" "database" { + instance = google_spanner_instance.instance.name + database = google_spanner_database.database.name + policy_data = data.google_iam_policy.database-reader-policy.policy_data +} + resource "google_spanner_instance" "instance" { name = var.spanner_config.instance_name # << be careful changing this in production config = var.spanner_config.configuration @@ -170,16 +219,6 @@ resource "kubernetes_namespace" "namespace" { } } -resource "kubernetes_secret" "service_account" { - metadata { - name = "sa-key" - namespace = kubernetes_namespace.namespace.metadata.0.name - } - data = { - "key.json" = file("${var.gke_config.service_account_json}") - } -} - resource "kubernetes_config_map" "configmap_jmeter_load_test" { metadata { labels = { @@ -292,17 +331,9 @@ resource "kubernetes_deployment" "jmeter-master" { name = "loadtest" sub_path = "load_test" } - volume_mount { - mount_path = "/var/secrets/google" - name = "google-cloud-key" - } port { container_port = 60000 } - env { - name = "GOOGLE_APPLICATION_CREDENTIALS" - value = "/var/secrets/google/key.json" - } } volume { name = "loadtest" @@ -315,12 +346,6 @@ resource "kubernetes_deployment" "jmeter-master" { } } } - volume { - name = "google-cloud-key" - secret { - secret_name = "sa-key" - } - } } } } @@ -349,14 +374,11 @@ resource "kubernetes_stateful_set" "jmeter-slave" { } } spec { + service_account_name = "${var.gke_config.kube_service_account}" container { image = "gcr.io/cloud-spanner-emulator/machmeter-jmeter-slave:latest" name = "jmslave" image_pull_policy = "Always" - volume_mount { - mount_path = "/var/secrets/google" - name = "google-cloud-key" - } volume_mount { mount_path = "/data" name = "jmeter-data" @@ -367,26 +389,7 @@ resource "kubernetes_stateful_set" "jmeter-slave" { port { container_port = 50000 } - resources { - limits = { - cpu = "1000m" - memory = "2Gi" - } - requests = { - cpu = "1000m" - memory = "2Gi" - } - } - env { - name = "GOOGLE_APPLICATION_CREDENTIALS" - value = "/var/secrets/google/key.json" - } - } - volume { - name = "google-cloud-key" - secret { - secret_name = "sa-key" - } + resources {} } volume { name = "jmeter-data" diff --git a/machmeter/usecases/entertainment/movies-database/sample-configs/setup.json b/machmeter/usecases/entertainment/movies-database/sample-configs/setup.json index 6595375..b2ddc1a 100644 --- a/machmeter/usecases/entertainment/movies-database/sample-configs/setup.json +++ b/machmeter/usecases/entertainment/movies-database/sample-configs/setup.json @@ -17,7 +17,8 @@ "subnetwork": "gke-subnet", "ipRangePodsName": "ip-range-pods", "ipRangeServicesName": "ip-range-services", - "service_account_json": "sa.json", + "google_service_account": "machmeter-test-gsa", + "kube_service_account": "machmeter-test-ksa", "machineType": "e2-standard-2", "nodeLocations": "us-central1-a,us-central1-b,us-central1-c", "minCount": 3, diff --git a/machmeter/usecases/finance/ledger/sample-configs/setup.json b/machmeter/usecases/finance/ledger/sample-configs/setup.json index 222d0b2..474c654 100644 --- a/machmeter/usecases/finance/ledger/sample-configs/setup.json +++ b/machmeter/usecases/finance/ledger/sample-configs/setup.json @@ -17,7 +17,8 @@ "subnetwork": "gke-subnet", "ipRangePodsName": "ip-range-pods", "ipRangeServicesName": "ip-range-services", - "service_account_json": "sa.json", + "google_service_account": "machmeter-test-gsa", + "kube_service_account": "machmeter-test-ksa", "machineType": "c2-standard-4", "nodeLocations": "us-central1-a,us-central1-b,us-central1-c", "minCount": 3, diff --git a/machmeter/usecases/finance/profile/README.md b/machmeter/usecases/finance/profile/README.md index eb519d3..2e9e760 100644 --- a/machmeter/usecases/finance/profile/README.md +++ b/machmeter/usecases/finance/profile/README.md @@ -53,7 +53,8 @@ cat profile-infra-config.json "subnetwork": "jaru-mm-subnet", "ipRangePodsName": "jaru-ip-range-pods", "ipRangeServicesName": "jaru-ip-range-services", - "service_account_json": "~/machmeter.json" + "google_service_account": "machmeter-test-gsa", + "kube_service_account": "machmeter-test-ksa", } }, "ddlConfig" : { diff --git a/machmeter/usecases/finance/profile/configs/profile-infra-config.json b/machmeter/usecases/finance/profile/configs/profile-infra-config.json index ed40304..6a713ae 100644 --- a/machmeter/usecases/finance/profile/configs/profile-infra-config.json +++ b/machmeter/usecases/finance/profile/configs/profile-infra-config.json @@ -21,7 +21,8 @@ "subnetwork": "jaru-mm-subnet", "ipRangePodsName": "jaru-ip-range-pods", "ipRangeServicesName": "jaru-ip-range-services", - "service_account_json": "~/machmeter.json" + "google_service_account": "machmeter-test-gsa", + "kube_service_account": "machmeter-test-ksa" } }, "ddlConfig" : { diff --git a/machmeter/usecases/shopping/cart/sample-configs/setup.json b/machmeter/usecases/shopping/cart/sample-configs/setup.json index cf5fd07..b251424 100644 --- a/machmeter/usecases/shopping/cart/sample-configs/setup.json +++ b/machmeter/usecases/shopping/cart/sample-configs/setup.json @@ -17,7 +17,8 @@ "subnetwork": "gke-subnet", "ipRangePodsName": "ip-range-pods", "ipRangeServicesName": "ip-range-services", - "service_account_json": "sa.json", + "google_service_account": "machmeter-test-gsa", + "kube_service_account": "machmeter-test-ksa", "machineType": "e2-standard-2", "nodeLocations": "us-central1-a,us-central1-b,us-central1-c", "minCount": 3,