Skip to content

Commit

Permalink
Fixing workload identity merge issue (#75)
Browse files Browse the repository at this point in the history 
There seems to be some versioning/merge issue due to which some of the
changes did not get added in the previous merge for enabling workload
identity. Adding the remaining changes here. Also updated some of the
usecases that were not using workload identity.

Tested the setup again by creating new set of resources, details of the
test for reference:
{
  "infraConfig": {
    "spannerInstanceConfig": {
      "projectId" : "span-cloud-testing",
      "instanceId": "machmeter-finance-ledger-fin",
      "dbName": "finance-ledger-fin",
      "configuration": "regional-us-east4",
      "displayName": "machmeter-finance-ledger-fin",
      "processingUnits": 1000,
      "environment": "testing"
    },
    "gkeConfig": {
      "clusterName": "machmeter-test-cluster-fin",
      "namespace": "spanner-test-fin",
      "region": "us-east4",
      "network": "gke-network-fin",
      "subnetwork": "gke-subnet-fin",
      "ipRangePodsName": "ip-range-pods-fin",
      "ipRangeServicesName": "ip-range-services-fin",
      "google_service_account": "machmeter-test-gsa-fin",
      "kube_service_account": "machmeter-test-ksa-fin",
      "machineType": "e2-standard-4",
      "nodeLocations": "us-east4-b,us-east4-a,us-east4-c",
      "minCount": 3,
      "maxCount": 3,
      "initialNodeCount": 3
    }
  },
  "ddlConfig" : {
    "spannerInstanceConfig" : {
      "instanceId" : "machmeter-finance-ledger-fin",
      "dbName" : "finance-ledger-fin",
      "configuration": "regional-us-east4",
      "projectId" : "span-cloud-testing"
    },
    "schemaFilePath" : "usecases/finance/ledger/templates/schema.sql"
  }
}
  • Loading branch information
@manu2
manu2 authored Jun 13, 2023
1 parent d11094b commit 3e42a6b
Show file tree
Hide file tree
Showing 11 changed files with 87 additions and 75 deletions.
4 changes: 1 addition & 3 deletions docs/requirements.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,7 @@ cd machmeter
cd machmeter

# Building the maven project

# You provide the path to service accounts key.
export GOOGLE_APPLICATION_CREDENTIALS=~/service-accounts.json
mvn clean package -P assembly

# Install the gcloud gke plugin
gcloud components install gke-gcloud-auth-plugin
Expand Down
33 changes: 21 additions & 12 deletions machmeter/src/main/java/com/google/cloud/machmeter/model/GKEConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,6 @@ public class GKEConfig {
@Expose
private String namespace;

@SerializedName(value = "service_account_json", alternate = "serviceAccountJson")
@Expose
private String serviceAccountJson;

@SerializedName(value = "machine_type", alternate = "machineType")
@Expose
private String machineType;
Expand All @@ -73,6 +69,27 @@ public class GKEConfig {
@Expose
private int initialNodeCount;

@SerializedName(value = "google_service_account", alternate = "googleServiceAccount")
@Expose
private String googleServiceAccount;

public void setGoogleServiceAccount(String googleServiceAccount) {
this.googleServiceAccount = googleServiceAccount;
}
public String getGoogleServiceAccount() {
return googleServiceAccount;
}

@SerializedName(value = "kube_service_account", alternate = "kubeServiceAccount")
@Expose
private String kubeServiceAccount;

public void setKubeServiceAccount(String kubeServiceAccount) {
this.kubeServiceAccount = kubeServiceAccount;
}
public String getKubeServiceAccount() {
return kubeServiceAccount;
}
public String getClusterName() {
return clusterName;
}
Expand Down Expand Up @@ -129,14 +146,6 @@ public void setNamespace(String namespace) {
this.namespace = namespace;
}

public String getServiceAccountJson() {
return serviceAccountJson;
}

public void setServiceAccountJson(String serviceAccountJson) {
this.serviceAccountJson = serviceAccountJson;
}

public String getMachineType() {
return machineType;
}
Expand Down
3 changes: 0 additions & 3 deletions machmeter/src/main/java/com/google/cloud/machmeter/plugins/CleanupPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@ public void execute(SetupConfig config) {
SpannerInstanceConfig spannerInstanceConfig =
config.getInfraConfig().getSpannerInstanceConfig();
GKEConfig gkeConfig = config.getInfraConfig().getGkeConfig();
if (gkeConfig.getServiceAccountJson().isEmpty()) {
gkeConfig.setServiceAccountJson(System.getenv("GOOGLE_APPLICATION_CREDENTIALS"));
}
String terraformDestroyCMD =
String.format(
"terraform destroy -var=gcp_project=%s -var='spanner_config=%s' -var='gke_config=%s'",
Expand Down
2 changes: 2 additions & 0 deletions machmeter/src/main/java/com/google/cloud/machmeter/plugins/ExecutePlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public void execute(ExecuteConfig config) {
try {
shellExecutor.run(kubectlStop, "machmeter_output/terraform");
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}));
Expand All @@ -87,6 +88,7 @@ public void execute(ExecuteConfig config) {
} catch (IOException | InterruptedException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
Expand Down
4 changes: 1 addition & 3 deletions machmeter/src/main/java/com/google/cloud/machmeter/plugins/InfraSetupPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@ public void execute(SetupConfig config) {
SpannerInstanceConfig spannerInstanceConfig =
config.getInfraConfig().getSpannerInstanceConfig();
GKEConfig gkeConfig = config.getInfraConfig().getGkeConfig();
if (gkeConfig.getServiceAccountJson().isEmpty()) {
gkeConfig.setServiceAccountJson(System.getenv("GOOGLE_APPLICATION_CREDENTIALS"));
}
String terraformPlanCMD =
String.format(
"terraform plan -var=gcp_project=%s -var='spanner_config=%s' -var='gke_config=%s'",
Expand All @@ -74,6 +71,7 @@ public void execute(SetupConfig config) {
} catch (IOException | InterruptedException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
Expand Down
101 changes: 52 additions & 49 deletions machmeter/src/main/resources/terraform/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,13 @@ variable "gke_config" {
subnetwork = string
ip_range_pods_name = string
ip_range_services_name = string
service_account_json = string
machine_type = string
node_locations = string
min_count = number
max_count = number
initial_node_count = number
google_service_account = string
kube_service_account = string
})
description = "The configuration specifications for the GKE cluster"
}
Expand Down Expand Up @@ -120,6 +121,9 @@ module "gke" {
grant_registry_access = true
remove_default_node_pool = false
create_service_account = false
node_metadata = "GKE_METADATA"
identity_namespace = "${var.gcp_project}.svc.id.goog"

node_pools = [
{
name = "default-node-pool"
Expand Down Expand Up @@ -149,6 +153,51 @@ module "gke" {
}
}

resource "google_service_account" "app-service-account" {
account_id = "${var.gke_config.google_service_account}"
project = var.gcp_project
}

resource "kubernetes_service_account" "k8s-service-account" {
automount_service_account_token = true
metadata {
name = "${var.gke_config.kube_service_account}"
namespace = var.gke_config.namespace
annotations = {
"iam.gke.io/gcp-service-account" : "${google_service_account.app-service-account.email}"
}
}
}

data "google_iam_policy" "spanner-policy" {
binding {
role = "roles/iam.workloadIdentityUser"
members = [
"serviceAccount:${var.gcp_project}.svc.id.goog[${var.gke_config.namespace}/${kubernetes_service_account.k8s-service-account.metadata[0].name}]"
]
}
}

resource "google_service_account_iam_policy" "app-service-account-iam" {
service_account_id = google_service_account.app-service-account.name
policy_data = data.google_iam_policy.spanner-policy.policy_data
}

data "google_iam_policy" "database-reader-policy" {
binding {
role = "roles/owner"
members = [
"serviceAccount:${google_service_account.app-service-account.email}"
]
}
}

resource "google_spanner_database_iam_policy" "database" {
instance = google_spanner_instance.instance.name
database = google_spanner_database.database.name
policy_data = data.google_iam_policy.database-reader-policy.policy_data
}

resource "google_spanner_instance" "instance" {
name = var.spanner_config.instance_name # << be careful changing this in production
config = var.spanner_config.configuration
Expand All @@ -170,16 +219,6 @@ resource "kubernetes_namespace" "namespace" {
}
}

resource "kubernetes_secret" "service_account" {
metadata {
name = "sa-key"
namespace = kubernetes_namespace.namespace.metadata.0.name
}
data = {
"key.json" = file("${var.gke_config.service_account_json}")
}
}

resource "kubernetes_config_map" "configmap_jmeter_load_test" {
metadata {
labels = {
Expand Down Expand Up @@ -292,17 +331,9 @@ resource "kubernetes_deployment" "jmeter-master" {
name = "loadtest"
sub_path = "load_test"
}
volume_mount {
mount_path = "/var/secrets/google"
name = "google-cloud-key"
}
port {
container_port = 60000
}
env {
name = "GOOGLE_APPLICATION_CREDENTIALS"
value = "/var/secrets/google/key.json"
}
}
volume {
name = "loadtest"
Expand All @@ -315,12 +346,6 @@ resource "kubernetes_deployment" "jmeter-master" {
}
}
}
volume {
name = "google-cloud-key"
secret {
secret_name = "sa-key"
}
}
}
}
}
Expand Down Expand Up @@ -349,14 +374,11 @@ resource "kubernetes_stateful_set" "jmeter-slave" {
}
}
spec {
service_account_name = "${var.gke_config.kube_service_account}"
container {
image = "gcr.io/cloud-spanner-emulator/machmeter-jmeter-slave:latest"
name = "jmslave"
image_pull_policy = "Always"
volume_mount {
mount_path = "/var/secrets/google"
name = "google-cloud-key"
}
volume_mount {
mount_path = "/data"
name = "jmeter-data"
Expand All @@ -367,26 +389,7 @@ resource "kubernetes_stateful_set" "jmeter-slave" {
port {
container_port = 50000
}
resources {
limits = {
cpu = "1000m"
memory = "2Gi"
}
requests = {
cpu = "1000m"
memory = "2Gi"
}
}
env {
name = "GOOGLE_APPLICATION_CREDENTIALS"
value = "/var/secrets/google/key.json"
}
}
volume {
name = "google-cloud-key"
secret {
secret_name = "sa-key"
}
resources {}
}
volume {
name = "jmeter-data"
Expand Down
3 changes: 2 additions & 1 deletion machmeter/usecases/entertainment/movies-database/sample-configs/setup.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
"subnetwork": "gke-subnet",
"ipRangePodsName": "ip-range-pods",
"ipRangeServicesName": "ip-range-services",
"service_account_json": "sa.json",
"google_service_account": "machmeter-test-gsa",
"kube_service_account": "machmeter-test-ksa",
"machineType": "e2-standard-2",
"nodeLocations": "us-central1-a,us-central1-b,us-central1-c",
"minCount": 3,
Expand Down
3 changes: 2 additions & 1 deletion machmeter/usecases/finance/ledger/sample-configs/setup.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
"subnetwork": "gke-subnet",
"ipRangePodsName": "ip-range-pods",
"ipRangeServicesName": "ip-range-services",
"service_account_json": "sa.json",
"google_service_account": "machmeter-test-gsa",
"kube_service_account": "machmeter-test-ksa",
"machineType": "c2-standard-4",
"nodeLocations": "us-central1-a,us-central1-b,us-central1-c",
"minCount": 3,
Expand Down
3 changes: 2 additions & 1 deletion machmeter/usecases/finance/profile/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ cat profile-infra-config.json
"subnetwork": "jaru-mm-subnet",
"ipRangePodsName": "jaru-ip-range-pods",
"ipRangeServicesName": "jaru-ip-range-services",
"service_account_json": "~/machmeter.json"
"google_service_account": "machmeter-test-gsa",
"kube_service_account": "machmeter-test-ksa",
}
},
"ddlConfig" : {
Expand Down
3 changes: 2 additions & 1 deletion machmeter/usecases/finance/profile/configs/profile-infra-config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
"subnetwork": "jaru-mm-subnet",
"ipRangePodsName": "jaru-ip-range-pods",
"ipRangeServicesName": "jaru-ip-range-services",
"service_account_json": "~/machmeter.json"
"google_service_account": "machmeter-test-gsa",
"kube_service_account": "machmeter-test-ksa"
}
},
"ddlConfig" : {
Expand Down
3 changes: 2 additions & 1 deletion machmeter/usecases/shopping/cart/sample-configs/setup.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
"subnetwork": "gke-subnet",
"ipRangePodsName": "ip-range-pods",
"ipRangeServicesName": "ip-range-services",
"service_account_json": "sa.json",
"google_service_account": "machmeter-test-gsa",
"kube_service_account": "machmeter-test-ksa",
"machineType": "e2-standard-2",
"nodeLocations": "us-central1-a,us-central1-b,us-central1-c",
"minCount": 3,
Expand Down

0 comments on commit 3e42a6b

Please sign in to comment.