From 4ce728afbd90aa8746a50d482ee3d78da03e32fe Mon Sep 17 00:00:00 2001
From: Tim Gross <tgross@hashicorp.com>
Date: Wed, 3 Apr 2024 08:45:01 -0400
Subject: [PATCH] E2E: make `vault.create_from_role` unique per cluster
 (#20267)

If a E2E cluster is destroyed after a different one has been created, the role
and policy we create in Vault for the cluster will be deleted and Vault-related
tests will fail. Note that before 1.9, we should figure out a way to give HCP
Vault access to the JWKS endpoint and have a different set of policies, but
we'll need to have a role-per-cluster in that case as well.

Fixes: https://github.com/hashicorp/nomad-e2e/issues/138 (internal)
---
 e2e/terraform/etc/acls/vault/nomad-policy.hcl |  8 +++---
 e2e/terraform/etc/nomad.d/vault.hcl           |  2 +-
 e2e/terraform/hcp_vault.tf                    | 25 ++++++++++---------
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/e2e/terraform/etc/acls/vault/nomad-policy.hcl b/e2e/terraform/etc/acls/vault/nomad-policy.hcl
index 8f3f1befdca..1059928967f 100644
--- a/e2e/terraform/etc/acls/vault/nomad-policy.hcl
+++ b/e2e/terraform/etc/acls/vault/nomad-policy.hcl
@@ -3,13 +3,13 @@
 
 # Allow creating tokens under "nomad-tasks" role. The role name should be
 # updated if "nomad-tasks" is not used.
-path "auth/token/create/nomad-tasks" {
+path "auth/token/create/${role}" {
   capabilities = ["update"]
 }
 
-# Allow looking up "nomad-tasks" role. The role name should be updated if
-# "nomad-tasks" is not used.
-path "auth/token/roles/nomad-tasks" {
+# Allow looking up "${role}" role. The role name should be updated if
+# "${role}" is not used.
+path "auth/token/roles/${role}" {
   capabilities = ["read"]
 }
 
diff --git a/e2e/terraform/etc/nomad.d/vault.hcl b/e2e/terraform/etc/nomad.d/vault.hcl
index a0f2a427f87..691f24de865 100644
--- a/e2e/terraform/etc/nomad.d/vault.hcl
+++ b/e2e/terraform/etc/nomad.d/vault.hcl
@@ -5,7 +5,7 @@ vault {
   enabled          = true
   address          = "${url}"
   task_token_ttl   = "1h"
-  create_from_role = "nomad-tasks"
+  create_from_role = "${role}"
   namespace        = "${namespace}"
   token            = "${token}"
 }
diff --git a/e2e/terraform/hcp_vault.tf b/e2e/terraform/hcp_vault.tf
index afcd0fb53e2..b148f25ef99 100644
--- a/e2e/terraform/hcp_vault.tf
+++ b/e2e/terraform/hcp_vault.tf
@@ -11,15 +11,14 @@ data "hcp_vault_cluster" "e2e_shared_vault" {
   cluster_id = var.hcp_vault_cluster_id
 }
 
-# Nomad servers configuration for Vault
-
+# Vault policy for the Nomad cluster, which allows it to mint derived tokens for
+# tasks. It's interpolated with the random cluster name to avoid collisions
+# between concurrent E2E clusters
 resource "vault_policy" "nomad" {
-  name   = "${local.random_name}-nomad-server"
-  policy = data.local_file.vault_policy_for_nomad.content
-}
-
-data "local_file" "vault_policy_for_nomad" {
-  filename = "${path.root}/etc/acls/vault/nomad-policy.hcl"
+  name = "${local.random_name}-nomad-server"
+  policy = templatefile("${path.root}/etc/acls/vault/nomad-policy.hcl", {
+    role = "nomad-tasks-${local.random_name}"
+  })
 }
 
 resource "vault_token" "nomad" {
@@ -29,11 +28,11 @@ resource "vault_token" "nomad" {
   ttl       = "72h"
 }
 
-# this is the role that Nomad will use for derived tokens. It's not
-# allowed access to nomad-policy so that only mint tokens for tasks,
-# not for new clusters
+# The default role that Nomad will use for derived tokens. It's not allowed
+# access to nomad-policy so that it can only mint tokens for tasks, not for new
+# clusters
 resource "vault_token_auth_backend_role" "nomad_cluster" {
-  role_name           = "nomad-tasks"
+  role_name           = "nomad-tasks-${local.random_name}"
   disallowed_policies = [vault_policy.nomad.name]
   orphan              = true
   token_period        = "259200"
@@ -41,11 +40,13 @@ resource "vault_token_auth_backend_role" "nomad_cluster" {
   token_max_ttl       = "0"
 }
 
+# Nomad agent configuration for Vault
 resource "local_sensitive_file" "nomad_config_for_vault" {
   content = templatefile("etc/nomad.d/vault.hcl", {
     token     = vault_token.nomad.client_token
     url       = data.hcp_vault_cluster.e2e_shared_vault.vault_private_endpoint_url
     namespace = var.hcp_vault_namespace
+    role      = "nomad-tasks-${local.random_name}"
   })
   filename        = "uploads/shared/nomad.d/vault.hcl"
   file_permission = "0600"