diff --git a/e2e/terraform/README.md b/e2e/terraform/README.md index c3e94fe7d40..b7d77a22bde 100644 --- a/e2e/terraform/README.md +++ b/e2e/terraform/README.md @@ -4,10 +4,10 @@ This folder contains Terraform resources for provisioning a Nomad cluster on EC2 instances on AWS to use as the target of end-to-end tests. -Terraform provisions the AWS infrastructure assuming that EC2 AMIs -have already been built via Packer and HCP Consul and HCP Vault -clusters are already running. It deploys a build of Nomad from your -local machine along with configuration files. +Terraform provisions the AWS infrastructure assuming that EC2 AMIs have already +been built via Packer and a HCP Vault cluster is already running. It deploys a +build of Nomad from your local machine along with configuration files, as well +as a single-node Consul server cluster. ## Setup @@ -30,8 +30,6 @@ team's vault under `nomad-e2e`. ``` export HCP_CLIENT_ID= export HCP_CLIENT_SECRET= -export CONSUL_HTTP_TOKEN= -export CONSUL_HTTP_ADDR= ``` The Vault admin token will expire after 6 hours. If you haven't @@ -57,6 +55,8 @@ client_count_ubuntu_jammy_amd64 = "4" client_count_windows_2016_amd64 = "1" ``` +You will also need a Consul Enterprise license file. + Optionally, edit the `nomad_local_binary` variable in the `terraform.tfvars` file to change the path to the local binary of Nomad you'd like to upload. diff --git a/e2e/terraform/compute.tf b/e2e/terraform/compute.tf index d340d0ca7fe..ddb101b85fe 100644 --- a/e2e/terraform/compute.tf +++ b/e2e/terraform/compute.tf @@ -58,6 +58,23 @@ resource "aws_instance" "client_windows_2016_amd64" { } } +resource "aws_instance" "consul_server" { + ami = data.aws_ami.ubuntu_jammy_amd64.image_id + instance_type = var.instance_type + key_name = module.keys.key_name + vpc_security_group_ids = [aws_security_group.consul_server.id] + iam_instance_profile = data.aws_iam_instance_profile.nomad_e2e_cluster.name + availability_zone = var.availability_zone + + # Instance tags + tags = { + Name = "${local.random_name}-consul-server-ubuntu-jammy-amd64" + ConsulAutoJoin = "auto-join-${local.random_name}" + User = data.aws_caller_identity.current.arn + } +} + + data "external" "packer_sha" { program = ["/bin/sh", "-c", </dev/null 2>&1 && pwd )" + +echo "waiting for Consul leader to be up..." +while true : +do + consul info && break + echo "Consul server not ready, waiting 5s" + sleep 5 +done + +consul acl bootstrap || echo "Consul ACLs already bootstrapped" + +if [ $(consul info | grep -q "version_metadata = ent") ]; then + echo "writing namespaces" + consul namespace create -name "prod" + consul namespace create -name "dev" +fi + +echo "writing Nomad cluster policy and token" +consul acl policy create -name nomad-cluster -rules @${DIR}/nomad-cluster-consul-policy.hcl +consul acl token create -policy-name=nomad-cluster -secret "$NOMAD_CLUSTER_CONSUL_TOKEN" + +echo "writing Consul cluster policy and token" +consul acl policy create -name consul-agents -rules @${DIR}/consul-agents-policy.hcl +consul acl token create -policy-name=consul-agents -secret "$CONSUL_AGENT_TOKEN" diff --git a/e2e/terraform/scripts/consul-agents-policy.hcl b/e2e/terraform/scripts/consul-agents-policy.hcl new file mode 100644 index 00000000000..28d74cf503d --- /dev/null +++ b/e2e/terraform/scripts/consul-agents-policy.hcl @@ -0,0 +1,12 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# Consul agents only need to register themselves and read services + +node "*" { + policy = "write" +} + +service_prefix "" { + policy = "read" +} diff --git a/e2e/terraform/scripts/nomad-cluster-consul-policy.hcl b/e2e/terraform/scripts/nomad-cluster-consul-policy.hcl new file mode 100644 index 00000000000..c07dc09b03a --- /dev/null +++ b/e2e/terraform/scripts/nomad-cluster-consul-policy.hcl @@ -0,0 +1,34 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +// The Nomad Client will be registering things into its buddy Consul Client. +// Note: because we also test the use of Consul namespaces, this token must be +// able to register services, read the keystore, and read node data for any +// namespace. +// The operator=write permission is required for creating config entries for +// connect ingress gateways. operator ACLs are not namespaced, though the +// config entries they can generate are. +operator = "write" + +agent_prefix "" { + policy = "read" +} + +namespace_prefix "" { + // The acl=write permission is required for generating Consul Service Identity + // tokens for consul connect services. Those services could be configured for + // any Consul namespace the job-submitter has access to. + acl = "write" + + key_prefix "" { + policy = "read" + } + + node_prefix "" { + policy = "read" + } + + service_prefix "" { + policy = "write" + } +} diff --git a/e2e/terraform/terraform.tfvars b/e2e/terraform/terraform.tfvars index f531e298327..324bb7c7ce3 100644 --- a/e2e/terraform/terraform.tfvars +++ b/e2e/terraform/terraform.tfvars @@ -8,5 +8,8 @@ nomad_local_binary = "../../pkg/linux_amd64/nomad" nomad_local_binary_client_windows_2016_amd64 = ["../../pkg/windows_amd64/nomad.exe"] -# For testing enterprise, set via --var: +# The Consul server is Consul Enterprise, so provide a license via --var: +# consul_license = + +# For testing Nomad enterprise, also set via --var: # nomad_license = diff --git a/e2e/terraform/tls_ca.tf b/e2e/terraform/tls_ca.tf index e30da79ca5f..992c165b5ca 100644 --- a/e2e/terraform/tls_ca.tf +++ b/e2e/terraform/tls_ca.tf @@ -22,12 +22,12 @@ resource "tls_self_signed_cert" "ca" { allowed_uses = ["cert_signing"] } -resource "local_file" "ca_key" { +resource "local_sensitive_file" "ca_key" { filename = "keys/tls_ca.key" content = tls_private_key.ca.private_key_pem } -resource "local_file" "ca_cert" { +resource "local_sensitive_file" "ca_cert" { filename = "keys/tls_ca.crt" content = tls_self_signed_cert.ca.cert_pem } diff --git a/e2e/terraform/variables.tf b/e2e/terraform/variables.tf index 5867f209c02..81c612c9bdc 100644 --- a/e2e/terraform/variables.tf +++ b/e2e/terraform/variables.tf @@ -53,7 +53,13 @@ variable "nomad_local_binary" { variable "nomad_license" { type = string - description = "If nomad_license is set, deploy a license to override the temporary license" + description = "If nomad_license is set, deploy a license" + default = "" +} + +variable "consul_license" { + type = string + description = "If consul_license is set, deploy a license" default = "" }