Merge pull request #73 from hashicorp/aws-provider-3

AWS Provider 3.x and other fixes

examples/nomad-consul-ami/nomad-consul-docker.json

@@ -4,12 +4,13 @@
     "aws_region": "us-east-1",
     "nomad_version": "0.9.1",
     "consul_module_version": "v0.7.4",
-    "consul_version": "1.5.1"
+    "consul_version": "1.5.1",
+    "ami_name_prefix": "nomad-consul"
   },
   "builders": [
     {
       "name": "ubuntu18-ami",
-      "ami_name": "nomad-consul-docker-ubuntu18-{{isotime | clean_ami_name}}",
+      "ami_name": "{{user `ami_name_prefix`}}-docker-ubuntu18-{{isotime | clean_ami_name}}",
       "ami_description": "An example of how to build an Ubuntu 18.04 AMI that has Nomad, Consul and Docker",
       "instance_type": "t2.micro",
       "region": "{{user `aws_region`}}",
@@ -31,7 +32,7 @@
     },
     {
       "name": "ubuntu16-ami",
-      "ami_name": "nomad-consul-docker-ubuntu16-{{isotime | clean_ami_name}}",
+      "ami_name": "{{user `ami_name_prefix`}}-docker-ubuntu16-{{isotime | clean_ami_name}}",
       "ami_description": "An Ubuntu 16.04 AMI that has Nomad, Consul and Docker installed.",
       "instance_type": "t2.micro",
       "region": "{{user `aws_region`}}",
@@ -52,7 +53,7 @@
       "ssh_username": "ubuntu"
     },
     {
-      "ami_name": "nomad-consul-docker-amazon-linux-2-{{isotime | clean_ami_name}}",
+      "ami_name": "{{user `ami_name_prefix`}}-docker-amazon-linux-2-{{isotime | clean_ami_name}}",
       "ami_description": "An Amazon Linux 2 AMI that has Nomad, Consul and Docker installed.",
       "instance_type": "t2.micro",
       "name": "amazon-linux-2-ami",

examples/nomad-consul-ami/nomad-consul.json

@@ -4,12 +4,13 @@
     "aws_region": "us-east-1",
     "nomad_version": "0.9.1",
     "consul_module_version": "v0.7.4",
-    "consul_version": "1.5.1"
+    "consul_version": "1.5.1",
+    "ami_name_prefix": "nomad-consul"
   },
   "builders": [
     {
       "name": "ubuntu18-ami",
-      "ami_name": "nomad-consul-ubuntu18-{{isotime | clean_resource_name}}",
+      "ami_name": "{{user `ami_name_prefix`}}-ubuntu18-{{isotime | clean_resource_name}}",
       "ami_description": "An example of how to build an Ubuntu 18.04 AMI that has Nomad and Consul installed",
       "instance_type": "t2.micro",
       "region": "{{user `aws_region`}}",
@@ -30,7 +31,7 @@
       "ssh_username": "ubuntu"
     },
     {
-      "ami_name": "nomad-consul-ubuntu-{{isotime | clean_resource_name}}",
+      "ami_name": "{{user `ami_name_prefix`}}-ubuntu-{{isotime | clean_resource_name}}",
       "ami_description": "An Ubuntu 16.04 AMI that has Nomad and Consul installed.",
       "instance_type": "t2.micro",
       "name": "ubuntu16-ami",
@@ -52,7 +53,7 @@
       "ssh_username": "ubuntu"
     },
     {
-      "ami_name": "nomad-consul-amazon-linux-2-{{isotime | clean_resource_name}}",
+      "ami_name": "{{user `ami_name_prefix`}}-amazon-linux-2-{{isotime | clean_resource_name}}",
       "ami_description": "An Amazon Linux 2 AMI that has Nomad and Consul installed.",
       "instance_type": "t2.micro",
       "name": "amazon-linux-2-ami",

examples/nomad-consul-separate-cluster/main.tf

@@ -95,7 +95,7 @@ module "nomad_servers" {
 # ---------------------------------------------------------------------------------------------------------------------
 
 module "consul_iam_policies_servers" {
-  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-iam-policies?ref=v0.7.0"
+  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-iam-policies?ref=v0.7.7"
 
   iam_role_id = module.nomad_servers.iam_role_id
 }
@@ -120,7 +120,7 @@ data "template_file" "user_data_nomad_server" {
 # ---------------------------------------------------------------------------------------------------------------------
 
 module "consul_servers" {
-  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-cluster?ref=v0.7.0"
+  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-cluster?ref=v0.7.7"
 
   cluster_name  = "${var.consul_cluster_name}-server"
   cluster_size  = var.num_consul_servers
@@ -206,7 +206,7 @@ module "nomad_clients" {
 # ---------------------------------------------------------------------------------------------------------------------
 
 module "consul_iam_policies_clients" {
-  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-iam-policies?ref=v0.7.0"
+  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-iam-policies?ref=v0.7.7"
 
   iam_role_id = module.nomad_clients.iam_role_id
 }

main.tf

@@ -58,7 +58,7 @@ data "aws_ami" "nomad_consul" {
 # ---------------------------------------------------------------------------------------------------------------------
 
 module "servers" {
-  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-cluster?ref=v0.7.0"
+  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-cluster?ref=v0.7.7"
 
   cluster_name  = "${var.cluster_name}-server"
   cluster_size  = var.num_servers
@@ -177,7 +177,7 @@ module "clients" {
 # ---------------------------------------------------------------------------------------------------------------------
 
 module "consul_iam_policies" {
-  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-iam-policies?ref=v0.7.0"
+  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-iam-policies?ref=v0.7.7"
 
   iam_role_id = module.clients.iam_role_id
 }

modules/nomad-cluster/variables.tf

@@ -62,13 +62,13 @@ variable "asg_name" {
 variable "subnet_ids" {
   description = "The subnet IDs into which the EC2 Instances should be deployed. We recommend one subnet ID per node in the cluster_size variable. At least one of var.subnet_ids or var.availability_zones must be non-empty."
   type        = list(string)
-  default     = []
+  default     = null
 }
 
 variable "availability_zones" {
   description = "The availability zones into which the EC2 Instances should be deployed. We recommend one availability zone per node in the cluster_size variable. At least one of var.subnet_ids or var.availability_zones must be non-empty."
   type        = list(string)
-  default     = []
+  default     = null
 }
 
 variable "ssh_key_name" {

modules/nomad-security-group-rules/README.md

@@ -11,7 +11,7 @@ servers that have both Nomad and Consul on each node:
 
 ```hcl
 module "consul_servers" {
-  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-cluster?ref=v0.7.0"
+  source = "github.com/hashicorp/terraform-aws-consul//modules/consul-cluster?ref=v0.7.7"
 
   # This AMI has both Nomad and Consul installed
   ami_id = "ami-1234abcd"

test/nomad_helpers.go

@@ -25,6 +25,7 @@ const VAR_AMI_ID = "ami_id"
 
 const CLUSTER_COLOCATED_EXAMPLE_PATH = "/"
 const CLUSTER_COLOCATED_EXAMPLE_VAR_CLUSTER_NAME = "cluster_name"
+const CLUSTER_COLOCATED_EXAMPLE_VAR_CLUSTER_TAG_VALUE = "cluster_tag_value"
 const CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_SERVERS = "num_servers"
 const CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_CLIENTS = "num_clients"
 const CLUSTER_COLOCATED_EXAMPLE_OUTPUT_SERVER_ASG_NAME = "asg_name_servers"
@@ -41,6 +42,7 @@ const DEFAULT_NUM_SERVERS = 3
 const DEFAULT_NUM_CLIENTS = 6
 
 const SAVED_AWS_REGION = "AwsRegion"
+const SAVED_UNIQUE_ID = "UniqueId"
 
 // Test the Nomad/Consul colocated cluster example by:
 //
@@ -65,21 +67,25 @@ func runNomadClusterColocatedTest(t *testing.T, packerBuildName string) {
 		awsRegion := getRandomRegion(t)
 		test_structure.SaveString(t, examplesDir, SAVED_AWS_REGION, awsRegion)
 
-		amiId := buildAmi(t, filepath.Join(examplesDir, "examples", "nomad-consul-ami", "nomad-consul.json"), packerBuildName, awsRegion)
+		uniqueId := random.UniqueId()
+		test_structure.SaveString(t, examplesDir, SAVED_UNIQUE_ID, uniqueId)
+
+		amiId := buildAmi(t, filepath.Join(examplesDir, "examples", "nomad-consul-ami", "nomad-consul.json"), packerBuildName, awsRegion, uniqueId)
 		test_structure.SaveAmiId(t, examplesDir, amiId)
 	})
 
 	test_structure.RunTestStage(t, "deploy", func() {
 		amiId := test_structure.LoadAmiId(t, examplesDir)
 		awsRegion := test_structure.LoadString(t, examplesDir, SAVED_AWS_REGION)
-		uniqueId := random.UniqueId()
+		uniqueId := test_structure.LoadString(t, examplesDir, SAVED_UNIQUE_ID)
 
 		terraformOptions := &terraform.Options{
 			TerraformDir: examplesDir,
 			Vars: map[string]interface{}{
-				CLUSTER_COLOCATED_EXAMPLE_VAR_CLUSTER_NAME: fmt.Sprintf("test-%s", uniqueId),
-				CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_SERVERS:  DEFAULT_NUM_SERVERS,
-				CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_CLIENTS:  DEFAULT_NUM_CLIENTS,
+				CLUSTER_COLOCATED_EXAMPLE_VAR_CLUSTER_NAME:      fmt.Sprintf("test-%s", uniqueId),
+				CLUSTER_COLOCATED_EXAMPLE_VAR_CLUSTER_TAG_VALUE: fmt.Sprintf("auto-join-%s", uniqueId),
+				CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_SERVERS:       DEFAULT_NUM_SERVERS,
+				CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_CLIENTS:       DEFAULT_NUM_CLIENTS,
 				VAR_AMI_ID: amiId,
 			},
 			EnvVars: map[string]string{
@@ -122,14 +128,17 @@ func runNomadClusterSeparateTest(t *testing.T, packerBuildName string) {
 		awsRegion := getRandomRegion(t)
 		test_structure.SaveString(t, examplesDir, SAVED_AWS_REGION, awsRegion)
 
-		amiId := buildAmi(t, filepath.Join(examplesDir, "examples", "nomad-consul-ami", "nomad-consul.json"), packerBuildName, awsRegion)
+		uniqueId := random.UniqueId()
+		test_structure.SaveString(t, examplesDir, SAVED_UNIQUE_ID, uniqueId)
+
+		amiId := buildAmi(t, filepath.Join(examplesDir, "examples", "nomad-consul-ami", "nomad-consul.json"), packerBuildName, awsRegion, uniqueId)
 		test_structure.SaveAmiId(t, examplesDir, amiId)
 	})
 
 	test_structure.RunTestStage(t, "deploy", func() {
 		amiId := test_structure.LoadAmiId(t, examplesDir)
 		awsRegion := test_structure.LoadString(t, examplesDir, SAVED_AWS_REGION)
-		uniqueId := random.UniqueId()
+		uniqueId := test_structure.LoadString(t, examplesDir, SAVED_UNIQUE_ID)
 
 		terraformOptions := &terraform.Options{
 			TerraformDir: filepath.Join(examplesDir, "examples", "nomad-consul-separate-cluster"),
@@ -171,7 +180,7 @@ func checkNomadClusterIsWorking(t *testing.T, asgNameOutputVar string, terraform
 // 2. The cluster has the expected number of server nodes
 // 2. The cluster has the expected number of client nodes
 func testNomadCluster(t *testing.T, nodeIpAddress string) {
-	maxRetries := 60
+	maxRetries := 90
 	sleepBetweenRetries := 10 * time.Second
 
 	response := retry.DoWithRetry(t, "Check Nomad cluster has expected number of servers and clients", maxRetries, sleepBetweenRetries, func() (string, error) {

test/terratest_helpers.go

@@ -1,19 +1,22 @@
 package test
 
 import (
+	"fmt"
 	"testing"
 	"github.com/gruntwork-io/terratest/modules/packer"
 )
 
 const CONSUL_AMI_TEMPLATE_VAR_REGION = "aws_region"
+const CONSUL_AMI_TEMPLATE_VAR_AMI_PREFIX = "ami_name_prefix"
 
 // Use Packer to build the AMI in the given packer template, with the given build name, and return the AMI's ID
-func buildAmi(t *testing.T, packerTemplatePath string, packerBuildName string, awsRegion string) string {
+func buildAmi(t *testing.T, packerTemplatePath string, packerBuildName string, awsRegion string, uniqueId string) string {
 	options := &packer.Options{
 		Template: packerTemplatePath,
 		Only:     packerBuildName,
 		Vars: map[string]string{
-			CONSUL_AMI_TEMPLATE_VAR_REGION: awsRegion,
+			CONSUL_AMI_TEMPLATE_VAR_REGION:     awsRegion,
+			CONSUL_AMI_TEMPLATE_VAR_AMI_PREFIX: fmt.Sprintf("nomad-consul-%s", uniqueId),
 		},
 	}