Included SPOT instance deployments for json and timeseries (#88)

* Added data-tiering r6gd 2xlarge setup (~=200GB)

* Included SPOT instance deployments for json and timeseries

terraform/oss-redisbloom-m5-spot-instances/bench-client-resources.tf

@@ -19,28 +19,6 @@ resource "aws_spot_instance_request" "client" {
     delete_on_termination = true
   }
 
-  volume_tags = {
-    Name           = "ebs_block_device-${var.setup_name}-CLIENT-${count.index + 1}"
-    setup          = "${var.setup_name}"
-    triggering_env = "${var.triggering_env}"
-    github_actor   = "${var.github_actor}"
-    github_org     = "${var.github_org}"
-    github_repo    = "${var.github_repo}"
-    github_sha     = "${var.github_sha}"
-    timeout_secs   = "${var.timeout_secs}"
-  }
-
-  tags = {
-    Name           = "${var.setup_name}-CLIENT-${count.index + 1}"
-    setup          = "${var.setup_name}"
-    triggering_env = "${var.triggering_env}"
-    github_actor   = "${var.github_actor}"
-    github_org     = "${var.github_org}"
-    github_repo    = "${var.github_repo}"
-    github_sha     = "${var.github_sha}"
-    timeout_secs   = "${var.timeout_secs}"
-  }
-
   ################################################################################
   # This will ensure we wait here until the instance is ready to receive the ssh connection 
   ################################################################################

terraform/oss-redisbloom-m5-spot-instances/db-resources.tf

@@ -30,16 +30,6 @@ resource "aws_spot_instance_request" "server" {
     timeout_secs   = "${var.timeout_secs}"
   }
 
-  tags = {
-    Name           = "${var.setup_name}-DB-${count.index + 1}"
-    setup          = "${var.setup_name}"
-    triggering_env = "${var.triggering_env}"
-    github_actor   = "${var.github_actor}"
-    github_org     = "${var.github_org}"
-    github_repo    = "${var.github_repo}"
-    github_sha     = "${var.github_sha}"
-    timeout_secs   = "${var.timeout_secs}"
-  }
 
   ################################################################################
   # This will ensure we wait here until the instance is ready to receive the ssh connection 

terraform/oss-redisbloom-m5-spot-instances/tags.tf

@@ -0,0 +1,41 @@
+
+
+locals {
+
+  tags_db = {
+    Name           = "${var.setup_name}-DB-1"
+    setup          = "${var.setup_name}"
+    triggering_env = "${var.triggering_env}"
+    github_actor   = "${var.github_actor}"
+    github_org     = "${var.github_org}"
+    github_repo    = "${var.github_repo}"
+    github_sha     = "${var.github_sha}"
+    timeout_secs   = "${var.timeout_secs}"
+  }
+
+  tags_client = {
+    Name           = "${var.setup_name}-CLIENT-1"
+    setup          = "${var.setup_name}"
+    triggering_env = "${var.triggering_env}"
+    github_actor   = "${var.github_actor}"
+    github_org     = "${var.github_org}"
+    github_repo    = "${var.github_repo}"
+    github_sha     = "${var.github_sha}"
+    timeout_secs   = "${var.timeout_secs}"
+  }
+}
+
+resource "aws_ec2_tag" "server" {
+  resource_id = aws_spot_instance_request.server[0].spot_instance_id
+  for_each    = local.tags_db
+  key         = each.key
+  value       = each.value
+}
+
+
+resource "aws_ec2_tag" "client" {
+  resource_id = aws_spot_instance_request.client[0].spot_instance_id
+  for_each    = local.tags_client
+  key         = each.key
+  value       = each.value
+}

terraform/oss-redisbloom-m5-spot-instances/variables.tf

@@ -4,7 +4,7 @@
 
 variable "setup_name" {
   description = "setup name"
-  default     = "oss-standalone-redisbloom-m5"
+  default     = "oss-redisbloom-m5-spot-instances"
 }
 variable "github_actor" {
   description = "The name of the person or app that initiated the deployment."

terraform/oss-redisjson-m5-spot-instances/README.md

@@ -0,0 +1,9 @@
+# oss-standalone-redisbloom-m5
+
+Deploy Multi-VM benchmark scenario, including 1 client and 1 DB machine.
+- Cloud provider: AWS
+- OS: Ubuntu 20.04
+- Client machine: c5.4xlarge
+- Benchmark machine: m5.8xlarge
+
+-------

terraform/oss-redisjson-m5-spot-instances/bench-client-resources.tf

@@ -0,0 +1,41 @@
+
+resource "aws_spot_instance_request" "client" {
+  wait_for_fulfillment        = true
+  spot_type                   = "one-time"
+  count                       = var.client_instance_count
+  ami                         = var.instance_ami
+  instance_type               = var.client_instance_type
+  subnet_id                   = data.terraform_remote_state.shared_resources.outputs.subnet_public_id
+  vpc_security_group_ids      = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"]
+  key_name                    = var.key_name
+  associate_public_ip_address = "true"
+  placement_group             = data.terraform_remote_state.shared_resources.outputs.perf_cto_pg_name
+  availability_zone           = "us-east-2a"
+
+  root_block_device {
+    volume_size           = var.instance_volume_size
+    volume_type           = var.instance_volume_type
+    encrypted             = var.instance_volume_encrypted
+    delete_on_termination = true
+  }
+
+  ################################################################################
+  # This will ensure we wait here until the instance is ready to receive the ssh connection 
+  ################################################################################
+  provisioner "remote-exec" {
+    script = "./../scripts/wait_for_instance.sh"
+    connection {
+      host        = self.public_ip # The `self` variable is like `this` in many programming languages
+      type        = "ssh"          # in this case, `self` is the resource (the server).
+      user        = var.ssh_user
+      private_key = file(var.private_key)
+      #need to increase timeout to larger then 5m for metal instances
+      timeout = "15m"
+      agent   = "false"
+    }
+  }
+
+  ################################################################################
+  # Deployment related
+  ################################################################################
+}

terraform/oss-redisjson-m5-spot-instances/common.tf

@@ -0,0 +1,11 @@
+
+################################################################################
+# This is the bucket holding this specific setup tfstate
+################################################################################
+terraform {
+  backend "s3" {
+    bucket = "performance-cto-group"
+    region = "us-east-1"
+  }
+}
+

terraform/oss-redisjson-m5-spot-instances/db-resources.tf

@@ -0,0 +1,53 @@
+
+resource "aws_spot_instance_request" "server" {
+  wait_for_fulfillment        = true
+  spot_type                   = "one-time"
+  count                       = var.server_instance_count
+  ami                         = var.instance_ami
+  instance_type               = var.server_instance_type
+  subnet_id                   = data.terraform_remote_state.shared_resources.outputs.subnet_public_id
+  vpc_security_group_ids      = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"]
+  key_name                    = var.key_name
+  associate_public_ip_address = "true"
+  placement_group             = data.terraform_remote_state.shared_resources.outputs.perf_cto_pg_name
+  availability_zone           = "us-east-2a"
+
+  root_block_device {
+    volume_size           = var.instance_volume_size
+    volume_type           = var.instance_volume_type
+    encrypted             = var.instance_volume_encrypted
+    delete_on_termination = true
+  }
+
+  volume_tags = {
+    Name           = "ebs_block_device-${var.setup_name}-DB-${count.index + 1}"
+    setup          = "${var.setup_name}"
+    triggering_env = "${var.triggering_env}"
+    github_actor   = "${var.github_actor}"
+    github_org     = "${var.github_org}"
+    github_repo    = "${var.github_repo}"
+    github_sha     = "${var.github_sha}"
+    timeout_secs   = "${var.timeout_secs}"
+  }
+
+
+  ################################################################################
+  # This will ensure we wait here until the instance is ready to receive the ssh connection 
+  ################################################################################
+  provisioner "remote-exec" {
+    script = "./../scripts/wait_for_instance.sh"
+    connection {
+      host        = self.public_ip # The `self` variable is like `this` in many programming languages
+      type        = "ssh"          # in this case, `self` is the resource (the server).
+      user        = var.ssh_user
+      private_key = file(var.private_key)
+      #need to increase timeout to larger then 5m for metal instances
+      timeout = "15m"
+      agent   = "false"
+    }
+  }
+
+  ################################################################################
+  # Deployment related
+  ################################################################################
+}

terraform/oss-redisjson-m5-spot-instances/output.tf

@@ -0,0 +1,15 @@
+output "server_public_ip" {
+  value = ["${aws_spot_instance_request.server[0].public_ip}"]
+}
+
+output "server_private_ip" {
+  value = ["${aws_spot_instance_request.server[0].private_ip}"]
+}
+
+output "client_public_ip" {
+  value = ["${aws_spot_instance_request.client[0].public_ip}"]
+}
+
+output "client_private_ip" {
+  value = ["${aws_spot_instance_request.client[0].private_ip}"]
+}

terraform/oss-redisjson-m5-spot-instances/shared_resources.tf

@@ -0,0 +1,17 @@
+# provider
+provider "aws" {
+  region = var.region
+}
+
+################################################################################
+# This is the shared resources bucket key -- you will need it across environments like security rules,etc...
+# !! do not change this !!
+################################################################################
+data "terraform_remote_state" "shared_resources" {
+  backend = "s3"
+  config = {
+    bucket = "performance-cto-group"
+    key    = "benchmarks/infrastructure/shared_resources.tfstate"
+    region = "us-east-1"
+  }
+}

terraform/oss-redisjson-m5-spot-instances/tags.tf

@@ -0,0 +1,41 @@
+
+
+locals {
+
+  tags_db = {
+    Name           = "${var.setup_name}-DB-1"
+    setup          = "${var.setup_name}"
+    triggering_env = "${var.triggering_env}"
+    github_actor   = "${var.github_actor}"
+    github_org     = "${var.github_org}"
+    github_repo    = "${var.github_repo}"
+    github_sha     = "${var.github_sha}"
+    timeout_secs   = "${var.timeout_secs}"
+  }
+
+  tags_client = {
+    Name           = "${var.setup_name}-CLIENT-1"
+    setup          = "${var.setup_name}"
+    triggering_env = "${var.triggering_env}"
+    github_actor   = "${var.github_actor}"
+    github_org     = "${var.github_org}"
+    github_repo    = "${var.github_repo}"
+    github_sha     = "${var.github_sha}"
+    timeout_secs   = "${var.timeout_secs}"
+  }
+}
+
+resource "aws_ec2_tag" "server" {
+  resource_id = aws_spot_instance_request.server[0].spot_instance_id
+  for_each    = local.tags_db
+  key         = each.key
+  value       = each.value
+}
+
+
+resource "aws_ec2_tag" "client" {
+  resource_id = aws_spot_instance_request.client[0].spot_instance_id
+  for_each    = local.tags_client
+  key         = each.key
+  value       = each.value
+}