removed env vars entirely and updated prometheus validator (#173)

aws-observability · Dec 4, 2020 · 27f1cb6 · 27f1cb6
1 parent 2b999b6
commit 27f1cb6
Show file tree

Hide file tree

Showing 14 changed files with 56 additions and 35 deletions.
diff --git a/terraform/basic_components/main.tf b/terraform/basic_components/main.tf
@@ -74,6 +74,8 @@ data "template_file" "otconfig" {
     grpc_port = module.common.grpc_port
     udp_port = module.common.udp_port
     cortex_instance_endpoint = var.cortex_instance_endpoint
+    sample_app_listen_address_host = var.sample_app_listen_address_host
+    sample_app_listen_address_port = var.sample_app_listen_address_port
 
     mock_endpoint = var.mocked_endpoint
   }

diff --git a/terraform/basic_components/variables.tf b/terraform/basic_components/variables.tf
@@ -32,3 +32,11 @@ variable "sample_app" {
 variable "cortex_instance_endpoint" {
   default = ""
 }
+
+variable "sample_app_listen_address_host" {
+  default = ""
+}
+
+variable "sample_app_listen_address_port" {
+  default = ""
+}
diff --git a/terraform/canary/amis.tf b/terraform/canary/amis.tf
@@ -7,7 +7,6 @@ variable "ami_family" {
       otconfig_destination = "/tmp/ot-default.yml"
       download_command_pattern = "wget %s"
       install_command = "sudo rpm -Uvh aws-otel-collector.rpm"
-      set_env_var_command = "sudo chmod 777 /opt/aws/aws-otel-collector/etc/.env && sudo echo 'SAMPLE_APP_HOST=%s' >> /opt/aws/aws-otel-collector/etc/.env && sudo echo 'SAMPLE_APP_PORT=%s' >> /opt/aws/aws-otel-collector/etc/.env"
       start_command = "sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c /tmp/ot-default.yml -a start"
       connection_type = "ssh"
       user_data = ""
@@ -19,7 +18,6 @@ variable "ami_family" {
       otconfig_destination = "C:\\ot-default.yml"
       download_command_pattern = "powershell -command \"Invoke-WebRequest -Uri %s -OutFile C:\\aws-otel-collector.msi\""
       install_command = "msiexec /i C:\\aws-otel-collector.msi"
-      set_env_var_command = "powershell \"[System.Environment]::SetEnvironmentVariable('SAMPLE_APP_HOST', '%s', [System.EnvironmentVariableTarget]::Machine); [System.Environment]::SetEnvironmentVariable('SAMPLE_APP_PORT', '%s', [System.EnvironmentVariableTarget]::Machine)\""
       start_command = "powershell \"& 'C:\\Program Files\\Amazon\\AwsOtelCollector\\aws-otel-collector-ctl.ps1' -ConfigLocation C:\\ot-default.yml -Action start\""
       connection_type = "winrm"
       user_data = <<EOF

diff --git a/terraform/ec2/amis.tf b/terraform/ec2/amis.tf
@@ -22,7 +22,6 @@ variable "ami_family" {
       otconfig_destination = "/tmp/ot-default.yml"
       download_command_pattern = "wget %s"
       install_command = "sudo dpkg -i aws-otel-collector.deb"
-      set_env_var_command = "echo \"no env vars to assign values %s and %s to\""
       start_command = "sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c /tmp/ot-default.yml -a start"
       connection_type = "ssh"
       user_data = ""
@@ -34,7 +33,6 @@ variable "ami_family" {
       otconfig_destination = "/tmp/ot-default.yml"
       download_command_pattern = "wget %s"
       install_command = "sudo rpm -Uvh aws-otel-collector.rpm"
-      set_env_var_command = "sudo chmod 777 /opt/aws/aws-otel-collector/etc/.env && sudo echo 'SAMPLE_APP_HOST=%s' >> /opt/aws/aws-otel-collector/etc/.env && sudo echo 'SAMPLE_APP_PORT=%s' >> /opt/aws/aws-otel-collector/etc/.env"
       start_command = "sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c /tmp/ot-default.yml -a start"
       connection_type = "ssh"
       user_data = ""
@@ -46,7 +44,6 @@ variable "ami_family" {
       otconfig_destination = "C:\\ot-default.yml"
       download_command_pattern = "powershell -command \"Invoke-WebRequest -Uri %s -OutFile C:\\aws-otel-collector.msi\""
       install_command = "msiexec /i C:\\aws-otel-collector.msi"
-      set_env_var_command = "powershell \"[System.Environment]::SetEnvironmentVariable('SAMPLE_APP_HOST', '%s', [System.EnvironmentVariableTarget]::Machine); [System.Environment]::SetEnvironmentVariable('SAMPLE_APP_PORT', '%s', [System.EnvironmentVariableTarget]::Machine)\""
       start_command = "powershell \"& 'C:\\Program Files\\Amazon\\AwsOtelCollector\\aws-otel-collector-ctl.ps1' -ConfigLocation C:\\ot-default.yml -Action start\""
       connection_type = "winrm"
       user_data = <<EOF

diff --git a/terraform/ec2/main.tf b/terraform/ec2/main.tf
@@ -33,6 +33,10 @@ module "basic_components" {
   sample_app = var.sample_app
 
   cortex_instance_endpoint = var.cortex_instance_endpoint
+
+  sample_app_listen_address_host = aws_instance.sidecar.public_ip
+
+  sample_app_listen_address_port = module.common.sample_app_lb_port
 }
 
 provider "aws" {
@@ -224,7 +228,6 @@ resource "null_resource" "start_collector" {
   provisioner "remote-exec" {
     inline = [
       local.ami_family["install_command"],
-      format(local.ami_family["set_env_var_command"], aws_instance.sidecar.public_ip, module.common.sample_app_lb_port),
       local.ami_family["start_command"],
     ]
 

diff --git a/terraform/ec2_setup/amis.tf b/terraform/ec2_setup/amis.tf
@@ -7,7 +7,6 @@ variable "ami_family" {
       otconfig_destination = "/tmp/ot-default.yml"
       download_command_pattern = "wget %s"
       install_command = "sudo rpm -Uvh aws-otel-collector.rpm && sudo chmod 777 /opt/aws/aws-otel-collector/etc/.env && sudo echo 'GODEBUG=madvdontneed=1' >> /opt/aws/aws-otel-collector/etc/.env"
-      set_env_var_command = "sudo chmod 777 /opt/aws/aws-otel-collector/etc/.env && sudo echo 'SAMPLE_APP_HOST=%s' >> /opt/aws/aws-otel-collector/etc/.env && sudo echo 'SAMPLE_APP_PORT=%s' >> /opt/aws/aws-otel-collector/etc/.env"
       start_command = "sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c /tmp/ot-default.yml -a start"
       connection_type = "ssh"
       user_data = ""
@@ -27,7 +26,6 @@ variable "ami_family" {
       otconfig_destination = "C:\\ot-default.yml"
       download_command_pattern = "powershell -command \"Invoke-WebRequest -Uri %s -OutFile C:\\aws-otel-collector.msi\""
       install_command = "msiexec /i C:\\aws-otel-collector.msi"
-      set_env_var_command = "powershell \"[System.Environment]::SetEnvironmentVariable('SAMPLE_APP_HOST', '%s', [System.EnvironmentVariableTarget]::Machine); [System.Environment]::SetEnvironmentVariable('SAMPLE_APP_PORT', '%s', [System.EnvironmentVariableTarget]::Machine)\""
       start_command = "powershell \"& 'C:\\Program Files\\Amazon\\AwsOtelCollector\\aws-otel-collector-ctl.ps1' -ConfigLocation C:\\ot-default.yml -Action start\""
       connection_type = "winrm"
       soaking_cwagent_config = "../templates/cwagent-config/soaking-linux.json.tpl"

diff --git a/terraform/eks/main.tf b/terraform/eks/main.tf
@@ -38,6 +38,10 @@ module "basic_components" {
   sample_app = var.sample_app
 
   cortex_instance_endpoint = var.cortex_instance_endpoint
+
+  sample_app_listen_address_host = var.sample_app_mode == "pull" ? kubernetes_service.pull_mode_sample_app_service[0].load_balancer_ingress.0.hostname : ""
+
+  sample_app_listen_address_port = module.common.sample_app_lb_port
 }
 
 locals {
@@ -389,13 +393,15 @@ resource "kubernetes_service" "mocked_server_service" {
 
 # create service upon the sample app
 resource "kubernetes_service" "sample_app_service" {
+  count = var.sample_app_mode == "push" ? 1 : 0
+
   metadata {
     name = "sample-app"
     namespace = kubernetes_namespace.aoc_ns.metadata[0].name
   }
   spec {
     selector = {
-      app = var.sample_app_mode == "push" ? kubernetes_deployment.sample_app_deployment[0].metadata[0].labels.app : kubernetes_deployment.pull_mode_sample_app_deployment[0].metadata[0].labels.app
+      app = kubernetes_deployment.sample_app_deployment[0].metadata[0].labels.app
     }
 
     type = "LoadBalancer"
@@ -417,7 +423,7 @@ module "validator" {
   region = var.region
   testing_id = module.common.testing_id
   metric_namespace = "${module.common.otel_service_namespace}/${module.common.otel_service_name}"
-  sample_app_endpoint = "http://${kubernetes_service.sample_app_service.load_balancer_ingress.0.hostname}:${module.common.sample_app_lb_port}"
+  sample_app_endpoint = "http://${var.sample_app_mode == "push" ? kubernetes_service.sample_app_service[0].load_balancer_ingress.0.hostname : kubernetes_service.pull_mode_sample_app_service[0].load_balancer_ingress.0.hostname}:${module.common.sample_app_lb_port}"
   mocked_server_validating_url = "http://${kubernetes_service.mocked_server_service.load_balancer_ingress.0.hostname}/check-data"
 
   cortex_instance_endpoint = var.cortex_instance_endpoint

diff --git a/terraform/eks/pull_mode_deployment.tf b/terraform/eks/pull_mode_deployment.tf
@@ -82,16 +82,6 @@ resource "kubernetes_deployment" "pull_mode_aoc_deployment" {
           args = [
             "--config=/aoc/aoc-config.yml"]
 
-          env {
-            name = "SAMPLE_APP_HOST"
-            value = kubernetes_service.sample_app_service.load_balancer_ingress.0.hostname
-          }
-
-          env {
-            name = "SAMPLE_APP_PORT"
-            value = module.common.sample_app_lb_port
-          }
-
           resources {
             requests {
               cpu = "0.2"
@@ -192,4 +182,26 @@ resource "kubernetes_deployment" "pull_mode_sample_app_deployment" {
       }
     }
   }
+}
+
+# create service upon the sample app
+resource "kubernetes_service" "pull_mode_sample_app_service" {
+  count = var.sample_app_mode == "pull" ? 1 : 0
+
+  metadata {
+    name = "sample-app"
+    namespace = kubernetes_namespace.aoc_ns.metadata[0].name
+  }
+  spec {
+    selector = {
+      app = kubernetes_deployment.pull_mode_sample_app_deployment[0].metadata[0].labels.app
+    }
+
+    type = "LoadBalancer"
+
+    port {
+      port = module.common.sample_app_lb_port
+      target_port = module.common.sample_app_listen_address_port
+    }
+  }
 }
diff --git a/terraform/mock/main.tf b/terraform/mock/main.tf
@@ -40,6 +40,8 @@ data "template_file" "otconfig" {
     grpc_port = module.common.grpc_port
     udp_port = module.common.udp_port
     mock_endpoint = local.mock_endpoint
+    sample_app_listen_address_host = "172.17.0.1"
+    sample_app_listen_address_port = module.common.sample_app_listen_address_port
   }
 }
 

diff --git a/terraform/templates/local/docker_compose.tpl b/terraform/templates/local/docker_compose.tpl
@@ -26,8 +26,6 @@ services:
       - AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
       - AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       - GODEBUG=x509ignoreCN=0
-      - SAMPLE_APP_PORT=${sample_app_external_port}
-      - SAMPLE_APP_HOST=172.17.0.1
     depends_on:
       - mocked-server
 

diff --git a/terraform/templates/local/docker_compose_from_source.tpl b/terraform/templates/local/docker_compose_from_source.tpl
@@ -26,8 +26,6 @@ services:
       - AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
       - AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       - GODEBUG=x509ignoreCN=0
-      - SAMPLE_APP_PORT=${sample_app_external_port}
-      - SAMPLE_APP_HOST=172.17.0.1
     depends_on:
       - mocked-server
 

diff --git a/terraform/testcases/prometheus_mock/otconfig.tpl b/terraform/testcases/prometheus_mock/otconfig.tpl
@@ -6,7 +6,7 @@ receivers:
       scrape_configs:
       - job_name: "test-pipeline-job"
         static_configs:
-        - targets: [ $SAMPLE_APP_HOST:$SAMPLE_APP_PORT ]
+        - targets: [ ${sample_app_listen_address_host}:${sample_app_listen_address_port} ]
 exporters:
   awsprometheusremotewrite:
     endpoint: "https://${mock_endpoint}"

diff --git a/terraform/testcases/prometheus_static/otconfig.tpl b/terraform/testcases/prometheus_static/otconfig.tpl
@@ -6,7 +6,7 @@ receivers:
       scrape_configs:
       - job_name: "test-prometheus-sample-app"
         static_configs:
-        - targets: [ $SAMPLE_APP_HOST:$SAMPLE_APP_PORT ]
+        - targets: [ ${sample_app_listen_address_host}:${sample_app_listen_address_port} ]
 exporters:
   awsprometheusremotewrite:
     endpoint: ${cortex_instance_endpoint}/api/v1/remote_write

diff --git a/validator/src/main/java/com/amazon/aoc/validators/PrometheusMetricValidator.java b/validator/src/main/java/com/amazon/aoc/validators/PrometheusMetricValidator.java
@@ -50,14 +50,18 @@ public class PrometheusMetricValidator implements IValidator {
   @Override
   public void validate() throws Exception {
     log.info("Start prometheus metric validating");
-    // get expected metrics
-    final List<PrometheusMetric> expectedMetricList = this.getExpectedMetricList(context);
+    log.info("sleeping to allow metrics to propogate through AOC");
+    TimeUnit.SECONDS.sleep(30);
+    log.info("resuming validation");
 
-    // get metric from cortex
-    CortexService cortexService = new CortexService(context);
     RetryHelper.retry(
         MAX_RETRY_COUNT,
         () -> {
+          // get expected metrics
+          List<PrometheusMetric> expectedMetricList = this.getExpectedMetricList(context);
+          // get metric from cortex
+          CortexService cortexService = new CortexService(context);
+
           List<PrometheusMetric> metricList =
                   this.listMetricFromPrometheus(cortexService, expectedMetricList);
 
@@ -103,11 +107,6 @@ private void compareMetricLists(List<PrometheusMetric> toBeCheckedMetricList,
 
   private List<PrometheusMetric> getExpectedMetricList(Context context) throws Exception {
     if (!validationConfig.getExpectedResultPath().isEmpty()) {
-      // add delay to allow sample app lb to start
-      log.info("sleeping to allow sample app lb to start");
-      TimeUnit.SECONDS.sleep(30);
-      log.info("resuming validation");
-
       log.info("getting expected metrics from sample endpoint");
       PullModeSampleAppClient<List<PrometheusMetric>> pullModeSampleAppClient =
               new PullModeSampleAppClient<>(context,