Merge branch 'main' into test/storage-limit-increment

kruskall · Jun 3, 2024 · 82b2b26 · 82b2b26
2 parents 5fbcb8f + c67adb9
commit 82b2b26
Show file tree

Hide file tree

Showing 13 changed files with 97 additions and 60 deletions.
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v1.1.0
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20240530134801-98202e442820
+Version: v7.0.0-alpha2.0.20240531144051-d77596ace183
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20240530134801-98202e442820/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20240531144051-d77596ace183/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -596,11 +596,11 @@ License Version 2.0.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-client/v7
-Version: v7.9.0
+Version: v7.10.0
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.9.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.10.0/LICENSE.txt:
 
 ELASTIC LICENSE AGREEMENT
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-44c17e84-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-b3ff27eb-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.15.0-44c17e84-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.15.0-b3ff27eb-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.15.0-44c17e84-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.15.0-b3ff27eb-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

diff --git a/go.mod b/go.mod
@@ -10,8 +10,8 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6
 	github.com/elastic/apm-data v1.1.0
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240530134801-98202e442820
-	github.com/elastic/elastic-agent-client/v7 v7.9.0
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240531144051-d77596ace183
+	github.com/elastic/elastic-agent-client/v7 v7.10.0
 	github.com/elastic/elastic-agent-libs v0.9.10
 	github.com/elastic/elastic-agent-system-metrics v0.10.2
 	github.com/elastic/gmux v0.3.2

diff --git a/go.sum b/go.sum
@@ -147,12 +147,12 @@ github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6 h1:Js+C3HE
 github.com/elastic/apm-aggregation v0.0.0-20230815024520-e75a37d9ddd6/go.mod h1:ba3gaJCuhxXN/O5AuiI56xxd6DukQdVOK0NfpzBntNo=
 github.com/elastic/apm-data v1.1.0 h1:5ahaTQwWb6+OmrXrc1dzfD2wjvTG7yJRaldcsc9dIc4=
 github.com/elastic/apm-data v1.1.0/go.mod h1:TLHPd2H8wYlf3FIqriQ018NZzE1FLC6VtsQ6DN0vRmY=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240530134801-98202e442820 h1:af3xXbz/B6snKZoVM9YujAmNTqp22jZRSUIU24QqloA=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240530134801-98202e442820/go.mod h1:Sgu+4L2XSFMQ0YvZLxbhKxkesTQkoetgeUhdkel9uIM=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240531144051-d77596ace183 h1:TZ0bnwJTCAMtEayuGvp15S+zBaaDQrZtUcfTJXfNKTk=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240531144051-d77596ace183/go.mod h1:Sgu+4L2XSFMQ0YvZLxbhKxkesTQkoetgeUhdkel9uIM=
 github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP+WioV73ia17Et+UlFbz8=
 github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8=
-github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE=
-github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss=
+github.com/elastic/elastic-agent-client/v7 v7.10.0 h1:qcz5EHOI+Jh8QHVGLAOQ9BRXORTYjcziXq1y4reESAk=
+github.com/elastic/elastic-agent-client/v7 v7.10.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss=
 github.com/elastic/elastic-agent-libs v0.9.10 h1:t6eM2s2EXv/AzXXdFy8eWS+/HAzE7AZL1yo3lSAPsfo=
 github.com/elastic/elastic-agent-libs v0.9.10/go.mod h1:CpBqDK8Aql/ou57UR4bWNQt0Cfr14rTRYBmWyiwDpW0=
 github.com/elastic/elastic-agent-system-metrics v0.10.2 h1:AVW+YqgezR0mNOZ80NxPLH3tiYMenNGZ8SC/bIUf4Uc=

diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml
@@ -3,7 +3,7 @@ kind: ApmServer
 metadata:
   name: apm-server
 spec:
-  version: 8.15.0-44c17e84-SNAPSHOT
+  version: 8.15.0-b3ff27eb-SNAPSHOT
   count: 1
   http:
     tls:

diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml
@@ -3,7 +3,7 @@ kind: Elasticsearch
 metadata:
   name: elasticsearch
 spec:
-  version: 8.15.0-44c17e84-SNAPSHOT
+  version: 8.15.0-b3ff27eb-SNAPSHOT
   auth:
     fileRealm:
       - secretName: elasticsearch-admin

diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml
@@ -3,7 +3,7 @@ kind: Kibana
 metadata:
   name: kibana
 spec:
-  version: 8.15.0-44c17e84-SNAPSHOT
+  version: 8.15.0-b3ff27eb-SNAPSHOT
   count: 1
   elasticsearchRef:
     name: elasticsearch

diff --git a/testing/infra/terraform/modules/standalone_apm_server/latest_apm_server.sh b/testing/infra/terraform/modules/standalone_apm_server/latest_apm_server.sh
@@ -8,4 +8,4 @@ if [[ -z ${VERSION} ]] || [[ "${VERSION}" == "latest" ]]; then
 fi
 LATEST_BUILD=$(curl -s "https://artifacts-api.elastic.co/v1/versions/${VERSION}/builds/" | jq -r '.builds[0]')
 
-curl -s "https://artifacts-api.elastic.co/v1/versions/${VERSION}/builds/${LATEST_BUILD}/projects/apm-server" | jq -r ".project.packages | {deb: .\"apm-server-${VERSION}-amd64.deb\".url, rpm: .\"apm-server-${VERSION}-x86_64.rpm\".url }"
+curl -s "https://artifacts-api.elastic.co/v1/versions/${VERSION}/builds/${LATEST_BUILD}/projects/apm-server" | jq -r ".project.packages | {deb_amd: .\"apm-server-${VERSION}-amd64.deb\".url, deb_arm: .\"apm-server-${VERSION}-arm64.deb\".url, rpm_amd: .\"apm-server-${VERSION}-x86_64.rpm\".url, rpm_arm: .\"apm-server-${VERSION}-aarch64.rpm\".url }"
diff --git a/testing/infra/terraform/modules/standalone_apm_server/latest_elastic_agent.sh b/testing/infra/terraform/modules/standalone_apm_server/latest_elastic_agent.sh
@@ -8,4 +8,4 @@ if [[ -z ${VERSION} ]] || [[ "${VERSION}" == "latest" ]]; then
 fi
 LATEST_BUILD=$(curl -s "https://artifacts-api.elastic.co/v1/versions/${VERSION}/builds/" | jq -r '.builds[0]')
 
-curl -s "https://artifacts-api.elastic.co/v1/versions/${VERSION}/builds/${LATEST_BUILD}/projects/elastic-agent-package" | jq -r ".project.packages | {deb: .\"elastic-agent-${VERSION}-amd64.deb\".url, rpm: .\"elastic-agent-${VERSION}-x86_64.rpm\".url }"
+curl -s "https://artifacts-api.elastic.co/v1/versions/${VERSION}/builds/${LATEST_BUILD}/projects/elastic-agent-package" | jq -r ".project.packages | {deb_amd: .\"elastic-agent-${VERSION}-amd64.deb\".url, deb_arm: .\"elastic-agent-${VERSION}-arm64.deb\".url, rpm_amd: .\"elastic-agent-${VERSION}-x86_64.rpm\".url, rpm_arm: .\"elastic-agent-${VERSION}-aarch64.rpm\".url }"
diff --git a/testing/infra/terraform/modules/standalone_apm_server/main.tf b/testing/infra/terraform/modules/standalone_apm_server/main.tf
@@ -1,43 +1,72 @@
 locals {
   image_owners = {
-    "ubuntu-bionic-18.04-amd64-server" = "099720109477" # canonical
-    "ubuntu-focal-20.04-amd64-server"  = "099720109477" # canonical
-    "ubuntu-jammy-22.04-amd64-server"  = "099720109477" # canonical
-    "debian-10-amd64"                  = "136693071363" # debian
-    "debian-11-amd64"                  = "136693071363" # debian
+    "ubuntu-bionic-18.04-arm64-server" = "099720109477" # canonical
+    "ubuntu-focal-20.04-arm64-server"  = "099720109477" # canonical
+    "ubuntu-jammy-22.04-arm64-server"  = "099720109477" # canonical
+    "debian-10-arm64"                  = "136693071363" # debian
+    "debian-11-arm64"                  = "136693071363" # debian
     "amzn2-ami-kernel-5.10"            = "137112412989" # amazon
     "RHEL-7"                           = "309956199498" # Red Hat
     "RHEL-8"                           = "309956199498" # Red Hat
     "RHEL-9"                           = "309956199498" # Red Hat
   }
+  instance_types = {
+    "ubuntu-bionic-18.04-arm64-server" = "t4g.nano"
+    "ubuntu-focal-20.04-arm64-server"  = "t4g.nano"
+    "ubuntu-jammy-22.04-arm64-server"  = "t4g.nano"
+    "debian-10-arm64"                  = "t4g.nano"
+    "debian-11-arm64"                  = "t4g.nano"
+    "amzn2-ami-kernel-5.10"            = "t4g.nano"
+    "RHEL-7"                           = "t3a.micro" # RHEL-7 doesn't support arm
+    "RHEL-8"                           = "t4g.micro" # RHEL doesn't support nano instances
+    "RHEL-9"                           = "t4g.micro" # RHEL doesn't support nano instances
+  }
+  instance_arch = {
+    "ubuntu-bionic-18.04-arm64-server" = "arm64"
+    "ubuntu-focal-20.04-arm64-server"  = "arm64"
+    "ubuntu-jammy-22.04-arm64-server"  = "arm64"
+    "debian-10-arm64"                  = "arm64"
+    "debian-11-arm64"                  = "arm64"
+    "amzn2-ami-kernel-5.10"            = "arm64"
+    "RHEL-7"                           = "x86_64" # RHEL-7 doesn't support arm
+    "RHEL-8"                           = "arm64"
+    "RHEL-9"                           = "arm64"
+  }
+  instance_ea_provision_cmd = {
+    "ubuntu-bionic-18.04-arm64-server" = "curl ${data.external.latest_elastic_agent.result.deb_arm} -o elastic-agent.deb && sudo dpkg -i elastic-agent.deb"
+    "ubuntu-focal-20.04-arm64-server"  = "curl ${data.external.latest_elastic_agent.result.deb_arm} -o elastic-agent.deb && sudo dpkg -i elastic-agent.deb"
+    "ubuntu-jammy-22.04-arm64-server"  = "curl ${data.external.latest_elastic_agent.result.deb_arm} -o elastic-agent.deb && sudo dpkg -i elastic-agent.deb"
+    "debian-10-arm64"                  = "curl ${data.external.latest_elastic_agent.result.deb_arm} -o elastic-agent.deb && sudo dpkg -i elastic-agent.deb"
+    "debian-11-arm64"                  = "curl ${data.external.latest_elastic_agent.result.deb_arm} -o elastic-agent.deb && sudo dpkg -i elastic-agent.deb"
+    "amzn2-ami-kernel-5.10"            = "curl ${data.external.latest_elastic_agent.result.rpm_arm} -o elastic-agent.rpm && sudo yum -y install elastic-agent.rpm"
+    "RHEL-7"                           = "curl ${data.external.latest_elastic_agent.result.rpm_amd} -o elastic-agent.rpm && sudo yum -y install elastic-agent.rpm"
+    "RHEL-8"                           = "curl ${data.external.latest_elastic_agent.result.rpm_arm} -o elastic-agent.rpm && sudo yum -y install elastic-agent.rpm"
+    "RHEL-9"                           = "curl ${data.external.latest_elastic_agent.result.rpm_arm} -o elastic-agent.rpm && sudo yum -y install elastic-agent.rpm"
+  }
+  instance_standalone_provision_cmd = {
+    "ubuntu-bionic-18.04-arm64-server" = "curl ${data.external.latest_apm_server.result.deb_arm} -o apm-server.deb && sudo dpkg -i apm-server.deb"
+    "ubuntu-focal-20.04-arm64-server"  = "curl ${data.external.latest_apm_server.result.deb_arm} -o apm-server.deb && sudo dpkg -i apm-server.deb"
+    "ubuntu-jammy-22.04-arm64-server"  = "curl ${data.external.latest_apm_server.result.deb_arm} -o apm-server.deb && sudo dpkg -i apm-server.deb"
+    "debian-10-arm64"                  = "curl ${data.external.latest_apm_server.result.deb_arm} -o apm-server.deb && sudo dpkg -i apm-server.deb"
+    "debian-11-arm64"                  = "curl ${data.external.latest_apm_server.result.deb_arm} -o apm-server.deb && sudo dpkg -i apm-server.deb"
+    "amzn2-ami-kernel-5.10"            = "curl ${data.external.latest_apm_server.result.rpm_arm} -o apm-server.rpm && sudo yum -y install apm-server.rpm"
+    "RHEL-7"                           = "curl ${data.external.latest_apm_server.result.rpm_amd} -o apm-server.rpm && sudo yum -y install apm-server.rpm"
+    "RHEL-8"                           = "curl ${data.external.latest_apm_server.result.rpm_arm} -o apm-server.rpm && sudo yum -y install apm-server.rpm"
+    "RHEL-9"                           = "curl ${data.external.latest_apm_server.result.rpm_arm} -o apm-server.rpm && sudo yum -y install apm-server.rpm"
+  }
   image_ssh_users = {
-    "ubuntu-bionic-18.04-amd64-server" = "ubuntu"
-    "ubuntu-focal-20.04-amd64-server"  = "ubuntu"
-    "ubuntu-jammy-22.04-amd64-server"  = "ubuntu"
-    "debian-10-amd64"                  = "admin"
-    "debian-11-amd64"                  = "admin"
+    "ubuntu-bionic-18.04-arm64-server" = "ubuntu"
+    "ubuntu-focal-20.04-arm64-server"  = "ubuntu"
+    "ubuntu-jammy-22.04-arm64-server"  = "ubuntu"
+    "debian-10-arm64"                  = "admin"
+    "debian-11-arm64"                  = "admin"
     "amzn2-ami-kernel-5.10"            = "ec2-user"
     "RHEL-7"                           = "ec2-user"
     "RHEL-8"                           = "ec2-user"
     "RHEL-9"                           = "ec2-user"
   }
   apm_port  = "8200"
   conf_path = "/tmp/local-apm-config.yml"
-  ea_provision_commands = [
-    "curl ${data.external.latest_elastic_agent.result.deb} -o elastic-agent.deb && curl ${data.external.latest_elastic_agent.result.rpm} -o elastic-agent.rpm",
-    "sudo dpkg -i elastic-agent.deb || sudo yum -y install elastic-agent.rpm",
-    "sudo elastic-agent install -n --unprivileged",
-    "sudo cp ${local.conf_path} /etc/elastic-agent/elastic-agent.yml",
-    "sudo systemctl start elastic-agent",
-    "sleep 1",
-  ]
-  standalone_provision_commands = [
-    "curl ${data.external.latest_apm_server.result.deb} -o apm-server.deb && curl ${data.external.latest_apm_server.result.rpm} -o apm-server.rpm",
-    "sudo dpkg -i apm-server.deb || sudo yum -y install apm-server.rpm",
-    "sudo cp ${local.conf_path} /etc/apm-server/apm-server.yml",
-    "sudo systemctl start apm-server",
-    "sleep 1",
-  ]
 }
 
 data "aws_ami" "os" {
@@ -50,7 +79,7 @@ data "aws_ami" "os" {
 
   filter {
     name   = "architecture"
-    values = ["x86_64"]
+    values = [local.instance_arch[var.aws_os]]
   }
 
   filter {
@@ -108,7 +137,7 @@ resource "aws_security_group" "main" {
 
 resource "aws_instance" "apm" {
   ami           = data.aws_ami.os.id
-  instance_type = "t3.micro"
+  instance_type = local.instance_types[var.aws_os]
   key_name      = aws_key_pair.provisioner_key.key_name
 
   connection {
@@ -131,10 +160,18 @@ resource "aws_instance" "apm" {
   }
 
   provisioner "remote-exec" {
-    inline = concat(
-      [for command in local.ea_provision_commands : command if var.ea_managed],
-      [for command in local.standalone_provision_commands : command if !var.ea_managed],
-    )
+    inline = var.ea_managed ? [
+      local.instance_ea_provision_cmd[var.aws_os],
+      "sudo elastic-agent install -n --unprivileged",
+      "sudo cp ${local.conf_path} /etc/elastic-agent/elastic-agent.yml",
+      "sudo systemctl start elastic-agent",
+      "sleep 1",
+      ] : [
+      local.instance_standalone_provision_cmd[var.aws_os],
+      "sudo cp ${local.conf_path} /etc/apm-server/apm-server.yml",
+      "sudo systemctl start apm-server",
+      "sleep 1",
+    ]
   }
 
   vpc_security_group_ids = [aws_security_group.main.id]

diff --git a/testing/smoke/managed/main.tf b/testing/smoke/managed/main.tf
@@ -29,12 +29,12 @@ module "ec_deployment" {
   source = "../../infra/terraform/modules/ec_deployment"
   region = var.region
 
-  deployment_template    = "gcp-compute-optimized-v3"
-  deployment_name_prefix = "smoke-upgrade"
+  deployment_template    = "gcp-vector-search-optimized"
+  deployment_name_prefix = "supported-os-managed"
 
   apm_server_size = "1g"
 
-  elasticsearch_size       = "4g"
+  elasticsearch_size       = "1g"
   elasticsearch_zone_count = 1
 
   stack_version = var.stack_version

diff --git a/testing/smoke/os_matrix.sh b/testing/smoke/os_matrix.sh
@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 
 os_names=(
-    "ubuntu-bionic-18.04-amd64-server"
-    "ubuntu-focal-20.04-amd64-server"
-    "ubuntu-jammy-22.04-amd64-server"
-    "debian-10-amd64"
-    "debian-11-amd64"
+    "ubuntu-bionic-18.04-arm64-server"
+    "ubuntu-focal-20.04-arm64-server"
+    "ubuntu-jammy-22.04-arm64-server"
+    "debian-10-arm64"
+    "debian-11-arm64"
     "amzn2-ami-kernel-5.10"
     "RHEL-7"
     "RHEL-8"

diff --git a/testing/smoke/supported-os/main.tf b/testing/smoke/supported-os/main.tf
@@ -29,12 +29,12 @@ module "ec_deployment" {
   source = "../../infra/terraform/modules/ec_deployment"
   region = var.region
 
-  deployment_template    = "gcp-compute-optimized-v3"
-  deployment_name_prefix = "smoke-upgrade"
+  deployment_template    = "gcp-vector-search-optimized"
+  deployment_name_prefix = "supported-os-standalone"
 
   apm_server_size = "1g"
 
-  elasticsearch_size       = "4g"
+  elasticsearch_size       = "1g"
   elasticsearch_zone_count = 1
 
   stack_version = var.stack_version