Ess terraform + run integration tests on BK agents

.buildkite/bk.integration.pipeline.yml

@@ -0,0 +1,160 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+  ASDF_MAGE_VERSION: 1.14.0
+  ASDF_GOLANG_VERSION: 1.22.6
+  ASDF_TERRAFORM_VERSION: 1.9.3
+
+steps:
+  - label: "Integration tests: packaging"
+    key: "package-it"
+    command: ".buildkite/scripts/steps/integration-package.sh"
+    artifact_paths:
+      - build/distributions/**
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+
+  - label: Start ESS stack for integration tests
+    key: integration-ess
+    depends_on:
+      - package-it
+    command: |
+      #!/usr/bin/env bash
+      set -euo pipefail
+      source .buildkite/scripts/steps/ess_start.sh
+    artifact_paths:
+      - test_infra/ess/*.tfstate
+      - test_infra/ess/*.lock.hcl
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+
+  - group: "Stateful IT (Sudo): Ubuntu"
+    key: integration-tests
+    depends_on:
+      - package-it
+      - integration-ess
+    steps:
+      - label: "Default"
+        key: stateful-ubuntu-default-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "default" "^(TestAPMConfig|TestDiagnosticsOptionalValues|TestIsolatedUnitsDiagnosticsOptionalValues|TestDiagnosticsCommand|TestIsolatedUnitsDiagnosticsCommand|TestEventLogFile|TestFakeComponent|TestFakeIsolatedUnitsComponent|TestOtelFileProcessing|TestOtelLogsIngestion|TestOtelAPMIngestion|TestPackageVersion)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Upgrade"
+        key: stateful-ubuntu-upgrade-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "upgrade" "^(TestUpgradeBrokenPackageVersion|TestStandaloneUpgradeWithGPGFallback|TestStandaloneUpgradeWithGPGFallbackOneRemoteFailing|TestStandaloneUpgradeRollback|TestStandaloneUpgradeRollbackOnRestarts|TestStandaloneUpgradeFailsWhenUpgradeIsInProgress|TestStandaloneUpgradeRetryDownload|TestStandaloneUpgradeSameCommit|TestStandaloneUpgrade|TestStandaloneUpgradeUninstallKillWatcher)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet"
+        key: stateful-ubuntu-fleet-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet" "^(TestLongRunningAgentForLeaks|TestDelayEnroll|TestDelayEnrollUnprivileged|TestInstallAndCLIUninstallWithEndpointSecurity|TestInstallAndUnenrollWithEndpointSecurity|TestInstallWithEndpointSecurityAndRemoveEndpointIntegration|TestEndpointSecurityNonDefaultBasePath|TestEndpointSecurityUnprivileged|TestEndpointSecurityCannotSwitchToUnprivileged|TestEndpointLogsAreCollectedInDiagnostics|TestForceInstallOverProtectedPolicy|TestSetLogLevelFleetManaged|TestLogIngestionFleetManaged|TestMetricsMonitoringCorrectBinaries|TestEndpointAgentServiceMonitoring|TestMonitoringPreserveTextConfig|TestMonitoringLivenessReloadable|TestComponentBuildHashInDiagnostics|TestProxyURL|TestFleetManagedUpgradeUnprivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "FQDN"
+        key: stateful-ubuntu-fqdn-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fqdn" "^(TestFQDN)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Deb"
+        key: stateful-ubuntu-deb-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "deb" "^(TestDebLogIngestFleetManaged|TestDebFleetUpgrade)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet Airgapped"
+        key: stateful-ubuntu-fleet-airgapped-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped" "^(TestFleetAirGappedUpgradeUnprivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet Privileged"
+        key: stateful-ubuntu-fleet-privileged-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-privileged" "^(TestInstallFleetServerBootstrap|TestFleetManagedUpgradePrivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet Airgapped Privileged"
+        key: stateful-ubuntu-fleet-airgapped-privileged-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped-privileged" "^(TestFleetAirGappedUpgradePrivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+  - label: ESS stack cleanup
+    depends_on: integration-tests
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
+      ls -lah test_infra/ess
+      .buildkite/scripts/steps/ess_down.sh
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true

.buildkite/scripts/common2.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ -z "${WORKSPACE-""}" ]]; then
+    WORKSPACE=$(git rev-parse --show-toplevel)
+    export WORKSPACE
+fi
+
+BEAT_VERSION=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' "${WORKSPACE}/version/version.go")
+export BEAT_VERSION
+
+getOSOptions() {
+  case $(uname | tr '[:upper:]' '[:lower:]') in
+    linux*)
+      export AGENT_OS_NAME=linux
+      ;;
+    darwin*)
+      export AGENT_OS_NAME=darwin
+      ;;
+    msys*)
+      export AGENT_OS_NAME=windows
+      ;;
+    *)
+      export AGENT_OS_NAME=notset
+      ;;
+  esac
+  case $(uname -m | tr '[:upper:]' '[:lower:]') in
+    aarch64*)
+      export AGENT_OS_ARCH=arm64
+      ;;
+    arm64*)
+      export AGENT_OS_ARCH=arm64
+      ;;
+    amd64*)
+      export AGENT_OS_ARCH=amd64
+      ;;
+    x86_64*)
+      export AGENT_OS_ARCH=amd64
+      ;;
+    *)
+      export AGENT_OS_ARCH=notset
+      ;;
+  esac
+}
+
+google_cloud_auth() {
+    local keyFile=$1
+
+    gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null
+
+    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
+}

.buildkite/scripts/retry.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -euo pipefail
+
+retry() {
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}

.buildkite/scripts/steps/ess.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+function ess_up() {
+  echo "~~~ Staring ESS Stack"  
+  local WORKSPACE=$(git rev-parse --show-toplevel)
+  local TF_DIR="${WORKSPACE}/test_infra/ess/"
+  local STACK_VERSION=$1
+  local ESS_REGION=${2:-"gcp-us-west2"}
+
+  if [ -z "$STACK_VERSION" ]; then
+    echo "Error: Specify stack version: ess_up [stack_version]" >&2
+    return 1
+  fi
+
+  export EC_API_KEY=$(retry 5 vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
+
+  if [[ -z "${EC_API_KEY}" ]]; then
+    echo "Error: Failed to get EC API key from vault" >&2
+    exit 1
+  fi
+
+  BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
+  BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
+  BUILDKITE_PIPELINE_SLUG="${BUILDKITE_PIPELINE_SLUG:-"elastic-agent-integration-tests"}"
+
+  pushd "${TF_DIR}"    
+  terraform init
+  terraform apply \
+    -auto-approve \
+    -var="stack_version=${STACK_VERSION}" \
+    -var="ess_region=${ESS_REGION}" \
+    -var="creator=${BUILDKITE_BUILD_CREATOR}" \
+    -var="buildkite_id=${BUILDKITE_BUILD_NUMBER}" \
+    -var="pipeline=${BUILDKITE_PIPELINE_SLUG}"
+
+  export ELASTICSEARCH_HOST=$(terraform output -raw es_host)
+  export ELASTICSEARCH_USERNAME=$(terraform output -raw es_username)
+  export ELASTICSEARCH_PASSWORD=$(terraform output -raw es_password)
+  export KIBANA_HOST=$(terraform output -raw kibana_endpoint)
+  export KIBANA_USERNAME=$ELASTICSEARCH_USERNAME
+  export KIBANA_PASSWORD=$ELASTICSEARCH_PASSWORD
+  popd
+}
+
+function ess_down() {
+  echo "~~~ Tearing down the ESS Stack"  
+  local WORKSPACE=$(git rev-parse --show-toplevel)
+  local TF_DIR="${WORKSPACE}/test_infra/ess/"
+  if [ -z "${EC_API_KEY:-}" ]; then
+    export EC_API_KEY=$(retry 5 vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)    
+  fi
+
+  pushd "${TF_DIR}"
+  terraform init
+  terraform destroy -auto-approve
+  popd
+}
+
+function get_git_user_email() {
+  if ! git rev-parse --is-inside-work-tree &>/dev/null; then
+    echo "unknown"  
+    return
+  fi
+
+  local email
+  email=$(git config --get user.email)
+
+  if [ -z "$email" ]; then
+    echo "unknown"  
+  else
+    echo "$email"
+  fi
+}
+

.buildkite/scripts/steps/ess_down.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source .buildkite/scripts/common2.sh
+
+source .buildkite/scripts/steps/ess.sh
+
+ess_down || echo "Failed to stop ESS stack" >&2

.buildkite/scripts/steps/ess_start.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source .buildkite/scripts/common2.sh
+
+source .buildkite/scripts/steps/ess.sh
+
+OVERRIDE_STACK_VERSION="$(cat .package-version)"
+OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT"
+
+ess_up $OVERRIDE_STACK_VERSION || echo "Failed to start ESS stack" >&2
+
+echo "ES_HOST: ${ELASTICSEARCH_HOST}"
+buildkite-agent meta-data set "es.host" $ELASTICSEARCH_HOST
+buildkite-agent meta-data set "es.username" $ELASTICSEARCH_USERNAME
+buildkite-agent meta-data set "es.pwd" $ELASTICSEARCH_PASSWORD
+buildkite-agent meta-data set "kibana.host" $KIBANA_HOST
+buildkite-agent meta-data set "kibana.username" $KIBANA_USERNAME
+buildkite-agent meta-data set "kibana.pwd" $KIBANA_PASSWORD

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source .buildkite/scripts/common2.sh
+source .buildkite/scripts/steps/ess.sh
+
+
+# Override the agent package version using a string with format <major>.<minor>.<patch>
+# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
+# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
+OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)"
+OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"
+
+echo "~~~ Bulding test binaries"
+mage build:testBinaries
+
+ess_up $OVERRIDE_TEST_AGENT_VERSION || echo "Failed to start ESS stack" >&2
+trap 'ess_down' EXIT
+
+echo "~~~ Running integration tests"
+AGENT_VERSION="8.16.0-SNAPSHOT" SNAPSHOT=true TEST_DEFINE_PREFIX=non_sudo_linux gotestsum --no-color -f standard-verbose --junitfile build/TEST-go-integration.xml --jsonfile build/TEST-go-integration.out.json -- -tags integration github.com/elastic/elastic-agent/testing/integration