Skip to content

Commit

Permalink
Merge branch 'main' into dependabot/go_modules/github.com/elastic/ela…
Browse files Browse the repository at this point in the history
…stic-agent-system-metrics-0.8.1
  • Loading branch information
mergify[bot] authored Nov 14, 2023
2 parents 614deba + d1fffb3 commit 40c1bdf
Show file tree
Hide file tree
Showing 90 changed files with 2,778 additions and 1,271 deletions.
5 changes: 2 additions & 3 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@ DOCKER_REGISTRY="docker.elastic.co"
DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
# CI_AGENT_QA_OBS_PATH="kv/ci-shared/observability-ingest/elastic-agent-ess-qa"
CI_ESS_STAGING_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-staging"
CI_ESS_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"


Expand Down Expand Up @@ -55,7 +54,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" ==
export TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE=$(realpath ./gcp.json)

# ESS credentials
export API_KEY_TOKEN=$(vault kv get -field apiKey ${CI_ESS_STAGING_PATH})
export API_KEY_TOKEN=$(vault kv get -field apiKey ${CI_ESS_PATH})
echo ${API_KEY_TOKEN} > ./apiKey
export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey)
fi
Expand Down
6 changes: 5 additions & 1 deletion .buildkite/hooks/pre-exit
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" ==

# Perform cleanup of integration tests resources
echo "--- Cleaning up integration test resources"
TEST_INTEG_AUTH_ESS_REGION=us-east-1 SNAPSHOT=true mage integration:clean
if [[ "$BUILDKITE_STEP_KEY" == "serverless-integration-tests" ]]; then
STACK_PROVISIONER=serverless SNAPSHOT=true mage integration:clean
else
SNAPSHOT=true mage integration:clean
fi
fi

if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
Expand Down
255 changes: 157 additions & 98 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,97 +4,138 @@ env:
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
DOCKER_REGISTRY: "docker.elastic.co"
steps:
- label: "Unit tests - Ubuntu 22.04"
key: "unit-tests-2204"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-ubuntu-2204"
retry:
manual:
allowed: true
- group: "Unit tests"
key: "unit-tests"
steps:
- label: "Unit tests - Ubuntu 22.04"
key: "unit-tests-2204"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-ubuntu-2204"
retry:
manual:
allowed: true

- label: "Unit tests - Ubuntu 22.04 ARM64"
key: "unit-tests-2204-arm64"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "aws"
imagePrefix: "core-ubuntu-2204-aarch64"
diskSizeGb: 200
instanceType: "m6g.4xlarge"
retry:
manual:
allowed: true
- label: "Unit tests - Ubuntu 22.04 ARM64"
key: "unit-tests-2204-arm64"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "aws"
imagePrefix: "core-ubuntu-2204-aarch64"
diskSizeGb: 200
instanceType: "m6g.xlarge"
retry:
manual:
allowed: true

- label: "Unit tests - Windows 2022"
key: "unit-tests-win2022"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2022"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true
- label: "Unit tests - Windows 2022"
key: "unit-tests-win2022"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2022"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true

- label: "Unit tests - Windows 2016"
key: "unit-tests-win2016"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2016"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true
- label: "Unit tests - Windows 2016"
key: "unit-tests-win2016"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-windows-2016"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
retry:
manual:
allowed: true

- label: "Unit tests - MacOS 13 ARM"
key: "unit-tests-macos-13-arm"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-arm
retry:
manual:
allowed: true
# Runs inly on the main branch
- label: "Unit tests - MacOS 13 ARM"
key: "unit-tests-macos-13-arm"
command: ".buildkite/scripts/steps/unit-tests.sh"
branches: main
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-arm
retry:
manual:
allowed: true

- label: "Unit tests - MacOS 13"
key: "unit-tests-macos-13"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-x64
retry:
manual:
allowed: true
- label: "Unit tests - MacOS 13"
key: "unit-tests-macos-13"
command: ".buildkite/scripts/steps/unit-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: orka
imagePrefix: generic-13-ventura-x64
retry:
manual:
allowed: true

- group: "Desktop Windows tests"
key: "extended-windows"
steps:
- label: "Unit tests - Windows 10"
key: "unit-tests-win10"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
# TODO create own image
image: "family/endpoint-windows-10-tester-rel"
machine_type: "n2-standard-8"
disk_type: "pd-ssd"
retry:
manual:
allowed: true
- label: "Unit tests - Windows 11"
key: "unit-tests-win11"
command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
# TODO create own image
image: "family/endpoint-windows-11-tester-rel"
machine_type: "n2-standard-8"
disk_type: "pd-ssd"
retry:
manual:
allowed: true

- label: "Merge coverage reports"
key: "merge-coverage"
Expand All @@ -107,21 +148,43 @@ steps:
unit-tests-win2016
unit-tests-win2022
unit-tests-macos-13
unit-tests-macos-13-arm
unit-tests-win10
unit-tests-win11
"
artifact_paths:
- "build/TEST-**"
agents:
image: "golang:1.20.10"
depends_on:
- unit-tests-2204
- unit-tests-2204-arm64
- unit-tests-win2022
- unit-tests-win2016
- unit-tests-macos-13
- unit-tests-macos-13-arm
- unit-tests
- extended-windows
allow_dependency_failure: true

- group: "K8s tests"
key: "k8s-tests"
steps:
- label: "K8s tests: {{matrix.k8s_version}}"
env:
K8S_VERSION: "v{{matrix.k8s_version}}"
KIND_VERSION: "v0.20.0"
command: ".buildkite/scripts/steps/k8s-tests.sh"
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
- "coverage.out"
agents:
provider: "gcp"
image: "family/core-ubuntu-2204"
matrix:
setup:
k8s_version:
- "1.28.0"
- "1.27.3"
- "1.26.6"
retry:
manual:
allowed: true

- label: ":sonarqube: Continuous Code Inspection"
env:
VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
Expand All @@ -138,9 +201,7 @@ steps:

- label: "Serverless integration test"
key: "serverless-integration-tests"
env:
TEST_INTEG_AUTH_ESS_REGION: us-east-1
command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestMonitoringLogsShipped" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
Expand All @@ -150,8 +211,6 @@ steps:

- label: "Integration tests"
key: "integration-tests"
env:
TEST_INTEG_AUTH_ESS_REGION: us-east-1
command: ".buildkite/scripts/steps/integration_tests.sh stateful"
artifact_paths:
- "build/TEST-**"
Expand Down
45 changes: 45 additions & 0 deletions .buildkite/scripts/install-kind.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -exuo pipefail

echo "--- Install Kind"

MSG="environment variable missing."
DEFAULT_HOME="/usr/local"
KIND_VERSION=${KIND_VERSION:?$MSG}
HOME=${HOME:?$DEFAULT_HOME}
KIND_CMD="${HOME}/bin/kind"

if command -v kind
then
set +e
echo "Found Kind. Checking version.."
FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ]
then
echo "Versions match. No need to install Kind. Exiting."
exit 0
fi
set -e
fi

echo "Installing Kind"

OS=$(uname -s| tr '[:upper:]' '[:lower:]')
ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
if [ "${ARCH}" == "aarch64" ] ; then
ARCH_SUFFIX=arm64
else
ARCH_SUFFIX=amd64
fi

mkdir -p "${HOME}/bin"

if curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then
chmod +x "${KIND_CMD}"
else
echo "Something bad with the download, let's delete the corrupted binary"
if [ -e "${KIND_CMD}" ] ; then
rm "${KIND_CMD}"
fi
exit 1
fi
Loading

0 comments on commit 40c1bdf

Please sign in to comment.