From 15ee50331e1207ab68f6a3e244d6de1f1f212849 Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Fri, 2 Feb 2024 17:41:29 +0200 Subject: [PATCH] migrate metricbeat pipeline to Buildkite (#37592) migrate the metricbeat pipeline to Buildkite --- .buildkite/hooks/post-checkout | 53 ++++ .buildkite/hooks/pre-command | 17 +- .buildkite/metricbeat/pipeline.yml | 45 +++- .buildkite/scripts/common.sh | 252 ++++++++++++++++++ .buildkite/scripts/crosscompile.sh | 8 + .../scripts/generate_metricbeat_pipeline.sh | 178 +++++++++++++ .buildkite/scripts/go_int_tests.sh | 12 + .buildkite/scripts/install_tools.sh | 48 ++++ .buildkite/scripts/packaging.sh | 12 + .buildkite/scripts/py_int_tests.sh | 12 + .buildkite/scripts/setenv.sh | 15 ++ .buildkite/scripts/unit_tests.sh | 12 + .buildkite/scripts/win_unit_tests.ps1 | 70 +++++ 13 files changed, 727 insertions(+), 7 deletions(-) create mode 100644 .buildkite/hooks/post-checkout create mode 100755 .buildkite/scripts/common.sh create mode 100755 .buildkite/scripts/crosscompile.sh create mode 100755 .buildkite/scripts/generate_metricbeat_pipeline.sh create mode 100755 .buildkite/scripts/go_int_tests.sh create mode 100644 .buildkite/scripts/install_tools.sh create mode 100755 .buildkite/scripts/packaging.sh create mode 100755 .buildkite/scripts/py_int_tests.sh create mode 100755 .buildkite/scripts/setenv.sh create mode 100755 .buildkite/scripts/unit_tests.sh create mode 100644 .buildkite/scripts/win_unit_tests.ps1 diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout new file mode 100644 index 000000000000..e10f15de7b65 --- /dev/null +++ b/.buildkite/hooks/post-checkout @@ -0,0 +1,53 @@ +#!/bin/bash + +set -euo pipefail + +checkout_merge() { + local target_branch=$1 + local pr_commit=$2 + local merge_branch=$3 + + if [[ -z "${target_branch}" ]]; then + echo "No pull request target branch" + exit 1 + fi + + git fetch -v origin "${target_branch}" + git checkout FETCH_HEAD + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + # create temporal branch to merge the PR with the target branch + git checkout -b ${merge_branch} + echo "New branch created: $(git rev-parse --abbrev-ref HEAD)" + + # set author identity so it can be run git merge + git config user.name "github-merged-pr-post-checkout" + git config user.email "auto-merge@buildkite" + + git merge --no-edit "${BUILDKITE_COMMIT}" || { + local merge_result=$? + echo "Merge failed: ${merge_result}" + git merge --abort + exit ${merge_result} + } +} + +pull_request="${BUILDKITE_PULL_REQUEST:-false}" + +if [[ "${pull_request}" == "false" ]]; then + echo "Not a pull request, skipping" + exit 0 +fi + +TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}" +PR_COMMIT="${BUILDKITE_COMMIT}" +PR_ID=${BUILDKITE_PULL_REQUEST} +MERGE_BRANCH="pr_merge_${PR_ID}" + +checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" + +echo "Commit information" +git --no-pager log --format=%B -n 1 + +# Ensure buildkite groups are rendered +echo "" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0a1567e53cd5..ef38478a4327 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,12 +2,11 @@ set -euo pipefail -source .buildkite/env-scripts/env.sh -source .buildkite/env-scripts/util.sh -source .buildkite/env-scripts/win-env.sh - - if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then + source .buildkite/env-scripts/env.sh + source .buildkite/env-scripts/util.sh + source .buildkite/env-scripts/win-env.sh + if [[ ${PLATFORM_TYPE} = MINGW* ]]; then install_python_win fi @@ -16,3 +15,11 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version") fi fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + source .buildkite/scripts/setenv.sh + if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then + echo "Skipped pre-command when running the Upload pipeline" + exit 0 + fi +fi diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 34321b61161b..0abc58a85ae5 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -1,5 +1,46 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GO_AGENT_IMAGE: "golang:${GO_VERSION}" + BEATS_PROJECT_NAME: "metricbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Metricbeat - runAllStages" + key: "runAllStages" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Metricbeat - runMacOsTests" + key: "UI_MACOS_TESTS" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic metricbeat pipeline" + key: "metricbeat-pipeline" + command: ".buildkite/scripts/generate_metricbeat_pipeline.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh new file mode 100755 index 000000000000..a27fa820a7ab --- /dev/null +++ b/.buildkite/scripts/common.sh @@ -0,0 +1,252 @@ +#!/bin/bash +set -euo pipefail + +WORKSPACE=${WORKSPACE:-"$(pwd)"} +BIN="${WORKSPACE}/bin" +platform_type="$(uname)" +platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') +arch_type="$(uname -m)" +GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} +ONLY_DOCS=${ONLY_DOCS:-"true"} +UI_MACOS_TESTS="$(buildkite-agent meta-data get UI_MACOS_TESTS --default ${UI_MACOS_TESTS:-"false"})" +runAllStages="$(buildkite-agent meta-data get runAllStages --default ${runAllStages:-"false"})" +metricbeat_changeset=( + "^metricbeat/.*" + "^go.mod" + "^pytest.ini" + "^dev-tools/.*" + "^libbeat/.*" + "^testing/.*" + ) +oss_changeset=( + "^go.mod" + "^pytest.ini" + "^dev-tools/.*" + "^libbeat/.*" + "^testing/.*" +) +ci_changeset=( + "^.buildkite/.*" +) +go_mod_changeset=( + "^go.mod" + ) +docs_changeset=( + ".*\\.(asciidoc|md)" + "deploy/kubernetes/.*-kubernetes\\.yaml" + ) +packaging_changeset=( + "^dev-tools/packaging/.*" + ".go-version" + ) + +with_docker_compose() { + local version=$1 + echo "Setting up the Docker-compose environment..." + create_workspace + retry 3 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" + chmod +x ${BIN}/docker-compose + export PATH="${BIN}:${PATH}" + docker-compose version +} + +create_workspace() { + if [[ ! -d "${BIN}" ]]; then + mkdir -p "${BIN}" + fi +} + +add_bin_path() { + echo "Adding PATH to the environment variables..." + create_workspace + export PATH="${BIN}:${PATH}" +} + +check_platform_architeture() { + case "${arch_type}" in + "x86_64") + go_arch_type="amd64" + ;; + "aarch64") + go_arch_type="arm64" + ;; + "arm64") + go_arch_type="arm64" + ;; + *) + echo "The current platform/OS type is unsupported yet" + ;; + esac +} + +with_mage() { + local install_packages=( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "gotest.tools/gotestsum" + ) + create_workspace + for pkg in "${install_packages[@]}"; do + go install "${pkg}@latest" + done +} + +with_go() { + echo "Setting up the Go environment..." + create_workspace + check_platform_architeture + retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${go_arch_type}" + chmod +x "${BIN}/gvm" + eval "$(gvm $GO_VERSION)" + go version + which go + local go_path="$(go env GOPATH):$(go env GOPATH)/bin" + export PATH="${go_path}:${PATH}" +} + +with_python() { + if [ "${platform_type}" == "Linux" ]; then + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo apt-get update + sudo apt-get install -y python3-pip python3-venv + elif [ "${platform_type}" == "Darwin" ]; then + brew update + pip3 install virtualenv + ulimit -Sn 10000 + fi +} + +with_dependencies() { + if [ "${platform_type}" == "Linux" ]; then + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo apt-get update + sudo apt-get install -y libsystemd-dev libpcap-dev + elif [ "${platform_type}" == "Darwin" ]; then + pip3 install libpcap + fi +} + +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +are_paths_changed() { + local patterns=("${@}") + local changelist=() + + for pattern in "${patterns[@]}"; do + changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) + if [ "${#changed_files[@]}" -gt 0 ]; then + changelist+=("${changed_files[@]}") + fi + done + + if [ "${#changelist[@]}" -gt 0 ]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + echo "No files changed within specified changeset:" + echo "${patterns[*]}" + return 1 + fi +} + +are_changed_only_paths() { + local patterns=("${@}") + local changelist=() + local changed_files=$(git diff --name-only HEAD@{1} HEAD) + if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + return 0 + else + return 1 + fi +} + +are_conditions_met_mandatory_tests() { + if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137 + if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + else + return 1 + fi + else + return 1 + fi +} + +are_conditions_met_extended_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + return 0 + else + return 1 + fi +} + +are_conditions_met_macos_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + else + return 1 + fi + else + return 1 + fi +} + +are_conditions_met_extended_windows_tests() { + if [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]] || [[ "${runAllStages}" == "true" ]]; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + else + return 1 + fi + else + return 1 + fi +} + +are_conditions_met_packaging() { + if are_conditions_met_extended_windows_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 + return 0 + else + return 1 + fi + else + return 1 + fi +} + +if ! are_changed_only_paths "${docs_changeset[@]}" ; then + ONLY_DOCS="false" + echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." +else + echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS." +fi + +if are_paths_changed "${go_mod_changeset[@]}" ; then + GO_MOD_CHANGES="true" +fi + +if are_paths_changed "${packaging_changeset[@]}" ; then + PACKAGING_CHANGES="true" +fi diff --git a/.buildkite/scripts/crosscompile.sh b/.buildkite/scripts/crosscompile.sh new file mode 100755 index 000000000000..12f0f6574ca9 --- /dev/null +++ b/.buildkite/scripts/crosscompile.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Crosscompile for $BEATS_PROJECT_NAME" +make -C "${BEATS_PROJECT_NAME}" crosscompile diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh new file mode 100755 index 000000000000..a15447ba4bf6 --- /dev/null +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.metricbeat-dynamic.yml" + +cat > $pipelineName <<- YAML + +steps: + +YAML + +if are_conditions_met_mandatory_tests; then + cat >> $pipelineName <<- YAML + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":go: Go Intergration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":negative_squared_cross_mark: Cross compile" + key: "mandatory-cross-compile" + command: ".buildkite/scripts/crosscompile.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_extended_tests && are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_extended_windows_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Windowds Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # Temporary disabled https://github.com/elastic/beats/issues/37841 + # - label: ":windows: Windows 10 Unit Tests" + # key: "extended-win-10-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machine_type: "n2-standard-8" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_extended_windows_tests; then + cat >> $pipelineName <<- YAML + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + + depends_on: + - step: "mandatory-tests" + allow_failure: false + - step: "extended-tests" + allow_failure: true + - step: "extended-win-tests" + allow_failure: true + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh new file mode 100755 index 000000000000..b4c519f45126 --- /dev/null +++ b/.buildkite/scripts/go_int_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Go Intergration Tests for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage goIntegTest + +popd > /dev/null diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh new file mode 100644 index 000000000000..796892341d30 --- /dev/null +++ b/.buildkite/scripts/install_tools.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +echo "--- Env preparation" + +# Temporary solution to fix the issues with "sudo apt get...." https://elastic.slack.com/archives/C0522G6FBNE/p1706003603442859?thread_ts=1706003209.424539&cid=C0522G6FBNE +# It could be removed when we use our own image for the BK agent. +if [ "${platform_type}" == "Linux" ]; then + DEBIAN_FRONTEND="noninteractive" + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo mkdir -p /etc/needrestart + echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null +fi + +add_bin_path + +if command -v docker-compose &> /dev/null +then + echo "Found docker-compose. Checking version.." + FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version | awk '{print $4}'|sed s/\,//) + if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then + echo "Versions match. No need to install docker-compose. Exiting." + elif [[ "${platform_type}" == "Linux" && "${arch_type}" == "aarch64" ]]; then + with_docker_compose "${DOCKER_COMPOSE_VERSION_AARCH64}" + elif [[ "${platform_type}" == "Linux" && "${arch_type}" == "x86_64" ]]; then + with_docker_compose "${DOCKER_COMPOSE_VERSION}" + fi +else + with_docker_compose "${DOCKER_COMPOSE_VERSION}" +fi + +with_go "${GO_VERSION}" +with_mage +with_python +with_dependencies + +#sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) +sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #TODO: Remove when the issue is solved https://github.com/elastic/beats/issues/37838 + +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +#TODO "umask 0022" has to be removed after our own image is ready (it has to be moved to the image) +umask 0022 # fix the filesystem permissions issue like this: https://buildkite.com/elastic/beats-metricbeat/builds/1329#018d3179-25a9-475b-a2c8-64329dfe092b/320-1696 + +popd > /dev/null diff --git a/.buildkite/scripts/packaging.sh b/.buildkite/scripts/packaging.sh new file mode 100755 index 000000000000..1539d3ab430c --- /dev/null +++ b/.buildkite/scripts/packaging.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Packaging for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage package + +popd > /dev/null diff --git a/.buildkite/scripts/py_int_tests.sh b/.buildkite/scripts/py_int_tests.sh new file mode 100755 index 000000000000..f43cc2021b5a --- /dev/null +++ b/.buildkite/scripts/py_int_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Python Intergration Tests for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage pythonIntegTest + +popd > /dev/null diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh new file mode 100755 index 000000000000..901ba9891c20 --- /dev/null +++ b/.buildkite/scripts/setenv.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SETUP_GVM_VERSION="v0.5.1" +DOCKER_COMPOSE_VERSION="1.21.0" +DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" +SETUP_WIN_PYTHON_VERSION="3.11.0" +GO_VERSION=$(cat .go-version) + +export SETUP_GVM_VERSION +export DOCKER_COMPOSE_VERSION +export DOCKER_COMPOSE_VERSION_AARCH64 +export SETUP_WIN_PYTHON_VERSION +export GO_VERSION diff --git a/.buildkite/scripts/unit_tests.sh b/.buildkite/scripts/unit_tests.sh new file mode 100755 index 000000000000..059b4166e296 --- /dev/null +++ b/.buildkite/scripts/unit_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Unit Tests" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage build unitTest + +popd > /dev/null diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 new file mode 100644 index 000000000000..34833d183ffa --- /dev/null +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -0,0 +1,70 @@ +$ErrorActionPreference = "Stop" # set -e +$WorkFolder = "metricbeat" +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF { + Write-Host "-- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} +function withChoco { + Write-Host "-- Configure Choco --" + $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" +} +function withGolang($version) { + Write-Host "-- Install golang $version --" + choco install -y golang --version=$version + refreshenv + go version +} +function withPython($version) { + Write-Host "-- Install Python $version --" + choco install python --version=$version + refreshenv + python --version +} +function withMinGW { + Write-Host "-- Install MinGW --" + choco install mingw -y + refreshenv +} +function installGoDependencies { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report/v2" + "gotest.tools/gotestsum" + ) + foreach ($pkg in $installPackages) { + go install "$pkg@latest" + } +} + +fixCRLF + +withChoco + +withGolang $env:GO_VERSION + +installGoDependencies + +withPython $env:SETUP_WIN_PYTHON_VERSION + +withMinGW + +$ErrorActionPreference = "Continue" # set +e + +Push-Location $WorkFolder + +New-Item -ItemType Directory -Force -Path "build" +mage build unitTest + +Pop-Location + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE