Merge branch 'main' into dependabot/go_modules/github.com/elastic/go-…

…elasticsearch/v8-8.11.0

.buildkite/pipeline.yml

@@ -4,6 +4,15 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
+  - label: "check-ci"
+    key: "check-ci"
+    command: ".buildkite/scripts/steps/check-ci.sh"
+    agents:
+      provider: "gcp"
+      image: "family/core-ubuntu-2204"
+    retry:
+      manual:
+        allowed: true
   - group: "Unit tests"
     key: "unit-tests"
     steps:
@@ -219,6 +228,18 @@ steps:
       provider: "gcp"
       machineType: "n1-standard-8"
 
+  - label: "Serverless Beats Tests"
+    key: "serverless-beats-integration-tests"
+    command: ".buildkite/scripts/steps/beats_tests.sh"
+    if: "build.env('CRON') == 'yes'"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    retry:
+      manual:
+        allowed: true
+
+
   - wait: ~
     continue_on_failure: true
   - label: "Processing test results"
@@ -227,3 +248,23 @@ steps:
     plugins:
       - junit-annotate#v2.4.1:
           artifacts: build/TEST-go-integration*.xml
+
+  # Triggers a dynamic step: Sync K8s
+  # Runs only on main and if k8s files are changed
+  - label: "Trigger k8s sync"
+    branches: main
+    plugins:
+      - monebag/monorepo-diff#v2.5.9:
+          diff: "git diff --name-only HEAD~1"
+          watch:
+            - path:
+                - deploy/kubernetes/*
+                - version/docs/version.asciidoc
+              config:
+                label: "Sync K8s"
+                command: ".buildkite/scripts/steps/sync-k8s.sh"
+                agents:
+                  provider: "gcp"
+                  image: "family/core-ubuntu-2204"
+                env:
+                  - GH_VERSION=2.4.0

.buildkite/scripts/common.sh

@@ -53,7 +53,7 @@ getOSOptions() {
 # Wrapper function for executing mage
 mage() {
     go version
-    if ! [ -x "$(type -p mage | sed 's/mage is //g')" ];
+    if ! [ -x "$(type -P mage | sed 's/mage is //g')" ];
     then
         echo "installing mage ${SETUP_MAGE_VERSION}"
         make mage
@@ -68,7 +68,7 @@ mage() {
 # Wrapper function for executing go
 go(){
     # Search for the go in the Path
-    if ! [ -x "$(type -p go | sed 's/go is //g')" ];
+    if ! [ -x "$(type -P go | sed 's/go is //g')" ];
     then
         getOSOptions
         echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" "

.buildkite/scripts/install-gh.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Required environment variables:
+# - GH_VERSION - the version of gh to install
+set -exuo pipefail
+
+echo "--- Install gh cli"
+
+MSG="environment variable missing."
+DEFAULT_HOME="/usr/local"
+GH_VERSION=${GH_VERSION:-$MSG}
+HOME=${HOME:-$DEFAULT_HOME}
+GH_CMD="${HOME}/bin/gh"
+
+if command -v gh
+then
+    set +e
+    echo "Found GH. Checking version.."
+    FOUND_GH_VERSION=$(gh --version 2>&1 >/dev/null | awk '{print $3}')
+    if [ "$FOUND_GH_VERSION" == "$GH_VERSION" ]
+    then
+        echo "GH Versions match: $GH_VERSION. No need to install gh. Exiting."
+        exit 0
+    else 
+        echo "GH Version mismatch. Desired version: $GH_VERSION, found version: $FOUND_GH_VERSION. Installing new version."    
+    fi
+    set -e
+fi
+
+source .buildkite/scripts/common.sh
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
+echo "Downloading gh : ${GH_VERSION}..."
+TMP_DIR=$(mktemp -d)
+if retry 5 curl -sL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" | tar xz -C $TMP_DIR ; then
+  mkdir -p "${HOME}/bin"
+  mv "${TMP_DIR}/gh_${GH_VERSION}_linux_amd64/bin/gh" "${GH_CMD}"
+  rm -rf "${TMP_DIR}"
+else    
+    echo "Something bad with the download, deleting the binary"
+    if [ -e "${GH_CMD}" ] ; then
+        rm "${GH_CMD}"
+    fi
+    exit 1
+fi  
+
+

.buildkite/scripts/install-kind.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -exuo pipefail
+set -euo pipefail
 
 echo "--- Install Kind"
 

.buildkite/scripts/steps/beats_tests.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+#=========================
+# NOTE: This entire script is a temporary hack until we have buildkite set up on the beats repo.
+# until then, we need some kind of serverless integration tests, hence this script, which just clones the beats repo,
+# and runs the serverless integration suite against different beats
+# After buildkite is set up on beats, this file/PR should be reverted.
+#==========================
+
+source .buildkite/scripts/common.sh
+STACK_PROVISIONER="${1:-"serverless"}"
+
+run_test_for_beat(){
+    local beat_name=$1
+
+    #build
+    export WORKSPACE="build/beats/x-pack/${beat_name}"
+    SNAPSHOT=true PLATFORMS=linux/amd64 PACKAGES=tar.gz,zip mage package
+
+    #run
+    export AGENT_BUILD_DIR="build/beats/x-pack/${beat_name}/build/distributions"
+    export WORKSPACE=$(pwd)
+
+    set +e
+    TEST_INTEG_CLEAN_ON_EXIT=true TEST_PLATFORMS="linux/amd64" STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage integration:testBeatServerless $beat_name
+    TESTS_EXIT_STATUS=$?
+    set -e
+
+    return $TESTS_EXIT_STATUS
+}
+#run mage before setup, since this will install go and mage
+#the setup scripts will do a few things that assume we're running out of elastic-agent and will break things for beats, so run before we do actual setup
+mage -l
+
+mkdir -p build
+cd build
+
+git clone --filter=tree:0 git@github.com:elastic/beats.git
+cd ..
+
+# export WORKSPACE=beats/x-pack/metricbeat
+
+# SNAPSHOT=true PLATFORMS=linux/amd64,windows/amd64 PACKAGES=tar.gz,zip mage package
+
+
+# cd ..
+
+# export AGENT_BUILD_DIR=build/beats/x-pack/metricbeat/build/distributions
+# export WORKSPACE=$(pwd)
+
+# set +e
+# TEST_INTEG_CLEAN_ON_EXIT=true TEST_PLATFORMS="linux/amd64" STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage integration:testBeatServerless metricbeat
+# TESTS_EXIT_STATUS=$?
+# set -e
+
+# exit $TESTS_EXIT_STATUS
+
+echo "testing metricbeat..."
+run_test_for_beat metricbeat
+
+
+
+echo "testing filebeat..."
+run_test_for_beat filebeat
+
+
+
+echo "testing auditbeat..."
+run_test_for_beat auditbeat

.buildkite/scripts/steps/check-ci.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+echo "--- Check CI"
+go version
+mage --version
+BEAT_VERSION=$(make get-version)
+echo "Beat version: $BEAT_VERSION"
+make check-ci

.buildkite/scripts/steps/sync-k8s.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -euo pipefail
+
+export PATH=$HOME/bin:${PATH}
+
+source .buildkite/scripts/install-gh.sh
+source .buildkite/scripts/common.sh
+
+export GITHUB_TOKEN=$(retry 5 vault kv get -field token kv/ci-shared/platform-ingest/github_token)
+
+cd deploy/kubernetes
+
+echo "--- [File Creation] Create-Needed-Manifest"
+WITHOUTCONFIG=true make generate-k8s
+./creator_k8s_manifest.sh .
+
+echo "--- [Clone] Kibana-Repository"
+make ci-clone-kibana-repository
+cp Makefile ./kibana
+cd kibana
+echo "--- Create Kibana PR"
+make ci-create-kubernetes-templates-pull-request