Update Elastic Agent Dashboards

.buildkite/hooks/post-checkout

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# ******************************* WARNING ******************************************
+# This post-checkout hook is not the same as in the rest of repositories e.g. beats
+# because some steps in this pipeline (in PR context) take a very long time and we
+# want to make sure that THE SAME COMMIT FROM TARGET BRANCH gets merged in every
+# pipeline step. Otherwise, HEAD (target branch) may have changed in the meantime
+# and therefore, some steps (e.g. sonarqube) may end up testing a different commit.
+#
+# Running builds from branches or tags (out of PR context) maintains the same behavior
+# as in the rest of the repositories.
+#
+# Reference: https://github.com/elastic/integrations/pull/10397
+# **********************************************************************************
+
+set -euo pipefail
+
+checkout_merge() {
+    local target_branch=$1
+    local pr_commit=$2
+    local merge_branch=$3
+
+    if [[ -z "${target_branch}" ]]; then
+        echo "No pull request target branch"
+        exit 1
+    fi
+
+    git fetch -v origin "${target_branch}"
+    if [[ ${REPOSITORY_TARGET_BRANCH_COMMIT} == "" ]]; then
+        git checkout FETCH_HEAD
+        echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+    else
+        # Use the same commit from target branch as in the other steps.
+        echo "Retrieved commit from meta-data: ${REPOSITORY_TARGET_BRANCH_COMMIT}"
+        git checkout "${REPOSITORY_TARGET_BRANCH_COMMIT}"
+        echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+    fi
+
+    # create temporal branch to merge the PR with the target branch
+    git checkout -b ${merge_branch}
+    echo "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # set author identity so it can be run git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit "${BUILDKITE_COMMIT}" || {
+        local merge_result=$?
+        echo "Merge failed: ${merge_result}"
+        git merge --abort
+        exit ${merge_result}
+    }
+}
+
+pull_request="${BUILDKITE_PULL_REQUEST:-false}"
+
+if [[ "${pull_request}" == "false" ]]; then
+    echo "Not a pull request, skipping"
+    exit 0
+fi
+
+TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}"
+PR_COMMIT="${BUILDKITE_COMMIT}"
+PR_ID=${BUILDKITE_PULL_REQUEST}
+MERGE_BRANCH="pr_merge_${PR_ID}"
+
+# This meta-data field is populated in the pre-command hook
+REPOSITORY_TARGET_BRANCH_COMMIT=$(buildkite-agent meta-data get "REPOSITORY_TARGET_BRANCH_COMMIT" --default "")
+
+checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
+
+echo "Commit information"
+
+git --no-pager log --format=%B -n 1
+
+# Ensure buildkite groups are rendered
+echo ""

.buildkite/hooks/pre-command

@@ -0,0 +1,161 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+# Avoid any pager when running git commands
+git config --global core.pager 'cat'
+
+# default values used in different pipelines
+export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
+export SERVERLESS=${SERVERLESS:-"false"}
+export STACK_VERSION=${STACK_VERSION:-""}
+export FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
+export PUBLISH_COVERAGE_REPORTS=${PUBLISH_COVERAGE_REPORTS:-"false"}
+
+BASE_DIR=$(pwd)
+export BASE_DIR
+
+GO_VERSION=$(cat .go-version)
+export GO_VERSION
+
+REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
+export REPO_NAME
+
+export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO_NAME}"
+export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
+
+REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
+export REPO_BUILD_TAG
+
+AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
+PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
+
+BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
+
+EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
+EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
+
+# variables required for terraform
+export ENVIRONMENT="ci"
+export REPO="${REPO_NAME}"
+
+branch_name_label() {
+    local branch="$1"
+
+    if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]] ; then
+        # remove fork from branch name
+        branch=$(echo $branch | cut -d : -f 2)
+    fi
+
+    # From Jenkins
+    # BRANCH_NAME_LOWER_CASE = "${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')}"
+    # to lower case and replace characters
+    branch=$(echo "$branch" | tr '[:upper:]' '[:lower:]' | tr '_/\:. ' '-')
+
+    # truncate up to 63 characters limit
+    echo $branch | head -c 63
+}
+
+BRANCH_NAME_LOWER_CASE=$(branch_name_label "$BUILDKITE_BRANCH")
+export BRANCH_NAME_LOWER_CASE
+# This variable contains the build number https://buildkite.com/elastic/elastic-package/<number>
+export BUILD_ID="${BUILDKITE_BUILD_NUMBER}"
+# get current timestamp in milliseconds
+# From Jenkins
+# CREATED_DATE = "${new Date().getTime()}"
+CREATED_DATE=$(date +%s%3N)
+export CREATED_DATE
+
+if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
+    # first upload pipeline does not have the environment variables defined in the YAML
+    export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
+fi
+
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" &&  "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then
+    # Get the commit from target branch in the first step (reference-target-branch).
+    # This step MUST be the first one and not run in parallel with any other step to ensure
+    # that there is just one value for this variable
+    if is_pr ; then
+        git fetch -v origin ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+        commit_main=$(git rev-parse --verify FETCH_HEAD)
+        buildkite-agent meta-data set "REPOSITORY_TARGET_BRANCH_COMMIT" "${commit_main}"
+    fi
+fi
+
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then
+    if [[ "${BUILDKITE_STEP_KEY}" == "trigger-publish" ]]; then
+        BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token ${BUILDKITE_API_TOKEN_PATH})
+        export BUILDKITE_API_TOKEN
+    fi
+fi
+
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
+    if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations" ]]; then
+        BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
+        export BUILDKITE_API_TOKEN
+    fi
+
+    if [[ "${BUILDKITE_STEP_KEY}" == "publish-benchmarks" ]]; then
+        BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
+        export BUILDKITE_API_TOKEN
+        GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
+        export GITHUB_TOKEN
+    fi
+
+    if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
+        ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
+        export ELASTIC_PACKAGE_AWS_SECRET_KEY
+        ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
+        export ELASTIC_PACKAGE_AWS_ACCESS_KEY
+
+        PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
+        export PRIVATE_CI_GCS_CREDENTIALS_SECRET
+        export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
+
+        # Environment variables required by the service deployer
+        export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
+        export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
+
+        BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
+        export BUILDKITE_API_TOKEN
+    fi
+fi
+
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
+    if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
+        # Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
+        # add the AWS credentials as in the integrations pipeline.
+
+        PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
+        export PRIVATE_CI_GCS_CREDENTIALS_SECRET
+        export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
+
+        BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
+        export BUILDKITE_API_TOKEN
+
+        EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_TOKEN_PATH}")
+        export EC_API_KEY_SECRET
+        EC_HOST_SECRET=$(retry 5 vault kv get -field url "${EC_TOKEN_PATH}")
+        export EC_HOST_SECRET
+        EC_REGION_SECRET=$(retry 5 vault read -field region_qa "${EC_DATA_PATH}")
+        export EC_REGION_SECRET
+    fi
+fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
+    GITHUB_USERNAME="elastic-vault-github-plugin-prod"
+    GITHUB_EMAIL="elasticmachine@elastic.co"
+    GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
+    export GITHUB_TOKEN GITHUB_EMAIL GITHUB_USERNAME
+  fi
+fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" || "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "report-failed-tests" ]]; then
+    export GITHUB_TOKEN="${VAULT_GITHUB_TOKEN}"
+  fi
+fi
+

.buildkite/hooks/pre-exit

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
+    # FIXME: update condition depending on the pipeline steps triggered
+    if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
+        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
+        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
+        unset AWS_ACCESS_KEY_ID
+        unset AWS_SECRET_ACCESS_KEY
+
+        # Ensure that kind cluster is deleted
+        delete_kind_cluster
+
+        # Ensure elastic stack is stopped
+        if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then
+            echo "--- Take down the Elastic stack"
+            ${ELASTIC_PACKAGE_BIN} stack down -v
+        fi
+    fi
+fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
+    if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
+        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
+        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
+        unset AWS_ACCESS_KEY_ID
+        unset AWS_SECRET_ACCESS_KEY
+
+        # Ensure that kind cluster is deleted
+        delete_kind_cluster
+
+        # Ensure elastic stack is stopped
+        if [ -f "${ELASTIC_PACKAGE_BIN}" ]; then
+            echo "--- Take down the Elastic stack"
+            EC_API_KEY=${EC_API_KEY_SECRET} EC_HOST=${EC_HOST_SECRET} ${ELASTIC_PACKAGE_BIN} stack down -v
+        fi
+    fi
+fi
+
+unset_secrets
+cleanup
+
+google_cloud_logout_active_account
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
+  cd "${WORKSPACE}"
+  git config remote.origin.url "https://github.com/elastic/integrations.git"
+fi

.buildkite/pipeline.backport.yml

@@ -0,0 +1,54 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+name: "integrations-backport"
+
+env:
+  YQ_VERSION: 'v4.35.2'
+
+steps:
+
+  - label: "Check that it runs from UI"
+    key: "check-ui"
+    command:
+      - "buildkite-agent annotate \"The $BUILDKITE_PIPELINE_SLUG is used only for running from UI!\" --style 'warning'"
+      - "exit 1"
+    if: "build.source != 'ui'"
+
+  - input: "Input values for the variables"
+    key: "input-variables"
+    fields:
+    - select: "DRY_RUN"
+      key: "DRY_RUN"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "true"
+    - text: "Enter base commit for the backport branch"
+      key: "BASE_COMMIT"
+      required: true
+      default: ""
+    - text: "Enter package name"
+      key: "PACKAGE_NAME"
+      required: true
+      default: ""
+    - text: "Enter package version (examples: 1.5.7, 1.0.0-beta1)"
+      key: "PACKAGE_VERSION"
+      required: true
+      default: ""
+    - select: "Include to backport-branch only one package?"
+      key: "REMOVE_OTHER_PACKAGES"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+
+  - label: "Creating the backport branch"
+    key: "create-backport-branch"
+    command: ".buildkite/scripts/backport_branch.sh"
+    depends_on:
+      - step: "input-variables"
+        allow_failure: false

.buildkite/pipeline.publish.yml

@@ -0,0 +1,49 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  SETUP_GVM_VERSION: "v0.5.2"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  DOCKER_COMPOSE_VERSION: "v2.24.1"
+  DOCKER_VERSION: "false"
+  YQ_VERSION: 'v4.35.2'
+  JQ_VERSION: '1.7'
+  # Elastic package settings
+  # Manage docker output/logs
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
+  # Default license to use by `elastic-package build`
+  ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
+  # Link definitions path (full path to be set in the corresponding step)
+  ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
+  # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
+  ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
+  NOTIFY_TO: "ecosystem-team@elastic.co"
+
+steps:
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: ":package: Build packages"
+    key: "build-packages"
+    command: ".buildkite/scripts/build_packages.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "8G"
+    env:
+      ARTIFACTS_FOLDER: "artifacts-to-sign"
+      # by default it will publish packages
+      DRY_RUN: "${DRY_RUN:-false}"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+    artifact_paths:
+      - artifacts-to-sign/*.zip
+
+notify:
+  - email: "$NOTIFY_TO"
+    if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'"