Merge branch 'main' into bump/systemtest

kruskall · Jun 5, 2024 · 8a79e21 · 8a79e21
2 parents 05d665e + 7306c14
commit 8a79e21
Show file tree

Hide file tree

Showing 133 changed files with 5,639 additions and 5,742 deletions.
diff --git a/.buildkite/package.yml b/.buildkite/package.yml
@@ -7,6 +7,7 @@ notify:
 env:
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  GENERATE_CHAINGUARD_IMAGES: true
 
 steps:
   - group: "Package"

diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh
@@ -5,7 +5,7 @@
 ##
 ##  It downloads the generated artifacts and run the DRA only if the branch is an active
 ##  branch, based on the Unified Release policy. Otherwise, it won't run the DRA but print
-##  some traces.
+##  some traces and fail unless it's a feature branch then it will list the DRA artifacts.
 ##
 
 set -eo pipefail
@@ -28,37 +28,64 @@ if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "true" ]]; then
   exit 0
 fi
 
-curl -s https://storage.googleapis.com/artifacts-api/snapshots/branches.json > active-branches.json
+# by default it uses the buildkite branch
+DRA_BRANCH="$BUILDKITE_BRANCH"
+# by default it publishes the DRA artifacts, for such it uses the collect command.
+dra_command=collect
+BRANCHES_URL=https://storage.googleapis.com/artifacts-api/snapshots/branches.json
+curl -s "${BRANCHES_URL}" > active-branches.json
 if ! grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then
   echo "--- :arrow_right: Release Manager only supports the current active branches, skipping"
   echo "BUILDKITE_BRANCH=$BUILDKITE_BRANCH"
   echo "BUILDKITE_COMMIT=$BUILDKITE_COMMIT"
   echo "VERSION=$VERSION"
   echo "Supported branches:"
   cat active-branches.json
-  exit 0
+  if [[ $BUILDKITE_BRANCH =~ "feature/" ]]; then
+    buildkite-agent annotate "${BUILDKITE_BRANCH} will list DRA artifacts. Feature branches are not supported. Look for the supported branches in ${BRANCHES_URL}" --style 'info' --context 'ctx-info'
+    dra_command=list
+
+    # use a different branch since DRA does not support feature branches but main/release branches
+    # for such we will use the VERSION and https://storage.googleapis.com/artifacts-api/snapshots/<major.minor>.json
+    # to know if the branch was branched out from main or the release branches.
+    MAJOR_MINOR=${VERSION%.*}
+    if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/main.json" | grep -q "$VERSION" ; then
+      DRA_BRANCH=main
+    else
+      if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/$MAJOR_MINOR.json" | grep -q "$VERSION" ; then
+        DRA_BRANCH="$MAJOR_MINOR"
+      else
+        buildkite-agent annotate "It was not possible to know the original base branch for ${BUILDKITE_BRANCH}. This won't fail - this is a feature branch." --style 'info' --context 'ctx-info-feature-branch'
+        exit 0
+      fi
+    fi
+  else
+    buildkite-agent annotate "${BUILDKITE_BRANCH} is not supported yet. Look for the supported branches in ${BRANCHES_URL}" --style 'warning' --context 'ctx-warn'
+    exit 1
+  fi
 fi
 
 dra() {
   local workflow=$1
-  echo "--- Run release manager $workflow"
+  local command=$2
+  echo "--- Run release manager $workflow (DRA command: $command)"
   docker run --rm \
     --name release-manager \
     -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
     -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
     -e VAULT_SECRET_ID="${VAULT_SECRET}" \
     --mount type=bind,readonly=false,src=$(pwd),target=/artifacts \
     docker.elastic.co/infra/release-manager:latest \
-      cli collect \
+      cli "$command" \
       --project apm-server \
-      --branch $BUILDKITE_BRANCH \
+      --branch $DRA_BRANCH \
       --commit $BUILDKITE_COMMIT \
       --workflow $workflow \
       --artifact-set main \
       --version $VERSION
 }
 
-dra "snapshot"
-if [[ "${BUILDKITE_BRANCH}" != "main" ]]; then
-  dra "staging"
+dra "snapshot" "$dra_command"
+if [[ "${DRA_BRANCH}" != "main" ]]; then
+  dra "staging" "$dra_command"
 fi
diff --git a/.ci/bump-elastic-stack-snapshot.yml b/.ci/bump-elastic-stack-snapshot.yml
@@ -20,13 +20,12 @@ scms:
   default:
     kind: github
     spec:
-      user: '{{ requiredEnv "GIT_USER" }}'
-      email: '{{ requiredEnv "GIT_EMAIL" }}'
+      user: '{{ requiredEnv "GITHUB_ACTOR" }}'
       owner: elastic
       repository: apm-server
       token: '{{ requiredEnv "GITHUB_TOKEN" }}'
-      username: '{{ requiredEnv "GIT_USER" }}'
       branch: '{{ requiredEnv "BRANCH" }}'
+      commitusingapi: true
 
 sources:
   latestVersion:

diff --git a/.ci/bump-golang.yml b/.ci/bump-golang.yml
@@ -23,13 +23,12 @@ scms:
   default:
     kind: github
     spec:
-      user: '{{ requiredEnv "GIT_USER" }}'
-      email: '{{ requiredEnv "GIT_EMAIL" }}'
+      user: '{{ requiredEnv "GITHUB_ACTOR" }}'
       owner: elastic
       repository: apm-server
       token: '{{ requiredEnv "GITHUB_TOKEN" }}'
-      username: '{{ requiredEnv "GIT_USER" }}'
       branch: main
+      commitusingapi: true
 
 sources:
   minor:
@@ -53,7 +52,7 @@ sources:
       owner: golang
       repository: go
       token: '{{ requiredEnv "GITHUB_TOKEN" }}'
-      username: '{{ requiredEnv "GIT_USER" }}'
+      username: '{{ requiredEnv "GITHUB_ACTOR" }}'
       versionfilter:
         kind: regex
         pattern: go1\.{{ source "minor" }}\.\d*$

diff --git a/.ci/update-beats.yml b/.ci/update-beats.yml
@@ -6,13 +6,12 @@ scms:
   default:
     kind: github
     spec:
-      user: '{{ requiredEnv "GIT_USER" }}'
-      email: '{{ requiredEnv "GIT_EMAIL" }}'
+      user: '{{ requiredEnv "GITHUB_ACTOR" }}'
       owner: elastic
       repository: apm-server
       token: '{{ requiredEnv "GITHUB_TOKEN" }}'
-      username: '{{ requiredEnv "GIT_USER" }}'
       branch: '{{ requiredEnv "BRANCH_NAME" }}'
+      commitusingapi: true
 
 actions:
   default:

diff --git a/.github/ISSUE_TEMPLATE/test-plan.md b/.github/ISSUE_TEMPLATE/test-plan.md
@@ -15,17 +15,14 @@ Testing can be started when the first build candidate (BC) is available in the C
 Thanks to https://github.com/elastic/apm-server/issues/8303 further smoke tests are run automatically on ESS now.
 **Consider extending the smoke tests to include more test cases which we'd like to cover**
 
-## Go agent
+## go-docappender library
 
-<!-- Add any issues / PRs which were worked on during the milestone release https://github.com/elastic/apm-agent-go/pulls-->
+<!-- Add any issues / PRs which were worked on during the milestone release https://github.com/elastic/go-docappender/pulls-->
 
-## Lambda extension
+## apm-data library
 
-<!-- Add any issues / PRs which were worked on during the milestone release https://github.com/elastic/apm-aws-lambda/pulls-->
+<!-- Add any issues / PRs which were worked on during the milestone release https://github.com/elastic/apm-data/pulls-->
 
-## APM Kubernetes Attacher
-
-<!-- Add any issues / PRs which were worked on during the milestone release -->
 
 ## Test cases from the GitHub board
 
@@ -37,4 +34,4 @@ Add yourself as _assignee_ on the PR before you start testing.
 
 ## Regressions
 
-Link any regressions to this issue.
+<!-- Link any regressions to this issue. -->
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -53,7 +53,7 @@ jobs:
         with:
           go-version-file: 'go.mod'
 
-      - uses: rlespinasse/github-slug-action@55f5982579285ce3138ca118a20247ee994619ee
+      - uses: rlespinasse/github-slug-action@797d68864753cbceedc271349d402da4590e6302
 
       - name: Set up env
         run: |
@@ -93,13 +93,13 @@ jobs:
             ${{ env.KIBANA_CREDENTIALS_PATH }} kibana_url | KIBANA_ENDPOINT ;
             ${{ env.KIBANA_CREDENTIALS_PATH }} kibana_dashboard_url | KIBANA_DASHBOARD_URL ;
 
-      - uses: elastic/apm-pipeline-library/.github/actions/docker-login@main
+      - name: Log in to the Elastic Container registry
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
         with:
-          registry: docker.elastic.co
-          secret: secret/observability-team/ci/docker-registry/prod
-          url: ${{ secrets.VAULT_ADDR }}
-          roleId: ${{ secrets.VAULT_ROLE_ID }}
-          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }}
+          username: ${{ secrets.ELASTIC_DOCKER_USERNAME }}
+          password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }}
+
       - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: 1.3.7

diff --git a/.github/workflows/bump-elastic-stack.yml b/.github/workflows/bump-elastic-stack.yml
@@ -30,11 +30,16 @@ jobs:
         with:
           ref: ${{ matrix.branch }}
 
-      - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current
+      - uses: elastic/oblt-actions/updatecli/run@v1.2.0
         with:
-          vaultUrl: ${{ secrets.VAULT_ADDR }}
-          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
-          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
-          pipeline: ./.ci/bump-elastic-stack-snapshot.yml
+          command: --experimental apply --config .ci/bump-elastic-stack-snapshot.yml
         env:
           BRANCH: ${{ matrix.branch }}
+          GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}
+
+      - if: ${{ failure()  }}
+        uses: elastic/oblt-actions/slack/send@v1.2.0
+        with:
+          channel-id: '#apm-server'
+          message: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, @robots-ci please look what's going on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
diff --git a/.github/workflows/bump-golang.yml b/.github/workflows/bump-golang.yml
@@ -15,9 +15,16 @@ jobs:
     steps:
 
       - uses: actions/checkout@v4
-      - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current
+
+      - uses: elastic/oblt-actions/updatecli/run@v1.2.0
+        with:
+          command: --experimental apply --config .ci/bump-golang.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}
+
+      - if: ${{ failure() }}
+        uses: elastic/oblt-actions/slack/send@v1.2.0
         with:
-          vaultUrl: ${{ secrets.VAULT_ADDR }}
-          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
-          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
-          pipeline: ./.ci/bump-golang.yml
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel-id: "#apm-server"
+          message: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, @robots-ci please look what's going on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -87,3 +87,27 @@ jobs:
           go-version-file: go.mod
           cache: false
       - run: make package-snapshot
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+  test-package-and-push:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false)
+    env:
+      GENERATE_CHAINGUARD_IMAGES: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: false
+      - name: Log in to the Elastic Container registry
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+        with:
+          registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }}
+          username: ${{ secrets.ELASTIC_DOCKER_USERNAME }}
+          password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }}
+      - run: make package-snapshot
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - run: make publish-docker-images
diff --git a/.github/workflows/microbenchmark.yml b/.github/workflows/microbenchmark.yml
@@ -19,21 +19,14 @@ permissions:
 jobs:
   microbenchmark:
     runs-on: ubuntu-latest
-    # wait up to 1 hour
-    timeout-minutes: 60
+    timeout-minutes: 5
     steps:
-      - id: buildkite
-        name: Run buildkite pipeline
-        uses: elastic/apm-pipeline-library/.github/actions/buildkite@current
+      - name: Run buildkite pipeline
+        uses: elastic/oblt-actions/buildkite/run@v1.4.0
         with:
-          vaultUrl: ${{ secrets.VAULT_ADDR }}
-          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
-          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
+          token: ${{ secrets.BUILDKITE_TOKEN }}
           pipeline: apm-agent-microbenchmark
-          triggerMessage: "${{ github.repository }}@${{ github.ref_name }}"
-          waitFor: true
-          printBuildLogs: true
-          buildEnvVars: |
+          env-vars: |
             script=.ci/scripts/bench.sh
             repo=apm-server
             sha=${{ github.sha }}

diff --git a/.github/workflows/opentelemetry.yml b/.github/workflows/opentelemetry.yml
diff --git a/.github/workflows/run-minor-release.yml b/.github/workflows/run-minor-release.yml
@@ -71,10 +71,16 @@ jobs:
       - name: Configure git user
         uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
         with:
-          username: ${{ env.GIT_USER }}
-          email: ${{ env.GIT_EMAIL }}
           token: ${{ env.GH_TOKEN }}
 
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4  # v6.1.0
+        with:
+          gpg_private_key: ${{ secrets.APM_SERVER_RELEASE_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.APM_SERVER_RELEASE_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+
       - run: make minor-release
 
       - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current

diff --git a/.github/workflows/run-patch-release.yml b/.github/workflows/run-patch-release.yml
@@ -62,10 +62,16 @@ jobs:
       - name: Configure git user
         uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
         with:
-          username: ${{ env.GIT_USER }}
-          email: ${{ env.GIT_EMAIL }}
           token: ${{ env.GH_TOKEN }}
 
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4  # v6.1.0
+        with:
+          gpg_private_key: ${{ secrets.APM_SERVER_RELEASE_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.APM_SERVER_RELEASE_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+
       - run: make patch-release
 
       - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current

diff --git a/.github/workflows/update-beats.yml b/.github/workflows/update-beats.yml
@@ -28,21 +28,21 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ matrix.branch }}
+
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
-      - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current
+
+      - uses: elastic/oblt-actions/updatecli/run@v1.2.0
+        with:
+          command: --experimental apply --config .ci/update-beats.yml
         env:
           BRANCH_NAME: ${{ matrix.branch }}
+          GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}
+
+      - if: ${{ failure()  }}
+        uses: elastic/oblt-actions/slack/send@v1.2.0
         with:
-          vaultUrl: ${{ secrets.VAULT_ADDR }}
-          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
-          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
-          pipeline: ./.ci/update-beats.yml
-      - if: failure()
-        uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current
-        with:
-          vaultUrl: ${{ secrets.VAULT_ADDR }}
-          vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
-          vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
-          slackChannel: "#apm-server"
+          channel-id: '#apm-server'
+          message: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, @robots-ci please look what's going on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
+          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}