Allow use of version qualifier for staging artifacts (#4328)

This commits adds support for an optional $VERSION_QUALIFIER env var/build option in the packaging pipeline to allow building prerelease artifacts from staging. Closes: - #4325 - https://github.com/elastic/ingest-dev/issues/4859 --------- Co-authored-by: Dimitrios Liappis <dimitrios.liappis@gmail.com> (cherry picked from commit 29c1d88)
elastic · Jan 17, 2025 · da9b4fe · da9b4fe
1 parent 127aa85
commit da9b4fe
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 38 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -38,7 +38,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
-  check_if_file_exist_in_repo "infra" "${BUILDKITE_BRANCH}"                  #TODO should be changed to "main" for rollback...
+  # TODO remove and replace _branch with BUILDKITE_BRANCH after PR tests
+  _branch="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
+
+  check_if_file_exist_in_repo "infra" "${_branch}"                  #TODO should be changed to "main" for rollback...
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" || "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-perf-tests" ]]; then
@@ -61,7 +64,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" || "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-arm* || "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
     export JOB_GCS_BUCKET
   fi
@@ -80,7 +83,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-arm* ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
     export JOB_GCS_BUCKET
   fi

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
@@ -16,7 +16,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" || "$BUILDKITE_STEP_KEY" == "dra-snapshot" && "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-arm* || "$BUILDKITE_STEP_KEY" == "dra-snapshot" && "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
     unset GOOGLE_APPLICATION_CREDENTIALS
     unset VAULT_ROLE_ID_SECRET
     unset VAULT_ADDR_SECRET

diff --git a/.buildkite/pipeline.package.mbp.yml b/.buildkite/pipeline.package.mbp.yml
@@ -8,55 +8,68 @@ env:
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
 
 steps:
-  - group: "Package and Publish"
-    key: "package-publish"
-    steps:
-    - label: "Package Ubuntu-20 x86_64"
-      key: "package-x86-64"
-      command: ".buildkite/scripts/package.sh {{matrix.type}}"
-      agents:
-        provider: "gcp"
-        image: "${IMAGE_UBUNTU_X86_64}"
-        machineType: "c2-standard-16"
-      matrix:
-        setup:
-          type:
-          - "snapshot"
-          - "staging"
+  - label: "Package x86_64 snapshot"
+    # skip building + packaging snapshot for pre-releases (flagged by a non-empty VERSION_QUALIFIER env var/BK param)
+    # as prereleases are only intended to be used with staging; details in https://github.com/elastic/ingest-dev/issues/4855
+    if: "build.env('VERSION_QUALIFIER') == null"
+    key: "package-x86-64-snapshot"
+    command: ".buildkite/scripts/package.sh snapshot"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "c2-standard-16"
+
+  - label: "Package x86_64 staging"
+    key: "package-x86-64-staging"
+    command: ".buildkite/scripts/package.sh snapshot"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "c2-standard-16"
 
-    - label: "Package Ubuntu-20 aarch64"
-      key: "package-arm"
-      command: ".buildkite/scripts/package.sh {{matrix.type}}"
-      agents:
-        provider: "aws"
-        imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-        instanceType: "t4g.2xlarge"
-      matrix:
-        setup:
-          type:
-          - "snapshot"
-          - "staging"
+  - label: "Package aarch64 snapshot"
+    if: "build.env('VERSION_QUALIFIER') == null"
+    key: "package-arm-snapshot"
+    command: ".buildkite/scripts/package.sh snapshot"
+    agents:
+      provider: "aws"
+      imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+      instanceType: "t4g.2xlarge"
+
+  - label: "Package aarch64 staging"
+    key: "package-arm-staging"
+    command: ".buildkite/scripts/package.sh staging"
+    agents:
+      provider: "aws"
+      imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+      instanceType: "t4g.2xlarge"
 
   - label: "DRA snapshot"
+    if: "${FILE_EXISTS_IN_REPO} && build.env('VERSION_QUALIFIER') == null"
     key: "dra-snapshot"
     command: ".buildkite/scripts/dra_release.sh snapshot"
     agents:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
-    if: ${FILE_EXISTS_IN_REPO}
     depends_on:
-      - step: "package-publish"
+      - step: "package-x86-64-snapshot"
+        allow_failure: false
+      - step: "package-arm-snapshot"
         allow_failure: false
 
   - label: "DRA release staging"
+    # we don't usually build staging from the main branch, but we exceptionally allow it for prereleases
+    # details in https://github.com/elastic/ingest-dev/issues/4855
+    if: "${FILE_EXISTS_IN_REPO} == true && (build.env('BUILDKITE_BRANCH') != 'main' || build.env('VERSION_QUALIFIER') != null)"
     key: "dra-staging"
     command: ".buildkite/scripts/dra_release.sh staging"
     agents:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
-    if: "${FILE_EXISTS_IN_REPO} == true && build.env('BUILDKITE_BRANCH') != 'main'"
     depends_on:
-      - step: "dra-snapshot"
+      - step: "package-x86-64-staging"
+        allow_failure: false
+      - step: "package-arm-staging"
         allow_failure: false
diff --git a/.buildkite/scripts/dra_release.sh b/.buildkite/scripts/dra_release.sh
@@ -9,7 +9,9 @@ BASE_DIR="${WORKSPACE}/${FOLDER_PATH}"
 DRA_OUTPUT="release-manager.out"
 export PROJECT="fleet-server"
 export TYPE=${1}
-export BRANCH="${BUILDKITE_BRANCH}"
+# DRA_BRANCH can be used for manually testing packaging with PRs
+# e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build
+export BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
 export VERSION="$(make get-version)"
 
 if [[ "${VERSION}" == *"-SNAPSHOT"* || "${VERSION}" == "" ]]; then

diff --git a/.buildkite/scripts/package.sh b/.buildkite/scripts/package.sh
@@ -4,10 +4,8 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
-VERSION=$(awk '/const DefaultVersion/{print $NF}' version/version.go | tr -d '"')
 PLATFORM_TYPE=$(uname -m)
 TYPE="$1"
-INFRA_REPO="https://github.com/repos/elastic/infra/contents"
 
 if [[ ${BUILDKITE_BRANCH} == "main" && ${TYPE} == "staging" ]]; then
     echo "INFO: staging artifacts for the main branch are not required."