Split fips integration tests into two steps (#17038)

* Split fips integration tests into two steps

The integration tests suite takes about 40 minutes. This is far too slow for
reasonable feedback on a PR. This commit follows the pattern for the non-fips
integration tests whereby the tests are split into two sections that can run in
parallel across two steps. This should halve the feedback time.

The logic for getting a list of specs files to run has been extracted to a
shared shell script for use here and in the integration tests shell script.

* Use shared function for splitting integration tests

The logic for getting a list of specs to run has been extracted so that it can
be shared across fips and non fips integration test modes. This commit updates
the non fips integration tests to use the shared function.

* fix typo in helper name (kebab case, not snake)

* Escape $ so buildkite upload does not try to interpolate

* Wrap integration tests in shell script to avoid BK interpolation

* Move entrypoint for running integration tests inside docker

.buildkite/pull_request_pipeline.yml

@@ -125,8 +125,8 @@ steps:
       manual:
         allowed: true
 
-  - label: ":lab_coat: Integration Tests - FIPS mode"
-    key: "integration-tests-fips"
+  - label: ":lab_coat: Integration Tests - FIPS mode / part 1"
+    key: "integration-tests-fips-part-1"
     agents:
       provider: gcp
       imageProject: elastic-images-prod
@@ -141,7 +141,25 @@ steps:
       set -euo pipefail
 
       docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
-      docker run test-runner-image ./gradlew --info --stacktrace -PrunTestsInFIPSMode=true runIntegrationTests
+      docker run test-runner-image ci/run-fips-integration-tests.sh 0
+
+  - label: ":lab_coat: Integration Tests - FIPS mode / part 2"
+    key: "integration-tests-fips-part-2"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+      # dont retry on failure while they are expected
+        - limit: 0
+    command: |
+      set -euo pipefail
+
+      docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
+      docker run test-runner-image ci/run-fips-integration-tests.sh 1
 
   - label: ":lab_coat: Integration Tests / part 1"
     key: "integration-tests-part-1"

ci/get-test-half.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# get_test_half returns either the first or second half of integration tests
+# Usage: get_test_half <half_number>
+# half_number: 0 for first half, 1 for second half
+get_test_half() {
+    local half_number=$1
+    # Ensure only spec files go to stdout
+    pushd qa/integration >/dev/null 2>&1
+
+    # Collect all spec files
+    local glob1=(specs/*spec.rb)
+    local glob2=(specs/**/*spec.rb)
+    local all_specs=("${glob1[@]}" "${glob2[@]}")
+
+    # Calculate the split point
+    local split_point=$((${#all_specs[@]} / 2))
+
+    # Get the requested half (:: is "up to", : is "from")
+    if [[ $half_number -eq 0 ]]; then
+        local specs="${all_specs[@]::$split_point}"
+    else
+        local specs="${all_specs[@]:$split_point}"
+    fi
+    popd >/dev/null 2>&1
+    echo "$specs"
+}

ci/integration_tests.sh

@@ -10,6 +10,9 @@ export GRADLE_OPTS="-Xmx2g -Dorg.gradle.jvmargs=-Xmx2g -Dorg.gradle.daemon=false
 export SPEC_OPTS="--order rand --format documentation"
 export CI=true
 
+# Source shared function for splitting integration tests
+source "$(dirname "${BASH_SOURCE[0]}")/get-test-half.sh"
+
 if [ -n "$BUILD_JAVA_HOME" ]; then
   GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"
 fi
@@ -19,20 +22,10 @@ if [[ $1 = "setup" ]]; then
  exit 0
 
 elif [[ $1 == "split" ]]; then
-    cd qa/integration
-    glob1=(specs/*spec.rb)
-    glob2=(specs/**/*spec.rb)
-    all_specs=("${glob1[@]}" "${glob2[@]}")
-
-    specs0=${all_specs[@]::$((${#all_specs[@]} / 2 ))}
-    specs1=${all_specs[@]:$((${#all_specs[@]} / 2 ))}
-    cd ../..
-    if [[ $2 == 0 ]]; then
-       echo "Running the first half of integration specs: $specs0"
-       ./gradlew runIntegrationTests -PrubyIntegrationSpecs="$specs0" --console=plain
-    elif [[ $2 == 1 ]]; then
-       echo "Running the second half of integration specs: $specs1"
-       ./gradlew runIntegrationTests -PrubyIntegrationSpecs="$specs1" --console=plain
+    if [[ $2 =~ ^[01]$ ]]; then
+        specs=$(get_test_half "$2")
+        echo "Running half $2 of integration specs: $specs"
+        ./gradlew runIntegrationTests -PrubyIntegrationSpecs="$specs" --console=plain
     else
        echo "Error, must specify 0 or 1 after the split. For example ci/integration_tests.sh split 0"
        exit 1

ci/run-fips-integration-tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# get_test_half returns either the first or second half of integration tests
+# Usage: get_test_half <half_number>
+# half_number: 0 for first half, 1 for second half
+
+half_number=$1
+source ci/get-test-half.sh
+specs=$(get_test_half "$half_number")
+./gradlew --info --stacktrace -PrunTestsInFIPSMode=true runIntegrationTests -PrubyIntegrationSpecs="$specs"