[CI] Pipeline resource for BK integration tests

.buildkite/bk.integration.pipeline.yml

@@ -0,0 +1,160 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+  ASDF_MAGE_VERSION: 1.14.0
+  ASDF_GOLANG_VERSION: 1.22.6
+  ASDF_TERRAFORM_VERSION: 1.9.3
+
+steps:
+  - label: "Integration tests: packaging"
+    key: "package-it"
+    command: ".buildkite/scripts/steps/integration-package.sh"
+    artifact_paths:
+      - build/distributions/**
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+
+  - label: Start ESS stack for integration tests
+    key: integration-ess
+    depends_on:
+      - package-it
+    command: |
+      #!/usr/bin/env bash
+      set -euo pipefail
+      source .buildkite/scripts/steps/ess_start.sh
+    artifact_paths:
+      - test_infra/ess/*.tfstate
+      - test_infra/ess/*.lock.hcl
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+
+  - group: "Stateful IT (Sudo): Ubuntu"
+    key: integration-tests
+    depends_on:
+      - package-it
+      - integration-ess
+    steps:
+      - label: "Default"
+        key: stateful-ubuntu-default-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "default" "^(TestAPMConfig|TestDiagnosticsOptionalValues|TestIsolatedUnitsDiagnosticsOptionalValues|TestDiagnosticsCommand|TestIsolatedUnitsDiagnosticsCommand|TestEventLogFile|TestFakeComponent|TestFakeIsolatedUnitsComponent|TestOtelFileProcessing|TestOtelLogsIngestion|TestOtelAPMIngestion|TestPackageVersion)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Upgrade"
+        key: stateful-ubuntu-upgrade-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "upgrade" "^(TestUpgradeBrokenPackageVersion|TestStandaloneUpgradeWithGPGFallback|TestStandaloneUpgradeWithGPGFallbackOneRemoteFailing|TestStandaloneUpgradeRollback|TestStandaloneUpgradeRollbackOnRestarts|TestStandaloneUpgradeFailsWhenUpgradeIsInProgress|TestStandaloneUpgradeRetryDownload|TestStandaloneUpgradeSameCommit|TestStandaloneUpgrade|TestStandaloneUpgradeUninstallKillWatcher)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet"
+        key: stateful-ubuntu-fleet-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet" "^(TestLongRunningAgentForLeaks|TestDelayEnroll|TestDelayEnrollUnprivileged|TestInstallAndCLIUninstallWithEndpointSecurity|TestInstallAndUnenrollWithEndpointSecurity|TestInstallWithEndpointSecurityAndRemoveEndpointIntegration|TestEndpointSecurityNonDefaultBasePath|TestEndpointSecurityUnprivileged|TestEndpointSecurityCannotSwitchToUnprivileged|TestEndpointLogsAreCollectedInDiagnostics|TestForceInstallOverProtectedPolicy|TestSetLogLevelFleetManaged|TestLogIngestionFleetManaged|TestMetricsMonitoringCorrectBinaries|TestEndpointAgentServiceMonitoring|TestMonitoringPreserveTextConfig|TestMonitoringLivenessReloadable|TestComponentBuildHashInDiagnostics|TestProxyURL|TestFleetManagedUpgradeUnprivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "FQDN"
+        key: stateful-ubuntu-fqdn-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fqdn" "^(TestFQDN)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Deb"
+        key: stateful-ubuntu-deb-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "deb" "^(TestDebLogIngestFleetManaged|TestDebFleetUpgrade)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet Airgapped"
+        key: stateful-ubuntu-fleet-airgapped-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped" "^(TestFleetAirGappedUpgradeUnprivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet Privileged"
+        key: stateful-ubuntu-fleet-privileged-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-privileged" "^(TestInstallFleetServerBootstrap|TestFleetManagedUpgradePrivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+      - label: "Fleet Airgapped Privileged"
+        key: stateful-ubuntu-fleet-airgapped-privileged-sudo
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          ls -lah build/distributions/
+          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped-privileged" "^(TestFleetAirGappedUpgradePrivileged)$$"
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          imageProject: elastic-images-qa
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+
+  - label: ESS stack cleanup
+    depends_on: integration-tests
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
+      ls -lah test_infra/ess
+      .buildkite/scripts/steps/ess_down.sh
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true

catalog-info.yaml

@@ -108,6 +108,45 @@ spec:
         everyone:
           access_level: BUILD_AND_READ
 
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elastic-agent-testing-bk
+  description: Elastic Agent extended testing using buildkite agents
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elastic-agent-extended-testing-bk
+
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: elastic-agent-extended-testing-bk
+      description: Elastic Agent extended testing using buildkite agents
+    spec:
+      repository: elastic/elastic-agent
+      pipeline_file: ".buildkite/bk.integration.pipeline.yml"
+      provider_settings:
+        trigger_mode: none # don't trigger jobs from github activity
+      cancel_intermediate_builds: false
+      skip_intermediate_builds: false
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
+        SLACK_NOTIFICATIONS_CHANNEL: "#ingest-notifications"
+        SLACK_NOTIFICATIONS_ALL_BRANCHES: "false"
+        SLACK_NOTIFICATIONS_ON_SUCCESS: "false"
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1