Add ability to run the same tests with multi docker image variants (#…

…5547)

* Add ability to run the same tests with multi docker image variants.

* Fix buildkite.

* Add back distro for kubernetes types.

* Refactor how kubernetes tests are selected.

* Allow specific docker variant.

* Add kubernetes testing to the dev guide.

* Add back K8S_VERSION to buildkite job.

* Fix docker image location.

* Don't create so many clusters.

* Ensure basic is always picked first.

* Fix cloud variant image name.

* revert builkite changes

* Move kubernetes integration tests in CI next to other integration tests.

* Rename wolfi-complete to complete-wolfi.

* Reduce docker variants on some tests.

* Only test basic and wolfi.

.buildkite/integration.pipeline.yml

@@ -82,7 +82,7 @@ steps:
           files: "build/TEST-*.xml"
           format: "junit"
           branches: "main"
-          debug: true    
+          debug: true
 
   - label: "Serverless Beats Tests"
     depends_on:
@@ -103,4 +103,20 @@ steps:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
 
-
+  - label: "Kubernetes Integration tests"
+    key: "k8s-integration-tests"
+    env:
+      K8S_VERSION: "v1.30.2"
+      KIND_VERSION: "v0.20.0"
+    command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
+    artifact_paths:
+      - "build/k8s-logs*/*"
+      - "build/k8s-logs*/**/*"
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      image: "family/core-ubuntu-2204"
+    notify:
+      - github_commit_status:
+          context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests"

.buildkite/pipeline.yml

@@ -198,23 +198,6 @@ steps:
         retry:
           manual:
             allowed: true
-      - label: "K8s integration tests"
-        key: "k8s-integration-tests"
-        env:
-          K8S_VERSION: "v1.30.2,v1.29.4,v1.28.9"
-          KIND_VERSION: "v0.20.0"
-        command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
-        artifact_paths:
-          - "build/k8s-logs*/*"
-          - "build/k8s-logs*/**/*"
-          - "build/TEST-**"
-          - "build/diagnostics/*"
-        agents:
-          provider: "gcp"
-          image: "family/core-ubuntu-2204"
-        retry:
-          manual:
-            allowed: true
 
   - label: ":sonarqube: Continuous Code Inspection"
     env:

.buildkite/scripts/steps/k8s-extended-tests.sh

@@ -26,7 +26,7 @@ else
 fi
 
 AGENT_PACKAGE_VERSION="8.16.0" DEV=true SNAPSHOT=true EXTERNAL=true PACKAGES=docker mage -v package
-AGENT_VERSION="8.16.0-SNAPSHOT" TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetes
+AGENT_VERSION="8.16.0-SNAPSHOT" TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetesMatrix
 TESTS_EXIT_STATUS=$?
 set -e
 

docs/test-framework-dev-guide.md

@@ -59,6 +59,10 @@ The test are run with mage using the `integration` namespace:
 
 - `mage integration:matrix` to run all tests on the complete matrix of supported operating systems and architectures of the Elastic Agent.
 
+- `mage integration:kubernetes` to run kubernetes tests for the default image on the default version of kubernetes (all previous commands will not run any kubernetes tests).
+
+- `mage integration:kubernetesMatrix` to run a matrix of kubernetes tests for all image types and supported versions of kubernetes.
+
 #### Selecting specific platform
 
 By default, the runner will deploy to every combination of operating system and architecture that the tests define
@@ -73,6 +77,7 @@ between, and it can be very specific or not very specific.
 - `TEST_PLATFORMS="linux/amd64/ubuntu/20.04 mage integration:test` to execute tests only on Ubuntu 20.04 ARM64.
 - `TEST_PLATFORMS="windows/amd64/2022 mage integration:test` to execute tests only on Windows Server 2022.
 - `TEST_PLATFORMS="linux/amd64 windows/amd64/2022 mage integration:test` to execute tests on Linux AMD64 and Windows Server 2022.
+- `TEST_PLATFORMS="kubernetes/arm64/1.30.2/wolfi" mage integration:kubernetes` to execute kubernetes tests on Kubernetes version 1.30.2 with wolfi docker variant.
 
 > **_NOTE:_**  This only filters down the tests based on the platform. It will not execute a tests on a platform unless
 > the test defines as supporting it.
@@ -184,7 +189,7 @@ credentials for the tests to succeed.
  - Docker
  - Delve
  - Mage
- 
+
 When called, it will show a menu to select a VM and then install the
 tools listed above. It will also create the `~/elastic-agent` folder
 containing the Git repository (required o package from within the VM)
@@ -336,6 +341,11 @@ want to use a local VM instead of a remote VM, you can use the [Multipass](https
 It is always best to run `mage integration:clean` before changing the provisioner because the change will
 not cause already provisioned resources to be replaced with an instance created by a different provisioner.
 
+### Kind Instance Provisioner
+Use only when running Kubernetes tests. Uses local installed kind to create Kubernetes clusters on the fly.
+
+- `INSTANCE_PROVISIONER="kind" mage integration:kubernetes`
+
 ## Troubleshooting Tips
 
 ### Error: GCE service token missing; run 'mage integration:auth'

magefile.go

@@ -1905,6 +1905,16 @@ func (Integration) Kubernetes(ctx context.Context) error {
 	return integRunner(ctx, false, "")
 }
 
+// KubernetesMatrix runs a matrix of kubernetes integration tests
+func (Integration) KubernetesMatrix(ctx context.Context) error {
+	// invoke integration tests
+	if err := os.Setenv("TEST_GROUPS", "kubernetes"); err != nil {
+		return err
+	}
+
+	return integRunner(ctx, true, "")
+}
+
 // UpdateVersions runs an update on the `.agent-versions.yml` fetching
 // the latest version list from the artifact API.
 func (Integration) UpdateVersions(ctx context.Context) error {
@@ -2605,11 +2615,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 	case multipass.Name:
 		instanceProvisioner = multipass.NewProvisioner()
 	case kind.Name:
-		k8sVersion := os.Getenv("K8S_VERSION")
-		if k8sVersion == "" {
-			return nil, errors.New("K8S_VERSION must be set to use kind instance provisioner")
-		}
-		instanceProvisioner = kind.NewProvisioner(k8sVersion)
+		instanceProvisioner = kind.NewProvisioner()
 	default:
 		return nil, fmt.Errorf("INSTANCE_PROVISIONER environment variable must be one of 'ogc' or 'multipass', not %s", instanceProvisionerMode)
 	}

pkg/testing/define/batch.go

@@ -150,25 +150,28 @@ func appendTest(batches []Batch, tar testActionResult, req Requirements) []Batch
 	for _, o := range req.OS {
 		if o.Arch == "" {
 			set = append(set, OS{
-				Type:    o.Type,
-				Arch:    AMD64,
-				Version: o.Version,
-				Distro:  o.Distro,
+				Type:          o.Type,
+				Arch:          AMD64,
+				Version:       o.Version,
+				Distro:        o.Distro,
+				DockerVariant: o.DockerVariant,
 			})
 			if o.Type != Windows {
 				set = append(set, OS{
-					Type:    o.Type,
-					Arch:    ARM64,
-					Version: o.Version,
-					Distro:  o.Distro,
+					Type:          o.Type,
+					Arch:          ARM64,
+					Version:       o.Version,
+					Distro:        o.Distro,
+					DockerVariant: o.DockerVariant,
 				})
 			}
 		} else {
 			set = append(set, OS{
-				Type:    o.Type,
-				Arch:    o.Arch,
-				Version: o.Version,
-				Distro:  o.Distro,
+				Type:          o.Type,
+				Arch:          o.Arch,
+				Version:       o.Version,
+				Distro:        o.Distro,
+				DockerVariant: o.DockerVariant,
 			})
 		}
 	}
@@ -197,6 +200,9 @@ func appendTest(batches []Batch, tar testActionResult, req Requirements) []Batch
 		if o.Version != "" {
 			batch.OS.Version = o.Version
 		}
+		if o.DockerVariant != "" {
+			batch.OS.DockerVariant = o.DockerVariant
+		}
 		if req.Stack != nil && batch.Stack == nil {
 			// assign the stack to this batch
 			batch.Stack = copyStack(req.Stack)
@@ -261,6 +267,12 @@ func findBatchIdx(batches []Batch, group string, os OS, stack *Stack) int {
 				continue
 			}
 		}
+		if os.DockerVariant != "" {
+			// must be the same docker image
+			if b.OS.DockerVariant != "" && b.OS.DockerVariant != os.DockerVariant {
+				continue
+			}
+		}
 		if stack == nil {
 			// don't care if the batch has a cloud or not
 			return i

pkg/testing/define/requirements.go

@@ -52,8 +52,12 @@ type OS struct {
 	// defined the test is run on a selected version for this operating system.
 	Version string `json:"version"`
 	// Distro allows in the Linux case for a specific distribution to be
-	// selected for running on. Example would be "ubuntu".
+	// selected for running on. Example would be "ubuntu". In the Kubernetes case
+	// for a specific distribution of kubernetes. Example would be "kind".
 	Distro string `json:"distro"`
+	// DockerVariant allows in the Kubernetes case for a specific variant to
+	// be selected for running with. Example would be "wolfi".
+	DockerVariant string `json:"docker_variant"`
 }
 
 // Validate returns an error if not valid.
@@ -75,6 +79,9 @@ func (o OS) Validate() error {
 	if o.Distro != "" && (o.Type != Linux && o.Type != Kubernetes) {
 		return errors.New("distro can only be set when type is linux or kubernetes")
 	}
+	if o.DockerVariant != "" && o.Type != Kubernetes {
+		return errors.New("docker variant can only be set when type is kubernetes")
+	}
 	return nil
 }
 

pkg/testing/kubernetes/image.go

@@ -16,11 +16,10 @@ import (
 	"path/filepath"
 	"strings"
 
-	devtools "github.com/elastic/elastic-agent/dev-tools/mage"
-	"github.com/elastic/elastic-agent/pkg/testing/runner"
-
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
+
+	devtools "github.com/elastic/elastic-agent/dev-tools/mage"
 )
 
 type DockerConfig struct {
@@ -46,8 +45,13 @@ type Endpoint struct {
 	Host string `json:"Host"`
 }
 
+type runnerLogger interface {
+	// Logf logs the message for this runner.
+	Logf(format string, args ...any)
+}
+
 // AddK8STestsToImage compiles and adds the k8s-inner-tests binary to the given image
-func AddK8STestsToImage(ctx context.Context, logger runner.Logger, baseImage string, arch string) (string, error) {
+func AddK8STestsToImage(ctx context.Context, logger runnerLogger, baseImage string, arch string) (string, error) {
 	// compile k8s test with tag kubernetes_inner
 	buildBase, err := filepath.Abs("build")
 	if err != nil {

pkg/testing/kubernetes/kind/provisioner.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"os/exec"
 	"runtime"
-	"slices"
 	"strings"
 
 	"github.com/elastic/elastic-agent/pkg/testing/define"
@@ -50,13 +49,12 @@ nodes:
         secure-port: "10257"
 `
 
-func NewProvisioner(versions string) runner.InstanceProvisioner {
-	return &provisioner{versions: strings.Split(versions, ",")}
+func NewProvisioner() runner.InstanceProvisioner {
+	return &provisioner{}
 }
 
 type provisioner struct {
-	logger   runner.Logger
-	versions []string
+	logger runner.Logger
 }
 
 func (p *provisioner) Name() string {
@@ -72,46 +70,36 @@ func (p *provisioner) SetLogger(l runner.Logger) {
 }
 
 func (p *provisioner) Supported(batch define.OS) bool {
-
-	supported := batch.Type == define.Kubernetes && batch.Arch == runtime.GOARCH && (batch.Distro == "" || batch.Distro == "kind")
-
-	if supported && batch.Version != "" {
-		supported = slices.Contains(p.versions, batch.Version)
+	if batch.Type != define.Kubernetes || batch.Arch != runtime.GOARCH {
+		return false
 	}
-
-	return supported
+	if batch.Distro != "" && batch.Distro != Name {
+		// not kind, don't run
+		return false
+	}
+	return true
 }
 
 func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches []runner.OSBatch) ([]runner.Instance, error) {
-
-	agentImageWithoutTests := fmt.Sprintf("docker.elastic.co/beats/elastic-agent-complete:%s", cfg.AgentVersion)
-	agentImage, err := kubernetes.AddK8STestsToImage(ctx, p.logger, agentImageWithoutTests, runtime.GOARCH)
-	if err != nil {
-		return nil, err
-	}
-
-	versionsMap := make(map[string]string)
-
+	var instances []runner.Instance
 	for _, batch := range batches {
-		k8sVersion := batch.OS.Version
-		if k8sVersion == "" {
-			for _, version := range p.versions {
-				versionsMap[version] = batch.ID
-			}
-			break
-		}
-
-		versionsMap[k8sVersion] = batch.ID
-	}
+		k8sVersion := fmt.Sprintf("v%s", batch.OS.Version)
+		instanceName := fmt.Sprintf("%s-%s", k8sVersion, batch.Batch.Group)
 
-	var instances []runner.Instance
-	for k8sVersion, instanceID := range versionsMap {
-		instanceName := fmt.Sprintf("%s-%s", k8sVersion, instanceID)
-		exists, err := p.clusterExists(instanceName)
+		agentImageName, err := kubernetes.VariantToImage(batch.OS.DockerVariant)
 		if err != nil {
 			return nil, err
 		}
+		agentImageName = fmt.Sprintf("%s:%s", agentImageName, cfg.AgentVersion)
+		agentImage, err := kubernetes.AddK8STestsToImage(ctx, p.logger, agentImageName, runtime.GOARCH)
+		if err != nil {
+			return nil, fmt.Errorf("failed to add k8s tests to image %s: %w", agentImageName, err)
+		}
 
+		exists, err := p.clusterExists(instanceName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to check if cluster exists: %w", err)
+		}
 		if !exists {
 			p.logger.Logf("Provisioning kind cluster %s", instanceName)
 			nodeImage := fmt.Sprintf("kindest/node:%s", k8sVersion)
@@ -153,7 +141,7 @@ func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches
 		}
 
 		instances = append(instances, runner.Instance{
-			ID:          instanceID,
+			ID:          batch.ID,
 			Name:        instanceName,
 			Provisioner: Name,
 			IP:          "",