Check for non-matching commit hash while testing upgrades (#4095)

Upgrade tests were failing because of the matching commit hashes in
picked snapshot builds. Multiple builds can have the same commit hash
for the agent which leads to file system collisions.

testing/integration/upgrade_downgrade_test.go

@@ -23,6 +23,10 @@ import (
 	"github.com/elastic/elastic-agent/testing/upgradetest"
 )
 
+const (
+	artifactElasticAgentProject = "elastic-agent-package"
+)
+
 func TestStandaloneDowngradeToSpecificSnapshotBuild(t *testing.T) {
 	define.Require(t, define.Requirements{
 		Group: Upgrade,
@@ -41,25 +45,45 @@ func TestStandaloneDowngradeToSpecificSnapshotBuild(t *testing.T) {
 	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
 	defer cancel()
 
-	// retrieve all the versions of agent from the artifact API
 	aac := tools.NewArtifactAPIClient()
 	latestSnapshotVersion, err := aac.GetLatestSnapshotVersion(ctx, t)
 	require.NoError(t, err)
 
-	// get all the builds of the snapshot version (need to pass x.y.z-SNAPSHOT format)
-	// 2 builds are required to be available for the test to work. This is because
-	// if we upgrade to the latest build there would be no difference then passing the version
-	// string without the buildid, being agent would pick that build anyway.
-	// We pick the build that is 2 builds back to upgrade to, to ensure that the buildid is
-	// working correctly and agent is not only picking the latest build.
-	builds, err := aac.GetBuildsForVersion(ctx, latestSnapshotVersion.VersionWithPrerelease())
+	// start at the build version as we want to test the retry
+	// logic that is in the build.
+	startFixture, err := define.NewFixture(t, define.Version())
+	require.NoError(t, err)
+	startVersion, err := startFixture.ExecVersion(ctx)
+	require.NoError(t, err)
+
+	// We need to find a build which is not the latest (so, we can make sure we address it by a build ID
+	// in the version prefix, e.g. x.y.z-SNAPSHOT-<buildid>) and does not have the same commit hash
+	// as the currently running binary (so, we don't have a file system collision).
+	// Multiple builds can have different IDs but the same commit hash.
+	preReleaseVersion := latestSnapshotVersion.VersionWithPrerelease()
+	resp, err := aac.GetBuildsForVersion(ctx, preReleaseVersion)
 	require.NoError(t, err)
-	if len(builds.Builds) < 2 {
-		t.Skipf("not enough SNAPSHOT builds exist for version %s. Found %d", latestSnapshotVersion.VersionWithPrerelease(), len(builds.Builds))
+
+	if len(resp.Builds) < 2 {
+		t.Skipf("need at least 2 builds in the version %s", latestSnapshotVersion.VersionWithPrerelease())
+		return
+	}
+
+	var upgradeVersionString string
+	for _, buildID := range resp.Builds[1:] {
+		details, err := aac.GetBuildDetails(ctx, preReleaseVersion, buildID)
+		require.NoError(t, err)
+		if details.Build.Projects[artifactElasticAgentProject].CommitHash != startVersion.Binary.Commit {
+			upgradeVersionString = buildID
+			break
+		}
+	}
+
+	if upgradeVersionString == "" {
+		t.Skipf("there is no other build with a non-matching commit hash in the given version %s", latestSnapshotVersion.VersionWithPrerelease())
+		return
 	}
 
-	// find the specific build to use for the test
-	upgradeVersionString := builds.Builds[1]
 	buildFragments := strings.Split(upgradeVersionString, "-")
 	require.Lenf(t, buildFragments, 2, "version %q returned by artifact api is not in format <version>-<buildID>", upgradeVersionString)
 	endParsedVersion := version.NewParsedSemVer(
@@ -70,11 +94,6 @@ func TestStandaloneDowngradeToSpecificSnapshotBuild(t *testing.T) {
 		buildFragments[1],
 	)
 
-	// Start at the build version as we want to test the retry
-	// logic that is in the build.
-	startFixture, err := define.NewFixture(t, define.Version())
-	require.NoError(t, err)
-
 	// Upgrade to the specific build.
 	endFixture, err := atesting.NewFixture(
 		t,

testing/integration/upgrade_fleet_test.go

@@ -182,6 +182,11 @@ func testUpgradeFleetManagedElasticAgent(
 	endVersionInfo, err := endFixture.ExecVersion(ctx)
 	require.NoError(t, err)
 
+	if startVersionInfo.Binary.Commit == endVersionInfo.Binary.Commit {
+		t.Skipf("target version has the same commit hash %q", endVersionInfo.Binary.Commit)
+		return
+	}
+
 	t.Log("Creating Agent policy...")
 	policyResp, err := kibClient.CreatePolicy(ctx, policy)
 	require.NoError(t, err, "failed creating policy")

testing/upgradetest/upgrader.go

@@ -174,6 +174,10 @@ func PerformUpgrade(
 		return fmt.Errorf("failed to get end agent build version info: %w", err)
 	}
 
+	if startVersionInfo.Binary.Commit == endVersionInfo.Binary.Commit {
+		return fmt.Errorf("target version has the same commit hash %q", endVersionInfo.Binary.Commit)
+	}
+
 	// For asserting on the effects of any Upgrade Watcher changes made in 8.12.0, we need
 	// the endVersion to be >= 8.12.0.  Otherwise, these assertions will fail as those changes
 	// won't be present in the Upgrade Watcher. So we disable these assertions if the endVersion
@@ -233,7 +237,7 @@ func PerformUpgrade(
 		}
 	}
 
-	logger.Logf("Upgrading from version %q (%s) to version %q (%s)", startParsedVersion, startVersionInfo.Binary.Commit, endVersionInfo.Binary.String(), endVersionInfo.Binary.Commit)
+	logger.Logf("Upgrading from version \"%s-%s\" to version \"%s-%s\"", startParsedVersion, startVersionInfo.Binary.Commit, endVersionInfo.Binary.String(), endVersionInfo.Binary.Commit)
 
 	upgradeCmdArgs := []string{"upgrade", endVersionInfo.Binary.String()}
 	if upgradeOpts.sourceURI == nil {