Merge branch 'main' into dharma

ee/agent/timemachine/timemachine_darwin.go

@@ -33,10 +33,17 @@ func AddExclusions(ctx context.Context, k types.Knapsack) {
 		launcher.pid
 	*/
 
-	exclusionPatterns := []string{
+	exclusionPatternsFirstBatch := []string{
 		"*.json",
 		"*.json.gz",
 		"*.db",
+	}
+
+	addExclusionsFromPathPatterns(ctx, k, exclusionPatternsFirstBatch)
+
+	// Attempting to run this with a single tmutil call we see a lot of tmutil failures logged with error "argument list too long".
+	// To avoid this we run in two separate batches, attempting to cut the post-glob argument list roughly in half
+	exclusionPatternsSecondBatch := []string{
 		"*.sqlite",
 		"desktop_*",
 		"*.pid",
@@ -45,6 +52,10 @@ func AddExclusions(ctx context.Context, k types.Knapsack) {
 		"osquery*",
 	}
 
+	addExclusionsFromPathPatterns(ctx, k, exclusionPatternsSecondBatch)
+}
+
+func addExclusionsFromPathPatterns(ctx context.Context, k types.Knapsack, exclusionPatterns []string) {
 	var exclusionPaths []string
 	for _, pattern := range exclusionPatterns {
 		matches, err := filepath.Glob(filepath.Join(k.RootDirectory(), pattern))

ee/tuf/library_lookup.go

@@ -230,7 +230,7 @@ func CheckOutLatest(ctx context.Context, binary autoupdatableBinary, rootDirecto
 
 	// If we can't find the specific release version that we should be on, then just return the executable
 	// with the most recent version in the library
-	return mostRecentVersion(ctx, binary, updateDirectory, channel)
+	return mostRecentVersion(ctx, slogger, binary, updateDirectory, channel)
 }
 
 // findExecutable looks at our local TUF repository to find the release for our
@@ -270,12 +270,12 @@ func findExecutable(ctx context.Context, binary autoupdatableBinary, tufReposito
 
 // mostRecentVersion returns the path to the most recent, valid version available in the library for the
 // given binary, along with its version.
-func mostRecentVersion(ctx context.Context, binary autoupdatableBinary, baseUpdateDirectory, channel string) (*BinaryUpdateInfo, error) {
+func mostRecentVersion(ctx context.Context, slogger *slog.Logger, binary autoupdatableBinary, baseUpdateDirectory, channel string) (*BinaryUpdateInfo, error) {
 	ctx, span := traces.StartSpan(ctx)
 	defer span.End()
 
 	// Pull all available versions from library
-	validVersionsInLibrary, _, err := sortedVersionsInLibrary(ctx, binary, baseUpdateDirectory)
+	validVersionsInLibrary, _, err := sortedVersionsInLibrary(ctx, slogger, binary, baseUpdateDirectory)
 	if err != nil {
 		return nil, fmt.Errorf("could not get sorted versions in library for %s: %w", binary, err)
 	}

ee/tuf/library_lookup_test.go

@@ -170,7 +170,7 @@ func Test_mostRecentVersion(t *testing.T) {
 			tufci.CopyBinary(t, secondVersionPath)
 			require.NoError(t, os.Chmod(secondVersionPath, 0755))
 
-			latest, err := mostRecentVersion(context.TODO(), binary, testBaseDir, "nightly")
+			latest, err := mostRecentVersion(context.TODO(), multislogger.NewNopLogger(), binary, testBaseDir, "nightly")
 			require.NoError(t, err, "did not expect error getting most recent version")
 			require.Equal(t, secondVersionPath, latest.Path)
 			require.Equal(t, secondVersion, latest.Version)
@@ -202,7 +202,7 @@ func Test_mostRecentVersion_DoesNotReturnInvalidExecutables(t *testing.T) {
 			require.NoError(t, os.MkdirAll(filepath.Dir(secondVersionPath), 0755))
 			os.WriteFile(secondVersionPath, []byte{}, 0755)
 
-			latest, err := mostRecentVersion(context.TODO(), binary, testBaseDir, "nightly")
+			latest, err := mostRecentVersion(context.TODO(), multislogger.NewNopLogger(), binary, testBaseDir, "nightly")
 			require.NoError(t, err, "did not expect error getting most recent version")
 			require.Equal(t, firstVersionPath, latest.Path)
 			require.Equal(t, firstVersion, latest.Version)
@@ -221,7 +221,7 @@ func Test_mostRecentVersion_ReturnsErrorOnNoUpdatesDownloaded(t *testing.T) {
 			// Create update directories
 			testBaseDir := t.TempDir()
 
-			_, err := mostRecentVersion(context.TODO(), binary, testBaseDir, "nightly")
+			_, err := mostRecentVersion(context.TODO(), multislogger.NewNopLogger(), binary, testBaseDir, "nightly")
 			require.Error(t, err, "should have returned error when there are no available updates")
 		})
 	}
@@ -239,7 +239,7 @@ func Test_mostRecentVersion_requiresLauncher_v1_4_1(t *testing.T) {
 	tufci.CopyBinary(t, firstVersionPath)
 	require.NoError(t, os.Chmod(firstVersionPath, 0755))
 
-	_, err := mostRecentVersion(context.TODO(), binaryLauncher, testBaseDir, "stable")
+	_, err := mostRecentVersion(context.TODO(), multislogger.NewNopLogger(), binaryLauncher, testBaseDir, "stable")
 	require.Error(t, err, "should not select launcher version under v1.4.1")
 }
 
@@ -255,7 +255,7 @@ func Test_mostRecentVersion_acceptsLauncher_v1_4_1(t *testing.T) {
 	tufci.CopyBinary(t, firstVersionPath)
 	require.NoError(t, os.Chmod(firstVersionPath, 0755))
 
-	latest, err := mostRecentVersion(context.TODO(), binaryLauncher, testBaseDir, "stable")
+	latest, err := mostRecentVersion(context.TODO(), multislogger.NewNopLogger(), binaryLauncher, testBaseDir, "stable")
 	require.NoError(t, err, "should be able to select launcher version equal to v1.4.1")
 	require.Equal(t, firstVersionPath, latest.Path)
 }

ee/tuf/library_manager.go

@@ -366,7 +366,7 @@ func (ulm *updateLibraryManager) TidyLibrary(binary autoupdatableBinary, current
 
 	const numberOfVersionsToKeep = 3
 
-	versionsInLibrary, invalidVersionsInLibrary, err := sortedVersionsInLibrary(context.Background(), binary, ulm.baseDir)
+	versionsInLibrary, invalidVersionsInLibrary, err := sortedVersionsInLibrary(context.Background(), ulm.slogger, binary, ulm.baseDir)
 	if err != nil {
 		ulm.slogger.Log(context.TODO(), slog.LevelWarn,
 			"could not get versions in library to tidy update library",
@@ -410,7 +410,7 @@ func (ulm *updateLibraryManager) TidyLibrary(binary autoupdatableBinary, current
 // sortedVersionsInLibrary looks through the update library for the given binary to validate and sort all
 // available versions. It returns a sorted list of the valid versions, a list of invalid versions, and
 // an error only when unable to glob for versions.
-func sortedVersionsInLibrary(ctx context.Context, binary autoupdatableBinary, baseUpdateDirectory string) ([]string, []string, error) {
+func sortedVersionsInLibrary(ctx context.Context, slogger *slog.Logger, binary autoupdatableBinary, baseUpdateDirectory string) ([]string, []string, error) {
 	ctx, span := traces.StartSpan(ctx)
 	defer span.End()
 
@@ -425,13 +425,27 @@ func sortedVersionsInLibrary(ctx context.Context, binary autoupdatableBinary, ba
 		rawVersion := filepath.Base(rawVersionWithPath)
 		v, err := semver.NewVersion(rawVersion)
 		if err != nil {
+			slogger.Log(ctx, slog.LevelWarn,
+				"detected invalid binary version while parsing raw version",
+				"raw_version", rawVersion,
+				"binary", binary,
+				"err", err,
+			)
+
 			invalidVersions = append(invalidVersions, rawVersion)
 			continue
 		}
 
 		versionDir := filepath.Join(updatesDirectory(binary, baseUpdateDirectory), rawVersion)
 		if err := CheckExecutable(ctx, executableLocation(versionDir, binary), "--version"); err != nil {
 			traces.SetError(span, err)
+			slogger.Log(ctx, slog.LevelWarn,
+				"detected invalid binary version while checking executable",
+				"version_dir", versionDir,
+				"binary", binary,
+				"err", err,
+			)
+
 			invalidVersions = append(invalidVersions, rawVersion)
 			continue
 		}

ee/tuf/library_manager_test.go

@@ -685,7 +685,7 @@ func Test_sortedVersionsInLibrary(t *testing.T) {
 	}
 
 	// Get sorted versions
-	validVersions, invalidVersions, err := sortedVersionsInLibrary(context.TODO(), binaryLauncher, testBaseDir)
+	validVersions, invalidVersions, err := sortedVersionsInLibrary(context.TODO(), multislogger.NewNopLogger(), binaryLauncher, testBaseDir)
 	require.NoError(t, err, "expected no error on sorting versions in library")
 
 	// Confirm invalid versions are the ones we expect
@@ -725,7 +725,7 @@ func Test_sortedVersionsInLibrary_devBuilds(t *testing.T) {
 	}
 
 	// Get sorted versions
-	validVersions, invalidVersions, err := sortedVersionsInLibrary(context.TODO(), binaryLauncher, testBaseDir)
+	validVersions, invalidVersions, err := sortedVersionsInLibrary(context.TODO(), multislogger.NewNopLogger(), binaryLauncher, testBaseDir)
 	require.NoError(t, err, "expected no error on sorting versions in library")
 
 	// Confirm we don't have any invalid versions