Merge branch 'main' into dependabot/go_modules/github.com/elastic/ela…

…stic-agent-system-metrics-0.10.1

.buildkite/scripts/steps/integration_tests.sh

@@ -9,9 +9,9 @@ MAGE_SUBTARGET="${3:-""}"
 
 
 # Override the agent package version using a string with format <major>.<minor>.<patch>
-# NOTE: use only after version bump when the new version is not yet available, for example:
-# OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3" otherwise OVERRIDE_AGENT_PACKAGE_VERSION="".
-OVERRIDE_AGENT_PACKAGE_VERSION="8.14.0"
+# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
+# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
+OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)"
 
 if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then
   OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"

.github/workflows/bump-agent-versions.sh

@@ -1,26 +1,27 @@
 #!/bin/bash
 set -e
 
+package_version=$(mage integration:updatePackageVersion)
 version_requirements=$(mage integration:updateVersions)
-changes=$(git status -s -uno .agent-versions.json)
+changes=$(git status -s -uno .agent-versions.json .package-version)
 if [ -z "$changes" ]
 then
-    echo "The versions file didn't change, skipping..."
+    echo "The version files didn't change, skipping..."
 else
-    echo "The versions file changed"
+    echo "The version file(s) changed"
+    git diff -p
     open=$(gh pr list --repo "$GITHUB_REPOSITORY" --label="update-versions" --limit 1 --state open --base "$GITHUB_REF_NAME")
     if [ -n "$open" ]
     then
         echo "Another PR for $GITHUB_REF_NAME is in review, skipping..."
         exit 0
     fi
-    git diff -p
-    git add ".agent-versions.json"
+    git add .agent-versions.json .package-version
 
     nl=$'\n' # otherwise the new line character is not recognized properly
-    commit_desc="This file is used for picking agent versions in integration tests.${nl}${nl}The file's content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`"
+    commit_desc="These files are used for picking agent versions in integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: ${package_version}${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`"
 
-    git commit -m "[$GITHUB_REF_NAME][Automation] Update .agent-versions.json" -m "$commit_desc"
+    git commit -m "[$GITHUB_REF_NAME][Automation] Update versions" -m "$commit_desc"
     git push --set-upstream origin "update-agent-versions-$GITHUB_RUN_ID"
     pr=$(gh pr create \
        --base "$GITHUB_REF_NAME" \
@@ -33,5 +34,5 @@ else
        --label 'backport-skip' \
        --repo $GITHUB_REPOSITORY)
     echo "pr=$pr" >> "$GITHUB_OUTPUT" # set the step output for Slack notifications
-    echo "Created a PR with the file update: $pr"
+    echo "Created a PR with the an update: $pr"
 fi

.github/workflows/bump-agent-versions.yml

@@ -39,7 +39,7 @@ jobs:
           version: v1.13.0
           install-only: true
 
-      - name: Update versions file
+      - name: Update versions
         id: update
         env:
           GH_TOKEN: ${{ env.GITHUB_TOKEN }}
@@ -51,7 +51,7 @@ jobs:
           url: ${{ secrets.VAULT_ADDR }}
           roleId: ${{ secrets.VAULT_ROLE_ID }}
           secretId: ${{ secrets.VAULT_SECRET_ID }}
-          message: ":traffic_cone: Elastic Agent versions file update failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          message: ":traffic_cone: Elastic Agent version update failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           channel: "#ingest-notifications"
 
         # if a PR was created as a result of this job, we notify on the Slack channel
@@ -61,5 +61,5 @@ jobs:
           url: ${{ secrets.VAULT_ADDR }}
           roleId: ${{ secrets.VAULT_ROLE_ID }}
           secretId: ${{ secrets.VAULT_SECRET_ID }}
-          message: "Update for Elastic Agent versions file has been created: ${{ steps.update.outputs.pr }}"
+          message: "Update for Elastic Agent versions has been created: ${{ steps.update.outputs.pr }}"
           channel: "#ingest-notifications"

.package-version

@@ -0,0 +1 @@
+8.14.0

internal/pkg/agent/application/actions/handlers/handler_action_settings.go

@@ -75,19 +75,21 @@ func (h *Settings) handleLogLevel(ctx context.Context, logLevel string, acker ac
 	} else if err := acker.Commit(ctx); err != nil {
 		h.log.Errorf("failed to commit acker after acknowledging action with id '%s'", action.ActionID)
 	}
-	h.log.Infof("Settings action done, setting agent log level to %s", logLevel)
 
 	if lvl != nil {
+		h.log.Infof("Settings action done, setting agent log level to %s", logLevel)
 		return h.logLevelSetter.SetLogLevel(ctx, lvl)
 	}
 
 	if h.fallbackLogLevel != nil {
+		h.log.Infof("Settings action done, setting agent log level to policy default %s", h.fallbackLogLevel)
 		// use fallback log level
 		return h.logLevelSetter.SetLogLevel(ctx, h.fallbackLogLevel)
 	}
 
 	// use default log level
 	defaultLogLevel := logger.DefaultLogLevel
+	h.log.Infof("Settings action done, setting agent log level to default %s", defaultLogLevel)
 	return h.logLevelSetter.SetLogLevel(ctx, &defaultLogLevel)
 }
 

magefile.go

@@ -2045,6 +2045,34 @@ func (Integration) UpdateVersions(ctx context.Context) error {
 	return nil
 }
 
+// UpdatePackageVersion update the file that contains the latest available snapshot version
+func (Integration) UpdatePackageVersion(ctx context.Context) error {
+	const packageVersionFilename = ".package-version"
+
+	sc := snapshots.NewSnapshotsClient()
+	versions, err := sc.FindLatestSnapshots(ctx, []string{"master"})
+	if err != nil {
+		return fmt.Errorf("failed to fetch a manifest for the latest snapshot: %w", err)
+	}
+	if len(versions) != 1 {
+		return fmt.Errorf("expected a single version, got %v", versions)
+	}
+	packageVersion := versions[0].CoreVersion()
+	file, err := os.OpenFile(packageVersionFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open %s for write: %w", packageVersionFilename, err)
+	}
+	defer file.Close()
+	_, err = file.WriteString(packageVersion)
+	if err != nil {
+		return fmt.Errorf("failed to write the package version file %s: %w", packageVersionFilename, err)
+	}
+
+	fmt.Println(packageVersion)
+
+	return nil
+}
+
 var stateDir = ".integration-cache"
 var stateFile = "state.yml"
 

pkg/testing/tools/estools/elasticsearch.go

@@ -375,21 +375,17 @@ func CheckForErrorsInLogs(ctx context.Context, client elastictransport.Interface
 // CheckForErrorsInLogsWithContext checks to see if any error-level lines exist
 // excludeStrings can be used to remove any particular error strings from logs
 func CheckForErrorsInLogsWithContext(ctx context.Context, client elastictransport.Interface, namespace string, excludeStrings []string) (Documents, error) {
-	queryRaw := map[string]interface{}{
-		"query": map[string]interface{}{
-			"bool": map[string]interface{}{
-				"must": []map[string]interface{}{
-					{
-						"match": map[string]interface{}{
-							"log.level": "error",
-						},
-					},
-					{
-						"term": map[string]interface{}{
-							"data_stream.namespace": map[string]interface{}{
-								"value": namespace,
-							},
-						},
+	filters := map[string]interface{}{
+		"must": []map[string]interface{}{
+			{
+				"match": map[string]interface{}{
+					"log.level": "error",
+				},
+			},
+			{
+				"term": map[string]interface{}{
+					"data_stream.namespace": map[string]interface{}{
+						"value": namespace,
 					},
 				},
 			},
@@ -405,27 +401,14 @@ func CheckForErrorsInLogsWithContext(ctx context.Context, client elastictranspor
 				},
 			})
 		}
-		queryRaw = map[string]interface{}{
-			"query": map[string]interface{}{
-				"bool": map[string]interface{}{
-					"must": []map[string]interface{}{
-						{
-							"match": map[string]interface{}{
-								"log.level": "error",
-							},
-						},
-						{
-							"term": map[string]interface{}{
-								"data_stream.namespace": map[string]interface{}{
-									"value": namespace,
-								},
-							},
-						},
-					},
-					"must_not": excludeStatements,
-				},
-			},
-		}
+
+		filters["must_not"] = excludeStatements
+	}
+
+	queryRaw := map[string]interface{}{
+		"query": map[string]interface{}{
+			"bool": filters,
+		},
 	}
 
 	var buf bytes.Buffer

testing/integration/logs_ingestion_test.go

@@ -274,21 +274,28 @@ func testMonitoringLogsAreShipped(
 			"Failed to initialize artifact",
 			"Failed to apply initial policy from on disk configuration",
 			"elastic-agent-client error: rpc error: code = Canceled desc = context canceled", // can happen on restart
-			"add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/instance-id\\\": dial tcp 169.254.169.254:443: connect: connection refused",                 // okay for the openstack metadata to not work
-			"add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/hostname\\\": dial tcp 169.254.169.254:443: connect: connection refused",                    // okay for the cloud metadata to not work
-			"add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/placement/availability-zone\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work
-			"add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/instance-type\\\": dial tcp 169.254.169.254:443: connect: connection refused",               // okay for the cloud metadata to not work
-			"add_cloud_metadata: received error failed with http status code 404",                                                                                                                             // okay for the cloud metadata to not work
-			"add_cloud_metadata: received error failed fetching EC2 Identity Document: not found, Signing",                                                                                                    // okay for the cloud metadata to not work
-			"add_cloud_metadata: received error failed fetching EC2 Identity Document: operation error ec2imds: GetInstanceIdentityDocument, http response error StatusCode: 404, request to EC2 IMDS failed", // okay for the cloud metadata to not work
-			"failed to invoke rollback watcher: failed to start Upgrade Watcher: fork/exec /var/lib/elastic-agent/elastic-agent: no such file or directory",                                                   // on debian this happens probably need to fix.
+			"add_cloud_metadata: received error failed requesting openstack metadata",        // okay for the cloud metadata to not work
+			"add_cloud_metadata: received error failed with http status code 404",            // okay for the cloud metadata to not work
+			"add_cloud_metadata: received error failed fetching EC2 Identity Document",       // okay for the cloud metadata to not work
+			"failed to invoke rollback watcher: failed to start Upgrade Watcher",             // on debian this happens probably need to fix.
+			"falling back to IMDSv1: operation error ec2imds: getToken",                      // okay for the cloud metadata to not work
 		})
 	})
 	t.Logf("error logs: Got %d documents", len(docs.Hits.Hits))
+	messages := make([]string, 0, len(docs.Hits.Hits))
 	for _, doc := range docs.Hits.Hits {
 		t.Logf("%#v", doc.Source)
+		message, ok := doc.Source["message"]
+		if !ok {
+			continue
+		}
+		messageStr, ok := message.(string)
+		if !ok {
+			continue
+		}
+		messages = append(messages, messageStr)
 	}
-	require.Empty(t, docs.Hits.Hits)
+	require.Emptyf(t, docs.Hits.Hits, "list of error messages is expected to be empty, found:\n%s", strings.Join(messages, ", \n"))
 
 	// Stage 3: Make sure we have message confirming central management is running
 	t.Log("Making sure we have message confirming central management is running")