From b4b3051d57fa93bb20aae9193294dcb7cecab8cd Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 16 May 2024 08:18:49 -0400 Subject: [PATCH 1/3] fix log message when settings log level from SETTINGS action (#4751) --- .../application/actions/handlers/handler_action_settings.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/pkg/agent/application/actions/handlers/handler_action_settings.go b/internal/pkg/agent/application/actions/handlers/handler_action_settings.go index 57dfe6f17f9..4c4c4c6a827 100644 --- a/internal/pkg/agent/application/actions/handlers/handler_action_settings.go +++ b/internal/pkg/agent/application/actions/handlers/handler_action_settings.go @@ -75,19 +75,21 @@ func (h *Settings) handleLogLevel(ctx context.Context, logLevel string, acker ac } else if err := acker.Commit(ctx); err != nil { h.log.Errorf("failed to commit acker after acknowledging action with id '%s'", action.ActionID) } - h.log.Infof("Settings action done, setting agent log level to %s", logLevel) if lvl != nil { + h.log.Infof("Settings action done, setting agent log level to %s", logLevel) return h.logLevelSetter.SetLogLevel(ctx, lvl) } if h.fallbackLogLevel != nil { + h.log.Infof("Settings action done, setting agent log level to policy default %s", h.fallbackLogLevel) // use fallback log level return h.logLevelSetter.SetLogLevel(ctx, h.fallbackLogLevel) } // use default log level defaultLogLevel := logger.DefaultLogLevel + h.log.Infof("Settings action done, setting agent log level to default %s", defaultLogLevel) return h.logLevelSetter.SetLogLevel(ctx, &defaultLogLevel) } From 302ec2cc10f306132a908ccc93c2e7f79cb1c429 Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 16 May 2024 15:29:14 +0200 Subject: [PATCH 2/3] Make error checks in TestLogIngestionFleetManaged more resilient (#4767) This check should not use exact matching but rather "contains" for accounting for know errors since details might change. --- pkg/testing/tools/estools/elasticsearch.go | 55 ++++++++-------------- testing/integration/logs_ingestion_test.go | 25 ++++++---- 2 files changed, 35 insertions(+), 45 deletions(-) diff --git a/pkg/testing/tools/estools/elasticsearch.go b/pkg/testing/tools/estools/elasticsearch.go index 67902a08e2c..fd0f6a68d3c 100644 --- a/pkg/testing/tools/estools/elasticsearch.go +++ b/pkg/testing/tools/estools/elasticsearch.go @@ -375,21 +375,17 @@ func CheckForErrorsInLogs(ctx context.Context, client elastictransport.Interface // CheckForErrorsInLogsWithContext checks to see if any error-level lines exist // excludeStrings can be used to remove any particular error strings from logs func CheckForErrorsInLogsWithContext(ctx context.Context, client elastictransport.Interface, namespace string, excludeStrings []string) (Documents, error) { - queryRaw := map[string]interface{}{ - "query": map[string]interface{}{ - "bool": map[string]interface{}{ - "must": []map[string]interface{}{ - { - "match": map[string]interface{}{ - "log.level": "error", - }, - }, - { - "term": map[string]interface{}{ - "data_stream.namespace": map[string]interface{}{ - "value": namespace, - }, - }, + filters := map[string]interface{}{ + "must": []map[string]interface{}{ + { + "match": map[string]interface{}{ + "log.level": "error", + }, + }, + { + "term": map[string]interface{}{ + "data_stream.namespace": map[string]interface{}{ + "value": namespace, }, }, }, @@ -405,27 +401,14 @@ func CheckForErrorsInLogsWithContext(ctx context.Context, client elastictranspor }, }) } - queryRaw = map[string]interface{}{ - "query": map[string]interface{}{ - "bool": map[string]interface{}{ - "must": []map[string]interface{}{ - { - "match": map[string]interface{}{ - "log.level": "error", - }, - }, - { - "term": map[string]interface{}{ - "data_stream.namespace": map[string]interface{}{ - "value": namespace, - }, - }, - }, - }, - "must_not": excludeStatements, - }, - }, - } + + filters["must_not"] = excludeStatements + } + + queryRaw := map[string]interface{}{ + "query": map[string]interface{}{ + "bool": filters, + }, } var buf bytes.Buffer diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index c9bd896e422..bec278afac1 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -274,21 +274,28 @@ func testMonitoringLogsAreShipped( "Failed to initialize artifact", "Failed to apply initial policy from on disk configuration", "elastic-agent-client error: rpc error: code = Canceled desc = context canceled", // can happen on restart - "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/instance-id\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the openstack metadata to not work - "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/hostname\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work - "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/placement/availability-zone\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work - "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/instance-type\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work - "add_cloud_metadata: received error failed with http status code 404", // okay for the cloud metadata to not work - "add_cloud_metadata: received error failed fetching EC2 Identity Document: not found, Signing", // okay for the cloud metadata to not work - "add_cloud_metadata: received error failed fetching EC2 Identity Document: operation error ec2imds: GetInstanceIdentityDocument, http response error StatusCode: 404, request to EC2 IMDS failed", // okay for the cloud metadata to not work - "failed to invoke rollback watcher: failed to start Upgrade Watcher: fork/exec /var/lib/elastic-agent/elastic-agent: no such file or directory", // on debian this happens probably need to fix. + "add_cloud_metadata: received error failed requesting openstack metadata", // okay for the cloud metadata to not work + "add_cloud_metadata: received error failed with http status code 404", // okay for the cloud metadata to not work + "add_cloud_metadata: received error failed fetching EC2 Identity Document", // okay for the cloud metadata to not work + "failed to invoke rollback watcher: failed to start Upgrade Watcher", // on debian this happens probably need to fix. + "falling back to IMDSv1: operation error ec2imds: getToken", // okay for the cloud metadata to not work }) }) t.Logf("error logs: Got %d documents", len(docs.Hits.Hits)) + messages := make([]string, 0, len(docs.Hits.Hits)) for _, doc := range docs.Hits.Hits { t.Logf("%#v", doc.Source) + message, ok := doc.Source["message"] + if !ok { + continue + } + messageStr, ok := message.(string) + if !ok { + continue + } + messages = append(messages, messageStr) } - require.Empty(t, docs.Hits.Hits) + require.Emptyf(t, docs.Hits.Hits, "list of error messages is expected to be empty, found:\n%s", strings.Join(messages, ", \n")) // Stage 3: Make sure we have message confirming central management is running t.Log("Making sure we have message confirming central management is running") From c5e9707328c49220b5aae41e58ce00ede3bff8f1 Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 16 May 2024 21:30:17 +0200 Subject: [PATCH 3/3] Automate fetching the latest package version (#4768) So, we don't have to manually edit the environment variable when the snapshot becomes available. --- .buildkite/scripts/steps/integration_tests.sh | 6 ++-- .github/workflows/bump-agent-versions.sh | 17 +++++------ .github/workflows/bump-agent-versions.yml | 6 ++-- .package-version | 1 + magefile.go | 28 +++++++++++++++++++ 5 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 .package-version diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh index 4936c62e6b6..c211fb74b44 100755 --- a/.buildkite/scripts/steps/integration_tests.sh +++ b/.buildkite/scripts/steps/integration_tests.sh @@ -9,9 +9,9 @@ MAGE_SUBTARGET="${3:-""}" # Override the agent package version using a string with format .. -# NOTE: use only after version bump when the new version is not yet available, for example: -# OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3" otherwise OVERRIDE_AGENT_PACKAGE_VERSION="". -OVERRIDE_AGENT_PACKAGE_VERSION="8.14.0" +# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically +# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready. +OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)" if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT" diff --git a/.github/workflows/bump-agent-versions.sh b/.github/workflows/bump-agent-versions.sh index ed51227645b..5a180e37d16 100755 --- a/.github/workflows/bump-agent-versions.sh +++ b/.github/workflows/bump-agent-versions.sh @@ -1,26 +1,27 @@ #!/bin/bash set -e +package_version=$(mage integration:updatePackageVersion) version_requirements=$(mage integration:updateVersions) -changes=$(git status -s -uno .agent-versions.json) +changes=$(git status -s -uno .agent-versions.json .package-version) if [ -z "$changes" ] then - echo "The versions file didn't change, skipping..." + echo "The version files didn't change, skipping..." else - echo "The versions file changed" + echo "The version file(s) changed" + git diff -p open=$(gh pr list --repo "$GITHUB_REPOSITORY" --label="update-versions" --limit 1 --state open --base "$GITHUB_REF_NAME") if [ -n "$open" ] then echo "Another PR for $GITHUB_REF_NAME is in review, skipping..." exit 0 fi - git diff -p - git add ".agent-versions.json" + git add .agent-versions.json .package-version nl=$'\n' # otherwise the new line character is not recognized properly - commit_desc="This file is used for picking agent versions in integration tests.${nl}${nl}The file's content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`" + commit_desc="These files are used for picking agent versions in integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: ${package_version}${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`" - git commit -m "[$GITHUB_REF_NAME][Automation] Update .agent-versions.json" -m "$commit_desc" + git commit -m "[$GITHUB_REF_NAME][Automation] Update versions" -m "$commit_desc" git push --set-upstream origin "update-agent-versions-$GITHUB_RUN_ID" pr=$(gh pr create \ --base "$GITHUB_REF_NAME" \ @@ -33,5 +34,5 @@ else --label 'backport-skip' \ --repo $GITHUB_REPOSITORY) echo "pr=$pr" >> "$GITHUB_OUTPUT" # set the step output for Slack notifications - echo "Created a PR with the file update: $pr" + echo "Created a PR with the an update: $pr" fi diff --git a/.github/workflows/bump-agent-versions.yml b/.github/workflows/bump-agent-versions.yml index 2736caf5a07..b8114c0f423 100644 --- a/.github/workflows/bump-agent-versions.yml +++ b/.github/workflows/bump-agent-versions.yml @@ -39,7 +39,7 @@ jobs: version: v1.13.0 install-only: true - - name: Update versions file + - name: Update versions id: update env: GH_TOKEN: ${{ env.GITHUB_TOKEN }} @@ -51,7 +51,7 @@ jobs: url: ${{ secrets.VAULT_ADDR }} roleId: ${{ secrets.VAULT_ROLE_ID }} secretId: ${{ secrets.VAULT_SECRET_ID }} - message: ":traffic_cone: Elastic Agent versions file update failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + message: ":traffic_cone: Elastic Agent version update failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" channel: "#ingest-notifications" # if a PR was created as a result of this job, we notify on the Slack channel @@ -61,5 +61,5 @@ jobs: url: ${{ secrets.VAULT_ADDR }} roleId: ${{ secrets.VAULT_ROLE_ID }} secretId: ${{ secrets.VAULT_SECRET_ID }} - message: "Update for Elastic Agent versions file has been created: ${{ steps.update.outputs.pr }}" + message: "Update for Elastic Agent versions has been created: ${{ steps.update.outputs.pr }}" channel: "#ingest-notifications" diff --git a/.package-version b/.package-version new file mode 100644 index 00000000000..68163fed754 --- /dev/null +++ b/.package-version @@ -0,0 +1 @@ +8.14.0 \ No newline at end of file diff --git a/magefile.go b/magefile.go index 5a06e405e9f..98637eb3191 100644 --- a/magefile.go +++ b/magefile.go @@ -2045,6 +2045,34 @@ func (Integration) UpdateVersions(ctx context.Context) error { return nil } +// UpdatePackageVersion update the file that contains the latest available snapshot version +func (Integration) UpdatePackageVersion(ctx context.Context) error { + const packageVersionFilename = ".package-version" + + sc := snapshots.NewSnapshotsClient() + versions, err := sc.FindLatestSnapshots(ctx, []string{"master"}) + if err != nil { + return fmt.Errorf("failed to fetch a manifest for the latest snapshot: %w", err) + } + if len(versions) != 1 { + return fmt.Errorf("expected a single version, got %v", versions) + } + packageVersion := versions[0].CoreVersion() + file, err := os.OpenFile(packageVersionFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open %s for write: %w", packageVersionFilename, err) + } + defer file.Close() + _, err = file.WriteString(packageVersion) + if err != nil { + return fmt.Errorf("failed to write the package version file %s: %w", packageVersionFilename, err) + } + + fmt.Println(packageVersion) + + return nil +} + var stateDir = ".integration-cache" var stateFile = "state.yml"