From a28bc90f14218bf92356480625b9864d717f024e Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 4 Jun 2024 03:49:31 -0400 Subject: [PATCH 1/5] Improve integration testing cloud client DeploymentIsReady. (#4798) Improves the logic in DeploymentIsReady to not return false and the error when it gets an error back from the API. Instead it will keep retrying until the timeout. --- pkg/testing/ess/deployment.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/testing/ess/deployment.go b/pkg/testing/ess/deployment.go index cbb15e40f5e..28a9a1b2c9d 100644 --- a/pkg/testing/ess/deployment.go +++ b/pkg/testing/ess/deployment.go @@ -9,6 +9,7 @@ import ( "context" _ "embed" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -251,20 +252,19 @@ func (c *Client) DeploymentIsReady(ctx context.Context, deploymentID string, tic ticker := time.NewTicker(tick) defer ticker.Stop() + var errs error statusCh := make(chan DeploymentStatus, 1) - errCh := make(chan error) - for { select { case <-ctx.Done(): - return false, ctx.Err() + return false, errors.Join(errs, ctx.Err()) case <-ticker.C: - statusCtx, statusCancel := context.WithTimeout(ctx, tick) - defer statusCancel() go func() { + statusCtx, statusCancel := context.WithTimeout(ctx, tick) + defer statusCancel() status, err := c.DeploymentStatus(statusCtx, deploymentID) if err != nil { - errCh <- err + errs = errors.Join(errs, err) return } statusCh <- status.Overall @@ -273,8 +273,6 @@ func (c *Client) DeploymentIsReady(ctx context.Context, deploymentID string, tic if status == DeploymentStatusStarted { return true, nil } - case err := <-errCh: - return false, err } } } From 0b99ccb922d26bcd26f6911534530b1e90149a8a Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 4 Jun 2024 16:38:40 +0200 Subject: [PATCH 2/5] [CI] Trigger improvements (#4836) * [CI] Added .github and .ci directories to skip_ci_on_only_changed * [CI] Removed sonar-project.properties from ignored PR changesets * [CI] skip labels and CI cheatsheet * Added alternative triggers description --- .buildkite/pull-requests.json | 11 ++++++----- .github/PULL_REQUEST_TEMPLATE.md | 10 ++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 001d7cd1709..628f46e6172 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -1,7 +1,7 @@ { "jobs": [ { - "enabled": true, + "enabled": true, "pipelineSlug": "elastic-agent", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], @@ -11,13 +11,14 @@ "build_on_comment": true, "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$", "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$", - "skip_ci_labels": [ ], + "skip_ci_labels": [ "skip-ci" ], "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ "changelog", "docs", "README.md", "sonar-project.properties", "docker-compose.yml", ".pre-commit-config.yaml", "skaffold.yaml", "Dockerfile.skaffold", "Dockerfile"], + "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"], "always_require_ci_on_changed": [ ] }, { "enabled": true, + "build_drafts": false, "pipelineSlug": "elastic-agent-extended-testing", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], @@ -27,9 +28,9 @@ "build_on_comment": true, "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:extended))|^/test extended$", "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:extended))|^/test extended$", - "skip_ci_labels": [ ], + "skip_ci_labels": [ "skip-ci", "skip-it" ], "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ "changelog", "docs", "README.md", "sonar-project.properties", "docker-compose.yml", ".pre-commit-config.yaml", "skaffold.yaml", "Dockerfile.skaffold", "Dockerfile"], + "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^changelog", "^docs/", "\\.md$", "^sonar-project.properties", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"], "always_require_ci_on_changed": [ ] }, { diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d4abe0c32ab..3335fee4631 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -69,3 +69,13 @@ Link related issues below. Insert the issue link or reference after the word "Cl - How are we going to debug this? - What are the metrics I should take care of? - ... + + \ No newline at end of file From 89542a7580e4961ba0437f5aeff9b6c3ba121cb5 Mon Sep 17 00:00:00 2001 From: Denis Date: Tue, 4 Jun 2024 19:46:04 +0200 Subject: [PATCH 3/5] Use higher tier ARM VM for integration testing (#4847) ARM cores are lower performance, to make it equal to our AMD64 VMs we should use higher tier instances. --- pkg/testing/ogc/supported.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/testing/ogc/supported.go b/pkg/testing/ogc/supported.go index 96bbc5cf520..4b72ec4464d 100644 --- a/pkg/testing/ogc/supported.go +++ b/pkg/testing/ogc/supported.go @@ -28,7 +28,7 @@ var ogcSupported = []LayoutOS{ Version: "22.04", }, Provider: Google, - InstanceSize: "e2-standard-2", // 2 amd64 cpus + InstanceSize: "e2-standard-2", // 2 amd64 cpus, 8 GB RAM RunsOn: "ubuntu-2204-lts", Username: "ubuntu", RemotePath: "/home/ubuntu/agent", @@ -41,7 +41,7 @@ var ogcSupported = []LayoutOS{ Version: "20.04", }, Provider: Google, - InstanceSize: "e2-standard-2", // 2 amd64 cpus + InstanceSize: "e2-standard-2", // 2 amd64 cpus, 8 GB RAM RunsOn: "ubuntu-2004-lts", Username: "ubuntu", RemotePath: "/home/ubuntu/agent", @@ -54,7 +54,7 @@ var ogcSupported = []LayoutOS{ Version: "22.04", }, Provider: Google, - InstanceSize: "t2a-standard-2", // 2 arm64 cpus + InstanceSize: "t2a-standard-4", // 4 arm64 cpus, 16 GB RAM RunsOn: "ubuntu-2204-lts-arm64", Username: "ubuntu", RemotePath: "/home/ubuntu/agent", @@ -67,7 +67,7 @@ var ogcSupported = []LayoutOS{ Version: "20.04", }, Provider: Google, - InstanceSize: "t2a-standard-2", // 2 arm64 cpus + InstanceSize: "t2a-standard-4", // 4 arm64 cpus, 16 GB RAM RunsOn: "ubuntu-2004-lts-arm64", Username: "ubuntu", RemotePath: "/home/ubuntu/agent", @@ -80,7 +80,7 @@ var ogcSupported = []LayoutOS{ Version: "8", }, Provider: Google, - InstanceSize: "e2-standard-2", // 2 amd64 cpus + InstanceSize: "e2-standard-2", // 2 amd64 cpus, 8 GB RAM RunsOn: "rhel-8", Username: "rhel", RemotePath: "/home/rhel/agent", @@ -92,7 +92,7 @@ var ogcSupported = []LayoutOS{ Version: "2022", }, Provider: Google, - InstanceSize: "e2-standard-4", // 4 amd64 cpus + InstanceSize: "e2-standard-4", // 4 amd64 cpus, 16 GB RAM RunsOn: "windows-2022", Username: "windows", RemotePath: "C:\\Users\\windows\\agent", @@ -104,7 +104,7 @@ var ogcSupported = []LayoutOS{ Version: "2022-core", }, Provider: Google, - InstanceSize: "e2-standard-4", // 4 amd64 cpus + InstanceSize: "e2-standard-4", // 4 amd64 cpus, 16 GB RAM RunsOn: "windows-2022-core", Username: "windows", RemotePath: "C:\\Users\\windows\\agent", @@ -116,7 +116,7 @@ var ogcSupported = []LayoutOS{ Version: "2019", }, Provider: Google, - InstanceSize: "e2-standard-4", // 4 amd64 cpus + InstanceSize: "e2-standard-4", // 4 amd64 cpus, 16 GB RAM RunsOn: "windows-2019", Username: "windows", RemotePath: "C:\\Users\\windows\\agent", @@ -128,7 +128,7 @@ var ogcSupported = []LayoutOS{ Version: "2019-core", }, Provider: Google, - InstanceSize: "e2-standard-4", // 4 amd64 cpus + InstanceSize: "e2-standard-4", // 4 amd64 cpus, 16 GB RAM RunsOn: "windows-2019-core", Username: "windows", RemotePath: "C:\\Users\\windows\\agent", @@ -140,7 +140,7 @@ var ogcSupported = []LayoutOS{ Version: "2016", }, Provider: Google, - InstanceSize: "e2-standard-4", // 4 amd64 cpus + InstanceSize: "e2-standard-4", // 4 amd64 cpus, 16 GB RAM RunsOn: "windows-2016", Username: "windows", RemotePath: "C:\\Users\\windows\\agent", @@ -152,7 +152,7 @@ var ogcSupported = []LayoutOS{ Version: "2016-core", }, Provider: Google, - InstanceSize: "e2-standard-4", // 4 amd64 cpus + InstanceSize: "e2-standard-4", // 4 amd64 cpus, 16 GB RAM RunsOn: "windows-2016-core", Username: "windows", RemotePath: "C:\\Users\\windows\\agent", From ecefbfd20d80c1f4c2dfec5cf4d177bcde455d9c Mon Sep 17 00:00:00 2001 From: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Date: Wed, 5 Jun 2024 00:19:43 -0500 Subject: [PATCH 4/5] Add changelog for adding benchmark input (#4849) * Add changelog for adding benchmark input --- .../1717523676-Add-benchmark-input.yaml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 changelog/fragments/1717523676-Add-benchmark-input.yaml diff --git a/changelog/fragments/1717523676-Add-benchmark-input.yaml b/changelog/fragments/1717523676-Add-benchmark-input.yaml new file mode 100644 index 00000000000..49a9ad52334 --- /dev/null +++ b/changelog/fragments/1717523676-Add-benchmark-input.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: enhancement + +# Change summary; a 80ish characters long description of the change. +summary: Add benchmark input + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; a word indicating the component this changeset affects. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/beats/pull/39789 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 From 3500a81f766393c955a3d899a94bca813a4638de Mon Sep 17 00:00:00 2001 From: Denis Date: Wed, 5 Jun 2024 12:59:02 +0200 Subject: [PATCH 5/5] Disable integration tests on ARM due to instability (#4852) The current ARM hardware is experimental and very unstable, we should wait until a new stable instance type is introduced. --- pkg/testing/ogc/supported.go | 55 +++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/pkg/testing/ogc/supported.go b/pkg/testing/ogc/supported.go index 4b72ec4464d..048f7a5ee24 100644 --- a/pkg/testing/ogc/supported.go +++ b/pkg/testing/ogc/supported.go @@ -46,32 +46,35 @@ var ogcSupported = []LayoutOS{ Username: "ubuntu", RemotePath: "/home/ubuntu/agent", }, - { - OS: define.OS{ - Type: define.Linux, - Arch: define.ARM64, - Distro: runner.Ubuntu, - Version: "22.04", - }, - Provider: Google, - InstanceSize: "t2a-standard-4", // 4 arm64 cpus, 16 GB RAM - RunsOn: "ubuntu-2204-lts-arm64", - Username: "ubuntu", - RemotePath: "/home/ubuntu/agent", - }, - { - OS: define.OS{ - Type: define.Linux, - Arch: define.ARM64, - Distro: runner.Ubuntu, - Version: "20.04", - }, - Provider: Google, - InstanceSize: "t2a-standard-4", // 4 arm64 cpus, 16 GB RAM - RunsOn: "ubuntu-2004-lts-arm64", - Username: "ubuntu", - RemotePath: "/home/ubuntu/agent", - }, + // These instance types are experimental on Google Cloud and very unstable + // We will wait until Google introduces new ARM instance types + // https://cloud.google.com/blog/products/compute/introducing-googles-new-arm-based-cpu + // { + // OS: define.OS{ + // Type: define.Linux, + // Arch: define.ARM64, + // Distro: runner.Ubuntu, + // Version: "22.04", + // }, + // Provider: Google, + // InstanceSize: "t2a-standard-4", // 4 arm64 cpus, 16 GB RAM + // RunsOn: "ubuntu-2204-lts-arm64", + // Username: "ubuntu", + // RemotePath: "/home/ubuntu/agent", + // }, + // { + // OS: define.OS{ + // Type: define.Linux, + // Arch: define.ARM64, + // Distro: runner.Ubuntu, + // Version: "20.04", + // }, + // Provider: Google, + // InstanceSize: "t2a-standard-4", // 4 arm64 cpus, 16 GB RAM + // RunsOn: "ubuntu-2004-lts-arm64", + // Username: "ubuntu", + // RemotePath: "/home/ubuntu/agent", + // }, { OS: define.OS{ Type: define.Linux,