Skip to content

Commit

Permalink
Merge branch 'main' into dependabot/go_modules/github.com/elastic/go-…
Browse files Browse the repository at this point in the history
…elasticsearch/v8-8.16.0
  • Loading branch information
andrzej-stencel authored Nov 15, 2024
2 parents d0deaef + 0c4e7a9 commit 620964c
Show file tree
Hide file tree
Showing 7 changed files with 274 additions and 83 deletions.
54 changes: 20 additions & 34 deletions .buildkite/scripts/integration-tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,11 @@ param (
[string]$GROUP_NAME
)

Write-Output "~~~ Switching to PowerShell 7"
pwsh
$PSVersionTable.PSVersion
echo "~~~ Preparing environment"

# TODO: dedicated ESS strack for retries
Write-Output "~~~ Receiving ESS stack metadata"
$env:ELASTICSEARCH_HOST = (buildkite-agent meta-data get "es.host")
$env:ELASTICSEARCH_USERNAME = (buildkite-agent meta-data get "es.username")
$env:ELASTICSEARCH_PASSWORD = (buildkite-agent meta-data get "es.pwd")
$env:KIBANA_HOST = (buildkite-agent meta-data get "kibana.host")
$env:KIBANA_USERNAME = (buildkite-agent meta-data get "kibana.username")
$env:KIBANA_PASSWORD = (buildkite-agent meta-data get "kibana.pwd")
$PSVersionTable.PSVersion

Write-Output "~~~ Running integration tests as $env:USERNAME"
Write-Output "~~~ Integration tests: $GROUP_NAME"
. "$PWD\.buildkite\scripts\steps\ess.ps1"

go install gotest.tools/gotestsum
gotestsum --version
Expand All @@ -26,30 +16,26 @@ $PACKAGE_VERSION = Get-Content .package-version -ErrorAction SilentlyContinue
if ($PACKAGE_VERSION) {
$PACKAGE_VERSION = "${PACKAGE_VERSION}-SNAPSHOT"
}

echo "~~~ Building test binaries"
mage build:testBinaries

# Run integration tests with gotestsum
echo "~~~ Running tests"
$env:TEST_BINARY_NAME = "elastic-agent"
$env:AGENT_VERSION = $PACKAGE_VERSION
$env:SNAPSHOT = $true

$ErrorActionPreference = 'Continue'

gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=true"
$TESTS_EXIT_STATUS = $LASTEXITCODE
$ErrorActionPreference = 'Stop'
echo "~~~ Building test binaries"
mage build:testBinaries

# Generate HTML report if XML output exists
$outputXML = "build/${GROUP_NAME}.integration.xml"
if (Test-Path $outputXML) {
# Install junit2html if not installed
go install github.com/alexec/junit2html@latest
Get-Content $outputXML | junit2html > "build/TEST-report.html"
} else {
Write-Output "Cannot generate HTML test report: $outputXML not found"
try {
Get-Ess-Stack -StackVersion $PACKAGE_VERSION
Write-Output "~~~ Running integration test group: $GROUP_NAME as user: $env:USERNAME"
gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=true"
} finally {
ess_down
# Generate HTML report if XML output exists
$outputXML = "build/${GROUP_NAME}.integration.xml"
if (Test-Path $outputXML) {
# Install junit2html if not installed
go install github.com/alexec/junit2html@latest
Get-Content $outputXML | junit2html > "build/TEST-report.html"
} else {
Write-Output "Cannot generate HTML test report: $outputXML not found"
}
}

exit $TESTS_EXIT_STATUS
131 changes: 131 additions & 0 deletions .buildkite/scripts/steps/ess.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
function ess_up {
param (
[string]$StackVersion,
[string]$EssRegion = "gcp-us-west2"
)

Write-Output "~~~ Starting ESS Stack"

$Workspace = & git rev-parse --show-toplevel
$TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/"

if (-not $StackVersion) {
Write-Error "Error: Specify stack version: ess_up [stack_version]"
return 1
}

$Env:EC_API_KEY = Retry-Command -ScriptBlock {
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
}

if (-not $Env:EC_API_KEY) {
Write-Error "Error: Failed to get EC API key from vault"
exit 1
}

$BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
$BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
$BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" }

Push-Location -Path $TfDir
& terraform init
& terraform apply -auto-approve `
-var="stack_version=$StackVersion" `
-var="ess_region=$EssRegion" `
-var="creator=$BuildkiteBuildCreator" `
-var="buildkite_id=$BuildkiteBuildNumber" `
-var="pipeline=$BuildkitePipelineSlug"

$Env:ELASTICSEARCH_HOST = & terraform output -raw es_host
$Env:ELASTICSEARCH_USERNAME = & terraform output -raw es_username
$Env:ELASTICSEARCH_PASSWORD = & terraform output -raw es_password
$Env:KIBANA_HOST = & terraform output -raw kibana_endpoint
$Env:KIBANA_USERNAME = $Env:ELASTICSEARCH_USERNAME
$Env:KIBANA_PASSWORD = $Env:ELASTICSEARCH_PASSWORD
Pop-Location
}

function ess_down {
$Workspace = & git rev-parse --show-toplevel
$TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/"
$stateFilePath = Join-Path -Path $TfDir -ChildPath "terraform.tfstate"

if (-not (Test-Path -Path $stateFilePath)) {
Write-Output "Terraform state file not found. Skipping ESS destroy."
return 0
}
Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
try {
$Env:EC_API_KEY = Retry-Command -ScriptBlock {
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
}
Push-Location -Path $TfDir
& terraform init
& terraform destroy -auto-approve
Pop-Location
} catch {
Write-Output "Error: Failed to destroy ESS stack(it will be auto-deleted later): $_"
}
}

function get_git_user_email {
if (!(git rev-parse --is-inside-work-tree *>&1)) {
return "unknown"
}

$email = & git config --get user.email

if (-not $email) {
return "unknown"
} else {
return $email
}
}

function Retry-Command {
param (
[scriptblock]$ScriptBlock,
[int]$MaxRetries = 3,
[int]$DelaySeconds = 5
)

$lastError = $null

for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) {
try {
$result = & $ScriptBlock
return $result
}
catch {
$lastError = $_
Write-Warning "Attempt $attempt failed: $($_.Exception.Message)"
Write-Warning "Retrying in $DelaySeconds seconds..."
Start-Sleep -Seconds $DelaySeconds
}
}

Write-Error "All $MaxRetries attempts failed. Original error: $($lastError.Exception.Message)"
throw $lastError.Exception
}

function Get-Ess-Stack {
param (
[string]$StackVersion
)

if ($Env:BUILDKITE_RETRY_COUNT -gt 0) {
Write-Output "The step is retried, starting the ESS stack again"
ess_up $StackVersion
Write-Output "ESS stack is up. ES_HOST: $Env:ELASTICSEARCH_HOST"
} else {
# For the first run, we retrieve ESS stack metadata
Write-Output "~~~ Receiving ESS stack metadata"
$Env:ELASTICSEARCH_HOST = & buildkite-agent meta-data get "es.host"
$Env:ELASTICSEARCH_USERNAME = & buildkite-agent meta-data get "es.username"
$Env:ELASTICSEARCH_PASSWORD = & buildkite-agent meta-data get "es.pwd"
$Env:KIBANA_HOST = & buildkite-agent meta-data get "kibana.host"
$Env:KIBANA_USERNAME = & buildkite-agent meta-data get "kibana.username"
$Env:KIBANA_PASSWORD = & buildkite-agent meta-data get "kibana.pwd"
Write-Output "Received ESS stack data from previous step. ES_HOST: $Env:ELASTICSEARCH_HOST"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Kind can be one of:
# - breaking-change: a change to previously-documented behavior
# - deprecation: functionality that is being removed in a later release
# - bug-fix: fixes a problem in a previous version
# - enhancement: extends functionality but does not break or fix existing behavior
# - feature: new functionality
# - known-issue: problems that we are aware of in a given version
# - security: impacts on the security of a product or a user’s deployment.
# - upgrade: important information for someone upgrading from a prior version
# - other: does not fit into any of the other categories
kind: enhancement

# Change summary; a 80ish characters long description of the change.
summary: removed endpoint security from linux containers

# Long description; in case the summary is not enough to describe the change
# this field accommodate a description without length limits.
# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
description: Removed elastic endpoint security from linux containers as it has a dependency on systemd.

# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
component: "elastic-agent"
# PR URL; optional; the PR number that added the changeset.
# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
# Please provide it if you are adding a fragment for a different PR.
pr: https://github.com/elastic/elastic-agent/pull/6016
# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
# If not present is automatically filled by the tooling with the issue linked to the PR number.
issue: https://github.com/elastic/elastic-agent/issues/5495
47 changes: 30 additions & 17 deletions dev-tools/mage/manifest/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/magefile/mage/mg"
"golang.org/x/sync/errgroup"

"github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon"
"github.com/elastic/elastic-agent/pkg/version"
)

Expand Down Expand Up @@ -78,8 +79,10 @@ var backoffSchedule = []time.Duration{
10 * time.Second,
}

var errorInvalidManifestURL = errors.New("invalid ManifestURL provided")
var errorNotAllowedManifestURL = errors.New("the provided ManifestURL is not allowed URL")
var (
errorInvalidManifestURL = errors.New("invalid ManifestURL provided")
errorNotAllowedManifestURL = errors.New("the provided ManifestURL is not allowed URL")
)

var AllowedManifestHosts = []string{"snapshots.elastic.co", "staging.elastic.co"}

Expand All @@ -95,22 +98,23 @@ var PlatformPackages = map[string]string{
// The project names are those used in the "projects" list in the unified release manifest.
// See the sample manifests in the testdata directory.
var ExpectedBinaries = []BinarySpec{
{BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms},
{BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}},
{BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
{BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true},
{BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms},
{BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms},
{BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
{BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
{BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
{BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms, PackageTypes: []pkgcommon.PackageType{pkgcommon.RPM, pkgcommon.Deb, pkgcommon.Zip, pkgcommon.TarGz}},
{BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes},
}

type BinarySpec struct {
BinaryName string
ProjectName string
Platforms []Platform
PythonWheel bool
BinaryName string
ProjectName string
Platforms []Platform
PythonWheel bool
PackageTypes []pkgcommon.PackageType
}

func (proj BinarySpec) SupportsPlatform(platform string) bool {
Expand All @@ -122,6 +126,15 @@ func (proj BinarySpec) SupportsPlatform(platform string) bool {
return false
}

func (proj BinarySpec) SupportsPackageType(pkgType pkgcommon.PackageType) bool {
for _, p := range proj.PackageTypes {
if p == pkgType {
return true
}
}
return false
}

func (proj BinarySpec) GetPackageName(version string, platform string) string {
if proj.PythonWheel {
return fmt.Sprintf("%s-%s.zip", proj.BinaryName, version)
Expand Down Expand Up @@ -153,7 +166,7 @@ func DownloadManifest(ctx context.Context, manifest string) (Build, error) {
if urlError != nil {
return Build{}, errorInvalidManifestURL
}
var valid = false
valid := false
for _, manifestHost := range AllowedManifestHosts {
if manifestHost == manifestUrl.Host {
valid = true
Expand Down Expand Up @@ -315,7 +328,7 @@ func DownloadPackage(ctx context.Context, downloadUrl string, target string) err
if errorUrl != nil {
return errorInvalidManifestURL
}
var valid = false
valid := false
for _, manifestHost := range AllowedManifestHosts {
if manifestHost == parsedURL.Host {
valid = true
Expand Down
25 changes: 25 additions & 0 deletions dev-tools/mage/pkgcommon/pkgcommon-types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License 2.0;
// you may not use this file except in compliance with the Elastic License 2.0.

package pkgcommon

// PackageType defines the file format of the package (e.g. zip, rpm, etc).
type PackageType int

// List of possible package types.
const (
RPM PackageType = iota + 1
Deb
Zip
TarGz
Docker
)

var AllPackageTypes = []PackageType{
RPM,
Deb,
Zip,
TarGz,
Docker,
}
19 changes: 11 additions & 8 deletions dev-tools/mage/pkgtypes.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import (
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"gopkg.in/yaml.v3"

"github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon"
)

const (
Expand Down Expand Up @@ -60,16 +62,17 @@ var (
componentConfigFilePattern = regexp.MustCompile(`.*beat\.spec\.yml$|.*beat\.yml$|apm-server\.yml$|apm-server\.spec\.yml$|elastic-agent\.yml$`)
)

// PackageType defines the file format of the package (e.g. zip, rpm, etc).
type PackageType int
// Alias for pkgcommon.PackageType. This type is moved to the pkgcommon to
// resolve circular dependency problems
type PackageType pkgcommon.PackageType

// List of possible package types.
const (
RPM PackageType = iota + 1
Deb
Zip
TarGz
Docker
var (
RPM PackageType = PackageType(pkgcommon.RPM)
Deb = PackageType(pkgcommon.Deb)
Zip = PackageType(pkgcommon.Zip)
TarGz = PackageType(pkgcommon.TarGz)
Docker = PackageType(pkgcommon.Docker)
)

// OSPackageArgs define a set of package types to build for an operating
Expand Down
Loading

0 comments on commit 620964c

Please sign in to comment.