From 7bd1196bbd1b99bca5a761e467f85d2184b151ec Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Fri, 2 Feb 2024 14:10:13 +0200 Subject: [PATCH] Filebeat pipeline migration to Buildkite (#37283) * added test scripts * added windows tests * added packaging step * updated packaging execution conditions * added arm packaging * fixed linting in filbeat test_crawler.py * added platforms for linux packaging --------- Co-authored-by: Pavel Zorin --- .buildkite/env-scripts/env.sh | 13 ++ .buildkite/env-scripts/linux-env.sh | 24 +++ .buildkite/env-scripts/macos-env.sh | 8 + .buildkite/env-scripts/util.sh | 91 +++++++++++ .buildkite/env-scripts/win-env.sh | 8 + .buildkite/filebeat/filebeat-pipeline.yml | 141 +++++++++++++++++- .../filebeat/scripts/integration-gotests.sh | 12 ++ .../filebeat/scripts/integration-pytests.sh | 12 ++ .buildkite/filebeat/scripts/package-step.sh | 46 ++++++ .buildkite/filebeat/scripts/package.sh | 12 ++ .../filebeat/scripts/unit-tests-win.ps1 | 51 +++++++ .buildkite/filebeat/scripts/unit-tests.sh | 12 ++ .buildkite/hooks/pre-command | 18 +++ .buildkite/pull-requests.json | 6 +- catalog-info.yaml | 8 +- dev-tools/packaging/package_test.go | 33 ++-- filebeat/filebeat_windows_amd64.syso | Bin 0 -> 1072 bytes .../filestream/internal/task/group_test.go | 6 +- filebeat/tests/system/test_crawler.py | 22 ++- 19 files changed, 487 insertions(+), 36 deletions(-) create mode 100644 .buildkite/env-scripts/env.sh create mode 100644 .buildkite/env-scripts/linux-env.sh create mode 100644 .buildkite/env-scripts/macos-env.sh create mode 100644 .buildkite/env-scripts/util.sh create mode 100644 .buildkite/env-scripts/win-env.sh create mode 100755 .buildkite/filebeat/scripts/integration-gotests.sh create mode 100755 .buildkite/filebeat/scripts/integration-pytests.sh create mode 100755 .buildkite/filebeat/scripts/package-step.sh create mode 100755 .buildkite/filebeat/scripts/package.sh create mode 100644 .buildkite/filebeat/scripts/unit-tests-win.ps1 create mode 100755 .buildkite/filebeat/scripts/unit-tests.sh create mode 100644 .buildkite/hooks/pre-command create mode 100644 filebeat/filebeat_windows_amd64.syso diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh new file mode 100644 index 000000000000..d94d03aad53b --- /dev/null +++ b/.buildkite/env-scripts/env.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +SETUP_GVM_VERSION="v0.5.1" +WORKSPACE="$(pwd)" +BIN="${WORKSPACE}/bin" +HW_TYPE="$(uname -m)" +PLATFORM_TYPE="$(uname)" + +export SETUP_GVM_VERSION +export WORKSPACE +export BIN +export HW_TYPE +export PLATFORM_TYPE diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh new file mode 100644 index 000000000000..edaf1a3100c2 --- /dev/null +++ b/.buildkite/env-scripts/linux-env.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +source .buildkite/env-scripts/util.sh + +DEBIAN_FRONTEND="noninteractive" + +export DEBIAN_FRONTEND + +sudo mkdir -p /etc/needrestart +echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + +# Remove this code once beats specific agent is set up +if [[ $PLATFORM_TYPE == "Linux" ]]; then + echo ":: Installing libs ::" + sudo apt-get update + sudo apt-get install -y libsystemd-dev + sudo apt install -y python3-pip + sudo apt-get install -y python3-venv +fi + +echo ":: Setting up environment ::" +add_bin_path +with_go +with_mage diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh new file mode 100644 index 000000000000..ac1486b64fdd --- /dev/null +++ b/.buildkite/env-scripts/macos-env.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [[ $PLATFORM_TYPE == Darwin* ]]; then + echo ":: Setting larger ulimit on MacOS ::" + # To bypass file descriptor errors like "Too many open files error" on MacOS + ulimit -Sn 50000 + echo ":: ULIMIT :: $(ulimit -n)" +fi diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh new file mode 100644 index 000000000000..157a5aff37af --- /dev/null +++ b/.buildkite/env-scripts/util.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +set -euo pipefail + +add_bin_path() { + echo "Adding PATH to the environment variables..." + create_bin + export PATH="${PATH}:${BIN}" +} + +with_go() { + local go_version="${GOLANG_VERSION}" + echo "Setting up the Go environment..." + create_bin + check_platform_architecture + retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}" + export PATH="${PATH}:${BIN}" + chmod +x ${BIN}/gvm + eval "$(gvm "$go_version")" + go version + which go + export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin" +} + +with_mage() { + local install_packages=( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "gotest.tools/gotestsum" + ) + create_bin + for pkg in "${install_packages[@]}"; do + go install "${pkg}@latest" + done +} + +create_bin() { + if [[ ! -d "${BIN}" ]]; then + mkdir -p ${BIN} + fi +} + +check_platform_architecture() { +# for downloading the GVM and Terraform packages + case "${HW_TYPE}" in + "x86_64") + arch_type="amd64" + ;; + "aarch64") + arch_type="arm64" + ;; + "arm64") + arch_type="arm64" + ;; + *) + echo "The current platform/OS type is unsupported yet" + ;; + esac +} + +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +are_files_changed() { + local changeset=$1 + + if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then + return 0; + else + echo "WARN! No files changed in $changeset" + return 1; + fi +} diff --git a/.buildkite/env-scripts/win-env.sh b/.buildkite/env-scripts/win-env.sh new file mode 100644 index 000000000000..aa5f67ca4cee --- /dev/null +++ b/.buildkite/env-scripts/win-env.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +install_python_win() { + if [[ ${PLATFORM_TYPE} = MINGW* ]]; then + choco install mingw -y + choco install python --version=3.11.0 -y + fi +} diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 34321b61161b..e3d7384a71ea 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -1,5 +1,142 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + steps: - - label: "Example test" - command: echo "Hello!" + - group: "Filebeat Mandatory Testing" + key: "mandatory-tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" + + steps: + - label: ":ubuntu: Unit Tests" + command: + - ".buildkite/filebeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Filebeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Go Integration Tests" + command: + - ".buildkite/filebeat/scripts/integration-gotests.sh" + notify: + - github_commit_status: + context: "Filebeat: Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Python Integration Tests" + command: + - ".buildkite/filebeat/scripts/integration-pytests.sh" + notify: + - github_commit_status: + context: "Filebeat: Python Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":windows:-{{matrix.image}} Unit Tests" + command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Filebeat: Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - group: "Extended Testing" + key: "extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support" + + steps: + - label: ":linux: ARM64 Unit Tests" + key: "arm-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: + - ".buildkite/filebeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Filebeat/Extended: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" + artifact_paths: "filebeat/build/*.xml" + + - label: ":mac: MacOS Unit Tests" + key: "macos-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: + - ".buildkite/filebeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Filebeat/Extended: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "filebeat/build/*.xml" + + - group: "Windows Extended Testing" + key: "extended-tests-win" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "win-extended-2019" + command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Filebeat/Extended: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + - "extended-tests" + - "extended-tests-win" + + steps: + - label: Package pipeline + commands: ".buildkite/filebeat/scripts/package-step.sh | buildkite-agent pipeline upload" diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh new file mode 100755 index 000000000000..a3eabf70c0d3 --- /dev/null +++ b/.buildkite/filebeat/scripts/integration-gotests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo ":: Execute Integration Tests ::" +sudo chmod -R go-w filebeat/ + +cd filebeat +umask 0022 +mage goIntegTest diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh new file mode 100755 index 000000000000..5e2e403dda87 --- /dev/null +++ b/.buildkite/filebeat/scripts/integration-pytests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo ":: Execute Integration Tests ::" +sudo chmod -R go-w filebeat/ + +cd filebeat +umask 0022 +mage pythonIntegTest diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh new file mode 100755 index 000000000000..a4127c3cd1d6 --- /dev/null +++ b/.buildkite/filebeat/scripts/package-step.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +changeset="^filebeat/ + ^go.mod + ^pytest.ini + ^dev-tools/ + ^libbeat/ + ^testing/ + ^\.buildkite/filebeat/" + +if are_files_changed "$changeset"; then + cat <<-EOF + steps: + - label: ":ubuntu: Packaging Linux X86" + key: "package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + command: + - ".buildkite/filebeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Filebeat/Packaging: Linux X86" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":linux: Packaging Linux ARM" + key: "package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + command: + - ".buildkite/filebeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Filebeat/Packaging: ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" +EOF +fi diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh new file mode 100755 index 000000000000..2ae226eb739c --- /dev/null +++ b/.buildkite/filebeat/scripts/package.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo ":: Evaluate Filebeat Changes ::" + +echo ":: Start Packaging ::" +cd filebeat +umask 0022 +mage package diff --git a/.buildkite/filebeat/scripts/unit-tests-win.ps1 b/.buildkite/filebeat/scripts/unit-tests-win.ps1 new file mode 100644 index 000000000000..8990eb30a093 --- /dev/null +++ b/.buildkite/filebeat/scripts/unit-tests-win.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" # set -e +$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually + +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF() { + Write-Host "-- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} + +function withGolang() { + Write-Host "-- Install golang $GoVersion --" + choco install golang -y --version $GoVersion + + $choco = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$choco\helpers\chocolateyProfile.psm1" + refreshenv + go version + go env +} + +function installGoDependencies() { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "github.com/tebeka/go2xunit" + ) + foreach ($pkg in $installPackages) { + go install "$pkg" + } +} + +fixCRLF + +$ErrorActionPreference = "Continue" # set +e + +Set-Location -Path filebeat +New-Item -ItemType Directory -Force -Path "build" +withGolang +installGoDependencies + +mage build unitTest + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh new file mode 100755 index 000000000000..cda1dd85aea2 --- /dev/null +++ b/.buildkite/filebeat/scripts/unit-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh +source .buildkite/env-scripts/macos-env.sh + +echo ":: Execute Unit Tests ::" +sudo chmod -R go-w filebeat/ + +umask 0022 +mage -d filebeat unitTest diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 000000000000..0a1567e53cd5 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/env.sh +source .buildkite/env-scripts/util.sh +source .buildkite/env-scripts/win-env.sh + + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then + if [[ ${PLATFORM_TYPE} = MINGW* ]]; then + install_python_win + fi + + if [[ -z "${GOLANG_VERSION-""}" ]]; then + export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version") + fi +fi diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index c1cfa299d079..76fdfb3e6cdc 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -25,12 +25,12 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test filebeat$", - "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test filebeat$", + "trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/packag[ing|e]$", + "always_trigger_comment_regex": "^/test filebeat(for (arm|macos|windows|extended support))?$|^/package filebeat$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ ] + "always_require_ci_on_changed": ["^filebeat/.*", ".buildkite/filebeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*" ] }, { "enabled": true, diff --git a/catalog-info.yaml b/catalog-info.yaml index c5679e1215ce..d145975ab7d3 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -130,9 +130,9 @@ spec: name: filebeat description: "Filebeat pipeline" spec: -# branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks + branch_configuration: "main 7.* 8.* v7.* v8.*" pipeline_file: ".buildkite/filebeat/filebeat-pipeline.yml" -# maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + maximum_timeout_in_minutes: 120 provider_settings: build_pull_request_forks: false build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot @@ -145,8 +145,8 @@ spec: cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" skip_intermediate_builds: true skip_intermediate_builds_branch_filter: "!main !7.* !8.*" - # env: - # ELASTIC_PR_COMMENTS_ENABLED: "true" TODO: uncomment when pipeline is ready + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index e01b6c566e5a..fff920b429c2 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -714,9 +714,11 @@ func readZip(t *testing.T, zipFile string, inspectors ...inspector) (*packageFil } func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { - // Read the manifest file first so that the config file and layer - // names are known in advance. - manifest, err := getDockerManifest(dockerFile) + var manifest *dockerManifest + var info *dockerInfo + layers := make(map[string]*packageFile) + + manifest, err := readManifest(dockerFile) if err != nil { return nil, nil, err } @@ -727,9 +729,6 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { } defer file.Close() - var info *dockerInfo - layers := make(map[string]*packageFile) - gzipReader, err := gzip.NewReader(file) if err != nil { return nil, nil, err @@ -770,11 +769,7 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { // Read layers in order and for each file keep only the entry seen in the later layer p := &packageFile{Name: filepath.Base(dockerFile), Contents: map[string]packageEntry{}} - for _, layer := range manifest.Layers { - layerFile, found := layers[layer] - if !found { - return nil, nil, fmt.Errorf("layer not found: %s", layer) - } + for _, layerFile := range layers { for name, entry := range layerFile.Contents { // Check only files in working dir and entrypoint if strings.HasPrefix("/"+name, workingDir) || "/"+name == entrypoint { @@ -799,22 +794,21 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { return p, info, nil } -// getDockerManifest opens a gzipped tar file to read the Docker manifest.json -// that it is expected to contain. -func getDockerManifest(file string) (*dockerManifest, error) { - f, err := os.Open(file) +func readManifest(dockerFile string) (*dockerManifest, error) { + var manifest *dockerManifest + + file, err := os.Open(dockerFile) if err != nil { return nil, err } - defer f.Close() + defer file.Close() - gzipReader, err := gzip.NewReader(f) + gzipReader, err := gzip.NewReader(file) if err != nil { return nil, err } defer gzipReader.Close() - var manifest *dockerManifest tarReader := tar.NewReader(gzipReader) for { header, err := tarReader.Next() @@ -833,8 +827,7 @@ func getDockerManifest(file string) (*dockerManifest, error) { break } } - - return manifest, nil + return manifest, err } type dockerManifest struct { diff --git a/filebeat/filebeat_windows_amd64.syso b/filebeat/filebeat_windows_amd64.syso new file mode 100644 index 0000000000000000000000000000000000000000..c52af94f8e059275dff851e701e42fafefdf4132 GIT binary patch literal 1072 zcmZvcPiqrV6vfY&fOb*nQd|w_QbN*yQBY7!i-l0dN^vLCB#i?x6Ot*k(1j~MPnT}| z7B2f8`VGY2dDEAF_$Kq-efOPr?!D)|N&jaewmS_v}vGgN)sxb-p=70Pf@Vy80h4-2mv0c}8F;8`(uk;7{gmt73I%a-Ee~9h& zyMFiA*=*`jn8z_p?z^g{h3EExcburyRJq0)!j@~K0nB5as{r~dV47Neq)R-B-huhf zGXP&|2EKRi$(g^_w)WLlrWV!?U|^3TYmq!ORo4LK(Bx;jLh;di~0fGJ>biqB-LT?#j`|Hx0ei~6oN+*`lHFHx{AzxVk%8;6vk~%W>futpr>sxm z&0($bd;zY777dhe8|f+dF1R5~qi`Y<;<>^htR)zq(yA|A)9HDr#EQ9QXO-$LH>1nA zJ&)Dlb|u`mirY8F5uULXtKIz vU72-VI_yjMVytfDT-<8unfw8NX4Q9SMjbx&h0c!a@6oNM<$r-Bc)a@`Kf{eH literal 0 HcmV?d00001 diff --git a/filebeat/input/filestream/internal/task/group_test.go b/filebeat/input/filestream/internal/task/group_test.go index 553070e5ec73..5ce15d455e3e 100644 --- a/filebeat/input/filestream/internal/task/group_test.go +++ b/filebeat/input/filestream/internal/task/group_test.go @@ -241,12 +241,14 @@ func TestGroup_Go(t *testing.T) { want := uint64(2) g := NewGroup(want, time.Second, logger, "errorPrefix") - wg.Add(2) + wg.Add(1) err := g.Go(workload(1)) require.NoError(t, err) + wg.Wait() + + wg.Add(1) err = g.Go(workload(2)) require.NoError(t, err) - wg.Wait() err = g.Stop() diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index fba8debcaea6..2bea57223fe8 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -197,7 +197,10 @@ def test_file_renaming(self): # expecting 6 more events self.wait_until( - lambda: self.output_has(lines=iterations1 + iterations2), max_timeout=10) + lambda: self.output_has( + lines=iterations1 + + iterations2), + max_timeout=10) filebeat.check_kill_and_wait() @@ -247,7 +250,10 @@ def test_file_disappear(self): # Let it read the file self.wait_until( - lambda: self.output_has(lines=iterations1 + iterations2), max_timeout=10) + lambda: self.output_has( + lines=iterations1 + + iterations2), + max_timeout=10) filebeat.check_kill_and_wait() @@ -317,7 +323,10 @@ def test_file_disappear_appear(self): # Let it read the file self.wait_until( - lambda: self.output_has(lines=iterations1 + iterations2), max_timeout=10) + lambda: self.output_has( + lines=iterations1 + + iterations2), + max_timeout=10) filebeat.check_kill_and_wait() @@ -468,7 +477,8 @@ def test_tail_files(self): f.write("hello world 2\n") f.flush() - # Sleep 1 second to make sure the file is persisted on disk and timestamp is in the past + # Sleep 1 second to make sure the file is persisted on disk and + # timestamp is in the past time.sleep(1) filebeat = self.start_beat() @@ -569,6 +579,7 @@ def test_encodings(self): with codecs.open(self.working_dir + "/log/test-{}".format(enc_py), "w", enc_py) as f: f.write(text + "\n") + f.close() # create the config file inputs = [] @@ -592,10 +603,11 @@ def test_encodings(self): with codecs.open(self.working_dir + "/log/test-{}".format(enc_py), "a", enc_py) as f: f.write(text + " 2" + "\n") + f.close() # wait again self.wait_until(lambda: self.output_has(lines=len(encodings) * 2), - max_timeout=15) + max_timeout=60) filebeat.check_kill_and_wait() # check that all outputs are present in the JSONs in UTF-8