From 75c5a4d5f0b7004091ee4dd1b1595d9daf64b572 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 21 Oct 2024 14:36:33 -0400 Subject: [PATCH 01/14] chore(ci): update actions/checkout to v4 (#21571) --- .github/workflows/changelog.yaml | 2 +- .github/workflows/changes.yml | 6 ++-- .github/workflows/ci-integration-review.yml | 4 +-- .github/workflows/cli.yml | 4 +-- .github/workflows/compilation-timings.yml | 10 +++--- .github/workflows/component_features.yml | 4 +-- .github/workflows/cross.yml | 4 +-- .github/workflows/deny.yml | 4 +-- .github/workflows/e2e.yml | 2 +- .github/workflows/environment.yml | 4 +-- .../gardener_remove_waiting_author.yml | 2 +- .github/workflows/install-sh.yml | 4 +-- .github/workflows/integration-test.yml | 4 +-- .github/workflows/k8s_e2e.yml | 8 ++--- .github/workflows/misc.yml | 4 +-- .github/workflows/msrv.yml | 2 +- .github/workflows/protobuf.yml | 2 +- .github/workflows/publish.yml | 34 +++++++++---------- .github/workflows/regression.yml | 16 ++++----- .github/workflows/unit_mac.yml | 4 +-- .github/workflows/unit_windows.yml | 4 +-- 21 files changed, 64 insertions(+), 64 deletions(-) diff --git a/.github/workflows/changelog.yaml b/.github/workflows/changelog.yaml index cc109b396ebee..fc73f73b183d8 100644 --- a/.github/workflows/changelog.yaml +++ b/.github/workflows/changelog.yaml @@ -26,7 +26,7 @@ jobs: # Checkout full depth because in the check_changelog_fragments script, we need to specify a # merge base. If we only shallow clone the repo, git may not have enough history to determine # the base. - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 6723580d93214..8b31b8136a252 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -140,7 +140,7 @@ jobs: install: ${{ steps.filter.outputs.install }} k8s: ${{ steps.filter.outputs.k8s }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter @@ -237,7 +237,7 @@ jobs: splunk: ${{ steps.filter.outputs.splunk }} webhdfs: ${{ steps.filter.outputs.webhdfs }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # creates a yaml file that contains the filters for each integration, # extracted from the output of the `vdev int ci-paths` command, which @@ -262,7 +262,7 @@ jobs: datadog-logs: ${{ steps.filter.outputs.datadog-logs }} datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # creates a yaml file that contains the filters for each test, # extracted from the output of the `vdev int ci-paths` command, which diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 7335e5058ff46..75f87e21b34ce 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -84,7 +84,7 @@ jobs: runs-on: ubuntu-20.04-4core timeout-minutes: 90 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} @@ -463,7 +463,7 @@ jobs: runs-on: ubuntu-20.04-8core timeout-minutes: 30 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 3b75c65a8c714..1f5bef3464867 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -21,13 +21,13 @@ jobs: - name: (PR review) Checkout review SHA if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache Cargo registry + index uses: actions/cache@v4 diff --git a/.github/workflows/compilation-timings.yml b/.github/workflows/compilation-timings.yml index 5b26604109093..1404abb64c107 100644 --- a/.github/workflows/compilation-timings.yml +++ b/.github/workflows/compilation-timings.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-20.04-8core steps: - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: cargo clean @@ -32,7 +32,7 @@ jobs: PROFILE: debug steps: - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: cargo clean @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-20.04-8core steps: - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: cargo clean @@ -54,7 +54,7 @@ jobs: runs-on: ubuntu-20.04-8core steps: - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: cargo clean @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-20.04-8core steps: - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: cargo clean diff --git a/.github/workflows/component_features.yml b/.github/workflows/component_features.yml index cca94298d1e13..b69e62b882b67 100644 --- a/.github/workflows/component_features.yml +++ b/.github/workflows/component_features.yml @@ -33,13 +33,13 @@ jobs: - name: (PR review) Checkout PR branch if: github.event_name == 'pull_request_review' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: github.event_name != 'pull_request_review' - uses: actions/checkout@v3 + uses: actions/checkout@v4 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index 5669142b62259..38acecdeac3bf 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -33,13 +33,13 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: actions/cache@v4 name: Cache Cargo registry + index diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index ac5ee9f353e79..7ed0ea5df328c 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -34,13 +34,13 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: actions/cache@v4 name: Cache Cargo registry + index diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3a6524f156c1b..6fbd1bee2160b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -60,7 +60,7 @@ jobs: ) ) steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index 05eea1f07d6f4..c041a08e3afc6 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -27,13 +27,13 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3.2.0 diff --git a/.github/workflows/gardener_remove_waiting_author.yml b/.github/workflows/gardener_remove_waiting_author.yml index 0317d27454358..e2599e596361f 100644 --- a/.github/workflows/gardener_remove_waiting_author.yml +++ b/.github/workflows/gardener_remove_waiting_author.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-ecosystem/action-remove-labels@v1 with: labels: "meta: awaiting author" diff --git a/.github/workflows/install-sh.yml b/.github/workflows/install-sh.yml index 9b0ed5550346c..587c957087082 100644 --- a/.github/workflows/install-sh.yml +++ b/.github/workflows/install-sh.yml @@ -25,13 +25,13 @@ jobs: - name: (PR comment) Checkout PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'issue_comment' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - run: sudo apt-get install --yes bc - run: bash distribution/install.sh -- -y diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 2d5833ee2b837..655985a22d0c0 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -41,13 +41,13 @@ jobs: - name: (PR comment) Checkout PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'issue_comment' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - run: sudo npm -g install @datadog/datadog-ci diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 687b865402ec6..7500cb0b4d43d 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -80,13 +80,13 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: actions/cache@v4 with: @@ -194,13 +194,13 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml index 3f34eeb5f23c9..d47c4f7fc3946 100644 --- a/.github/workflows/misc.yml +++ b/.github/workflows/misc.yml @@ -21,13 +21,13 @@ jobs: - name: (PR review) Checkout review SHA if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: actions/cache@v4 name: Cache Cargo registry + index diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index b485698dd9ca4..15dbd9da1a74f 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - run: cargo install cargo-msrv --version 0.15.1 - run: cargo msrv verify diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 5be147bf3004c..7f8ed6972df7e 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -19,7 +19,7 @@ jobs: timeout-minutes: 5 steps: # Run `git checkout` - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Install the `buf` CLI - uses: bufbuild/buf-setup-action@v1.45.0 # Perform breaking change detection against the `master` branch diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7cd8901a699d4..22761ba5257a2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -36,7 +36,7 @@ jobs: vector_release_channel: ${{ steps.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Generate publish metadata @@ -54,7 +54,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -80,7 +80,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -106,7 +106,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -134,7 +134,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -162,7 +162,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -190,7 +190,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -218,7 +218,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -245,7 +245,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -273,7 +273,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (macOS-specific) @@ -304,7 +304,7 @@ jobs: RELEASE_BUILDER: "true" steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Windows-specific) @@ -460,7 +460,7 @@ jobs: runner: [macos-14, macos-15] steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-apple-darwin) @@ -493,7 +493,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Login to DockerHub @@ -581,7 +581,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) @@ -666,7 +666,7 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) @@ -742,7 +742,7 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Publish update to Homebrew tap @@ -770,7 +770,7 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 79db51541b539..8406b3ae42948 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -54,7 +54,7 @@ jobs: source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }} comment_valid: ${{ steps.comment.outputs.isTeamMember }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Collect file changes id: changes @@ -250,9 +250,9 @@ jobs: steps: - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ needs.compute-metadata.outputs.baseline-sha }} path: baseline-vector @@ -288,9 +288,9 @@ jobs: steps: - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ needs.compute-metadata.outputs.comparison-sha }} path: comparison-vector @@ -439,7 +439,7 @@ jobs: -f context='Regression Detection Suite / submission' \ -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ needs.compute-metadata.outputs.comparison-sha }} @@ -553,7 +553,7 @@ jobs: - submit-job - compute-metadata steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4.0.2 @@ -643,7 +643,7 @@ jobs: -f context='Regression Detection Suite / analyze-experiment' \ -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ needs.compute-metadata.outputs.comparison-sha }} diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml index 728c74c4894d4..b5d6a7c2f187b 100644 --- a/.github/workflows/unit_mac.yml +++ b/.github/workflows/unit_mac.yml @@ -21,13 +21,13 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: actions/cache@v4 name: Cache Cargo registry + index diff --git a/.github/workflows/unit_windows.yml b/.github/workflows/unit_windows.yml index 07c2a78211a4f..10c238d07e683 100644 --- a/.github/workflows/unit_windows.yml +++ b/.github/workflows/unit_windows.yml @@ -20,13 +20,13 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - run: .\scripts\environment\bootstrap-windows-2022.ps1 - run: make test From a8c0b6ee596999b28fb0bb136fdc55dce7eb361d Mon Sep 17 00:00:00 2001 From: Ivan Trubach Date: Mon, 21 Oct 2024 21:40:18 +0300 Subject: [PATCH 02/14] feat(config): support loading secrets from files and directories (#21282) * feat(config): Support loading secrets from files * feat(config): Support loading secrets from directories This change allows loading secrets from files in directories. In particular, this is useful for loading secrets from systemd credentials directory (see https://systemd.io/CREDENTIALS) and similar mechanisms. * chore: Update spelling dictionary * chore: Add changelog entry for file/directory secrets backends * fix: Move test data from tests/behavior to tests/data directory --- .github/actions/spelling/expect.txt | 2 + changelog.d/add-file-secrets.enhancement.md | 4 ++ src/secrets/directory.rs | 52 ++++++++++++++++ src/secrets/file.rs | 46 ++++++++++++++ src/secrets/mod.rs | 10 +++ tests/behavior/config/secret.toml | 13 ++++ .../secret-backends/directory-secrets/jkl | 1 + tests/data/secret-backends/file-secrets.json | 1 + website/cue/reference/configuration.cue | 62 +++++++++++++++++++ 9 files changed, 191 insertions(+) create mode 100644 changelog.d/add-file-secrets.enhancement.md create mode 100644 src/secrets/directory.rs create mode 100644 src/secrets/file.rs create mode 100644 tests/data/secret-backends/directory-secrets/jkl create mode 100644 tests/data/secret-backends/file-secrets.json diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 433ab88124d09..458168dd4cf11 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -388,6 +388,8 @@ Fomichev fonttbl foob foobarbaz +foobarbazqux +foobarbazquxquux foobarfoobarfoo FOOBARy foobaz diff --git a/changelog.d/add-file-secrets.enhancement.md b/changelog.d/add-file-secrets.enhancement.md new file mode 100644 index 0000000000000..3934e9890a4f7 --- /dev/null +++ b/changelog.d/add-file-secrets.enhancement.md @@ -0,0 +1,4 @@ +Vector now supports two additional back-ends for loading secrets: `file`, for reading a set of +secrets from a JSON file, and `directory`, for loading secrets from a list of files. + +authors: tie diff --git a/src/secrets/directory.rs b/src/secrets/directory.rs new file mode 100644 index 0000000000000..e9ebaebbdd771 --- /dev/null +++ b/src/secrets/directory.rs @@ -0,0 +1,52 @@ +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +use vector_lib::configurable::{component::GenerateConfig, configurable_component}; + +use crate::{config::SecretBackend, signal}; + +/// Configuration for the `directory` secrets backend. +#[configurable_component(secrets("directory"))] +#[derive(Clone, Debug)] +pub struct DirectoryBackend { + /// Directory path to read secrets from. + pub path: PathBuf, + + /// Remove trailing whitespace from file contents. + #[serde(default)] + pub remove_trailing_whitespace: bool, +} + +impl GenerateConfig for DirectoryBackend { + fn generate_config() -> toml::Value { + toml::Value::try_from(DirectoryBackend { + path: PathBuf::from("/path/to/secrets"), + remove_trailing_whitespace: false, + }) + .unwrap() + } +} + +impl SecretBackend for DirectoryBackend { + async fn retrieve( + &mut self, + secret_keys: HashSet, + _: &mut signal::SignalRx, + ) -> crate::Result> { + let mut secrets = HashMap::new(); + for k in secret_keys.into_iter() { + let file_path = self.path.join(&k); + let contents = tokio::fs::read_to_string(&file_path).await?; + let secret = if self.remove_trailing_whitespace { + contents.trim_end() + } else { + &contents + }; + if secret.is_empty() { + return Err(format!("secret in file '{}' was empty", k).into()); + } + secrets.insert(k, secret.to_string()); + } + Ok(secrets) + } +} diff --git a/src/secrets/file.rs b/src/secrets/file.rs new file mode 100644 index 0000000000000..221a293622cb6 --- /dev/null +++ b/src/secrets/file.rs @@ -0,0 +1,46 @@ +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +use vector_lib::configurable::{component::GenerateConfig, configurable_component}; + +use crate::{config::SecretBackend, signal}; + +/// Configuration for the `file` secrets backend. +#[configurable_component(secrets("file"))] +#[derive(Clone, Debug)] +pub struct FileBackend { + /// File path to read secrets from. + pub path: PathBuf, +} + +impl GenerateConfig for FileBackend { + fn generate_config() -> toml::Value { + toml::Value::try_from(FileBackend { + path: PathBuf::from("/path/to/secret"), + }) + .unwrap() + } +} + +impl SecretBackend for FileBackend { + async fn retrieve( + &mut self, + secret_keys: HashSet, + _: &mut signal::SignalRx, + ) -> crate::Result> { + let contents = tokio::fs::read_to_string(&self.path).await?; + let output = serde_json::from_str::>(&contents)?; + let mut secrets = HashMap::new(); + for k in secret_keys.into_iter() { + if let Some(secret) = output.get(&k) { + if secret.is_empty() { + return Err(format!("secret for key '{}' was empty", k).into()); + } + secrets.insert(k, secret.to_string()); + } else { + return Err(format!("secret for key '{}' was not retrieved", k).into()); + } + } + Ok(secrets) + } +} diff --git a/src/secrets/mod.rs b/src/secrets/mod.rs index 97b66c64cd3f5..febe708f252d5 100644 --- a/src/secrets/mod.rs +++ b/src/secrets/mod.rs @@ -8,7 +8,9 @@ use crate::{config::SecretBackend, signal}; #[cfg(feature = "secrets-aws-secrets-manager")] mod aws_secrets_manager; +mod directory; mod exec; +mod file; mod test; /// Configurable secret backends in Vector. @@ -18,6 +20,12 @@ mod test; #[enum_dispatch(SecretBackend)] #[serde(tag = "type", rename_all = "snake_case")] pub enum SecretBackends { + /// File. + File(file::FileBackend), + + /// Directory. + Directory(directory::DirectoryBackend), + /// Exec. Exec(exec::ExecBackend), @@ -34,6 +42,8 @@ pub enum SecretBackends { impl NamedComponent for SecretBackends { fn get_component_name(&self) -> &'static str { match self { + Self::File(config) => config.get_component_name(), + Self::Directory(config) => config.get_component_name(), Self::Exec(config) => config.get_component_name(), #[cfg(feature = "secrets-aws-secrets-manager")] Self::AwsSecretsManager(config) => config.get_component_name(), diff --git a/tests/behavior/config/secret.toml b/tests/behavior/config/secret.toml index 110c414bfa561..a4066e5ca5e3c 100644 --- a/tests/behavior/config/secret.toml +++ b/tests/behavior/config/secret.toml @@ -6,12 +6,23 @@ type = "exec" command = ["./target/debug/secret-backend-example"] +[secret.file_backend] + type = "file" + path = "tests/data/secret-backends/file-secrets.json" + +[secret.directory_backend] + type = "directory" + path = "tests/data/secret-backends/directory-secrets" + remove_trailing_whitespace = true + [transforms.add_field_from_secret] inputs = [] type = "remap" source = ''' .foobar = "SECRET[test_backend.abc]" .foobarbaz = "SECRET[exec_backend.def]" + .foobarbazqux = "SECRET[file_backend.ghi]" + .foobarbazquxquux = "SECRET[directory_backend.jkl]" ''' [[tests]] @@ -27,4 +38,6 @@ source = ''' .foobar == "this_is_a_secret_value" .foobarbaz == "def.retrieved" + .foobarbazqux == "ghi.retrieved" + .foobarbazquxquux == "jkl.retrieved" ''' diff --git a/tests/data/secret-backends/directory-secrets/jkl b/tests/data/secret-backends/directory-secrets/jkl new file mode 100644 index 0000000000000..50cf4de5ac92f --- /dev/null +++ b/tests/data/secret-backends/directory-secrets/jkl @@ -0,0 +1 @@ +jkl.retrieved diff --git a/tests/data/secret-backends/file-secrets.json b/tests/data/secret-backends/file-secrets.json new file mode 100644 index 0000000000000..8de9e8b673504 --- /dev/null +++ b/tests/data/secret-backends/file-secrets.json @@ -0,0 +1 @@ +{"ghi":"ghi.retrieved"} diff --git a/website/cue/reference/configuration.cue b/website/cue/reference/configuration.cue index 23142e3f3b41c..b1ec7a925676d 100644 --- a/website/cue/reference/configuration.cue +++ b/website/cue/reference/configuration.cue @@ -480,6 +480,68 @@ configuration: { """ required: false type: object: options: { + file: { + required: true + description: """ + Retrieve secrets from a file path. + + The secret must be a JSON text string with key/value pairs. For example: + ```json + { + "username": "test", + "password": "example-password" + } + ``` + + If an error occurs while reading the file, Vector will log the error and exit. + + Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its + configuration reload process. + """ + type: object: options: { + path: { + description: "The file path to read." + required: true + type: string: { + examples: ["/path/to/secret.json"] + } + } + } + } + directory: { + required: true + description: """ + Retrieve secrets from file contents in a directory. + + The directory must contain files with names corresponding to secret keys. + + If an error occurs while reading the file, Vector will log the error and exit. + + Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its + configuration reload process. + """ + type: object: options: { + path: { + description: """ + The path of the directory with secrets. + """ + required: true + type: string: { + examples: [ + "${CREDENTIALS_DIRECTORY}", // https://systemd.io/CREDENTIALS + "/path/to/secrets-directory", + ] + } + } + remove_trailing_whitespace: { + description: """ + Remove trailing whitespace from file contents. + """ + required: false + type: bool: default: false + } + } + } exec: { required: true description: """ From dd39ab35290316e34cbeebba8e1b26d455bc9d0a Mon Sep 17 00:00:00 2001 From: Brian Deutsch Date: Mon, 21 Oct 2024 15:32:43 -0400 Subject: [PATCH 03/14] chore: upgrade `typesense-sync` package (#21572) Updates typesense sync package Signed-off-by: Brian Deutsch --- website/package.json | 2 +- website/yarn.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/website/package.json b/website/package.json index f4a0619937296..80316741cc408 100644 --- a/website/package.json +++ b/website/package.json @@ -49,7 +49,7 @@ "react-spring": "^9.2.3", "react-use": "^17.2.4", "regenerator-runtime": "^0.13.7", - "typesense-sync": "https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/typesense-sync-v1.0.0.tgz", + "typesense-sync": "https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/typesense-sync-v1.0.4.tgz", "tailwindcss": "^2.2.4", "tocbot": "^4.12.2", "topojson-client": "^3.1.0", diff --git a/website/yarn.lock b/website/yarn.lock index 0d8bc159f8220..3f0d778b58340 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -4991,9 +4991,9 @@ typescript@^4.1.3: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3" integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA== -"typesense-sync@https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/typesense-sync-v1.0.0.tgz": - version "1.0.0" - resolved "https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/typesense-sync-v1.0.0.tgz#f750afba4b130377f3fbcb63b0a3fe1fe24b2248" +"typesense-sync@https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/typesense-sync-v1.0.4.tgz": + version "1.0.4" + resolved "https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/typesense-sync-v1.0.4.tgz#f84c4d78140f8213d0ca5cad0ddb190bd6d90a01" dependencies: "@babel/runtime" "^7.25.0" crypto "^1.0.1" From 86fb0740f124b88eb08dd87d5f9383ed89916ad0 Mon Sep 17 00:00:00 2001 From: genadipost Date: Mon, 21 Oct 2024 22:33:36 +0300 Subject: [PATCH 04/14] fix(sinks/gcp/pubsub.rs): remove pubsub log only restriction (#21557) * fix(sinks/gcp/pubsub.rs): remove pubsub log only restriction * add changelog entry * fix fmt --------- Co-authored-by: Genady Postrilko --- ..._remove_pubsub_log_only_restriction.fix.md | 3 ++ src/sinks/gcp/pubsub.rs | 42 +++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 changelog.d/21557_remove_pubsub_log_only_restriction.fix.md diff --git a/changelog.d/21557_remove_pubsub_log_only_restriction.fix.md b/changelog.d/21557_remove_pubsub_log_only_restriction.fix.md new file mode 100644 index 0000000000000..aa340b2226408 --- /dev/null +++ b/changelog.d/21557_remove_pubsub_log_only_restriction.fix.md @@ -0,0 +1,3 @@ +Remove pubsub log only restriction. This allows using pubsub sink/source for metric and trace events. + +authors: genadipost diff --git a/src/sinks/gcp/pubsub.rs b/src/sinks/gcp/pubsub.rs index 3421b1bab975a..3703962f00d9d 100644 --- a/src/sinks/gcp/pubsub.rs +++ b/src/sinks/gcp/pubsub.rs @@ -11,7 +11,7 @@ use vector_lib::configurable::configurable_component; use crate::{ codecs::{Encoder, EncodingConfig, Transformer}, - config::{AcknowledgementsConfig, DataType, GenerateConfig, Input, SinkConfig, SinkContext}, + config::{AcknowledgementsConfig, GenerateConfig, Input, SinkConfig, SinkContext}, event::Event, gcp::{GcpAuthConfig, GcpAuthenticator, Scope, PUBSUB_URL}, http::HttpClient, @@ -144,7 +144,7 @@ impl SinkConfig for PubsubConfig { } fn input(&self) -> Input { - Input::new(self.encoding.config().input_type() & DataType::Log) + Input::new(self.encoding.config().input_type()) } fn acknowledgements(&self) -> &AcknowledgementsConfig { @@ -278,7 +278,7 @@ mod integration_tests { use crate::test_util::components::{run_and_assert_sink_error, COMPONENT_ERROR_TAGS}; use crate::test_util::{ components::{run_and_assert_sink_compliance, HTTP_SINK_TAGS}, - random_events_with_stream, random_string, trace_init, + random_events_with_stream, random_metrics_with_stream, random_string, trace_init, }; const PROJECT: &str = "testproject"; @@ -305,6 +305,34 @@ mod integration_tests { config(topic).build(cx).await.expect("Building sink failed") } + #[tokio::test] + async fn publish_metrics() { + trace_init(); + + let (topic, subscription) = create_topic_subscription().await; + let (sink, healthcheck) = config_build(&topic).await; + + healthcheck.await.expect("Health check failed"); + + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + let (input, events) = random_metrics_with_stream(100, Some(batch), None); + run_and_assert_sink_compliance(sink, events, &HTTP_SINK_TAGS).await; + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Delivered)); + + let response = pull_messages(&subscription, 1000).await; + let messages = response + .receivedMessages + .as_ref() + .expect("Response is missing messages"); + assert_eq!(input.len(), messages.len()); + for i in 0..input.len() { + let data = messages[i].message.decode_data_as_value(); + let data = serde_json::to_value(data).unwrap(); + let expected = serde_json::to_value(input[i].as_metric()).unwrap(); + assert_eq!(data, expected); + } + } + #[tokio::test] async fn publish_events() { trace_init(); @@ -434,6 +462,14 @@ mod integration_tests { let data = String::from_utf8_lossy(&data); serde_json::from_str(&data).expect("Invalid message structure") } + + fn decode_data_as_value(&self) -> Value { + let data = BASE64_STANDARD + .decode(&self.data) + .expect("Invalid base64 data"); + let data = String::from_utf8_lossy(&data); + serde_json::from_str(&data).expect("Invalid json") + } } #[derive(Debug, Deserialize, Serialize)] From eb699ef64dd40c3c2ce4d7366bae44e44b68b1f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 20:46:32 +0000 Subject: [PATCH 05/14] chore(deps): Bump serde_json from 1.0.128 to 1.0.132 (#21565) Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.128 to 1.0.132. - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/1.0.128...1.0.132) --- updated-dependencies: - dependency-name: serde_json dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e278eccf6dba..e8c04d48ee1cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4436,7 +4436,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.7", "tokio", "tower-service", "tracing 0.1.40", @@ -7242,7 +7242,7 @@ checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes 1.7.2", "heck 0.5.0", - "itertools 0.11.0", + "itertools 0.12.1", "log", "multimap", "once_cell", @@ -7275,7 +7275,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.12.1", "proc-macro2 1.0.88", "quote 1.0.37", "syn 2.0.79", @@ -7288,7 +7288,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.13.0", "proc-macro2 1.0.88", "quote 1.0.37", "syn 2.0.79", @@ -8569,9 +8569,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "indexmap 2.6.0", "itoa", diff --git a/Cargo.toml b/Cargo.toml index f976bb1361241..23122679c125a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,7 +149,7 @@ prost = { version = "0.12", default-features = false, features = ["std"] } prost-build = { version = "0.12", default-features = false } prost-reflect = { version = "0.14", features = ["serde"], default-features = false } prost-types = { version = "0.12", default-features = false } -serde_json = { version = "1.0.128", default-features = false, features = ["raw_value", "std"] } +serde_json = { version = "1.0.132", default-features = false, features = ["raw_value", "std"] } serde = { version = "1.0.210", default-features = false, features = ["alloc", "derive", "rc"] } toml = { version = "0.8.19", default-features = false, features = ["display", "parse"] } tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } From 75d59e957a716622fec2cb5590b5d71cbb659a52 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Mon, 21 Oct 2024 17:11:51 -0600 Subject: [PATCH 06/14] chore(deps): Properly hand off the `proptest` feature to `vrl` (#21574) When `vrl` is used through the `vector-lib` import, there is no direct way of specifying the `proptest` feature, but that should be activated. This corrects that omission. --- Cargo.toml | 2 +- lib/vector-lib/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 23122679c125a..108e6bf15c72c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -958,7 +958,7 @@ transform-benches = ["transforms-filter", "transforms-dedupe", "transforms-reduc codecs-benches = [] loki-benches = ["sinks-loki"] enrichment-tables-benches = ["enrichment-tables-geoip", "enrichment-tables-mmdb"] -proptest = ["dep:proptest", "dep:proptest-derive"] +proptest = ["dep:proptest", "dep:proptest-derive", "vrl/proptest"] [[bench]] name = "default" diff --git a/lib/vector-lib/Cargo.toml b/lib/vector-lib/Cargo.toml index 76bc0fe2b1171..d274b123635c3 100644 --- a/lib/vector-lib/Cargo.toml +++ b/lib/vector-lib/Cargo.toml @@ -28,7 +28,7 @@ lua = ["vector-core/lua"] file-source = ["dep:file-source"] opentelemetry = ["dep:opentelemetry-proto"] prometheus = ["dep:prometheus-parser"] -proptest = ["vector-lookup/proptest"] +proptest = ["vector-lookup/proptest", "vrl/proptest"] syslog = ["codecs/syslog"] test = ["vector-core/test"] vrl = ["vector-core/vrl", "dep:vrl"] From 5d8736f1a3ec40165f027138bd8913eb94df59cf Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 22 Oct 2024 08:20:47 -0700 Subject: [PATCH 07/14] chore(ci): rework Semantic check workflow (#14723) * chore: Try adding semantic check Signed-off-by: Jesse Szwedko * ci * temp run on PRs Signed-off-by: Jesse Szwedko * Bump version Signed-off-by: Jesse Szwedko * Didn't mean to commit this Signed-off-by: Jesse Szwedko * Remove old semantic pr file Signed-off-by: Jesse Szwedko * add comments and more descriptive name --------- Signed-off-by: Jesse Szwedko Signed-off-by: Jesse Szwedko Co-authored-by: Pavlos Rontidis --- .github/semantic.yml | 253 --------------------------------- .github/workflows/semantic.yml | 238 +++++++++++++++++++++++++++++++ 2 files changed, 238 insertions(+), 253 deletions(-) delete mode 100644 .github/semantic.yml create mode 100644 .github/workflows/semantic.yml diff --git a/.github/semantic.yml b/.github/semantic.yml deleted file mode 100644 index 825848de985e0..0000000000000 --- a/.github/semantic.yml +++ /dev/null @@ -1,253 +0,0 @@ -# WARNING! THIS FILE IS AUTOGENERATED! DO NOT EDIT! -# Original file: https://github.com/timberio/resources/tree/master/vector/github/semantic_pull_requests.tf -# ------------------------------------------------------------------------------ -# -# Configuration for probot/semantic-pull-requests GitHub check bot -# -# Installation/configuration docs here -# https://github.com/probot/semantic-pull-requests#configuration -# -# Always validate the PR title, and ignore the commits -titleOnly: true - -types: - - chore # An internal change this is not observable by users. - - enhancement # Any user observable enhancement to an existing feature. - - feat # Any user observable new feature, such as a new platform, source, transform, or sink. - - fix # Any user observable bug fix. - - docs # A documentation change. - - revert # A revert of a previous change. - -scopes: - - new source - - new transform - - new sink - - # domains - - ARC # Anything related to adaptive request concurrency - - administration # Anything related to administration/operation - - api # Anything related to Vector's GraphQL API - - architecture # Anything related to architecture - - auth # Anything related to authentication/authorization - - buffers # Anything related to Vector's memory/disk buffers - - ci # Anything related to Vector's CI environment - - cli # Anything related to Vector's CLI - - codecs # Anything related to Vector's codecs (encoding/decoding) - - compression # Anything related to compressing data within Vector - - config # Anything related to configuring Vector - - core # Anything related to core crates i.e. vector-core, core-common, etc - - data model # Anything related to Vector's internal data model - - delivery # Anything related to delivering events within Vector such as end-to-end acknowledgements - - deployment # Anything related to deploying Vector - - deps # Anything related to Vector's dependencies - - dev # Anything related to Vector's development environment - - durability # Anything related to the durability of Vector events - - enriching # Anything related to enriching Vector's events with context data - - enrichment_tables # Anything related to the Vector's enrichment tables - - exceptions # Anything related to Vector's exception events. - - external docs # Anything related to Vector's external, public documentation - - filtering # Anything related to filtering within Vector - - healthchecks # Anything related to Vector's healthchecks - - internal docs # Anything related to Vector's internal documentation - - logs # Anything related to Vector's log events - - metrics # Anything related to Vector's metrics events - - networking # Anything related to Vector's networking - - observability # Anything related to monitoring/observing Vector - - parsing # Anything related to parsing within Vector - - performance # Anything related to Vector's performance - - platforms # Anything related to Vector's supported platforms - - privacy # Anything related to privacy/compliance - - processing # Anything related to processing Vector's events (parsing, merging, reducing, etc.) - - releasing # Anything related to releasing Vector - - reliability # Anything related to Vector's reliability - - reload # Anything related to reloading Vector (updating configuration) - - replay # Anything related to replaying data - - schemas # Anything related to internal Vector event schemas - - security # Anything related to security - - setup # Anything related to setting up or installing Vector - - shutdown # Anything related to the shutdown of Vector - - sinks # Anything related to the Vector's sinks - - soak tests # Anything related to Vector's soak tests - - sources # Anything related to the Vector's sources - - startup # Anything related to the startup of Vector - - templating # Anything related to templating Vector's configuration values - - tests # Anything related to Vector's internal tests - - topology # Anything related to Vector's topology code - - traces # Anything related to Vectors' trace events - - transforms # Anything related to Vector's transform components - - unit tests # Anything related to Vector's unit testing feature - - vdev # Anything related to the vdev tooling - - vrl # Anything related to the Vector Remap Language - - # platforms - - amazon-linux platform # Anything `amazon-linux` platform related - - apt platform # Anything `apt` platform related - - arm platform # Anything `arm` platform related - - arm64 platform # Anything `arm64` platform related - - centos platform # Anything `centos` platform related - - debian platform # Anything `debian` platform related - - docker platform # Anything `docker` platform related - - dpkg platform # Anything `dpkg` platform related - - helm platform # Anything `helm` platform related - - heroku platform # Anything `heroku` platform related - - homebrew platform # Anything `homebrew` platform related - - kubernetes platform # Anything `kubernetes` platform related - - macos platform # Anything `macos` platform related - - msi platform # Anything `msi` platform related - - nix platform # Anything `nix` platform related - - nixos platform # Anything `nixos` platform related - - raspbian platform # Anything `raspbian` platform related - - rhel platform # Anything `rhel` platform related - - rpm platform # Anything `rpm` platform related - - ubuntu platform # Anything `ubuntu` platform related - - windows platform # Anything `windows` platform related - - x86_64 platform # Anything `x86_64` platform related - - yum platform # Anything `yum` platform related - - # service providers - - aws service # Anything `aws` service provider related - - azure service # Anything `azure` service provider related - - confluent service # Anything `confluent` service provider related - - datadog service # Anything `datadog` service provider related - - elastic service # Anything `elastic` service provider related - - gcp service # Anything `gcp` service provider related - - grafana service # Anything `grafana` service provider related - - heroku service # Anything `heroku` service provider related - - honeycomb service # Anything `honeycomb` service provider related - - humio service # Anything `humio` service provider related - - influxdata service # Anything `influxdata` service provider related - - mezmo service # Anything `mezmo` service provider related - - new relic service # Anything `new relic` service provider related - - papertrail service # Anything `papertrail` service provider related - - sematext service # Anything `sematext` service provider related - - splunk service # Anything `splunk` service provider related - - yandex service # Anything `yandex` service provider related - - # sources - - amqp source # Anything `amqp` source related - - apache_metrics source # Anything `apache_metrics` source related - - aws_ecs_metrics source # Anything `aws_ecs_metrics` source related - - aws_kinesis_firehose source # Anything `aws_kinesis_firehose` source related - - aws_s3 source # Anything `aws_s3` source related - - aws_sqs source # Anything `aws_sqs` source related - - datadog_agent source # Anything `datadog_agent` source related - - demo_logs source # Anything `demo_logs` source related - - dnstap source # Anything `dnstap` source related - - docker_logs source # Anything `docker_logs` source related - - eventstoredb_metrics source # Anything `eventstoredb_metrics` source related - - exec source # Anything `exec` source related - - file source # Anything `file` source related - - file_descriptor source # Anything `file_descriptor` source related - - fluent source # Anything `fluent` source related - - gcp_pubsub source # Anything `gcp_pubsub` source related - - heroku_logs source # Anything `heroku_logs` source related - - host_metrics source # Anything `host_metrics` source related - - http_client source # Anything `http_client` source related - - http_server source # Anything `http_server` source related - - internal_logs source # Anything `internal_logs` source related - - internal_metrics source # Anything `internal_metrics` source related - - journald source # Anything `journald` source related - - kafka source # Anything `kafka` source related - - kubernetes_logs source # Anything `kubernetes_logs` source related - - logstash source # Anything `logstash` source related - - mongodb_metrics source # Anything `mongodb_metrics` source related - - new source # A request for a new source - - nginx_metrics source # Anything `nginx_metrics` source related - - opentelemetry source # Anything `opentelemetry` source related - - postgresql_metrics source # Anything `postgresql_metrics` source related - - prometheus_remote_write source # Anything `prometheus_remote_write` source related - - prometheus_scrape source # Anything `prometheus_scrape` source related - - pulsar source # Anything `pulsar` source related - - redis source # Anything `redis` source related - - socket source # Anything `socket` source related - - splunk_hec source # Anything `splunk_hec` source related - - statsd source # Anything `statsd` source related - - stdin source # Anything `stdin` source related - - syslog source # Anything `syslog` source related - - vector source # Anything `vector` source related - - # transforms - - aggregate transform # Anything `aggregate` transform related - - aws_ec2_metadata transform # Anything `aws_ec2_metadata` transform related - - dedupe transform # Anything `dedupe` transform related - - filter transform # Anything `filter` transform related - - log_to_metric transform # Anything `log_to_metric` transform related - - lua transform # Anything `lua` transform related - - metric_to_log transform # Anything `metric_to_log` transform related - - new transform # A request for a new transform - - pipelines transform # Anything `pipelines` transform related - - reduce transform # Anything `reduce` transform related - - remap transform # Anything `remap` transform related - - route transform # Anything `route` transform related - - sample transform # Anything `sample` transform related - - tag_cardinality_limit transform # Anything `tag_cardinality_limit` transform related - - throttle transform # Anything `throttle` transform related - - # sinks - - amqp sink # Anything `amqp` sink related - - appsignal sink # Anything `appsignal` sink related - - aws_cloudwatch_logs sink # Anything `aws_cloudwatch_logs` sink related - - aws_cloudwatch_metrics sink # Anything `aws_cloudwatch_metrics` sink related - - aws_kinesis_firehose sink # Anything `aws_kinesis_firehose` sink related - - aws_kinesis_streams sink # Anything `aws_kinesis_streams` sink related - - aws_s3 sink # Anything `aws_s3` sink related - - aws_sqs sink # Anything `aws_sqs` sink related - - axiom sink # Anything `axiom` sink related - - azure_blob sink # Anything `azure_blob` sink related - - azure_monitor_logs sink # Anything `azure_monitor_logs` sink related - - blackhole sink # Anything `blackhole` sink related - - clickhouse sink # Anything `clickhouse` sink related - - console sink # Anything `console` sink related - - databend sink # Anything `databend` sink related - - datadog_events sink # Anything `datadog_events` sink related - - datadog_logs sink # Anything `datadog_logs` sink related - - datadog_metrics sink # Anything `datadog_metrics` sink related - - elasticsearch sink # Anything `elasticsearch` sink related - - file sink # Anything `file` sink related - - gcp_chronicle sink # Anything `gcp_chronicle` sink related - - gcp_cloud_storage sink # Anything `gcp_cloud_storage` sink related - - gcp_pubsub sink # Anything `gcp_pubsub` sink related - - gcp_stackdriver_logs sink # Anything `gcp_stackdriver_logs` sink related - - gcp_stackdriver_metrics sink # Anything `gcp_stackdriver_metrics` sink related - - honeycomb sink # Anything `honeycomb` sink related - - http sink # Anything `http` sink related - - humio_logs sink # Anything `humio_logs` sink related - - humio_metrics sink # Anything `humio_metrics` sink related - - influxdb_logs sink # Anything `influxdb_logs` sink related - - influxdb_metrics sink # Anything `influxdb_metrics` sink related - - kafka sink # Anything `kafka` sink related - - loki sink # Anything `loki` sink related - - mezmo sink # Anything `mezmo` sink related - - nats sink # Anything `nats` sink related - - new sink # A request for a new sink - - new_relic sink # Anything `new_relic` sink related - - new_relic_logs sink # Anything `new_relic_logs` sink related - - opentelemetry sink # Anything `opentelemetry` sink related - - papertrail sink # Anything `papertrail` sink related - - prometheus_exporter sink # Anything `prometheus_exporter` sink related - - prometheus_remote_write sink # Anything `prometheus_remote_write` sink related - - pulsar sink # Anything `pulsar` sink related - - redis sink # Anything `redis` sink related - - sematext_logs sink # Anything `sematext_logs` sink related - - sematext_metrics sink # Anything `sematext_metrics` sink related - - socket sink # Anything `socket` sink related - - splunk_hec sink # Anything `splunk_hec` sink related - - statsd sink # Anything `statsd` sink related - - vector sink # Anything `vector` sink related - - websocket sink # Anything `websocket` sink related - - # website - - SEO website # Anything related to search engine optimization (SEO) - - analytics website # Anything related to website analytics - - blog website # Anything related to the Vector blog - - copy website # Anything related to website copy - - css website # Anything related to vector.dev's CSS and aesthetics - - guides website # Anything related to Vector's guides - - highlights website # Anything related to vector.dev highlights - - javascript website # Anything related to the JavaScript functionality on vector.dev - - navigation website # Anything related to navigating the site - - operations website # Anything related to site operations: deploying, etc - - releases website # Anything related to the press page - - search website # Anything related to the website's Algolia search indexing/config - - sitemap website # Anything related to the press page - - template website # Anything related to website HTML and other templates diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml new file mode 100644 index 0000000000000..361034749a5ce --- /dev/null +++ b/.github/workflows/semantic.yml @@ -0,0 +1,238 @@ +# This is a GitHub Action that ensures that the PR titles match the Conventional Commits spec. +# See: https://www.conventionalcommits.org/en/v1.0.0/ + +name: "PR Title Semantic Check" + +on: + pull_request: + types: + - opened + - edited + - synchronize + +jobs: + main: + name: Check Semantic PR + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + requireScope: true + types: | + chore + enhancement + feat + fix + docs + revert + + scopes: | + new source + new transform + new sink + ARC + administration + api + architecture + auth + buffers + ci + cli + codecs + compression + config + core + data model + delivery + deployment + deps + dev + durability + enriching + enterprise + exceptions + external docs + filtering + healthchecks + internal docs + logs + metrics + networking + observability + parsing + performance + platforms + privacy + processing + releasing + reliability + reload + replay + schemas + security + setup + shutdown + sinks + soak tests + sources + startup + templating + tests + topology + traces + transforms + unit tests + vrl + amazon-linux platform + apt platform + arm platform + arm64 platform + centos platform + debian platform + docker platform + dpkg platform + helm platform + heroku platform + homebrew platform + kubernetes platform + macos platform + msi platform + nix platform + nixos platform + raspbian platform + rhel platform + rpm platform + ubuntu platform + windows platform + x86_64 platform + yum platform + + service providers + aws service + azure service + confluent service + datadog service + elastic service + gcp service + grafana service + heroku service + honeycomb service + humio service + influxdata service + logdna service + new relic service + papertrail service + sematext service + splunk service + yandex service + apache_metrics source + aws_ecs_metrics source + aws_kinesis_firehose source + aws_s3 source + aws_sqs source + datadog_agent source + demo_logs source + dnstap source + docker_logs source + exec source + file source + file_descriptor source + fluent source + gcp_pubsub source + heroku_logs source + host_metrics source + http source + http_scrape source + internal_logs source + internal_metrics source + journald source + kafka source + kubernetes_logs source + logstash source + mongodb_metrics source + new source + nginx_metrics source + opentelemetry source + postgresql_metrics source + prometheus_remote_write source + prometheus_scrape source + redis source + socket source + splunk_hec source + statsd source + stdin source + syslog source + vector source + aws_ec2_metadata transform + dedupe transform + filter transform + geoip transform + log_to_metric transform + lua transform + metric_to_log transform + new transform + pipelines transform + reduce transform + remap transform + route transform + sample transform + tag_cardinality_limit transform + throttle transform + apex sink + aws_cloudwatch_logs sink + aws_cloudwatch_metrics sink + aws_kinesis_firehose sink + aws_kinesis_streams sink + aws_s3 sink + aws_sqs sink + azure_blob sink + azure_monitor_logs sink + blackhole sink + clickhouse sink + console sink + datadog_archives sink + datadog_events sink + datadog_logs sink + datadog_metrics sink + elasticsearch sink + file sink + gcp_chronicle sink + gcp_cloud_storage sink + gcp_pubsub sink + gcp_stackdriver_logs sink + gcp_stackdriver_metrics sink + honeycomb sink + http sink + humio_logs sink + humio_metrics sink + influxdb_logs sink + influxdb_metrics sink + kafka sink + logdna sink + loki sink + nats sink + new sink + new_relic sink + new_relic_logs sink + opentelemetry sink + papertrail sink + prometheus_exporter sink + prometheus_remote_write sink + pulsar sink + redis sink + sematext_logs sink + sematext_metrics sink + socket sink + splunk_hec sink + statsd sink + vector sink + websocket sink + blog website + css website + guides website + highlights website + javascript website + search website + template website From 4a44e41ac1929408a4142c63b6f10e17812be50e Mon Sep 17 00:00:00 2001 From: Devin Ford Date: Tue, 22 Oct 2024 10:54:56 -0400 Subject: [PATCH 08/14] fix(gh_action): Prevent workflow from triggering unless previous workflow succeeds (#21579) * fix: add conditional based on trigger success * chore: update trigger check * fix: add syncronize to trigger list * chore: debug * chore: remove debugging * fix: address feedback * fix: remove trailing space --- .github/workflows/build_preview_sites.yml | 4 +++- .github/workflows/preview_site_trigger.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_preview_sites.yml b/.github/workflows/build_preview_sites.yml index e3478ee2711d2..dd2a52302961f 100644 --- a/.github/workflows/build_preview_sites.yml +++ b/.github/workflows/build_preview_sites.yml @@ -8,6 +8,7 @@ on: jobs: deploy_vector_preview_site: + if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.workflow_run.head_branch, 'website') }} uses: ./.github/workflows/create_preview_sites.yml with: APP_ID: "d1a7j77663uxsc" @@ -18,6 +19,7 @@ jobs: ENDPOINT: ${{ secrets.BUILDER_ENDPOINT }} deploy_rust_doc_preview_site: + if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.workflow_run.head_branch, 'website') }} uses: ./.github/workflows/create_preview_sites.yml with: APP_ID: "d1hoyoksbulg25" @@ -28,6 +30,7 @@ jobs: ENDPOINT: ${{ secrets.BUILDER_ENDPOINT }} deploy_vrl_playground_preview_site: + if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.workflow_run.head_branch, 'website') }} uses: ./.github/workflows/create_preview_sites.yml with: APP_ID: "d2lr4eds605rpz" @@ -36,4 +39,3 @@ jobs: REQUEST_TOKEN: ${{ secrets.REQUEST_TOKEN }} REQUEST_MESSAGE: ${{ secrets.REQUEST_MESSAGE }} ENDPOINT: ${{ secrets.BUILDER_ENDPOINT }} - diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index 7c2a960e1f88d..3676a64554e03 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -2,7 +2,7 @@ name: Call Build Preview on: pull_request: - types: [opened, reopened] + types: [opened, reopened, synchronize] jobs: approval_check: From 2c24f7e41a39363b23ac33a9a7108e3d0034e31e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 11:02:27 -0400 Subject: [PATCH 09/14] chore(deps): Bump bytes from 1.7.2 to 1.8.0 (#21576) Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.7.2 to 1.8.0. - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/compare/v1.7.2...v1.8.0) --- updated-dependencies: - dependency-name: bytes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 176 +++++++++++++++-------------- Cargo.toml | 2 +- lib/file-source/Cargo.toml | 2 +- lib/loki-logproto/Cargo.toml | 2 +- lib/opentelemetry-proto/Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- 8 files changed, 96 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8c04d48ee1cc..6a54605452bb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -459,7 +459,7 @@ dependencies = [ "async-stream", "async-trait", "base64 0.22.1", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "fnv", "futures-util", @@ -513,7 +513,7 @@ version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69117c43c01d81a69890a9f5dd6235f2f027ca8d1ec62d6d3c5e01ca0edb4f2b" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "indexmap 2.6.0", "serde", "serde_json", @@ -599,7 +599,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc1f1a75fd07f0f517322d103211f12d757658e91676def9a2e688774656c60" dependencies = [ "base64 0.21.7", - "bytes 1.7.2", + "bytes 1.8.0", "futures 0.3.31", "http 0.2.9", "memchr", @@ -765,7 +765,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "fastrand 2.1.1", "hex", "http 0.2.9", @@ -798,7 +798,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "http-body 0.4.5", "pin-project-lite", @@ -866,7 +866,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "fastrand 2.1.1", "http 0.2.9", "regex", @@ -889,7 +889,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "regex", "tracing 0.1.40", @@ -911,7 +911,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "regex", "tracing 0.1.40", @@ -933,7 +933,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "regex", "tracing 0.1.40", @@ -959,7 +959,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "http-body 0.4.5", "once_cell", @@ -985,7 +985,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "fastrand 2.1.1", "http 0.2.9", "regex", @@ -1031,7 +1031,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "regex", "tracing 0.1.40", @@ -1053,7 +1053,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "regex", "tracing 0.1.40", @@ -1075,7 +1075,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "regex", "tracing 0.1.40", @@ -1115,7 +1115,7 @@ dependencies = [ "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.7.2", + "bytes 1.8.0", "form_urlencoded", "hex", "hmac", @@ -1147,7 +1147,7 @@ checksum = "c5a373ec01aede3dd066ec018c1bc4e8f5dd11b2c11c59c8eef1a5c68101f397" dependencies = [ "aws-smithy-http", "aws-smithy-types", - "bytes 1.7.2", + "bytes 1.8.0", "crc32c", "crc32fast", "hex", @@ -1167,7 +1167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" dependencies = [ "aws-smithy-types", - "bytes 1.7.2", + "bytes 1.8.0", "crc32fast", ] @@ -1180,7 +1180,7 @@ dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.7.2", + "bytes 1.8.0", "bytes-utils", "futures-core", "http 0.2.9", @@ -1221,7 +1221,7 @@ dependencies = [ "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.7.2", + "bytes 1.8.0", "fastrand 2.1.1", "h2 0.3.26", "http 0.2.9", @@ -1246,7 +1246,7 @@ checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96" dependencies = [ "aws-smithy-async", "aws-smithy-types", - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "http 1.1.0", "pin-project-lite", @@ -1262,7 +1262,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" dependencies = [ "base64-simd", - "bytes 1.7.2", + "bytes 1.8.0", "bytes-utils", "futures-core", "http 0.2.9", @@ -1313,7 +1313,7 @@ dependencies = [ "async-trait", "axum-core 0.3.4", "bitflags 1.3.2", - "bytes 1.7.2", + "bytes 1.8.0", "futures-util", "http 0.2.9", "http-body 0.4.5", @@ -1341,7 +1341,7 @@ checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core 0.4.5", - "bytes 1.7.2", + "bytes 1.8.0", "futures-util", "http 1.1.0", "http-body 1.0.0", @@ -1367,7 +1367,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.7.2", + "bytes 1.8.0", "futures-util", "http 0.2.9", "http-body 0.4.5", @@ -1384,7 +1384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", - "bytes 1.7.2", + "bytes 1.8.0", "futures-util", "http 1.1.0", "http-body 1.0.0", @@ -1405,7 +1405,7 @@ checksum = "4ccd63c07d1fbfb3d4543d7ea800941bf5a30db1911b9b9e4db3b2c4210a434f" dependencies = [ "async-trait", "base64 0.21.7", - "bytes 1.7.2", + "bytes 1.8.0", "dyn-clone", "futures 0.3.31", "getrandom 0.2.15", @@ -1454,7 +1454,7 @@ dependencies = [ "RustyXML", "async-trait", "azure_core", - "bytes 1.7.2", + "bytes 1.8.0", "futures 0.3.31", "hmac", "log", @@ -1476,7 +1476,7 @@ dependencies = [ "RustyXML", "azure_core", "azure_storage", - "bytes 1.7.2", + "bytes 1.8.0", "futures 0.3.31", "log", "serde", @@ -1683,7 +1683,7 @@ checksum = "0aed08d3adb6ebe0eff737115056652670ae290f177759aac19c30456135f94c" dependencies = [ "base64 0.22.1", "bollard-stubs", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "futures-core", "futures-util", @@ -1867,9 +1867,9 @@ dependencies = [ [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" dependencies = [ "serde", ] @@ -1880,7 +1880,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "either", ] @@ -2178,7 +2178,7 @@ name = "codecs" version = "0.1.0" dependencies = [ "apache-avro", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "csv-core", "derivative", @@ -2256,7 +2256,7 @@ version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-core", "memchr", "pin-project-lite", @@ -3070,7 +3070,7 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64008666d9f3b6a88a63cd28ad8f3a5a859b8037e11bfb680c1b24945ea1c28d" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-util", "moka", "octseq", @@ -3480,7 +3480,7 @@ name = "file-source" version = "0.1.0" dependencies = [ "bstr 1.10.0", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "crc", "criterion", @@ -3968,7 +3968,7 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "fnv", "futures-core", "futures-sink", @@ -3988,7 +3988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", - "bytes 1.7.2", + "bytes 1.8.0", "fnv", "futures-core", "futures-sink", @@ -4071,7 +4071,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ "base64 0.21.7", - "bytes 1.7.2", + "bytes 1.8.0", "headers-core", "http 0.2.9", "httpdate", @@ -4315,7 +4315,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "fnv", "itoa", ] @@ -4326,7 +4326,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "fnv", "itoa", ] @@ -4337,7 +4337,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "http 0.2.9", "pin-project-lite", ] @@ -4348,7 +4348,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "http 1.1.0", ] @@ -4358,7 +4358,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-util", "http 1.1.0", "http-body 1.0.0", @@ -4425,7 +4425,7 @@ version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-channel", "futures-core", "futures-util", @@ -4449,7 +4449,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-channel", "futures-util", "h2 0.4.6", @@ -4503,7 +4503,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca815a891b24fdfb243fa3239c86154392b0953ee584aa1a2a1f66d20cbe75cc" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures 0.3.31", "headers", "http 0.2.9", @@ -4580,7 +4580,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "hyper 0.14.28", "native-tls", "tokio", @@ -4593,7 +4593,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-channel", "futures-util", "http 1.1.0", @@ -4751,7 +4751,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22fa7ee6be451ea0b1912b962c91c8380835e97cf1584a77e18264e908448dcb" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "log", "nom", "smallvec", @@ -4999,7 +4999,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d9455388f4977de4d0934efa9f7d36296295537d774574113a20f6082de03da" dependencies = [ "base64 0.13.1", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "serde", "serde-value", @@ -5013,7 +5013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd990069640f9db34b3b0f7a1afc62a05ffaa3be9b66aa3c313f58346df7f788" dependencies = [ "base64 0.21.7", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "http 0.2.9", "percent-encoding", @@ -5091,7 +5091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "544339f1665488243f79080441cacb09c997746fd763342303e66eebb9d3ba13" dependencies = [ "base64 0.20.0", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "dirs-next", "either", @@ -5348,7 +5348,7 @@ checksum = "879777f0cc6f3646a044de60e4ab98c75617e3f9580f7a2032e6ad7ea0cd3054" name = "loki-logproto" version = "0.1.0" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "prost 0.12.6", "prost-build 0.12.6", @@ -5773,7 +5773,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a15d522be0a9c3e46fd2632e272d178f56387bdb5c9fbb3a36c649062e9b5219" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "encoding_rs", "futures-util", "http 1.1.0", @@ -6242,7 +6242,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "126c3ca37c9c44cec575247f43a3e4374d8927684f129d2beeb0d2cef262fe12" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "serde", "smallvec", ] @@ -6306,7 +6306,7 @@ dependencies = [ "async-trait", "backon", "base64 0.21.7", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "flagset", "futures 0.3.31", @@ -6414,7 +6414,7 @@ dependencies = [ name = "opentelemetry-proto" version = "0.1.0" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "hex", "ordered-float 4.4.0", @@ -6933,7 +6933,7 @@ checksum = "acda0ebdebc28befa84bee35e651e4c5f09073d668c7aed4cf7e23c3cda84b23" dependencies = [ "base64 0.22.1", "byteorder", - "bytes 1.7.2", + "bytes 1.8.0", "fallible-iterator", "hmac", "md-5", @@ -6949,7 +6949,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f66ea23a2d0e5734297357705193335e0a957696f34bed2f2faefacb2fec336f" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "fallible-iterator", "postgres-protocol", @@ -7188,7 +7188,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "prost-derive 0.11.9", ] @@ -7198,7 +7198,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "prost-derive 0.12.6", ] @@ -7208,7 +7208,7 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "prost-derive 0.13.3", ] @@ -7218,7 +7218,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "heck 0.4.1", "itertools 0.10.5", "lazy_static", @@ -7240,7 +7240,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "heck 0.5.0", "itertools 0.12.1", "log", @@ -7388,7 +7388,7 @@ checksum = "d7f3541ff84e39da334979ac4bf171e0f277f4f782603aeae65bf5795dc7275a" dependencies = [ "async-trait", "bit-vec", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "crc", "data-url", @@ -7724,7 +7724,7 @@ checksum = "c580d9cbbe1d1b479e8d67cf9daf6a62c957e6846048408b80b43ac3f6af84cd" dependencies = [ "arc-swap", "async-trait", - "bytes 1.7.2", + "bytes 1.8.0", "combine 4.6.6", "futures 0.3.31", "futures-util", @@ -7856,7 +7856,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" dependencies = [ "base64 0.21.7", - "bytes 1.7.2", + "bytes 1.8.0", "encoding_rs", "futures-core", "futures-util", @@ -7902,7 +7902,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ "base64 0.22.1", - "bytes 1.7.2", + "bytes 1.8.0", "futures-core", "futures-util", "http 1.1.0", @@ -7981,7 +7981,7 @@ checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" dependencies = [ "bitvec", "bytecheck", - "bytes 1.7.2", + "bytes 1.8.0", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -8114,7 +8114,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "flume 0.11.0", "futures-util", "log", @@ -8134,7 +8134,7 @@ checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4" dependencies = [ "arrayvec", "borsh", - "bytes 1.7.2", + "bytes 1.8.0", "num-traits", "rand 0.8.5", "rkyv", @@ -9509,7 +9509,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", - "bytes 1.7.2", + "bytes 1.8.0", "libc", "mio 1.0.1", "parking_lot", @@ -9582,7 +9582,7 @@ checksum = "3b5d3742945bc7d7f210693b0c58ae542c6fd47b17adbbda0885f3dcb34a6bdb" dependencies = [ "async-trait", "byteorder", - "bytes 1.7.2", + "bytes 1.8.0", "fallible-iterator", "futures-channel", "futures-util", @@ -9651,7 +9651,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" dependencies = [ "async-stream", - "bytes 1.7.2", + "bytes 1.8.0", "futures-core", "tokio", "tokio-stream", @@ -9687,7 +9687,7 @@ name = "tokio-util" version = "0.7.11" source = "git+https://github.com/vectordotdev/tokio?branch=tokio-util-0.7.11-framed-read-continue-on-error#156dcaacdfa53f530a39eb91b1ceb731a9908986" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-core", "futures-io", "futures-sink", @@ -9762,7 +9762,7 @@ dependencies = [ "async-trait", "axum 0.6.20", "base64 0.21.7", - "bytes 1.7.2", + "bytes 1.8.0", "flate2", "h2 0.3.26", "http 0.2.9", @@ -9795,7 +9795,7 @@ dependencies = [ "async-trait", "axum 0.7.5", "base64 0.22.1", - "bytes 1.7.2", + "bytes 1.8.0", "h2 0.4.6", "http 1.1.0", "http-body 1.0.0", @@ -9870,7 +9870,7 @@ dependencies = [ "async-compression", "base64 0.21.7", "bitflags 2.4.1", - "bytes 1.7.2", + "bytes 1.8.0", "futures-core", "futures-util", "http 0.2.9", @@ -10131,7 +10131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" dependencies = [ "byteorder", - "bytes 1.7.2", + "bytes 1.8.0", "data-encoding", "http 0.2.9", "httparse", @@ -10150,7 +10150,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", - "bytes 1.7.2", + "bytes 1.8.0", "data-encoding", "http 1.1.0", "httparse", @@ -10507,7 +10507,7 @@ dependencies = [ "base64 0.22.1", "bloomy", "bollard", - "bytes 1.7.2", + "bytes 1.8.0", "bytesize", "chrono", "chrono-tz", @@ -10684,7 +10684,7 @@ dependencies = [ "async-stream", "async-trait", "bytecheck", - "bytes 1.7.2", + "bytes 1.8.0", "clap", "crc32fast", "criterion", @@ -10726,7 +10726,7 @@ name = "vector-common" version = "0.1.0" dependencies = [ "async-stream", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "chrono-tz", "crossbeam-utils", @@ -10816,7 +10816,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "bitmask-enum", - "bytes 1.7.2", + "bytes 1.8.0", "chrono", "chrono-tz", "criterion", @@ -11043,7 +11043,7 @@ dependencies = [ "base16", "base62", "base64 0.22.1", - "bytes 1.7.2", + "bytes 1.8.0", "cbc", "cfb-mode", "cfg-if", @@ -11093,6 +11093,8 @@ dependencies = [ "pest_derive", "prettydiff", "prettytable-rs", + "proptest", + "proptest-derive", "prost 0.13.3", "prost-reflect", "psl", @@ -11194,7 +11196,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-channel", "futures-util", "headers", diff --git a/Cargo.toml b/Cargo.toml index 108e6bf15c72c..02efbafb690a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -282,7 +282,7 @@ axum = { version = "0.6.20", default-features = false } base64 = { version = "0.22.1", default-features = false, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } bollard = { version = "0.16.1", default-features = false, features = ["ssl", "chrono"], optional = true } -bytes = { version = "1.7.2", default-features = false, features = ["serde"] } +bytes = { version = "1.8.0", default-features = false, features = ["serde"] } bytesize = { version = "1.3.0", default-features = false } chrono.workspace = true chrono-tz.workspace = true diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 67684445721f6..fdeef6a88bf68 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -24,7 +24,7 @@ default-features = false features = [] [dependencies.bytes] -version = "1.7.2" +version = "1.8.0" default-features = false features = [] diff --git a/lib/loki-logproto/Cargo.toml b/lib/loki-logproto/Cargo.toml index 5ed1a66d0d08a..2cdf8f03109af 100644 --- a/lib/loki-logproto/Cargo.toml +++ b/lib/loki-logproto/Cargo.toml @@ -10,7 +10,7 @@ publish = false [dependencies] prost.workspace = true prost-types.workspace = true -bytes = { version = "1.7.2", default-features = false } +bytes = { version = "1.8.0", default-features = false } snap = { version = "1.1.1", default-features = false } [dev-dependencies] diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index a782978a0cc1f..6ac71e7e3e4d3 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -10,7 +10,7 @@ prost-build.workspace = true tonic-build.workspace = true [dependencies] -bytes = { version = "1.7.2", default-features = false, features = ["serde"] } +bytes = { version = "1.8.0", default-features = false, features = ["serde"] } chrono.workspace = true hex = { version = "0.4.3", default-features = false, features = ["std"] } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false } diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index ef591267f6986..7f9dd03a78ab1 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -13,7 +13,7 @@ async-recursion = "1.1.1" async-stream = "0.3.6" async-trait = { version = "0.1", default-features = false } bytecheck = { version = "0.6.9", default-features = false, features = ["std"] } -bytes = { version = "1.7.2", default-features = false } +bytes = { version = "1.8.0", default-features = false } crc32fast = { version = "1.4.2", default-features = false } crossbeam-queue = { version = "0.3.11", default-features = false, features = ["std"] } crossbeam-utils = { version = "0.8.20", default-features = false } diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 8cfd257a94e48..02d0eaa227d3f 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -40,7 +40,7 @@ tokenize = [ [dependencies] async-stream = "0.3.6" -bytes = { version = "1.7.2", default-features = false, optional = true } +bytes = { version = "1.8.0", default-features = false, optional = true } chrono.workspace = true chrono-tz.workspace = true crossbeam-utils = { version = "0.8.20", default-features = false } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index e2299fccead8d..1a875b2ed1d77 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -12,7 +12,7 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(ddsketch_extended)'] } async-graphql = { version = "7.0.7", default-features = false, features = ["playground" ], optional = true } async-trait = { version = "0.1", default-features = false } bitmask-enum = { version = "2.2.4", default-features = false } -bytes = { version = "1.7.2", default-features = false, features = ["serde"] } +bytes = { version = "1.8.0", default-features = false, features = ["serde"] } chrono.workspace = true chrono-tz.workspace = true crossbeam-utils = { version = "0.8.20", default-features = false } From 25fe9b897b4076cd1df3adc630518cd8903328f6 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 22 Oct 2024 14:35:18 -0400 Subject: [PATCH 10/14] chore(ci): fix ignore filter (#21580) --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 8406b3ae42948..5407e033df959 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -68,7 +68,7 @@ jobs: all_changed: - added|deleted|modified: "**" ignore: - - "./.github/!(regression.yml)" + - "./.github/**/!(regression.yml)" - "./.gitignore" - "distribution/**" - "rust-doc/**" From 5d517b9f75c38a33670f5ecb1c6374cd9d0ac9e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:23:41 +0000 Subject: [PATCH 11/14] chore(deps): Bump syn from 2.0.79 to 2.0.82 (#21560) Bumps [syn](https://github.com/dtolnay/syn) from 2.0.79 to 2.0.82. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.79...2.0.82) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 100 ++++++++++++++++++++++++++--------------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a54605452bb2..c1fe2d82fa222 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -491,7 +491,7 @@ dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", "strum 0.26.2", - "syn 2.0.79", + "syn 2.0.82", "thiserror", ] @@ -674,7 +674,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -714,7 +714,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -731,7 +731,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1617,7 +1617,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afb15541e888071f64592c0b4364fdff21b7cb0a247f984296699351963a8721" dependencies = [ "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1747,7 +1747,7 @@ dependencies = [ "proc-macro-crate 2.0.0", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", "syn_derive", ] @@ -2146,7 +2146,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2706,7 +2706,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2754,7 +2754,7 @@ dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", "strsim 0.10.0", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2776,7 +2776,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core 0.20.8", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2909,7 +2909,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2930,7 +2930,7 @@ dependencies = [ "darling 0.20.8", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2940,7 +2940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3239,7 +3239,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3251,7 +3251,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3271,7 +3271,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3708,7 +3708,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -5687,7 +5687,7 @@ dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", "regex", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -6168,7 +6168,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -6180,7 +6180,7 @@ dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -6379,7 +6379,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -6662,7 +6662,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -6750,7 +6750,7 @@ checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -7029,7 +7029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2 1.0.88", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -7251,7 +7251,7 @@ dependencies = [ "prost 0.12.6", "prost-types 0.12.6", "regex", - "syn 2.0.79", + "syn 2.0.82", "tempfile", ] @@ -7278,7 +7278,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -7291,7 +7291,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -8104,7 +8104,7 @@ dependencies = [ "regex", "relative-path", "rustc_version 0.4.1", - "syn 2.0.79", + "syn 2.0.82", "unicode-ident", ] @@ -8553,7 +8553,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -8564,7 +8564,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -8627,7 +8627,7 @@ checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -8700,7 +8700,7 @@ dependencies = [ "darling 0.20.8", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -8980,7 +8980,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -9050,7 +9050,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ff9eaf853dec4c8802325d8b6d3dffa86cc707fd7a1a4cdbf416e13b061787a" dependencies = [ "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -9146,7 +9146,7 @@ dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", "rustversion", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -9159,7 +9159,7 @@ dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", "rustversion", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -9211,9 +9211,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", @@ -9229,7 +9229,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -9401,7 +9401,7 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -9550,7 +9550,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -9838,7 +9838,7 @@ dependencies = [ "proc-macro2 1.0.88", "prost-build 0.12.6", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -9941,7 +9941,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -10200,7 +10200,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -10230,7 +10230,7 @@ checksum = "70b20a22c42c8f1cd23ce5e34f165d4d37038f5b663ad20fb6adbdf029172483" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -10790,7 +10790,7 @@ dependencies = [ "quote 1.0.37", "serde", "serde_json", - "syn 2.0.79", + "syn 2.0.82", "tracing 0.1.40", ] @@ -10803,7 +10803,7 @@ dependencies = [ "quote 1.0.37", "serde", "serde_derive_internals", - "syn 2.0.79", + "syn 2.0.82", "vector-config", "vector-config-common", ] @@ -11258,7 +11258,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", "wasm-bindgen-shared", ] @@ -11292,7 +11292,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -11794,7 +11794,7 @@ checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2 1.0.88", "quote 1.0.37", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] From 86c313f7fecff3c7300296f70f7667398e145eed Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Tue, 22 Oct 2024 12:38:56 -0600 Subject: [PATCH 12/14] chore(core): Wrap event metadata in `Arc` for performance (#21188) * enhancement(core): Wrap event metadata in `Arc` for clone performance The `EventMetadata` structure contains a number of sub-structures that can cause extensive deep copies when an event is cloned, most notably `value: vrl::value::Value`. To improve this, wrap the actual inner event metadata in an `Arc`, and use the standard tools from that type to only clone it when needed to provide a mutable or owned copy. * chore(releasing): Prepare v0.42.0 release Co-authored-by: Bryce Eadie Signed-off-by: Jesse Szwedko --------- Signed-off-by: Jesse Szwedko Co-authored-by: Jesse Szwedko Co-authored-by: Bryce Eadie --- lib/vector-core/src/event/metadata.rs | 115 +++++++++++++++----------- lib/vector-core/src/event/proto.rs | 10 +-- 2 files changed, 71 insertions(+), 54 deletions(-) diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index 5d44b8236aafb..c1dd5790236bb 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -21,11 +21,16 @@ use crate::{ const DATADOG_API_KEY: &str = "datadog_api_key"; const SPLUNK_HEC_TOKEN: &str = "splunk_hec_token"; -/// The top-level metadata structure contained by both `struct Metric` +/// The event metadata structure is a `Arc` wrapper around the actual metadata to avoid cloning the +/// underlying data until it becomes necessary to provide a `mut` copy. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct EventMetadata(pub(super) Arc); + +/// The actual metadata structure contained by both `struct Metric` /// and `struct LogEvent` types. -#[derive(Clone, Debug, Deserialize, Serialize, Derivative)] +#[derive(Clone, Debug, Derivative, Deserialize, Serialize)] #[derivative(PartialEq)] -pub struct EventMetadata { +pub(super) struct Inner { /// Arbitrary data stored with an event #[serde(default = "default_metadata_value")] pub(crate) value: Value, @@ -124,84 +129,92 @@ fn default_metadata_value() -> Value { impl EventMetadata { /// Creates `EventMetadata` with the given `Value`, and the rest of the fields with default values pub fn default_with_value(value: Value) -> Self { - Self { + Self(Arc::new(Inner { value, ..Default::default() - } + })) + } + + fn get_mut(&mut self) -> &mut Inner { + Arc::make_mut(&mut self.0) + } + + pub(super) fn into_owned(self) -> Inner { + Arc::unwrap_or_clone(self.0) } /// Returns a reference to the metadata value pub fn value(&self) -> &Value { - &self.value + &self.0.value } /// Returns a mutable reference to the metadata value pub fn value_mut(&mut self) -> &mut Value { - &mut self.value + &mut self.get_mut().value } /// Returns a reference to the secrets pub fn secrets(&self) -> &Secrets { - &self.secrets + &self.0.secrets } /// Returns a mutable reference to the secrets pub fn secrets_mut(&mut self) -> &mut Secrets { - &mut self.secrets + &mut self.get_mut().secrets } /// Returns a reference to the metadata source id. #[must_use] pub fn source_id(&self) -> Option<&Arc> { - self.source_id.as_ref() + self.0.source_id.as_ref() } /// Returns a reference to the metadata source type. #[must_use] pub fn source_type(&self) -> Option<&str> { - self.source_type.as_deref() + self.0.source_type.as_deref() } /// Returns a reference to the metadata parent id. This is the `OutputId` /// of the previous component the event was sent through (if any). #[must_use] pub fn upstream_id(&self) -> Option<&OutputId> { - self.upstream_id.as_deref() + self.0.upstream_id.as_deref() } /// Sets the `source_id` in the metadata to the provided value. pub fn set_source_id(&mut self, source_id: Arc) { - self.source_id = Some(source_id); + self.get_mut().source_id = Some(source_id); } /// Sets the `source_type` in the metadata to the provided value. pub fn set_source_type>>(&mut self, source_type: S) { - self.source_type = Some(source_type.into()); + self.get_mut().source_type = Some(source_type.into()); } /// Sets the `upstream_id` in the metadata to the provided value. pub fn set_upstream_id(&mut self, upstream_id: Arc) { - self.upstream_id = Some(upstream_id); + self.get_mut().upstream_id = Some(upstream_id); } /// Return the datadog API key, if it exists pub fn datadog_api_key(&self) -> Option> { - self.secrets.get(DATADOG_API_KEY).cloned() + self.0.secrets.get(DATADOG_API_KEY).cloned() } /// Set the datadog API key to passed value pub fn set_datadog_api_key(&mut self, secret: Arc) { - self.secrets.insert(DATADOG_API_KEY, secret); + self.get_mut().secrets.insert(DATADOG_API_KEY, secret); } /// Return the splunk hec token, if it exists pub fn splunk_hec_token(&self) -> Option> { - self.secrets.get(SPLUNK_HEC_TOKEN).cloned() + self.0.secrets.get(SPLUNK_HEC_TOKEN).cloned() } /// Set the splunk hec token to passed value pub fn set_splunk_hec_token(&mut self, secret: Arc) { - self.secrets.insert(SPLUNK_HEC_TOKEN, secret); + self.get_mut().secrets.insert(SPLUNK_HEC_TOKEN, secret); } /// Adds the value to the dropped fields list. @@ -209,26 +222,26 @@ impl EventMetadata { /// and then the field is re-added with a new value - the dropped value will still be /// retrieved. pub fn add_dropped_field(&mut self, meaning: KeyString, value: Value) { - self.dropped_fields.insert(meaning, value); + self.get_mut().dropped_fields.insert(meaning, value); } /// Fetches the dropped field by meaning. pub fn dropped_field(&self, meaning: impl AsRef) -> Option<&Value> { - self.dropped_fields.get(meaning.as_ref()) + self.0.dropped_fields.get(meaning.as_ref()) } /// Returns a reference to the `DatadogMetricOriginMetadata`. pub fn datadog_origin_metadata(&self) -> Option<&DatadogMetricOriginMetadata> { - self.datadog_origin_metadata.as_ref() + self.0.datadog_origin_metadata.as_ref() } /// Returns a reference to the event id. pub fn source_event_id(&self) -> Option { - self.source_event_id + self.0.source_event_id } } -impl Default for EventMetadata { +impl Default for Inner { fn default() -> Self { Self { value: Value::Object(ObjectMap::new()), @@ -245,6 +258,12 @@ impl Default for EventMetadata { } } +impl Default for EventMetadata { + fn default() -> Self { + Self(Arc::new(Inner::default())) + } +} + fn default_schema_definition() -> Arc { Arc::new(schema::Definition::new_with_default_metadata( Kind::any(), @@ -257,7 +276,7 @@ impl ByteSizeOf for EventMetadata { // NOTE we don't count the `str` here because it's allocated somewhere // else. We're just moving around the pointer, which is already captured // by `ByteSizeOf::size_of`. - self.finalizers.allocated_bytes() + self.0.finalizers.allocated_bytes() } } @@ -265,14 +284,14 @@ impl EventMetadata { /// Replaces the existing event finalizers with the given one. #[must_use] pub fn with_finalizer(mut self, finalizer: EventFinalizer) -> Self { - self.finalizers = EventFinalizers::new(finalizer); + self.get_mut().finalizers = EventFinalizers::new(finalizer); self } /// Replaces the existing event finalizers with the given ones. #[must_use] pub fn with_finalizers(mut self, finalizers: EventFinalizers) -> Self { - self.finalizers = finalizers; + self.get_mut().finalizers = finalizers; self } @@ -294,28 +313,28 @@ impl EventMetadata { /// Replace the schema definition with the given one. #[must_use] pub fn with_schema_definition(mut self, schema_definition: &Arc) -> Self { - self.schema_definition = Arc::clone(schema_definition); + self.get_mut().schema_definition = Arc::clone(schema_definition); self } /// Replaces the existing `source_type` with the given one. #[must_use] pub fn with_source_type>>(mut self, source_type: S) -> Self { - self.source_type = Some(source_type.into()); + self.get_mut().source_type = Some(source_type.into()); self } /// Replaces the existing `DatadogMetricOriginMetadata` with the given one. #[must_use] pub fn with_origin_metadata(mut self, origin_metadata: DatadogMetricOriginMetadata) -> Self { - self.datadog_origin_metadata = Some(origin_metadata); + self.get_mut().datadog_origin_metadata = Some(origin_metadata); self } /// Replaces the existing `source_event_id` with the given one. #[must_use] pub fn with_source_event_id(mut self, source_event_id: Option) -> Self { - self.source_event_id = source_event_id; + self.get_mut().source_event_id = source_event_id; self } @@ -323,16 +342,18 @@ impl EventMetadata { /// If a Datadog API key is not set in `self`, the one from `other` will be used. /// If a Splunk HEC token is not set in `self`, the one from `other` will be used. pub fn merge(&mut self, other: Self) { - self.finalizers.merge(other.finalizers); - self.secrets.merge(other.secrets); + let inner = self.get_mut(); + let other = other.into_owned(); + inner.finalizers.merge(other.finalizers); + inner.secrets.merge(other.secrets); // Update `source_event_id` if necessary. - match (self.source_event_id, other.source_event_id) { + match (inner.source_event_id, other.source_event_id) { (None, Some(id)) => { - self.source_event_id = Some(id); + inner.source_event_id = Some(id); } (Some(uuid1), Some(uuid2)) if uuid2 < uuid1 => { - self.source_event_id = Some(uuid2); + inner.source_event_id = Some(uuid2); } _ => {} // Keep the existing value. }; @@ -340,42 +361,42 @@ impl EventMetadata { /// Update the finalizer(s) status. pub fn update_status(&self, status: EventStatus) { - self.finalizers.update_status(status); + self.0.finalizers.update_status(status); } /// Update the finalizers' sources. pub fn update_sources(&mut self) { - self.finalizers.update_sources(); + self.get_mut().finalizers.update_sources(); } /// Gets a reference to the event finalizers. pub fn finalizers(&self) -> &EventFinalizers { - &self.finalizers + &self.0.finalizers } /// Add a new event finalizer to the existing list of event finalizers. pub fn add_finalizer(&mut self, finalizer: EventFinalizer) { - self.finalizers.add(finalizer); + self.get_mut().finalizers.add(finalizer); } /// Consumes all event finalizers and returns them, leaving the list of event finalizers empty. pub fn take_finalizers(&mut self) -> EventFinalizers { - std::mem::take(&mut self.finalizers) + std::mem::take(&mut self.get_mut().finalizers) } /// Merges the given event finalizers into the existing list of event finalizers. pub fn merge_finalizers(&mut self, finalizers: EventFinalizers) { - self.finalizers.merge(finalizers); + self.get_mut().finalizers.merge(finalizers); } /// Get the schema definition. pub fn schema_definition(&self) -> &Arc { - &self.schema_definition + &self.0.schema_definition } /// Set the schema definition. pub fn set_schema_definition(&mut self, definition: &Arc) { - self.schema_definition = Arc::clone(definition); + self.get_mut().schema_definition = Arc::clone(definition); } /// Helper function to add a semantic meaning to the schema definition. @@ -394,7 +415,7 @@ impl EventMetadata { /// .set_schema_definition(new_schema); /// ```` pub fn add_schema_meaning(&mut self, target_path: OwnedTargetPath, meaning: &str) { - let schema = Arc::make_mut(&mut self.schema_definition); + let schema = Arc::make_mut(&mut self.get_mut().schema_definition); schema.add_meaning(target_path, meaning); } } @@ -543,13 +564,13 @@ mod test { { let mut merged = m1.clone(); merged.merge(m2.clone()); - assert_eq!(merged.source_event_id, m1.source_event_id); + assert_eq!(merged.source_event_id(), m1.source_event_id()); } { let mut merged = m2.clone(); merged.merge(m1.clone()); - assert_eq!(merged.source_event_id, m1.source_event_id); + assert_eq!(merged.source_event_id(), m1.source_event_id()); } } } diff --git a/lib/vector-core/src/event/proto.rs b/lib/vector-core/src/event/proto.rs index 0c05448931efa..f1c926b49e1cd 100644 --- a/lib/vector-core/src/event/proto.rs +++ b/lib/vector-core/src/event/proto.rs @@ -618,7 +618,7 @@ impl From for crate::config::OutputId { impl From for Metadata { fn from(value: EventMetadata) -> Self { - let EventMetadata { + let super::metadata::Inner { value, secrets, source_id, @@ -627,13 +627,9 @@ impl From for Metadata { datadog_origin_metadata, source_event_id, .. - } = value; + } = value.into_owned(); - let secrets = if secrets.is_empty() { - None - } else { - Some(secrets.into()) - }; + let secrets = (!secrets.is_empty()).then(|| secrets.into()); Self { value: Some(encode_value(value)), From 21ae5d007d693e74e539cc584fdc41cd72c83437 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 22 Oct 2024 15:50:57 -0400 Subject: [PATCH 13/14] chore(ci): set regression workflow timeouts to 70 mins (#21582) --- .github/workflows/regression.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 5407e033df959..e9ea53bb71e73 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -420,7 +420,7 @@ jobs: submit-job: name: Submit regression job runs-on: ubuntu-22.04 - timeout-minutes: 60 + timeout-minutes: 70 needs: - compute-metadata - upload-baseline-image-to-ecr @@ -479,7 +479,7 @@ jobs: path: ${{ runner.temp }}/submission-metadata - name: Await job - timeout-minutes: 120 + timeout-minutes: 70 env: RUST_LOG: info run: | @@ -488,7 +488,7 @@ jobs: ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \ --wait \ --wait-delay-seconds 60 \ - --wait-timeout-minutes 90 \ + --wait-timeout-minutes 70 \ --submission-metadata ${{ runner.temp }}/submission-metadata - name: Handle cancellation if necessary From 8c4e27651f554db8c543bc29d184d03a7e5b69e2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 22 Oct 2024 18:19:17 -0400 Subject: [PATCH 14/14] feat(ci): regression detection overhaul (#21567) * --no-edit * remove trailing spaces * pulled in fixes: changed timeouts to 70, fixed ignore glob --- .github/workflows/regression_v2.yml | 703 ++++++++++++++++++++++++++++ 1 file changed, 703 insertions(+) create mode 100644 .github/workflows/regression_v2.yml diff --git a/.github/workflows/regression_v2.yml b/.github/workflows/regression_v2.yml new file mode 100644 index 0000000000000..1b9001519bd26 --- /dev/null +++ b/.github/workflows/regression_v2.yml @@ -0,0 +1,703 @@ +# Regression Detection Suite +# +# This workflow runs under the following conditions: +# - Once per day, based on a schedule +# - On demand, triggered from https://github.com/vectordotdev/vector/actions/workflows/regression_v2.yml +# +# The workflow accepts two optional inputs: +# - The baseline SHA: +# - If not specified, the SHA from 24 hours ago on origin/master is used. +# - The comparison SHA: +# - If not specified, the current HEAD of origin/master is used. +# +# This workflow runs regression detection experiments, performing relative +# evaluations of the baseline SHA and comparison SHA. The exact SHAs are determined +# by how the workflow is triggered. +# +# The goal is to provide quick feedback on Vector's performance across a variety +# of configurations, checking if throughput performance has degraded or become +# more variable in the comparison SHA relative to the baseline SHA. +# +# Docker image tags are based on the resolved SHAs. + +name: Regression Detection Suite (new) + +on: + workflow_dispatch: + inputs: + baseline-sha: + description: "The SHA to use as the baseline (optional). If not provided, it defaults to the SHA from 24 hours ago." + required: false + comparison-sha: + description: "The SHA to use for comparison (optional). If not provided, it defaults to the current HEAD of the origin/master branch." + required: false + schedule: + - cron: '0 6 * * 1-5' # Runs at 6 AM UTC on weekdays (Monday to Friday) + +env: + SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }} + SMP_WARMUP_SECONDS: 70 # default is 45 seconds + +jobs: + + resolve-inputs: + runs-on: ubuntu-latest + outputs: + baseline-sha: ${{ steps.set_and_validate_shas.outputs.BASELINE_SHA }} + comparison-sha: ${{ steps.set_and_validate_shas.outputs.COMPARISON_SHA }} + baseline-tag: ${{ steps.set_and_validate_shas.outputs.BASELINE_TAG }} + comparison-tag: ${{ steps.set_and_validate_shas.outputs.COMPARISON_TAG }} + smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # need to pull repository history to find merge bases + + - name: Set and Validate SHAs + id: set_and_validate_shas + run: | + # Set baseline SHA + if [ -z "${{ github.event.inputs.baseline-sha }}" ]; then + BASELINE_SHA=$(git rev-list -n 1 --before="24 hours ago" origin/master) + echo "Using baseline SHA from 24 hours ago: ${BASELINE_SHA}" + else + BASELINE_SHA="${{ github.event.inputs.baseline-sha }}" + echo "Using provided baseline SHA: ${BASELINE_SHA}" + fi + + # Validate baseline SHA + if [ -n "${BASELINE_SHA}" ] && git cat-file -e "${BASELINE_SHA}^{commit}"; then + echo "Baseline SHA is valid." + else + echo "Invalid baseline SHA: ${BASELINE_SHA}." + exit 1 + fi + + # Set comparison SHA + if [ -z "${{ github.event.inputs.comparison-sha }}" ]; then + COMPARISON_SHA=$(git rev-parse origin/master) + echo "Using current HEAD for comparison SHA: ${COMPARISON_SHA}" + else + COMPARISON_SHA="${{ github.event.inputs.comparison-sha }}" + echo "Using provided comparison SHA: ${COMPARISON_SHA}" + fi + + # Validate comparison SHA + if [ -n "${COMPARISON_SHA}" ] && git cat-file -e "${COMPARISON_SHA}^{commit}"; then + echo "Comparison SHA is valid." + else + echo "Invalid comparison SHA: ${COMPARISON_SHA}." + exit 1 + fi + + # Set tags and export them + BASELINE_TAG="workflow_dispatch-${COMPARISON_SHA}-${BASELINE_SHA}" + COMPARISON_TAG="workflow_dispatch-${COMPARISON_SHA}-${COMPARISON_SHA}" + + echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT + echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT + + echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT + echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT + + - name: Set SMP version + id: experimental-meta + run: | + export SMP_CRATE_VERSION="0.16.1" + echo "smp crate version: ${SMP_CRATE_VERSION}" + echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT + + # Only run this workflow if files changed in areas that could possibly introduce a regression. + check-source-changed: + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: resolve-inputs + outputs: + source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }} + steps: + - uses: actions/checkout@v3 + + - name: Collect file changes + id: changes + uses: dorny/paths-filter@v3 + with: + base: ${{ needs.resolve-inputs.outputs.baseline-sha }} + ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} + list-files: shell + filters: | + all_changed: + - added|deleted|modified: "**" + ignore: + - "./.github/**/!(regression_v2.yml)" + - "./.gitignore" + - "distribution/**" + - "rust-doc/**" + - "docs/**" + - "rfcs/**" + - "testing/**" + - "tilt/**" + - "website/**" + - "*.md" + - "Tiltfile" + - "netlify.toml" + - "NOTICE" + - "LICENSE-3rdparty.csv" + - "LICENSE" + - "lib/codecs/tests/data/**" + + # This step allows us to conservatively run the tests if we added a new + # file or directory for source code, but forgot to add it to this workflow. + # Instead, we may unnecessarily run the test on new file or dir additions that + # wouldn't likely introduce regressions. + - name: Determine if should not run due to irrelevant file changes + id: filter + if: github.event_name == 'merge_group' + env: + ALL: ${{ steps.changes.outputs.all_changed_files }} + IGNORE: ${{ steps.changes.outputs.ignore_files }} + run: | + echo "ALL='${{ env.ALL }}'" + echo "IGNORE='${{ env.IGNORE }}'" + export SOURCE_CHANGED=$(comm -2 -3 <(printf "%s\n" "${{ env.ALL }}") <(printf "%s\n" "${{ env.IGNORE }}")) + echo "SOURCE_CHANGED='${SOURCE_CHANGED}'" + + if [ "${SOURCE_CHANGED}" == "" ]; then + export SOURCE_CHANGED="false" + else + export SOURCE_CHANGED="true" + fi + + echo "SOURCE_CHANGED='${SOURCE_CHANGED}'" + echo "SOURCE_CHANGED=${SOURCE_CHANGED}" >> $GITHUB_OUTPUT + + should-run-gate: + runs-on: ubuntu-latest + needs: check-source-changed + if: ${{ needs.check-source-changed.outputs.source_changed }} + + ## + ## BUILD + ## + + build-baseline: + name: Build baseline Vector container + runs-on: ubuntu-20.04-4core + timeout-minutes: 30 + needs: + - should-run-gate + - resolve-inputs + steps: + - uses: colpal/actions-clean@v1 + + - uses: actions/checkout@v3 + + - uses: actions/checkout@v3 + with: + ref: ${{ needs.resolve-inputs.outputs.baseline-sha }} + path: baseline-vector + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3.7.1 + + - name: Build 'vector' target image + uses: docker/build-push-action@v6.9.0 + with: + context: baseline-vector/ + cache-from: type=gha + cache-to: type=gha,mode=max + file: regression/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar + tags: | + vector:${{ needs.resolve-inputs.outputs.baseline-tag }} + + - name: Upload image as artifact + uses: actions/upload-artifact@v3 + with: + name: baseline-image + path: "${{ runner.temp }}/baseline-image.tar" + + build-comparison: + name: Build comparison Vector container + runs-on: ubuntu-20.04-4core + timeout-minutes: 30 + needs: + - should-run-gate + - resolve-inputs + steps: + - uses: colpal/actions-clean@v1 + + - uses: actions/checkout@v3 + + - uses: actions/checkout@v3 + with: + ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} + path: comparison-vector + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3.7.1 + + - name: Build 'vector' target image + uses: docker/build-push-action@v6.9.0 + with: + context: comparison-vector/ + cache-from: type=gha + cache-to: type=gha,mode=max + file: regression/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar + tags: | + vector:${{ needs.resolve-inputs.outputs.comparison-tag }} + + - name: Upload image as artifact + uses: actions/upload-artifact@v3 + with: + name: comparison-image + path: "${{ runner.temp }}/comparison-image.tar" + + confirm-valid-credentials: + name: Confirm AWS credentials are minimally valid + runs-on: ubuntu-22.04 + timeout-minutes: 5 + needs: + - should-run-gate + - resolve-inputs + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Download SMP binary + run: | + aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + + ## + ## SUBMIT + ## + + upload-baseline-image-to-ecr: + name: Upload baseline images to ECR + runs-on: ubuntu-22.04 + timeout-minutes: 5 + needs: + - should-run-gate + - resolve-inputs + - confirm-valid-credentials + - build-baseline + steps: + - name: 'Download baseline image' + uses: actions/download-artifact@v3 + with: + name: baseline-image + + - name: Load baseline image + run: | + docker load --input baseline-image.tar + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Docker Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ steps.login-ecr.outputs.registry }} + + - name: Tag & push baseline image + run: | + docker tag vector:${{ needs.resolve-inputs.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }} + docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }} + + upload-comparison-image-to-ecr: + name: Upload comparison images to ECR + runs-on: ubuntu-22.04 + timeout-minutes: 5 + needs: + - should-run-gate + - resolve-inputs + - confirm-valid-credentials + - build-comparison + steps: + - name: 'Download comparison image' + uses: actions/download-artifact@v3 + with: + name: comparison-image + + - name: Load comparison image + run: | + docker load --input comparison-image.tar + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Docker Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ steps.login-ecr.outputs.registry }} + + - name: Tag & push comparison image + run: | + docker tag vector:${{ needs.resolve-inputs.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }} + docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }} + + submit-job: + name: Submit regression job + runs-on: ubuntu-22.04 + timeout-minutes: 70 + needs: + - should-run-gate + - resolve-inputs + - upload-baseline-image-to-ecr + - upload-comparison-image-to-ecr + steps: + - name: Check status, in-progress + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='pending' \ + -f description='Experiments submitted to the Regression Detection cluster.' \ + -f context='Regression Detection Suite / submission' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - uses: actions/checkout@v3 + with: + ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Download SMP binary + run: | + aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + + - name: Submit job + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \ + --baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }} \ + --comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }} \ + --baseline-sha ${{ needs.resolve-inputs.outputs.baseline-sha }} \ + --comparison-sha ${{ needs.resolve-inputs.outputs.comparison-sha }} \ + --target-config-dir ${{ github.workspace }}/regression/ \ + --warmup-seconds ${{ env.SMP_WARMUP_SECONDS }} \ + --submission-metadata ${{ runner.temp }}/submission-metadata + + - uses: actions/upload-artifact@v3 + with: + name: vector-submission-metadata + path: ${{ runner.temp }}/submission-metadata + + - name: Await job + timeout-minutes: 70 + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \ + --wait \ + --wait-delay-seconds 60 \ + --wait-timeout-minutes 70 \ + --submission-metadata ${{ runner.temp }}/submission-metadata + + - name: Handle cancellation if necessary + if: ${{ cancelled() }} + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \ + --submission-metadata ${{ runner.temp }}/submission-metadata + + - name: Check status, cancelled + if: ${{ cancelled() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Experiments submitted to the Regression Detection cluster cancelled.' \ + -f context='Regression Detection Suite / submission' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, success + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='success' \ + -f description='Experiments submitted to the Regression Detection cluster successfully.' \ + -f context='Regression Detection Suite / submission' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='success' \ + -f description='Experiments submitted to the Regression Detection Suite failed.' \ + -f context='Regression Detection Suite / submission' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + ## + ## ANALYZE + ## + + detect-regression: + name: Determine regression status + runs-on: ubuntu-22.04 + timeout-minutes: 5 + needs: + - submit-job + - should-run-gate + - resolve-inputs + steps: + - uses: actions/checkout@v3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Download SMP binary + run: | + aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + + - name: Download submission metadata + uses: actions/download-artifact@v3 + with: + name: vector-submission-metadata + path: ${{ runner.temp }}/ + + - name: Determine SMP job result + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \ + --submission-metadata ${{ runner.temp }}/submission-metadata + + - name: Check status, cancelled + if: ${{ cancelled() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Analyze experimental results from Regression Detection Suite cancelled.' \ + -f context='Regression Detection Suite / detect-regression' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, success + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='success' \ + -f description='Analyze experimental results from Regression Detection Suite succeeded.' \ + -f context='Regression Detection Suite / detect-regression' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Analyze experimental results from Regression Detection Suite failed.' \ + -f context='Regression Detection Suite / detect-regression' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + analyze-experiment: + name: Download regression analysis & upload report + runs-on: ubuntu-22.04 + timeout-minutes: 5 + needs: + - should-run-gate + - submit-job + - resolve-inputs + steps: + - name: Check status, in-progress + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='pending' \ + -f description='Analyze experimental results from Regression Detection Suite.' \ + -f context='Regression Detection Suite / analyze-experiment' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - uses: actions/checkout@v3 + with: + ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Download SMP binary + run: | + aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + + - name: Download submission metadata + uses: actions/download-artifact@v3 + with: + name: vector-submission-metadata + path: ${{ runner.temp }}/ + + - name: Sync regression report to local system + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \ + --submission-metadata ${{ runner.temp }}/submission-metadata \ + --output-path "${{ runner.temp }}/outputs" + + - name: Read regression report + id: read-analysis + uses: juliangruber/read-file-action@v1 + with: + path: ${{ runner.temp }}/outputs/report.md + + - name: Upload regression report to artifacts + uses: actions/upload-artifact@v3 + with: + name: capture-artifacts + path: ${{ runner.temp }}/outputs/* + + - name: Check status, cancelled + if: ${{ cancelled() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Analyze experimental results from Regression Detection Suite cancelled.' \ + -f context='Regression Detection Suite / analyze-experiment' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, success + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='success' \ + -f description='Analyze experimental results from Regression Detection Suite succeeded.' \ + -f context='Regression Detection Suite / analyze-experiment' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Analyze experimental results from Regression Detection Suite failed.' \ + -f context='Regression Detection Suite / analyze-experiment' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + regression-detection-suite: + name: Set final result + runs-on: ubuntu-latest + timeout-minutes: 5 + if: always() + needs: + - should-run-gate + - resolve-inputs + - build-baseline + - build-comparison + - confirm-valid-credentials + - upload-baseline-image-to-ecr + - upload-comparison-image-to-ecr + - submit-job + - detect-regression + - analyze-experiment + env: + FAILED: ${{ contains(needs.*.result, 'failure') }} + steps: + - name: exit + run: | + echo "failed=${{ env.FAILED }}" + if [[ "$FAILED" == "true" ]] ; then + exit 1 + else + exit 0 + fi