From b7788e812694e2f22cc7869d67d1793891e0721e Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Tue, 4 Jun 2024 14:00:06 -0500 Subject: [PATCH 01/61] mobile: Android Bazel build rules cleanup (#34521) This PR renames the kotlin_*.bzl to envoy_mobile_android_*.bzl since they are not always specific to Kotlin. Risk Level: low Testing: CI Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a Signed-off-by: Fredy Wijaya --- .../{kotlin_lib.bzl => envoy_mobile_android_jni.bzl} | 0 ...{kotlin_test.bzl => envoy_mobile_android_test.bzl} | 11 +---------- mobile/test/java/integration/BUILD | 2 +- .../test/java/io/envoyproxy/envoymobile/engine/BUILD | 2 +- .../io/envoyproxy/envoymobile/engine/testing/BUILD | 2 +- mobile/test/java/io/envoyproxy/envoymobile/jni/BUILD | 2 +- .../java/io/envoyproxy/envoymobile/utilities/BUILD | 2 +- mobile/test/java/org/chromium/net/BUILD | 2 +- mobile/test/java/org/chromium/net/impl/BUILD | 2 +- mobile/test/java/org/chromium/net/testing/BUILD | 2 +- mobile/test/java/org/chromium/net/urlconnection/BUILD | 2 +- mobile/test/jni/BUILD | 2 +- mobile/test/kotlin/integration/BUILD | 2 +- mobile/test/kotlin/integration/proxying/BUILD | 2 +- mobile/test/kotlin/io/envoyproxy/envoymobile/BUILD | 2 +- .../test/kotlin/io/envoyproxy/envoymobile/stats/BUILD | 2 +- 16 files changed, 15 insertions(+), 24 deletions(-) rename mobile/bazel/{kotlin_lib.bzl => envoy_mobile_android_jni.bzl} (100%) rename mobile/bazel/{kotlin_test.bzl => envoy_mobile_android_test.bzl} (89%) diff --git a/mobile/bazel/kotlin_lib.bzl b/mobile/bazel/envoy_mobile_android_jni.bzl similarity index 100% rename from mobile/bazel/kotlin_lib.bzl rename to mobile/bazel/envoy_mobile_android_jni.bzl diff --git a/mobile/bazel/kotlin_test.bzl b/mobile/bazel/envoy_mobile_android_test.bzl similarity index 89% rename from mobile/bazel/kotlin_test.bzl rename to mobile/bazel/envoy_mobile_android_test.bzl index 808fdef89f8b..b847c7ad430e 100644 --- a/mobile/bazel/kotlin_test.bzl +++ b/mobile/bazel/envoy_mobile_android_test.bzl @@ -1,6 +1,6 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library", "android_local_test") load("@io_bazel_rules_kotlin//kotlin:android.bzl", "kt_android_local_test") -load("//bazel:kotlin_lib.bzl", "native_lib_name") +load("//bazel:envoy_mobile_android_jni.bzl", "native_lib_name") # A simple macro to define the JVM flags. def jvm_flags(lib_name): @@ -27,15 +27,6 @@ def _contains_all(srcs, extension): # A basic macro to run android based (robolectric) tests with native dependencies def envoy_mobile_android_test(name, srcs, native_lib_name = "", deps = [], native_deps = [], repository = "", exec_properties = {}): - android_library( - name = name + "_test_lib", - custom_package = "io.envoyproxy.envoymobile.test", - manifest = repository + "//bazel:test_manifest.xml", - visibility = ["//visibility:public"], - data = native_deps, - exports = deps, - testonly = True, - ) dependencies = deps + [ repository + "//bazel:envoy_mobile_test_suite", "@maven//:androidx_annotation_annotation", diff --git a/mobile/test/java/integration/BUILD b/mobile/test/java/integration/BUILD index b94e05a160ac..a45c135d28a9 100644 --- a/mobile/test/java/integration/BUILD +++ b/mobile/test/java/integration/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/BUILD b/mobile/test/java/io/envoyproxy/envoymobile/engine/BUILD index bd1e14e503ae..e441b7a9e42d 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/BUILD +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD index a58af01812d3..367f4261dc73 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD @@ -1,6 +1,6 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/java/io/envoyproxy/envoymobile/jni/BUILD b/mobile/test/java/io/envoyproxy/envoymobile/jni/BUILD index 00d488778572..5e236b4fd137 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/jni/BUILD +++ b/mobile/test/java/io/envoyproxy/envoymobile/jni/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/java/io/envoyproxy/envoymobile/utilities/BUILD b/mobile/test/java/io/envoyproxy/envoymobile/utilities/BUILD index de8657c4d0da..4e7fe7e835d5 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/utilities/BUILD +++ b/mobile/test/java/io/envoyproxy/envoymobile/utilities/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/java/org/chromium/net/BUILD b/mobile/test/java/org/chromium/net/BUILD index 19b6d3e91d64..f22d692ee86c 100644 --- a/mobile/test/java/org/chromium/net/BUILD +++ b/mobile/test/java/org/chromium/net/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/java/org/chromium/net/impl/BUILD b/mobile/test/java/org/chromium/net/impl/BUILD index 8bde99407885..9605525ebffd 100644 --- a/mobile/test/java/org/chromium/net/impl/BUILD +++ b/mobile/test/java/org/chromium/net/impl/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/java/org/chromium/net/testing/BUILD b/mobile/test/java/org/chromium/net/testing/BUILD index d086f967686e..ff20d29a8295 100644 --- a/mobile/test/java/org/chromium/net/testing/BUILD +++ b/mobile/test/java/org/chromium/net/testing/BUILD @@ -1,6 +1,6 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/java/org/chromium/net/urlconnection/BUILD b/mobile/test/java/org/chromium/net/urlconnection/BUILD index c5424f48828a..2872d85568cd 100644 --- a/mobile/test/java/org/chromium/net/urlconnection/BUILD +++ b/mobile/test/java/org/chromium/net/urlconnection/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/jni/BUILD b/mobile/test/jni/BUILD index 31b628a3230e..22afd615eef8 100644 --- a/mobile/test/jni/BUILD +++ b/mobile/test/jni/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("//bazel:kotlin_lib.bzl", "envoy_mobile_so_to_jni_lib") +load("//bazel:envoy_mobile_android_jni.bzl", "envoy_mobile_so_to_jni_lib") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/kotlin/integration/BUILD b/mobile/test/kotlin/integration/BUILD index 9fb1c3f6e28b..a648eb74c43e 100644 --- a/mobile/test/kotlin/integration/BUILD +++ b/mobile/test/kotlin/integration/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") load("@io_bazel_rules_kotlin//kotlin:android.bzl", "kt_android_library") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/kotlin/integration/proxying/BUILD b/mobile/test/kotlin/integration/proxying/BUILD index 492baa548d90..ef4c24c53327 100644 --- a/mobile/test/kotlin/integration/proxying/BUILD +++ b/mobile/test/kotlin/integration/proxying/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/BUILD b/mobile/test/kotlin/io/envoyproxy/envoymobile/BUILD index 48853e4413c8..bd3c32617858 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/BUILD +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 diff --git a/mobile/test/kotlin/io/envoyproxy/envoymobile/stats/BUILD b/mobile/test/kotlin/io/envoyproxy/envoymobile/stats/BUILD index 267a6a3a5c04..1f46f947bb59 100644 --- a/mobile/test/kotlin/io/envoyproxy/envoymobile/stats/BUILD +++ b/mobile/test/kotlin/io/envoyproxy/envoymobile/stats/BUILD @@ -1,5 +1,5 @@ load("@envoy//bazel:envoy_build_system.bzl", "envoy_mobile_package") -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test") +load("@envoy_mobile//bazel:envoy_mobile_android_test.bzl", "envoy_mobile_android_test") licenses(["notice"]) # Apache 2 From 5603ca2a0b299070eb7f9aaa129e61134bdf27d2 Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Tue, 4 Jun 2024 16:26:45 -0400 Subject: [PATCH 02/61] xds-failover: adding support for legacy SotW and delta (#34437) This is a breakdown of #34417 to multiple PRs. In this PR we introduce the envoy.restart_features.xds_failover_support runtime flag which defaults to false. We also modify the legacy GrpcMux SotW and Delta implementations to use the GrpcMuxFailover object instead of the GrpcStream object when the runtime flag is set to true. Next is updating the Unified Sotw/Delta GrpcMux implementation, and plumbing the API through to use the xDS-Failover. Risk Level: medium if the runtime flag is set to true. Testing: Updated the test cases (parameterized). Signed-off-by: Adi Suissa-Peleg --- envoy/config/subscription_factory.h | 5 +- source/common/runtime/runtime_features.cc | 2 + .../common/upstream/cluster_manager_impl.cc | 12 +-- .../extensions/config_subscription/grpc/BUILD | 2 + .../grpc_collection_subscription_factory.cc | 1 + .../grpc/grpc_mux_context.h | 1 + .../grpc/grpc_mux_failover.h | 27 ++++-- .../config_subscription/grpc/grpc_mux_impl.cc | 53 +++++++++--- .../config_subscription/grpc/grpc_mux_impl.h | 30 +++++-- .../grpc/grpc_subscription_factory.cc | 2 + .../grpc/new_grpc_mux_impl.cc | 52 +++++++++--- .../grpc/new_grpc_mux_impl.h | 30 +++++-- .../grpc/xds_mux/grpc_mux_impl.cc | 10 ++- .../config/grpc_subscription_test_harness.h | 1 + .../upstream/cluster_manager_impl_test.cc | 10 +-- .../extensions/clusters/eds/eds_speed_test.cc | 1 + .../grpc/delta_subscription_impl_test.cc | 1 + .../grpc/delta_subscription_test_harness.h | 1 + .../grpc/grpc_mux_impl_test.cc | 83 +++++++++++-------- .../grpc/new_grpc_mux_impl_test.cc | 24 ++++-- .../grpc/xds_grpc_mux_impl_test.cc | 16 ++-- 21 files changed, 258 insertions(+), 106 deletions(-) diff --git a/envoy/config/subscription_factory.h b/envoy/config/subscription_factory.h index de9d02100ef3..1471e5f1752a 100644 --- a/envoy/config/subscription_factory.h +++ b/envoy/config/subscription_factory.h @@ -115,8 +115,9 @@ class MuxFactory : public Config::UntypedFactory { std::string category() const override { return "envoy.config_mux"; } virtual void shutdownAll() PURE; virtual std::shared_ptr - create(std::unique_ptr&& async_client, Event::Dispatcher& dispatcher, - Random::RandomGenerator& random, Stats::Scope& scope, + create(std::unique_ptr&& async_client, + std::unique_ptr&& async_failover_client, + Event::Dispatcher& dispatcher, Random::RandomGenerator& random, Stats::Scope& scope, const envoy::config::core::v3::ApiConfigSource& ads_config, const LocalInfo::LocalInfo& local_info, std::unique_ptr&& config_validators, diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 8d3a4e7065d4..91ecb569ee5f 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -159,6 +159,8 @@ FALSE_RUNTIME_GUARD(envoy_reloadable_features_prefer_quic_client_udp_gro); FALSE_RUNTIME_GUARD(envoy_reloadable_features_reresolve_null_addresses); // TODO(alyssar) evaluate and either make this a config knob or remove. FALSE_RUNTIME_GUARD(envoy_reloadable_features_reresolve_if_no_connections); +// TODO(adisuissa): flip to true after this is out of alpha mode. +FALSE_RUNTIME_GUARD(envoy_restart_features_xds_failover_support); // A flag to set the maximum TLS version for google_grpc client to TLS1.2, when needed for // compliance restrictions. diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 28c7fb53e3e5..64de15c7c196 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -450,11 +450,11 @@ ClusterManagerImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bo auto factory_or_error = Config::Utility::factoryForGrpcApiConfigSource( *async_client_manager_, dyn_resources.ads_config(), *stats_.rootScope(), false); THROW_IF_STATUS_NOT_OK(factory_or_error, throw); - ads_mux_ = - factory->create(factory_or_error.value()->createUncachedRawAsyncClient(), dispatcher_, - random_, *stats_.rootScope(), dyn_resources.ads_config(), local_info_, - std::move(custom_config_validators), std::move(backoff_strategy), - makeOptRefFromPtr(xds_config_tracker_.get()), {}, use_eds_cache); + ads_mux_ = factory->create(factory_or_error.value()->createUncachedRawAsyncClient(), nullptr, + dispatcher_, random_, *stats_.rootScope(), + dyn_resources.ads_config(), local_info_, + std::move(custom_config_validators), std::move(backoff_strategy), + makeOptRefFromPtr(xds_config_tracker_.get()), {}, use_eds_cache); } else { absl::Status status = Config::Utility::checkTransportVersion(dyn_resources.ads_config()); RETURN_IF_NOT_OK(status); @@ -474,7 +474,7 @@ ClusterManagerImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bo *async_client_manager_, dyn_resources.ads_config(), *stats_.rootScope(), false); THROW_IF_STATUS_NOT_OK(factory_or_error, throw); ads_mux_ = factory->create( - factory_or_error.value()->createUncachedRawAsyncClient(), dispatcher_, random_, + factory_or_error.value()->createUncachedRawAsyncClient(), nullptr, dispatcher_, random_, *stats_.rootScope(), dyn_resources.ads_config(), local_info_, std::move(custom_config_validators), std::move(backoff_strategy), makeOptRefFromPtr(xds_config_tracker_.get()), xds_delegate_opt_ref, use_eds_cache); diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD index 581be4213417..9114b4a5213f 100644 --- a/source/extensions/config_subscription/grpc/BUILD +++ b/source/extensions/config_subscription/grpc/BUILD @@ -29,6 +29,7 @@ envoy_cc_extension( deps = [ ":eds_resources_cache_lib", ":grpc_mux_context_lib", + ":grpc_mux_failover_lib", ":grpc_stream_lib", ":xds_source_id_lib", "//envoy/config:custom_config_validators_interface", @@ -62,6 +63,7 @@ envoy_cc_library( ":delta_subscription_state_lib", ":eds_resources_cache_lib", ":grpc_mux_context_lib", + ":grpc_mux_failover_lib", ":grpc_stream_lib", ":pausable_ack_queue_lib", ":watch_map_lib", diff --git a/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc index a356e7a8f6c3..5b8433fd989c 100644 --- a/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc +++ b/source/extensions/config_subscription/grpc/grpc_collection_subscription_factory.cc @@ -31,6 +31,7 @@ SubscriptionPtr DeltaGrpcCollectionConfigSubscriptionFactory::create( THROW_IF_STATUS_NOT_OK(rate_limit_settings_or_error, throw); GrpcMuxContext grpc_mux_context{ factory_or_error.value()->createUncachedRawAsyncClient(), + /*failover_async_client_*/ nullptr, /*dispatcher_=*/data.dispatcher_, /*service_method_=*/deltaGrpcMethod(data.type_url_), /*local_info_=*/data.local_info_, diff --git a/source/extensions/config_subscription/grpc/grpc_mux_context.h b/source/extensions/config_subscription/grpc/grpc_mux_context.h index 254cb18b9534..03f7e4c7d119 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_context.h +++ b/source/extensions/config_subscription/grpc/grpc_mux_context.h @@ -19,6 +19,7 @@ namespace Config { // These are parameters needed for the creation of all GrpcMux objects. struct GrpcMuxContext { Grpc::RawAsyncClientPtr async_client_; + Grpc::RawAsyncClientPtr failover_async_client_; Event::Dispatcher& dispatcher_; const Protobuf::MethodDescriptor& service_method_; const LocalInfo::LocalInfo& local_info_; diff --git a/source/extensions/config_subscription/grpc/grpc_mux_failover.h b/source/extensions/config_subscription/grpc/grpc_mux_failover.h index 7ab3e944f084..60cba40cbd18 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_failover.h +++ b/source/extensions/config_subscription/grpc/grpc_mux_failover.h @@ -47,7 +47,8 @@ namespace Config { * defined, and may be converted to a config field in the future. */ template -class GrpcMuxFailover : public Logger::Loggable { +class GrpcMuxFailover : public GrpcStreamInterface, + public Logger::Loggable { public: // A GrpcStream creator function that receives the stream callbacks and returns a // GrpcStream object. This is introduced to facilitate dependency injection for @@ -79,7 +80,7 @@ class GrpcMuxFailover : public Logger::Loggable { virtual ~GrpcMuxFailover() = default; // Attempts to establish a new stream to the either the primary or failover source. - void establishNewStream() { + void establishNewStream() override { // Attempt establishing a connection to the primary source. // This method may be called multiple times, even if the primary stream is already // established or in the process of being established. @@ -111,7 +112,7 @@ class GrpcMuxFailover : public Logger::Loggable { } // Returns the availability of the underlying stream. - bool grpcStreamAvailable() const { + bool grpcStreamAvailable() const override { if (connectingToOrConnectedToFailover()) { return failover_grpc_stream_->grpcStreamAvailable(); } @@ -120,7 +121,7 @@ class GrpcMuxFailover : public Logger::Loggable { } // Sends a message using the underlying stream. - void sendMessage(const RequestType& request) { + void sendMessage(const RequestType& request) override { if (connectingToOrConnectedToFailover()) { failover_grpc_stream_->sendMessage(request); return; @@ -130,7 +131,7 @@ class GrpcMuxFailover : public Logger::Loggable { } // Updates the queue size of the underlying stream. - void maybeUpdateQueueSizeStat(uint64_t size) { + void maybeUpdateQueueSizeStat(uint64_t size) override { if (connectingToOrConnectedToFailover()) { failover_grpc_stream_->maybeUpdateQueueSizeStat(size); return; @@ -140,7 +141,7 @@ class GrpcMuxFailover : public Logger::Loggable { } // Returns true if the rate-limit allows draining. - bool checkRateLimitAllowsDrain() { + bool checkRateLimitAllowsDrain() override { if (connectingToOrConnectedToFailover()) { return failover_grpc_stream_->checkRateLimitAllowsDrain(); } @@ -355,6 +356,20 @@ class GrpcMuxFailover : public Logger::Loggable { } } + // The following method overrides are to allow GrpcMuxFailover to extend the + // GrpcStreamInterface. Once envoy.restart_features.xds_failover_support is deprecated, + // the class will no longer need to extend the interface, and these can be removed. + void onCreateInitialMetadata(Http::RequestHeaderMap&) override { PANIC("not implemented"); } + void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&&) override { PANIC("not implemented"); } + void onReceiveMessage(std::unique_ptr&&) override { PANIC("not implemented"); } + void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override { + PANIC("not implemented"); + } + void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override { + PANIC("not implemented"); + } + void closeStream() override { PANIC("not implemented"); } + // The stream callbacks that will be invoked on the GrpcMux object, to notify // about the state of the underlying primary/failover stream. GrpcStreamCallbacks& grpc_mux_callbacks_; diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc index 5d7faa7012d9..8091835caa6e 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc @@ -60,10 +60,7 @@ std::string convertToWildcard(const std::string& resource_name) { } // namespace GrpcMuxImpl::GrpcMuxImpl(GrpcMuxContext& grpc_mux_context, bool skip_subsequent_node) - : grpc_stream_(this, std::move(grpc_mux_context.async_client_), - grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, - grpc_mux_context.scope_, std::move(grpc_mux_context.backoff_strategy_), - grpc_mux_context.rate_limit_settings_), + : grpc_stream_(createGrpcStreamObject(grpc_mux_context)), local_info_(grpc_mux_context.local_info_), skip_subsequent_node_(skip_subsequent_node), config_validators_(std::move(grpc_mux_context.config_validators_)), xds_config_tracker_(grpc_mux_context.xds_config_tracker_), @@ -81,6 +78,37 @@ GrpcMuxImpl::GrpcMuxImpl(GrpcMuxContext& grpc_mux_context, bool skip_subsequent_ AllMuxes::get().insert(this); } +std::unique_ptr> +GrpcMuxImpl::createGrpcStreamObject(GrpcMuxContext& grpc_mux_context) { + if (Runtime::runtimeFeatureEnabled("envoy.restart_features.xds_failover_support")) { + return std::make_unique>( + /*primary_stream_creator=*/ + [&grpc_mux_context]( + GrpcStreamCallbacks* callbacks) + -> GrpcStreamInterfacePtr { + return std::make_unique>( + callbacks, std::move(grpc_mux_context.async_client_), + grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, + grpc_mux_context.scope_, std::move(grpc_mux_context.backoff_strategy_), + grpc_mux_context.rate_limit_settings_); + }, + /*failover_stream_creator=*/ + // TODO(adisuissa): implement when failover is fully plumbed. + absl::nullopt, + /*grpc_mux_callbacks=*/*this, + /*dispatch=*/grpc_mux_context.dispatcher_); + } + return std::make_unique>( + this, std::move(grpc_mux_context.async_client_), grpc_mux_context.service_method_, + grpc_mux_context.dispatcher_, grpc_mux_context.scope_, + std::move(grpc_mux_context.backoff_strategy_), grpc_mux_context.rate_limit_settings_); +} + GrpcMuxImpl::~GrpcMuxImpl() { AllMuxes::get().erase(this); } void GrpcMuxImpl::shutdownAll() { AllMuxes::get().shutdownAll(); } @@ -100,7 +128,7 @@ void GrpcMuxImpl::start() { return; } started_ = true; - grpc_stream_.establishNewStream(); + grpc_stream_->establishNewStream(); } void GrpcMuxImpl::sendDiscoveryRequest(absl::string_view type_url) { @@ -131,7 +159,7 @@ void GrpcMuxImpl::sendDiscoveryRequest(absl::string_view type_url) { request.clear_node(); } ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url, request.ShortDebugString()); - grpc_stream_.sendMessage(request); + grpc_stream_->sendMessage(request); first_stream_request_ = false; // clear error_detail after the request is sent if it exists. @@ -470,7 +498,7 @@ void GrpcMuxImpl::onWriteable() { drainRequests(); } void GrpcMuxImpl::onStreamEstablished() { first_stream_request_ = true; - grpc_stream_.maybeUpdateQueueSizeStat(0); + grpc_stream_->maybeUpdateQueueSizeStat(0); clearNonce(); request_queue_ = std::make_unique>(); for (const auto& type_url : subscriptions_) { @@ -498,7 +526,7 @@ void GrpcMuxImpl::onEstablishmentFailure() { } void GrpcMuxImpl::queueDiscoveryRequest(absl::string_view queue_item) { - if (!grpc_stream_.grpcStreamAvailable()) { + if (!grpc_stream_->grpcStreamAvailable()) { ENVOY_LOG(debug, "No stream available to queueDiscoveryRequest for {}", queue_item); return; // Drop this request; the reconnect will enqueue a new one. } @@ -551,12 +579,12 @@ GrpcMuxImpl::ApiState& GrpcMuxImpl::apiStateFor(absl::string_view type_url) { } void GrpcMuxImpl::drainRequests() { - while (!request_queue_->empty() && grpc_stream_.checkRateLimitAllowsDrain()) { + while (!request_queue_->empty() && grpc_stream_->checkRateLimitAllowsDrain()) { // Process the request, if rate limiting is not enabled at all or if it is under rate limit. sendDiscoveryRequest(request_queue_->front()); request_queue_->pop(); } - grpc_stream_.maybeUpdateQueueSizeStat(request_queue_->size()); + grpc_stream_->maybeUpdateQueueSizeStat(request_queue_->size()); } // A factory class for creating GrpcMuxImpl so it does not have to be @@ -566,8 +594,8 @@ class GrpcMuxFactory : public MuxFactory { std::string name() const override { return "envoy.config_mux.grpc_mux_factory"; } void shutdownAll() override { return GrpcMuxImpl::shutdownAll(); } std::shared_ptr - create(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - Random::RandomGenerator&, Stats::Scope& scope, + create(Grpc::RawAsyncClientPtr&& async_client, Grpc::RawAsyncClientPtr&& failover_async_client, + Event::Dispatcher& dispatcher, Random::RandomGenerator&, Stats::Scope& scope, const envoy::config::core::v3::ApiConfigSource& ads_config, const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, BackOffStrategyPtr&& backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, @@ -577,6 +605,7 @@ class GrpcMuxFactory : public MuxFactory { THROW_IF_STATUS_NOT_OK(rate_limit_settings_or_error, throw); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::move(async_client), + /*failover_async_client_=*/std::move(failover_async_client), /*dispatcher_=*/dispatcher, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/grpc_mux_impl.h index f2a5b1fb3f9f..8766ce96935a 100644 --- a/source/extensions/config_subscription/grpc/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.h @@ -27,7 +27,7 @@ #include "source/common/config/xds_context_params.h" #include "source/common/config/xds_resource.h" #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" -#include "source/extensions/config_subscription/grpc/grpc_stream.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" #include "absl/container/node_hash_map.h" #include "xds/core/v3/resource_name.pb.h" @@ -84,13 +84,28 @@ class GrpcMuxImpl : public GrpcMux, ControlPlaneStats& control_plane_stats) override; void onWriteable() override; - GrpcStream& + GrpcStreamInterface& grpcStreamForTest() { - return grpc_stream_; + // TODO(adisuissa): Once envoy.restart_features.xds_failover_support is deprecated, + // return grpc_stream_.currentStreamForTest() directly (defined in GrpcMuxFailover). + if (Runtime::runtimeFeatureEnabled("envoy.restart_features.xds_failover_support")) { + return dynamic_cast*>( + grpc_stream_.get()) + ->currentStreamForTest(); + } + return *grpc_stream_.get(); } private: + // Helper function to create the grpc_stream_ object. + // TODO(adisuissa): this should be removed when envoy.restart_features.xds_failover_support + // is deprecated. + std::unique_ptr> + createGrpcStreamObject(GrpcMuxContext& grpc_mux_context); + void drainRequests(); void setRetryTimer(); void sendDiscoveryRequest(absl::string_view type_url); @@ -257,8 +272,11 @@ class GrpcMuxImpl : public GrpcMux, ApiState& api_state, const std::string& type_url, const std::string& version_info, bool call_delegate); - GrpcStream + // Multiplexes the stream to the primary and failover sources. + // TODO(adisuissa): Once envoy.restart_features.xds_failover_support is deprecated, + // convert from unique_ptr to GrpcMuxFailover directly. + std::unique_ptr> grpc_stream_; const LocalInfo::LocalInfo& local_info_; const bool skip_subsequent_node_; diff --git a/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc b/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc index 20efc1d418b4..e3792b49b42f 100644 --- a/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc +++ b/source/extensions/config_subscription/grpc/grpc_subscription_factory.cc @@ -34,6 +34,7 @@ GrpcConfigSubscriptionFactory::create(ConfigSubscriptionFactory::SubscriptionDat THROW_IF_STATUS_NOT_OK(rate_limit_settings_or_error, throw); GrpcMuxContext grpc_mux_context{ /*async_client_=*/factory_or_error.value()->createUncachedRawAsyncClient(), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/data.dispatcher_, /*service_method_=*/sotwGrpcMethod(data.type_url_), /*local_info_=*/data.local_info_, @@ -82,6 +83,7 @@ DeltaGrpcConfigSubscriptionFactory::create(ConfigSubscriptionFactory::Subscripti THROW_IF_STATUS_NOT_OK(rate_limit_settings_or_error, throw); GrpcMuxContext grpc_mux_context{ /*async_client_=*/factory_or_error.value()->createUncachedRawAsyncClient(), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/data.dispatcher_, /*service_method_=*/deltaGrpcMethod(data.type_url_), /*local_info_=*/data.local_info_, diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc index 229a4865ed55..c06685f13af6 100644 --- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc @@ -36,10 +36,7 @@ using AllMuxes = ThreadSafeSingleton; } // namespace NewGrpcMuxImpl::NewGrpcMuxImpl(GrpcMuxContext& grpc_mux_context) - : grpc_stream_(this, std::move(grpc_mux_context.async_client_), - grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, - grpc_mux_context.scope_, std::move(grpc_mux_context.backoff_strategy_), - grpc_mux_context.rate_limit_settings_), + : grpc_stream_(createGrpcStreamObject(grpc_mux_context)), local_info_(grpc_mux_context.local_info_), config_validators_(std::move(grpc_mux_context.config_validators_)), dynamic_update_callback_handle_( @@ -54,6 +51,38 @@ NewGrpcMuxImpl::NewGrpcMuxImpl(GrpcMuxContext& grpc_mux_context) AllMuxes::get().insert(this); } +std::unique_ptr> +NewGrpcMuxImpl::createGrpcStreamObject(GrpcMuxContext& grpc_mux_context) { + if (Runtime::runtimeFeatureEnabled("envoy.restart_features.xds_failover_support")) { + return std::make_unique>( + /*primary_stream_creator=*/ + [&grpc_mux_context]( + GrpcStreamCallbacks* callbacks) + -> GrpcStreamInterfacePtr { + return std::make_unique< + GrpcStream>( + callbacks, std::move(grpc_mux_context.async_client_), + grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, + grpc_mux_context.scope_, std::move(grpc_mux_context.backoff_strategy_), + grpc_mux_context.rate_limit_settings_); + }, + /*failover_stream_creator=*/ + // TODO(adisuissa): implement when failover is fully plumbed. + absl::nullopt, + /*grpc_mux_callbacks=*/*this, + /*dispatch=*/grpc_mux_context.dispatcher_); + } + return std::make_unique>( + this, std::move(grpc_mux_context.async_client_), grpc_mux_context.service_method_, + grpc_mux_context.dispatcher_, grpc_mux_context.scope_, + std::move(grpc_mux_context.backoff_strategy_), grpc_mux_context.rate_limit_settings_); +} + NewGrpcMuxImpl::~NewGrpcMuxImpl() { AllMuxes::get().erase(this); } void NewGrpcMuxImpl::shutdownAll() { AllMuxes::get().shutdownAll(); } @@ -165,7 +194,7 @@ void NewGrpcMuxImpl::start() { return; } started_ = true; - grpc_stream_.establishNewStream(); + grpc_stream_->establishNewStream(); } GrpcMuxWatchPtr NewGrpcMuxImpl::addWatch(const std::string& type_url, @@ -297,9 +326,9 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { } else { request = sub->second->sub_state_.getNextRequestAckless(); } - grpc_stream_.sendMessage(request); + grpc_stream_->sendMessage(request); } - grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); + grpc_stream_->maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); } // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check @@ -311,10 +340,10 @@ bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", type_url)); - if (!grpc_stream_.grpcStreamAvailable()) { + if (!grpc_stream_->grpcStreamAvailable()) { ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); return false; - } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { + } else if (!grpc_stream_->checkRateLimitAllowsDrain()) { ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); return false; } @@ -352,8 +381,8 @@ class NewGrpcMuxFactory : public MuxFactory { std::string name() const override { return "envoy.config_mux.new_grpc_mux_factory"; } void shutdownAll() override { return NewGrpcMuxImpl::shutdownAll(); } std::shared_ptr - create(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - Random::RandomGenerator&, Stats::Scope& scope, + create(Grpc::RawAsyncClientPtr&& async_client, Grpc::RawAsyncClientPtr&& failover_async_client, + Event::Dispatcher& dispatcher, Random::RandomGenerator&, Stats::Scope& scope, const envoy::config::core::v3::ApiConfigSource& ads_config, const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, BackOffStrategyPtr&& backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, @@ -363,6 +392,7 @@ class NewGrpcMuxFactory : public MuxFactory { THROW_IF_STATUS_NOT_OK(rate_limit_settings_or_error, throw); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::move(async_client), + /*failover_async_client_=*/std::move(failover_async_client), /*dispatcher_=*/dispatcher, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h index 788879136560..26a949ae4dbb 100644 --- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h @@ -17,7 +17,7 @@ #include "source/common/runtime/runtime_features.h" #include "source/extensions/config_subscription/grpc/delta_subscription_state.h" #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" -#include "source/extensions/config_subscription/grpc/grpc_stream.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" #include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" #include "source/extensions/config_subscription/grpc/watch_map.h" @@ -78,10 +78,18 @@ class NewGrpcMuxImpl // TODO(fredlas) remove this from the GrpcMux interface. void start() override; - GrpcStream& + GrpcStreamInterface& grpcStreamForTest() { - return grpc_stream_; + // TODO(adisuissa): Once envoy.restart_features.xds_failover_support is deprecated, + // return grpc_stream_.currentStreamForTest() directly (defined in GrpcMuxFailover). + if (Runtime::runtimeFeatureEnabled("envoy.restart_features.xds_failover_support")) { + return dynamic_cast*>( + grpc_stream_.get()) + ->currentStreamForTest(); + } + return *grpc_stream_.get(); } struct SubscriptionStuff { @@ -140,6 +148,13 @@ class NewGrpcMuxImpl const SubscriptionOptions options_; }; + // Helper function to create the grpc_stream_ object. + // TODO(adisuissa): this should be removed when envoy.restart_features.xds_failover_support + // is deprecated. + std::unique_ptr> + createGrpcStreamObject(GrpcMuxContext& grpc_mux_context); + void removeWatch(const std::string& type_url, Watch* watch); // Updates the list of resource names watched by the given watch. If an added name is new across @@ -181,8 +196,11 @@ class NewGrpcMuxImpl // the order of Envoy's dependency ordering). std::list subscription_ordering_; - GrpcStream + // Multiplexes the stream to the primary and failover sources. + // TODO(adisuissa): Once envoy.restart_features.xds_failover_support is deprecated, + // convert from unique_ptr to GrpcMuxFailover directly. + std::unique_ptr> grpc_stream_; const LocalInfo::LocalInfo& local_info_; diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc index 7de99ac446fd..544a813e7a26 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc @@ -410,8 +410,8 @@ class DeltaGrpcMuxFactory : public MuxFactory { std::string name() const override { return "envoy.config_mux.delta_grpc_mux_factory"; } void shutdownAll() override { return GrpcMuxDelta::shutdownAll(); } std::shared_ptr - create(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - Random::RandomGenerator&, Stats::Scope& scope, + create(Grpc::RawAsyncClientPtr&& async_client, Grpc::RawAsyncClientPtr&& failover_async_client, + Event::Dispatcher& dispatcher, Random::RandomGenerator&, Stats::Scope& scope, const envoy::config::core::v3::ApiConfigSource& ads_config, const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, BackOffStrategyPtr&& backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, @@ -421,6 +421,7 @@ class DeltaGrpcMuxFactory : public MuxFactory { THROW_IF_STATUS_NOT_OK(rate_limit_settings_or_error, throw); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::move(async_client), + /*failover_async_client=*/std::move(failover_async_client), /*dispatcher_=*/dispatcher, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( @@ -448,8 +449,8 @@ class SotwGrpcMuxFactory : public MuxFactory { std::string name() const override { return "envoy.config_mux.sotw_grpc_mux_factory"; } void shutdownAll() override { return GrpcMuxSotw::shutdownAll(); } std::shared_ptr - create(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - Random::RandomGenerator&, Stats::Scope& scope, + create(Grpc::RawAsyncClientPtr&& async_client, Grpc::RawAsyncClientPtr&& failover_async_client, + Event::Dispatcher& dispatcher, Random::RandomGenerator&, Stats::Scope& scope, const envoy::config::core::v3::ApiConfigSource& ads_config, const LocalInfo::LocalInfo& local_info, CustomConfigValidatorsPtr&& config_validators, BackOffStrategyPtr&& backoff_strategy, XdsConfigTrackerOptRef xds_config_tracker, @@ -459,6 +460,7 @@ class SotwGrpcMuxFactory : public MuxFactory { THROW_IF_STATUS_NOT_OK(rate_limit_settings_or_error, throw); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::move(async_client), + /*failover_async_client_=*/std::move(failover_async_client), /*dispatcher_=*/dispatcher, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 6467981a8510..13c02090c0f2 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -61,6 +61,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/*method_descriptor_, /*local_info_=*/local_info_, diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 03e59f08245d..2b9206cbe056 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -144,11 +144,11 @@ class MockGrpcMuxFactory : public Config::MuxFactory { std::string name() const override { return "envoy.config_mux.grpc_mux_factory"; } void shutdownAll() override {} std::shared_ptr - create(std::unique_ptr&&, Event::Dispatcher&, Random::RandomGenerator&, - Stats::Scope&, const envoy::config::core::v3::ApiConfigSource&, - const LocalInfo::LocalInfo&, std::unique_ptr&&, - BackOffStrategyPtr&&, OptRef, - OptRef, bool) override { + create(std::unique_ptr&&, std::unique_ptr&&, + Event::Dispatcher&, Random::RandomGenerator&, Stats::Scope&, + const envoy::config::core::v3::ApiConfigSource&, const LocalInfo::LocalInfo&, + std::unique_ptr&&, BackOffStrategyPtr&&, + OptRef, OptRef, bool) override { return std::make_shared>(); } }; diff --git a/test/extensions/clusters/eds/eds_speed_test.cc b/test/extensions/clusters/eds/eds_speed_test.cc index d87d22a5bfaf..bfdd7d51173a 100644 --- a/test/extensions/clusters/eds/eds_speed_test.cc +++ b/test/extensions/clusters/eds/eds_speed_test.cc @@ -54,6 +54,7 @@ class EdsSpeedTest { Config::SubscriptionFactory::RetryMaxDelayMs, random_); Config::GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/server_context_.dispatcher_, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( diff --git a/test/extensions/config_subscription/grpc/delta_subscription_impl_test.cc b/test/extensions/config_subscription/grpc/delta_subscription_impl_test.cc index c74e53cd5b27..694f44cf0e44 100644 --- a/test/extensions/config_subscription/grpc/delta_subscription_impl_test.cc +++ b/test/extensions/config_subscription/grpc/delta_subscription_impl_test.cc @@ -156,6 +156,7 @@ TEST_P(DeltaSubscriptionNoGrpcStreamTest, NoGrpcStream) { GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher, /*service_method_=*/*method_descriptor, /*local_info_=*/local_info, diff --git a/test/extensions/config_subscription/grpc/delta_subscription_test_harness.h b/test/extensions/config_subscription/grpc/delta_subscription_test_harness.h index 95ae79c6f042..57196a1ff8c1 100644 --- a/test/extensions/config_subscription/grpc/delta_subscription_test_harness.h +++ b/test/extensions/config_subscription/grpc/delta_subscription_test_harness.h @@ -52,6 +52,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/*method_descriptor_, /*local_info_=*/local_info_, diff --git a/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc index 418eab7506be..93e5d13c5457 100644 --- a/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/grpc_mux_impl_test.cc @@ -27,6 +27,7 @@ #include "test/test_common/logging.h" #include "test/test_common/resources.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/test_time.h" #include "test/test_common/utility.h" @@ -49,7 +50,7 @@ namespace { // We test some mux specific stuff below, other unit test coverage for singleton use of GrpcMuxImpl // is provided in [grpc_]subscription_impl_test.cc. -class GrpcMuxImplTestBase : public testing::Test { +class GrpcMuxImplTestBase : public testing::TestWithParam { public: GrpcMuxImplTestBase() : async_client_(new Grpc::MockAsyncClient()), @@ -58,15 +59,19 @@ class GrpcMuxImplTestBase : public testing::Test { control_plane_connected_state_( stats_.gauge("control_plane.connected_state", Stats::Gauge::ImportMode::NeverImport)), control_plane_pending_requests_( - stats_.gauge("control_plane.pending_requests", Stats::Gauge::ImportMode::NeverImport)) - - {} + stats_.gauge("control_plane.pending_requests", Stats::Gauge::ImportMode::NeverImport)) { + // Once "envoy.restart_features.xds_failover_support" is deprecated, the + // test should no longer be parameterized. + scoped_runtime_.mergeValues( + {{"envoy.restart_features.xds_failover_support", GetParam() ? "true" : "false"}}); + } void setup() { setup(rate_limit_settings_); } void setup(const RateLimitSettings& custom_rate_limit_settings) { GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( @@ -111,6 +116,7 @@ class GrpcMuxImplTestBase : public testing::Test { EXPECT_CALL(async_stream_, sendMessageRaw_(Grpc::ProtoBufferEq(expected_request), false)); } + TestScopedRuntime scoped_runtime_; NiceMock dispatcher_; NiceMock random_; NiceMock local_info_; @@ -132,9 +138,11 @@ class GrpcMuxImplTest : public GrpcMuxImplTestBase { Event::SimulatedTimeSystem time_system_; }; +INSTANTIATE_TEST_SUITE_P(GrpcMuxImpl, GrpcMuxImplTest, ::testing::Bool()); + // Validate behavior when multiple type URL watches are maintained, watches are created/destroyed // (via RAII). -TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { +TEST_P(GrpcMuxImplTest, MultipleTypeUrlStreams) { setup(); InSequence s; auto foo_sub = grpc_mux_->addWatch("foo", {"x", "y"}, callbacks_, resource_decoder_, {}); @@ -154,7 +162,7 @@ TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { } // Validate behavior when dynamic context parameters are updated. -TEST_F(GrpcMuxImplTest, DynamicContextParameters) { +TEST_P(GrpcMuxImplTest, DynamicContextParameters) { setup(); InSequence s; auto foo_sub = grpc_mux_->addWatch("foo", {"x", "y"}, callbacks_, resource_decoder_, {}); @@ -179,7 +187,7 @@ TEST_F(GrpcMuxImplTest, DynamicContextParameters) { } // Validate behavior when xdstp naming is used. -TEST_F(GrpcMuxImplTest, Xdstp) { +TEST_P(GrpcMuxImplTest, Xdstp) { setup(); InSequence s; @@ -203,7 +211,7 @@ TEST_F(GrpcMuxImplTest, Xdstp) { } // Validate behavior when xdstp naming is used with context parameters in the URI. -TEST_F(GrpcMuxImplTest, XdstpWithContextParams) { +TEST_P(GrpcMuxImplTest, XdstpWithContextParams) { setup(); InSequence s; @@ -234,7 +242,7 @@ TEST_F(GrpcMuxImplTest, XdstpWithContextParams) { } // Validate behavior when multiple type URL watches are maintained and the stream is reset. -TEST_F(GrpcMuxImplTest, ResetStream) { +TEST_P(GrpcMuxImplTest, ResetStream) { InSequence s; auto* timer = new Event::MockTimer(&dispatcher_); @@ -278,7 +286,7 @@ TEST_F(GrpcMuxImplTest, ResetStream) { } // Validate cached nonces are cleared on reconnection. -TEST_F(GrpcMuxImplTest, ReconnectionResetsNonceAndAcks) { +TEST_P(GrpcMuxImplTest, ReconnectionResetsNonceAndAcks) { OpaqueResourceDecoderSharedPtr resource_decoder( std::make_shared>("cluster_name")); @@ -334,7 +342,7 @@ TEST_F(GrpcMuxImplTest, ReconnectionResetsNonceAndAcks) { } // Validate pause-resume behavior. -TEST_F(GrpcMuxImplTest, PauseResume) { +TEST_P(GrpcMuxImplTest, PauseResume) { setup(); InSequence s; GrpcMuxWatchPtr foo_sub; @@ -368,7 +376,7 @@ TEST_F(GrpcMuxImplTest, PauseResume) { } // Validate behavior when type URL mismatches occur. -TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { +TEST_P(GrpcMuxImplTest, TypeUrlMismatch) { setup(); auto invalid_response = std::make_unique(); @@ -406,7 +414,7 @@ TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { expectSendMessage("foo", {}, ""); } -TEST_F(GrpcMuxImplTest, RpcErrorMessageTruncated) { +TEST_P(GrpcMuxImplTest, RpcErrorMessageTruncated) { setup(); auto invalid_response = std::make_unique(); InSequence s; @@ -459,7 +467,7 @@ resourceWithTtl(std::chrono::milliseconds ttl, return resource; } // Validates the behavior when the TTL timer expires. -TEST_F(GrpcMuxImplTest, ResourceTTL) { +TEST_P(GrpcMuxImplTest, ResourceTTL) { setup(); time_system_.setSystemTime(std::chrono::seconds(0)); @@ -590,7 +598,7 @@ TEST_F(GrpcMuxImplTest, ResourceTTL) { } // Checks that the control plane identifier is logged -TEST_F(GrpcMuxImplTest, LogsControlPlaneIndentifier) { +TEST_P(GrpcMuxImplTest, LogsControlPlaneIndentifier) { setup(); std::string type_url = "foo"; auto foo_sub = grpc_mux_->addWatch(type_url, {}, callbacks_, resource_decoder_, {}); @@ -624,7 +632,7 @@ TEST_F(GrpcMuxImplTest, LogsControlPlaneIndentifier) { } // Validate behavior when watches has an unknown resource name. -TEST_F(GrpcMuxImplTest, WildcardWatch) { +TEST_P(GrpcMuxImplTest, WildcardWatch) { setup(); InSequence s; @@ -660,7 +668,7 @@ TEST_F(GrpcMuxImplTest, WildcardWatch) { } // Validate behavior when watches specify resources (potentially overlapping). -TEST_F(GrpcMuxImplTest, WatchDemux) { +TEST_P(GrpcMuxImplTest, WatchDemux) { setup(); InSequence s; OpaqueResourceDecoderSharedPtr resource_decoder( @@ -748,7 +756,7 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { } // Validate behavior when we have multiple watchers that send empty updates. -TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { +TEST_P(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { setup(); InSequence s; const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; @@ -771,7 +779,7 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { } // Validate behavior when we have Single Watcher that sends Empty updates. -TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { +TEST_P(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { setup(); const std::string& type_url = Config::TypeUrl::get().Cluster; NiceMock foo_callbacks; @@ -801,8 +809,11 @@ class GrpcMuxImplTestWithMockTimeSystem : public GrpcMuxImplTestBase { Event::DelegatingTestTimeSystem mock_time_system_; }; +INSTANTIATE_TEST_SUITE_P(GrpcMuxImplTestWithMockTimeSystem, GrpcMuxImplTestWithMockTimeSystem, + ::testing::Bool()); + // Verifies that rate limiting is not enforced with defaults. -TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { +TEST_P(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { auto ttl_timer = new Event::MockTimer(&dispatcher_); // Retry timer, @@ -840,7 +851,7 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { } // Verifies that default rate limiting is enforced with empty RateLimitSettings. -TEST_F(GrpcMuxImplTest, TooManyRequestsWithEmptyRateLimitSettings) { +TEST_P(GrpcMuxImplTest, TooManyRequestsWithEmptyRateLimitSettings) { // Validate that request drain timer is created. auto ttl_timer = new Event::MockTimer(&dispatcher_); @@ -897,7 +908,7 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithEmptyRateLimitSettings) { } // Verifies that rate limiting is enforced with custom RateLimitSettings. -TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { +TEST_P(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { // Validate that request drain timer is created. // TTL timer. @@ -950,7 +961,7 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { } // Verifies that a message with no resources is accepted. -TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { +TEST_P(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { setup(); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); @@ -986,7 +997,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { } // Verifies that a message with some resources is rejected when there are no watches. -TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { +TEST_P(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { setup(); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); @@ -1016,10 +1027,11 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response))); } -TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { +TEST_P(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { EXPECT_CALL(local_info_, clusterName()).WillOnce(ReturnRef(EMPTY_STRING)); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( @@ -1041,10 +1053,11 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { "--service-node and --service-cluster options."); } -TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { +TEST_P(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { EXPECT_CALL(local_info_, nodeName()).WillOnce(ReturnRef(EMPTY_STRING)); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( @@ -1067,20 +1080,20 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { } // Validates that the EDS cache getter returns the cache. -TEST_F(GrpcMuxImplTest, EdsResourcesCacheForEds) { +TEST_P(GrpcMuxImplTest, EdsResourcesCacheForEds) { eds_resources_cache_ = new NiceMock(); setup(); EXPECT_TRUE(grpc_mux_->edsResourcesCache().has_value()); } // Validates that the EDS cache getter returns empty if there is no cache. -TEST_F(GrpcMuxImplTest, EdsResourcesCacheForEdsNoCache) { +TEST_P(GrpcMuxImplTest, EdsResourcesCacheForEdsNoCache) { setup(); EXPECT_FALSE(grpc_mux_->edsResourcesCache().has_value()); } // Validate that an EDS resource is cached if there's a cache. -TEST_F(GrpcMuxImplTest, CacheEdsResource) { +TEST_P(GrpcMuxImplTest, CacheEdsResource) { // Create the cache that will also be passed to the GrpcMux object via setup(). eds_resources_cache_ = new NiceMock(); setup(); @@ -1122,7 +1135,7 @@ TEST_F(GrpcMuxImplTest, CacheEdsResource) { // Validate that an update to an EDS resource watcher is reflected in the cache, // if there's a cache. -TEST_F(GrpcMuxImplTest, UpdateCacheEdsResource) { +TEST_P(GrpcMuxImplTest, UpdateCacheEdsResource) { // Create the cache that will also be passed to the GrpcMux object via setup(). eds_resources_cache_ = new NiceMock(); setup(); @@ -1170,7 +1183,7 @@ TEST_F(GrpcMuxImplTest, UpdateCacheEdsResource) { // Validate that adding and removing watchers reflects on the cache changes, // if there's a cache. -TEST_F(GrpcMuxImplTest, AddRemoveSubscriptions) { +TEST_P(GrpcMuxImplTest, AddRemoveSubscriptions) { // Create the cache that will also be passed to the GrpcMux object via setup(). eds_resources_cache_ = new NiceMock(); setup(); @@ -1248,7 +1261,7 @@ TEST_F(GrpcMuxImplTest, AddRemoveSubscriptions) { // Validate that a cached resource is removed only after the last subscription // (watch) that needs it is removed. -TEST_F(GrpcMuxImplTest, RemoveCachedResourceOnLastSubscription) { +TEST_P(GrpcMuxImplTest, RemoveCachedResourceOnLastSubscription) { // Create the cache that will also be passed to the GrpcMux object via setup(). eds_resources_cache_ = new NiceMock(); setup(); @@ -1398,9 +1411,9 @@ TEST(GrpcMuxFactoryTest, InvalidRateLimit) { ads_config.mutable_rate_limit_settings()->mutable_max_tokens()->set_value(100); ads_config.mutable_rate_limit_settings()->mutable_fill_rate()->set_value( std::numeric_limits::quiet_NaN()); - EXPECT_THROW(factory->create(std::make_unique(), dispatcher, random, scope, - ads_config, local_info, nullptr, nullptr, absl::nullopt, - absl::nullopt, false), + EXPECT_THROW(factory->create(std::make_unique(), nullptr, dispatcher, + random, scope, ads_config, local_info, nullptr, nullptr, + absl::nullopt, absl::nullopt, false), EnvoyException); } diff --git a/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc index dec6a2542512..71f33b93118b 100644 --- a/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/new_grpc_mux_impl_test.cc @@ -28,6 +28,7 @@ #include "test/test_common/logging.h" #include "test/test_common/resources.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/test_time.h" #include "test/test_common/utility.h" @@ -50,7 +51,7 @@ enum class LegacyOrUnified { Legacy, Unified }; // We test some mux specific stuff below, other unit test coverage for singleton use of // NewGrpcMuxImpl is provided in [grpc_]subscription_impl_test.cc. -class NewGrpcMuxImplTestBase : public testing::TestWithParam { +class NewGrpcMuxImplTestBase : public testing::TestWithParam> { public: NewGrpcMuxImplTestBase(LegacyOrUnified legacy_or_unified) : async_client_(new Grpc::MockAsyncClient()), @@ -60,13 +61,19 @@ class NewGrpcMuxImplTestBase : public testing::TestWithParam { control_plane_stats_(Utility::generateControlPlaneStats(*stats_.rootScope())), control_plane_connected_state_( stats_.gauge("control_plane.connected_state", Stats::Gauge::ImportMode::NeverImport)), - should_use_unified_(legacy_or_unified == LegacyOrUnified::Unified) {} + should_use_unified_(legacy_or_unified == LegacyOrUnified::Unified) { + // Once "envoy.restart_features.xds_failover_support" is deprecated, the + // test should no longer be parameterized on the bool value. + scoped_runtime_.mergeValues({{"envoy.restart_features.xds_failover_support", + std::get<1>(GetParam()) ? "true" : "false"}}); + } void setup() { auto backoff_strategy = std::make_unique( SubscriptionFactory::RetryInitialDelayMs, SubscriptionFactory::RetryMaxDelayMs, random_); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( @@ -163,6 +170,7 @@ class NewGrpcMuxImplTestBase : public testing::TestWithParam { bool isUnifiedMuxTest() const { return should_use_unified_; } + TestScopedRuntime scoped_runtime_; NiceMock dispatcher_; NiceMock random_; Grpc::MockAsyncClient* async_client_; @@ -182,12 +190,14 @@ class NewGrpcMuxImplTestBase : public testing::TestWithParam { class NewGrpcMuxImplTest : public NewGrpcMuxImplTestBase { public: - NewGrpcMuxImplTest() : NewGrpcMuxImplTestBase(GetParam()) {} + NewGrpcMuxImplTest() : NewGrpcMuxImplTestBase(std::get<0>(GetParam())) {} Event::SimulatedTimeSystem time_system_; }; INSTANTIATE_TEST_SUITE_P(NewGrpcMuxImplTest, NewGrpcMuxImplTest, - testing::ValuesIn({LegacyOrUnified::Legacy, LegacyOrUnified::Unified})); + testing::Combine(testing::ValuesIn({LegacyOrUnified::Legacy, + LegacyOrUnified::Unified}), + testing::Bool())); // Validate behavior when dynamic context parameters are updated. TEST_P(NewGrpcMuxImplTest, DynamicContextParameters) { @@ -777,9 +787,9 @@ TEST(NewGrpcMuxFactoryTest, InvalidRateLimit) { ads_config.mutable_rate_limit_settings()->mutable_max_tokens()->set_value(100); ads_config.mutable_rate_limit_settings()->mutable_fill_rate()->set_value( std::numeric_limits::quiet_NaN()); - EXPECT_THROW(factory->create(std::make_unique(), dispatcher, random, scope, - ads_config, local_info, nullptr, nullptr, absl::nullopt, - absl::nullopt, false), + EXPECT_THROW(factory->create(std::make_unique(), nullptr, dispatcher, + random, scope, ads_config, local_info, nullptr, nullptr, + absl::nullopt, absl::nullopt, false), EnvoyException); } diff --git a/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc index b36e2c6f958c..2e9ea564397c 100644 --- a/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc @@ -64,6 +64,7 @@ class GrpcMuxImplTestBase : public testing::Test { void setup(const RateLimitSettings& custom_rate_limit_settings) { GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( @@ -911,6 +912,7 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { EXPECT_CALL(local_info_, clusterName()).WillOnce(ReturnRef(EMPTY_STRING)); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( @@ -936,6 +938,7 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { EXPECT_CALL(local_info_, nodeName()).WillOnce(ReturnRef(EMPTY_STRING)); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( @@ -1062,6 +1065,7 @@ TEST_F(GrpcMuxImplTest, AllMuxesStateTest) { setup(); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(), + /*failover_async_client_=*/nullptr, /*dispatcher_=*/dispatcher_, /*service_method_=*/ *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( @@ -1312,9 +1316,9 @@ TEST(UnifiedSotwGrpcMuxFactoryTest, InvalidRateLimit) { ads_config.mutable_rate_limit_settings()->mutable_max_tokens()->set_value(100); ads_config.mutable_rate_limit_settings()->mutable_fill_rate()->set_value( std::numeric_limits::quiet_NaN()); - EXPECT_THROW(factory->create(std::make_unique(), dispatcher, random, scope, - ads_config, local_info, nullptr, nullptr, absl::nullopt, - absl::nullopt, false), + EXPECT_THROW(factory->create(std::make_unique(), nullptr, dispatcher, + random, scope, ads_config, local_info, nullptr, nullptr, + absl::nullopt, absl::nullopt, false), EnvoyException); } @@ -1330,9 +1334,9 @@ TEST(UnifiedDeltaGrpcMuxFactoryTest, InvalidRateLimit) { ads_config.mutable_rate_limit_settings()->mutable_max_tokens()->set_value(100); ads_config.mutable_rate_limit_settings()->mutable_fill_rate()->set_value( std::numeric_limits::quiet_NaN()); - EXPECT_THROW(factory->create(std::make_unique(), dispatcher, random, scope, - ads_config, local_info, nullptr, nullptr, absl::nullopt, - absl::nullopt, false), + EXPECT_THROW(factory->create(std::make_unique(), nullptr, dispatcher, + random, scope, ads_config, local_info, nullptr, nullptr, + absl::nullopt, absl::nullopt, false), EnvoyException); } From afadef6193c7ea44694eed35bffa656582c52e63 Mon Sep 17 00:00:00 2001 From: code Date: Wed, 5 Jun 2024 07:55:56 +0800 Subject: [PATCH 03/61] generic proxy: change the encoding interface to sync mode (#34452) generic proxy: change the encode interface to sync mode Signed-off-by: wbpcode Co-authored-by: wbpcode --- .../network/source/codecs/dubbo/config.h | 16 +- .../network/source/codecs/http1/config.cc | 40 ++-- .../network/source/codecs/http1/config.h | 4 +- .../network/source/codecs/kafka/config.cc | 33 ++- .../network/source/codecs/kafka/config.h | 9 +- .../filters/network/source/interface/codec.h | 35 +-- .../source/interface/codec_callbacks.h | 22 +- .../filters/network/source/interface/filter.h | 4 +- .../filters/network/source/interface/stream.h | 16 +- .../filters/network/source/proxy.cc | 81 +++---- .../filters/network/source/proxy.h | 25 +-- .../filters/network/source/router/router.cc | 85 ++++---- .../filters/network/source/router/router.h | 15 +- .../network/test/codecs/dubbo/config_test.cc | 18 +- .../network/test/codecs/http1/config_test.cc | 176 +++++++++------ .../network/test/codecs/kafka/config_test.cc | 44 ++-- .../filters/network/test/fake_codec.h | 16 +- .../filters/network/test/integration_test.cc | 201 +++++++++++------- .../filters/network/test/mocks/codec.h | 33 ++- .../filters/network/test/mocks/filter.h | 4 +- .../filters/network/test/proxy_test.cc | 118 +++++----- .../network/test/router/router_test.cc | 119 ++--------- 22 files changed, 589 insertions(+), 525 deletions(-) diff --git a/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.h b/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.h index 2480bd98282c..c9b4da229373 100644 --- a/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.h +++ b/contrib/generic_proxy/filters/network/source/codecs/dubbo/config.h @@ -165,17 +165,25 @@ class DubboDecoderBase : public DubboCodecBase, public CodecType { } } - void encode(const StreamFrame& frame, EncodingCallbacks& callbacks) override { + EncodingResult encode(const StreamFrame& frame, EncodingContext&) override { ASSERT(dynamic_cast(&frame) != nullptr); const auto* typed_message = static_cast(&frame); - Buffer::OwnedImpl buffer; - codec_->encode(buffer, *typed_message->inner_metadata_); - callbacks.onEncodingSuccess(buffer, true); + codec_->encode(encoding_buffer_, *typed_message->inner_metadata_); + const uint64_t encoded_size = encoding_buffer_.length(); + + // Write the encoded data to the connection and clean the buffer for the next encoding. + callback_->writeToConnection(encoding_buffer_); + encoding_buffer_.drain(encoding_buffer_.length()); + + return encoded_size; } Common::Dubbo::MessageMetadataSharedPtr metadata_; CallBackType* callback_{}; + +private: + Buffer::OwnedImpl encoding_buffer_; }; class DubboServerCodec diff --git a/contrib/generic_proxy/filters/network/source/codecs/http1/config.cc b/contrib/generic_proxy/filters/network/source/codecs/http1/config.cc index 61761c07b787..c7e61180387e 100644 --- a/contrib/generic_proxy/filters/network/source/codecs/http1/config.cc +++ b/contrib/generic_proxy/filters/network/source/codecs/http1/config.cc @@ -434,12 +434,12 @@ Http::Http1::CallbackResult Http1ServerCodec::onMessageCompleteImpl() { return Http::Http1::CallbackResult::Success; } -void Http1ServerCodec::encode(const StreamFrame& frame, EncodingCallbacks& callbacks) { +EncodingResult Http1ServerCodec::encode(const StreamFrame& frame, EncodingContext&) { const bool response_end_stream = frame.frameFlags().endStream(); if (auto* headers = dynamic_cast(&frame); headers != nullptr) { - ENVOY_LOG(debug, "encoding response headers (end_stream={}):\n{}", response_end_stream, - *headers->response_); + ENVOY_LOG(debug, "Generic proxy HTTP1 codec: encoding response headers (end_stream={}):\n{}", + response_end_stream, *headers->response_); active_request_->response_chunk_encoding_ = Utility::isChunked(*headers->response_, response_end_stream); @@ -449,8 +449,7 @@ void Http1ServerCodec::encode(const StreamFrame& frame, EncodingCallbacks& callb if (!status.ok()) { ENVOY_LOG(error, "Generic proxy HTTP1 codec: failed to encode response headers: {}", status.message()); - callbacks_->connection()->close(Network::ConnectionCloseType::FlushWrite); - return; + return status; } // Encode the optional buffer if it exists. This is used for local response or for the @@ -462,27 +461,31 @@ void Http1ServerCodec::encode(const StreamFrame& frame, EncodingCallbacks& callb } } else if (auto* body = dynamic_cast(&frame); body != nullptr) { - ENVOY_LOG(debug, "encoding response body (end_stream={} size={})", response_end_stream, - body->buffer().length()); + ENVOY_LOG(debug, "Generic proxy HTTP1 codec: encoding response body (end_stream={} size={})", + response_end_stream, body->buffer().length()); Utility::encodeBody(encoding_buffer_, body->buffer(), active_request_->response_chunk_encoding_, response_end_stream); } - callbacks.onEncodingSuccess(encoding_buffer_, response_end_stream); + const uint64_t encoded_size = encoding_buffer_.length(); + + // Send the encoded data to the connection and reset the buffer for the next frame. + callbacks_->writeToConnection(encoding_buffer_); + ASSERT(encoding_buffer_.length() == 0); if (response_end_stream) { if (active_request_->request_complete_) { active_request_.reset(); - return; + return encoded_size; } ENVOY_LOG(debug, "Generic proxy HTTP1 server codec: response complete before request complete"); - if (callbacks_->connection().has_value()) { - callbacks_->connection()->close(Network::ConnectionCloseType::FlushWrite); - } + return absl::InvalidArgumentError("response complete before request complete"); } + + return encoded_size; } -void Http1ClientCodec::encode(const StreamFrame& frame, EncodingCallbacks& callbacks) { +EncodingResult Http1ClientCodec::encode(const StreamFrame& frame, EncodingContext&) { const bool request_end_stream = frame.frameFlags().endStream(); if (auto* headers = dynamic_cast(&frame); headers != nullptr) { @@ -501,8 +504,7 @@ void Http1ClientCodec::encode(const StreamFrame& frame, EncodingCallbacks& callb if (!status.ok()) { ENVOY_LOG(error, "Generic proxy HTTP1 codec: failed to encode request headers: {}", status.message()); - callbacks_->connection()->close(Network::ConnectionCloseType::FlushWrite); - return; + return status; } // Encode the optional buffer if it exists. This is used for local response or for the @@ -524,7 +526,13 @@ void Http1ClientCodec::encode(const StreamFrame& frame, EncodingCallbacks& callb expect_response_->request_complete_ = true; } - callbacks.onEncodingSuccess(encoding_buffer_, request_end_stream); + const uint64_t encoded_size = encoding_buffer_.length(); + + // Send the encoded data to the connection and reset the buffer for the next frame. + callbacks_->writeToConnection(encoding_buffer_); + ASSERT(encoding_buffer_.length() == 0); + + return encoded_size; } Http::Http1::CallbackResult Http1ClientCodec::onMessageBeginImpl() { diff --git a/contrib/generic_proxy/filters/network/source/codecs/http1/config.h b/contrib/generic_proxy/filters/network/source/codecs/http1/config.h index 92838bd0897e..253665c62926 100644 --- a/contrib/generic_proxy/filters/network/source/codecs/http1/config.h +++ b/contrib/generic_proxy/filters/network/source/codecs/http1/config.h @@ -291,7 +291,7 @@ class Http1ServerCodec : public Http1CodecBase, public ServerCodec { callbacks_->onDecodingFailure(); } } - void encode(const StreamFrame& frame, EncodingCallbacks& callbacks) override; + EncodingResult encode(const StreamFrame& frame, EncodingContext& ctx) override; ResponsePtr respond(absl::Status status, absl::string_view data, const Request&) override { auto response = Http::ResponseHeaderMapImpl::create(); response->setStatus(std::to_string(Utility::statusToHttpStatus(status.code()))); @@ -352,7 +352,7 @@ class Http1ClientCodec : public Http1CodecBase, public ClientCodec { callbacks_->onDecodingFailure(); } } - void encode(const StreamFrame& frame, EncodingCallbacks& callbacks) override; + EncodingResult encode(const StreamFrame& frame, EncodingContext& ctx) override; void onDecodingSuccess(RequestHeaderFramePtr, absl::optional) override {} void onDecodingSuccess(ResponseHeaderFramePtr response_header_frame, diff --git a/contrib/generic_proxy/filters/network/source/codecs/kafka/config.cc b/contrib/generic_proxy/filters/network/source/codecs/kafka/config.cc index 5cf98e2f6935..35c6d9572311 100644 --- a/contrib/generic_proxy/filters/network/source/codecs/kafka/config.cc +++ b/contrib/generic_proxy/filters/network/source/codecs/kafka/config.cc @@ -22,23 +22,28 @@ void KafkaServerCodec::decode(Envoy::Buffer::Instance& buffer, bool) { request_buffer_.drain(request_buffer_.length()); } -void KafkaServerCodec::encode(const GenericProxy::StreamFrame& frame, - GenericProxy::EncodingCallbacks& callbacks) { +GenericProxy::EncodingResult KafkaServerCodec::encode(const GenericProxy::StreamFrame& frame, + GenericProxy::EncodingContext&) { auto* typed_response = dynamic_cast(&frame); if (typed_response == nullptr) { ENVOY_LOG(error, "Kafka codec: invalid response frame type and cannot encode"); - return; + return absl::InvalidArgumentError("Invalid response frame type"); } if (typed_response->response_ != nullptr) { response_encoder_.encode(*typed_response->response_); } else { - ENVOY_LOG(error, "Kafka codec: invalid empty response frame type and close connection"); - request_callbacks_->callbacks_.connection()->close(Network::ConnectionCloseType::FlushWrite); - return; + ENVOY_LOG(error, "Kafka codec: invalid empty response frame"); + return absl::InvalidArgumentError("Invalid empty response frame"); } - callbacks.onEncodingSuccess(response_buffer_, true); + + const uint64_t encoded_size = response_buffer_.length(); + + // Write the encoded data to the connection and clean the buffer for the next encoding. + request_callbacks_->callbacks_.writeToConnection(response_buffer_); // All data should be consumed by the generic proxy and send to the network. ASSERT(response_buffer_.length() == 0); + + return encoded_size; } GenericProxy::ResponsePtr KafkaServerCodec::respond(absl::Status, absl::string_view, const GenericProxy::Request&) { @@ -60,20 +65,26 @@ void KafkaClientCodec::decode(Envoy::Buffer::Instance& buffer, bool) { response_buffer_.drain(response_buffer_.length()); } -void KafkaClientCodec::encode(const GenericProxy::StreamFrame& frame, - GenericProxy::EncodingCallbacks& callbacks) { +GenericProxy::EncodingResult KafkaClientCodec::encode(const GenericProxy::StreamFrame& frame, + GenericProxy::EncodingContext&) { auto* typed_request = dynamic_cast(&frame); if (typed_request == nullptr) { ENVOY_LOG(error, "Kafka codec: invalid request frame type and cannot encode"); - return; + return absl::InvalidArgumentError("Invalid request frame type"); } response_decoder_->expectResponse(typed_request->request_->request_header_.correlation_id_, typed_request->request_->request_header_.api_key_, typed_request->request_->request_header_.api_version_); request_encoder_.encode(*typed_request->request_); - callbacks.onEncodingSuccess(request_buffer_, true); + + const uint64_t encoded_size = request_buffer_.length(); + + // Write the encoded data to the connection and clean the buffer for the next encoding. + response_callbacks_->callbacks_.writeToConnection(request_buffer_); // All data should be consumed by the generic proxy and send to the network. ASSERT(request_buffer_.length() == 0); + + return encoded_size; } CodecFactoryPtr diff --git a/contrib/generic_proxy/filters/network/source/codecs/kafka/config.h b/contrib/generic_proxy/filters/network/source/codecs/kafka/config.h index d694ac53ce00..1c8e72aa6d76 100644 --- a/contrib/generic_proxy/filters/network/source/codecs/kafka/config.h +++ b/contrib/generic_proxy/filters/network/source/codecs/kafka/config.h @@ -91,7 +91,6 @@ class KafkaResponseCallbacks : public NetworkFilters::Kafka::ResponseCallback, callbacks_.onDecodingFailure(); } -private: GenericProxy::ClientCodecCallbacks& callbacks_; }; @@ -102,8 +101,8 @@ class KafkaServerCodec : public GenericProxy::ServerCodec, void setCodecCallbacks(GenericProxy::ServerCodecCallbacks& callbacks) override; void decode(Envoy::Buffer::Instance& buffer, bool end_stream) override; - void encode(const GenericProxy::StreamFrame& frame, - GenericProxy::EncodingCallbacks& callbacks) override; + GenericProxy::EncodingResult encode(const GenericProxy::StreamFrame& frame, + GenericProxy::EncodingContext& ctx) override; GenericProxy::ResponsePtr respond(absl::Status, absl::string_view, const GenericProxy::Request&) override; @@ -123,8 +122,8 @@ class KafkaClientCodec : public GenericProxy::ClientCodec, void setCodecCallbacks(GenericProxy::ClientCodecCallbacks& callbacks) override; void decode(Envoy::Buffer::Instance& buffer, bool end_stream) override; - void encode(const GenericProxy::StreamFrame& frame, - GenericProxy::EncodingCallbacks& callbacks) override; + GenericProxy::EncodingResult encode(const GenericProxy::StreamFrame& frame, + GenericProxy::EncodingContext& ctx) override; Envoy::Buffer::OwnedImpl request_buffer_; Envoy::Buffer::OwnedImpl response_buffer_; diff --git a/contrib/generic_proxy/filters/network/source/interface/codec.h b/contrib/generic_proxy/filters/network/source/interface/codec.h index eedb24862213..e9ca99af5674 100644 --- a/contrib/generic_proxy/filters/network/source/interface/codec.h +++ b/contrib/generic_proxy/filters/network/source/interface/codec.h @@ -23,8 +23,8 @@ class ServerCodec { /** * Set callbacks of server codec. - * @param callbacks callbacks of server codec. This callback will have same lifetime - * as the server codec. + * @param callbacks callbacks of server codec. This callback will have same or longer + * lifetime as the server codec. */ virtual void setCodecCallbacks(ServerCodecCallbacks& callbacks) PURE; @@ -36,20 +36,26 @@ class ServerCodec { virtual void decode(Buffer::Instance& buffer, bool end_stream) PURE; /** - * Encode response frame. - * @param frame response frame to encode. - * @param callbacks callbacks of encoding. This callback should be used to notify the - * generic proxy filter that the response is encoded and should be called only once. + * Encode response frame and send it to upstream connection by the writeToConnection() + * method of the codec callbacks. + * @param frame response frame to encode. NOTE: the generic proxy will assume this is + * sync encoding and the frame may be destroyed after this method is called. + * @param ctx context of encoding that will be used to provide additional information + * to the codec. Like the route that the downstream request is matched to. + * @return the size of the encoded data or error message if encoding failed. */ - virtual void encode(const StreamFrame& frame, EncodingCallbacks& callbacks) PURE; + virtual EncodingResult encode(const StreamFrame& frame, EncodingContext& ctx) PURE; /** * Create a response frame with specified status and flags. * @param status status of the response. * @param data any data that generic proxy filter wants to tell the codec. * @param request origin request that the response is created for. + * @return ResponseHeaderFramePtr the response frame. Only single frame is allowed for + * local response. */ - virtual ResponsePtr respond(Status status, absl::string_view data, const Request& request) PURE; + virtual ResponseHeaderFramePtr respond(Status status, absl::string_view data, + const RequestHeaderFrame& request) PURE; }; /** @@ -75,12 +81,15 @@ class ClientCodec { virtual void decode(Buffer::Instance& buffer, bool end_stream) PURE; /** - * Encode request frame. - * @param frame request frame to encode. - * @param callbacks callbacks of encoding. This callbacks should be used to notify the - * generic proxy filter that the request is encoded and should be called only once. + * Encode request frame and send it to upstream connection by the writeToConnection() + * method of the codec callbacks. + * @param frame request frame to encode. NOTE: the generic proxy will assume this is + * sync encoding and the frame may be destroyed after this method is called. + * @param ctx context of encoding that will be used to provide additional information + * to the codec. Like the route that the request is matched to. + * @return the size of the encoded data or error message if encoding failed. */ - virtual void encode(const StreamFrame& frame, EncodingCallbacks& callbacks) PURE; + virtual EncodingResult encode(const StreamFrame& frame, EncodingContext& ctx) PURE; }; using ServerCodecPtr = std::unique_ptr; diff --git a/contrib/generic_proxy/filters/network/source/interface/codec_callbacks.h b/contrib/generic_proxy/filters/network/source/interface/codec_callbacks.h index 1cd87e432648..367cb8783e4b 100644 --- a/contrib/generic_proxy/filters/network/source/interface/codec_callbacks.h +++ b/contrib/generic_proxy/filters/network/source/interface/codec_callbacks.h @@ -149,25 +149,11 @@ class ClientCodecCallbacks { }; /** - * Callback of request/response frame. + * Context of stream that will be used to encode request/response frame. */ -class EncodingCallbacks { +class EncodingContext { public: - virtual ~EncodingCallbacks() = default; - - /** - * If encoding success then this method will be called to write the data to upstream - * or downstream connection and notify the generic proxy if the encoding is completed. - * @param buffer encoding result buffer. - * @param end_stream if last frame is encoded. - */ - virtual void onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) PURE; - - /** - * If encoding failure then this method will be called. - * @param reason the reason of encoding failure. - */ - virtual void onEncodingFailure(absl::string_view reason = {}) PURE; + virtual ~EncodingContext() = default; /** * The route that the request is matched to. This is optional when encoding the response @@ -179,6 +165,8 @@ class EncodingCallbacks { virtual OptRef routeEntry() const PURE; }; +using EncodingResult = absl::StatusOr; + } // namespace GenericProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/contrib/generic_proxy/filters/network/source/interface/filter.h b/contrib/generic_proxy/filters/network/source/interface/filter.h index ec2c3b042a75..bb651594e7a3 100644 --- a/contrib/generic_proxy/filters/network/source/interface/filter.h +++ b/contrib/generic_proxy/filters/network/source/interface/filter.h @@ -104,13 +104,13 @@ class DecoderFilterCallback : public virtual StreamFilterCallbacks { * Called when the upstream response frame is received. This should only be called once. * @param frame supplies the upstream response frame. */ - virtual void onResponseStart(ResponseHeaderFramePtr frame) PURE; + virtual void onResponseHeaderFrame(ResponseHeaderFramePtr frame) PURE; /** * Called when the upstream response frame is received. * @param frame supplies the upstream frame. */ - virtual void onResponseFrame(ResponseCommonFramePtr frame) PURE; + virtual void onResponseCommonFrame(ResponseCommonFramePtr frame) PURE; /** * Register a request frames handler to used to handle the request frames (except the special diff --git a/contrib/generic_proxy/filters/network/source/interface/stream.h b/contrib/generic_proxy/filters/network/source/interface/stream.h index 4c7100257e72..8ca9e12e2c6d 100644 --- a/contrib/generic_proxy/filters/network/source/interface/stream.h +++ b/contrib/generic_proxy/filters/network/source/interface/stream.h @@ -132,9 +132,21 @@ class StreamFrame { virtual FrameFlags frameFlags() const { return {}; } }; +/** + * Common frame that used to represent any data or structure of L7 protocols. No specific + * interface is provided for the common frame. + */ class CommonFrame : public StreamFrame {}; using CommonFramePtr = std::unique_ptr; +/** + * Header frame of generic request or response. This provide some basic interfaces that are + * used to get/set attributes of the request or response. + * NOTE: Header frame should always be the first frame of the request or response. And there + * has no requirement that the header frame could only contain the 'header' of L7 protocols. + * For example, for short HTTP request, the header frame could contain the whole request + * header map, body, and even trailer. + */ class HeaderFrame : public StreamFrame { public: using IterateCallback = std::function; @@ -183,10 +195,6 @@ using StreamBase = HeaderFrame; /** * Interface of generic request. This is derived from StreamFrame that contains the request * specific information. First frame of the request MUST be a RequestHeaderFrame. - * - * NOTE: using interface that provided by the TraceContext as the interface of generic request - * here to simplify the tracing integration. This is not a good design. This should be changed - * in the future. */ class RequestHeaderFrame : public HeaderFrame { public: diff --git a/contrib/generic_proxy/filters/network/source/proxy.cc b/contrib/generic_proxy/filters/network/source/proxy.cc index eeca48c4a028..6b17f371cfab 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.cc +++ b/contrib/generic_proxy/filters/network/source/proxy.cc @@ -128,15 +128,13 @@ void ActiveStream::resetStream(DownstreamStreamResetReason reason) { completeRequest(); } -void ActiveStream::sendResponseStartToDownstream() { +void ActiveStream::sendHeaderFrameToDownstream() { ASSERT(response_stream_ != nullptr); response_filter_chain_complete_ = true; - // The first frame of response is sent. - stream_info_.downstreamTiming().onFirstDownstreamTxByteSent(parent_.time_source_); - parent_.sendFrameToDownstream(*response_stream_, *this); + sendFrameToDownstream(*response_stream_, true); } -void ActiveStream::sendResponseFrameToDownstream() { +void ActiveStream::sendCommonFrameToDownstream() { if (!response_filter_chain_complete_) { // Wait for the response header frame to be sent first. It may be blocked by // the filter chain. @@ -149,10 +147,41 @@ void ActiveStream::sendResponseFrameToDownstream() { response_stream_frames_.pop_front(); // Send the frame to downstream. - parent_.sendFrameToDownstream(*frame, *this); + if (!sendFrameToDownstream(*frame, false)) { + break; + } } } +bool ActiveStream::sendFrameToDownstream(const StreamFrame& frame, bool header_frame) { + const bool end_stream = frame.frameFlags().endStream(); + + const auto result = parent_.server_codec_->encode(frame, *this); + if (!result.ok()) { + ENVOY_LOG(error, "Generic proxy: response encoding failure: {}", result.status().message()); + resetStream(DownstreamStreamResetReason::ProtocolError); + return false; + } + + ENVOY_LOG(debug, "Generic proxy: send {} bytes to client, complete: {}", result.value(), + end_stream); + + if (header_frame) { + stream_info_.downstreamTiming().onFirstDownstreamTxByteSent(parent_.time_source_); + } + + // If the request is fully sent, record the last downstream tx byte sent time and clean + // up the stream. + if (end_stream) { + stream_info_.downstreamTiming().onLastDownstreamTxByteSent(parent_.time_source_); + + ASSERT(response_stream_end_); + ASSERT(response_stream_frames_.empty()); + completeRequest(); + } + return true; +} + void ActiveStream::sendRequestFrameToUpstream() { if (!request_filter_chain_complete_) { // Wait for the request header frame to be sent first. It may be blocked by @@ -196,7 +225,7 @@ void ActiveStream::sendLocalReply(Status status, absl::string_view data, // Set the response code to the stream info. stream_info_.setResponseCode(response_stream_->status().code()); - sendResponseStartToDownstream(); + sendHeaderFrameToDownstream(); } void ActiveStream::continueDecoding() { @@ -251,7 +280,7 @@ void ActiveStream::onRequestFrame(RequestCommonFramePtr request_common_frame) { sendRequestFrameToUpstream(); } -void ActiveStream::onResponseStart(ResponsePtr response) { +void ActiveStream::onResponseHeaderFrame(ResponsePtr response) { ASSERT(response_stream_ == nullptr); response_stream_ = std::move(response); ASSERT(response_stream_ != nullptr); @@ -265,11 +294,11 @@ void ActiveStream::onResponseStart(ResponsePtr response) { continueEncoding(); } -void ActiveStream::onResponseFrame(ResponseCommonFramePtr response_common_frame) { +void ActiveStream::onResponseCommonFrame(ResponseCommonFramePtr response_common_frame) { response_stream_end_ = response_common_frame->frameFlags().endStream(); response_stream_frames_.emplace_back(std::move(response_common_frame)); // Try to send the frame to downstream immediately. - sendResponseFrameToDownstream(); + sendCommonFrameToDownstream(); } void ActiveStream::completeDirectly() { @@ -298,33 +327,9 @@ void ActiveStream::continueEncoding() { if (next_encoder_filter_index_ == encoder_filters_.size()) { ENVOY_LOG(debug, "Complete encoder filters"); - sendResponseStartToDownstream(); - sendResponseFrameToDownstream(); - } -} - -void ActiveStream::onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) { - ASSERT(parent_.downstreamConnection().state() == Network::Connection::State::Open); - parent_.downstreamConnection().write(buffer, false); - - if (!end_stream) { - return; + sendHeaderFrameToDownstream(); + sendCommonFrameToDownstream(); } - - // The response is fully sent. - stream_info_.downstreamTiming().onLastDownstreamTxByteSent(parent_.time_source_); - - ENVOY_LOG(debug, "Generic proxy: downstream response complete"); - - ASSERT(response_stream_end_); - ASSERT(response_stream_frames_.empty()); - - completeRequest(); -} - -void ActiveStream::onEncodingFailure(absl::string_view reason) { - ENVOY_LOG(error, "Generic proxy: response encoding failure: {}", reason); - resetStream(DownstreamStreamResetReason::ProtocolError); } void ActiveStream::initializeFilterChain(FilterChainFactory& factory) { @@ -440,10 +445,6 @@ OptRef Filter::connection() { return {downstreamConnection()}; } -void Filter::sendFrameToDownstream(StreamFrame& frame, EncodingCallbacks& callbacks) { - server_codec_->encode(frame, callbacks); -} - void Filter::registerFrameHandler(uint64_t stream_id, ActiveStream* raw_stream) { // If the stream expects variable length frames, then add it to the frame // handler map. diff --git a/contrib/generic_proxy/filters/network/source/proxy.h b/contrib/generic_proxy/filters/network/source/proxy.h index 1f3c161fe9e5..f4e24df9ed45 100644 --- a/contrib/generic_proxy/filters/network/source/proxy.h +++ b/contrib/generic_proxy/filters/network/source/proxy.h @@ -127,7 +127,7 @@ class FilterConfigImpl : public FilterConfig { class ActiveStream : public FilterChainManager, public LinkedObject, public Envoy::Event::DeferredDeletable, - public EncodingCallbacks, + public EncodingContext, public Tracing::Config, Logger::Loggable { public: @@ -173,11 +173,11 @@ class ActiveStream : public FilterChainManager, parent_.sendLocalReply(status, data, std::move(func)); } void continueDecoding() override { parent_.continueDecoding(); } - void onResponseStart(ResponseHeaderFramePtr frame) override { - parent_.onResponseStart(std::move(frame)); + void onResponseHeaderFrame(ResponseHeaderFramePtr frame) override { + parent_.onResponseHeaderFrame(std::move(frame)); } - void onResponseFrame(ResponseCommonFramePtr frame) override { - parent_.onResponseFrame(std::move(frame)); + void onResponseCommonFrame(ResponseCommonFramePtr frame) override { + parent_.onResponseCommonFrame(std::move(frame)); } void setRequestFramesHandler(RequestFramesHandler& handler) override { ASSERT(parent_.request_stream_frames_handler_ == nullptr, @@ -249,8 +249,8 @@ class ActiveStream : public FilterChainManager, void sendLocalReply(Status status, absl::string_view data, ResponseUpdateFunction func); void continueDecoding(); - void onResponseStart(ResponseHeaderFramePtr response_header_frame); - void onResponseFrame(ResponseCommonFramePtr response_common_frame); + void onResponseHeaderFrame(ResponseHeaderFramePtr response_header_frame); + void onResponseCommonFrame(ResponseCommonFramePtr response_common_frame); void completeDirectly(); void continueEncoding(); @@ -261,9 +261,7 @@ class ActiveStream : public FilterChainManager, factory(callbacks); } - // ResponseEncoderCallback - void onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) override; - void onEncodingFailure(absl::string_view reason = {}) override; + // EncodingContext OptRef routeEntry() const override { return makeOptRefFromPtr(cached_route_entry_.get()); } @@ -309,8 +307,9 @@ class ActiveStream : public FilterChainManager, void sendRequestFrameToUpstream(); - void sendResponseStartToDownstream(); - void sendResponseFrameToDownstream(); + void sendHeaderFrameToDownstream(); + void sendCommonFrameToDownstream(); + bool sendFrameToDownstream(const StreamFrame& frame, bool header_frame); bool active_stream_reset_{false}; @@ -395,8 +394,6 @@ class Filter : public Envoy::Network::ReadFilter, void onAboveWriteBufferHighWatermark() override {} void onBelowWriteBufferLowWatermark() override {} - void sendFrameToDownstream(StreamFrame& frame, EncodingCallbacks& callbacks); - Network::Connection& downstreamConnection() { ASSERT(callbacks_ != nullptr); return callbacks_->connection(); diff --git a/contrib/generic_proxy/filters/network/source/router/router.cc b/contrib/generic_proxy/filters/network/source/router/router.cc index 3850fb870737..238c2341208a 100644 --- a/contrib/generic_proxy/filters/network/source/router/router.cc +++ b/contrib/generic_proxy/filters/network/source/router/router.cc @@ -130,16 +130,12 @@ void UpstreamRequest::deferredDelete() { } } -void UpstreamRequest::sendRequestStartToUpstream() { +void UpstreamRequest::sendHeaderFrameToUpstream() { request_stream_header_sent_ = true; - ASSERT(generic_upstream_ != nullptr); - - // The first frame of request is sent. - upstream_info_->upstreamTiming().onFirstUpstreamTxByteSent(parent_.time_source_); - generic_upstream_->clientCodec().encode(*parent_.request_stream_, *this); + sendFrameToUpstream(*parent_.request_stream_, true); } -void UpstreamRequest::sendRequestFrameToUpstream() { +void UpstreamRequest::sendCommonFrameToUpstream() { if (!request_stream_header_sent_) { // Do not send request frame to upstream until the request header is sent. It may be blocked // by the upstream connecting. @@ -149,36 +145,43 @@ void UpstreamRequest::sendRequestFrameToUpstream() { while (!parent_.request_stream_frames_.empty()) { auto frame = std::move(parent_.request_stream_frames_.front()); parent_.request_stream_frames_.pop_front(); - - ASSERT(generic_upstream_ != nullptr); - generic_upstream_->clientCodec().encode(*frame, *this); + if (!sendFrameToUpstream(*frame, false)) { + break; + } } } -void UpstreamRequest::onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) { - encodeBufferToUpstream(buffer); +bool UpstreamRequest::sendFrameToUpstream(const StreamFrame& frame, bool header_frame) { + ASSERT(generic_upstream_ != nullptr); + const bool end_stream = frame.frameFlags().endStream(); - if (!end_stream) { - return; + const auto result = generic_upstream_->clientCodec().encode(frame, *this); + if (!result.ok()) { + ENVOY_LOG(error, "Generic proxy: request encoding failure: {}", result.status().message()); + // The request encoding failure is treated as a protocol error. + resetStream(StreamResetReason::ProtocolError, result.status().message()); + return false; } - // The request is fully sent. - upstream_info_->upstreamTiming().onLastUpstreamTxByteSent(parent_.time_source_); - - // Request is complete. - ENVOY_LOG(debug, "upstream request encoding success"); + ENVOY_LOG(debug, "Generic proxy: send {} bytes to server, complete: {}", result.value(), + end_stream); - // Need not to wait for the upstream response and complete directly. - if (!expects_response_) { - clearStream(false); - parent_.completeDirectly(); - return; + if (header_frame) { + upstream_info_->upstreamTiming().onFirstUpstreamTxByteSent(parent_.time_source_); } -} -void UpstreamRequest::onEncodingFailure(absl::string_view reason) { - // The request encoding failure is treated as a protocol error. - resetStream(StreamResetReason::ProtocolError, reason); + // If the request is fully sent, record the last downstream tx byte sent time and clean + // up the stream. + if (end_stream) { + upstream_info_->upstreamTiming().onLastUpstreamTxByteSent(parent_.time_source_); + + // Oneway requests need not to wait for the upstream response and complete directly. + if (!expects_response_) { + clearStream(false); + parent_.completeDirectly(); + } + } + return true; } OptRef UpstreamRequest::routeEntry() const { @@ -215,8 +218,8 @@ void UpstreamRequest::onUpstreamSuccess() { parent_.callbacks_->activeSpan().injectContext(trace_context, upstream_context); } - sendRequestStartToUpstream(); - sendRequestFrameToUpstream(); + sendHeaderFrameToUpstream(); + sendCommonFrameToUpstream(); } void UpstreamRequest::onUpstreamResponseComplete(bool drain_close) { @@ -244,7 +247,7 @@ void UpstreamRequest::onDecodingSuccess(ResponseHeaderFramePtr response_header_f onUpstreamResponseComplete(response_header_frame->frameFlags().streamFlags().drainClose()); } - parent_.onResponseStart(std::move(response_header_frame)); + parent_.onResponseHeaderFrame(std::move(response_header_frame)); } void UpstreamRequest::onDecodingSuccess(ResponseCommonFramePtr response_common_frame) { @@ -258,7 +261,7 @@ void UpstreamRequest::onDecodingSuccess(ResponseCommonFramePtr response_common_f onUpstreamResponseComplete(response_common_frame->frameFlags().streamFlags().drainClose()); } - parent_.onResponseFrame(std::move(response_common_frame)); + parent_.onResponseCommonFrame(std::move(response_common_frame)); } void UpstreamRequest::onDecodingFailure(absl::string_view reason) { @@ -311,26 +314,18 @@ void UpstreamRequest::onUpstreamConnectionReady() { connecting_start_time_.value(), parent_.time_source_); } -void UpstreamRequest::encodeBufferToUpstream(Buffer::Instance& buffer) { - ENVOY_LOG(trace, "proxying {} bytes", buffer.length()); - - ASSERT(generic_upstream_ != nullptr); - ASSERT(generic_upstream_->upstreamConnection().has_value()); - generic_upstream_->upstreamConnection()->write(buffer, false); -} - -void RouterFilter::onResponseStart(ResponseHeaderFramePtr response_header_frame) { +void RouterFilter::onResponseHeaderFrame(ResponseHeaderFramePtr response_header_frame) { if (response_header_frame->frameFlags().endStream()) { onFilterComplete(); } - callbacks_->onResponseStart(std::move(response_header_frame)); + callbacks_->onResponseHeaderFrame(std::move(response_header_frame)); } -void RouterFilter::onResponseFrame(ResponseCommonFramePtr response_common_frame) { +void RouterFilter::onResponseCommonFrame(ResponseCommonFramePtr response_common_frame) { if (response_common_frame->frameFlags().endStream()) { onFilterComplete(); } - callbacks_->onResponseFrame(std::move(response_common_frame)); + callbacks_->onResponseCommonFrame(std::move(response_common_frame)); } void RouterFilter::completeDirectly() { @@ -468,7 +463,7 @@ void RouterFilter::onRequestCommonFrame(RequestCommonFramePtr frame) { return; } - upstream_requests_.front()->sendRequestFrameToUpstream(); + upstream_requests_.front()->sendCommonFrameToUpstream(); } FilterStatus RouterFilter::onStreamDecoded(StreamRequest& request) { diff --git a/contrib/generic_proxy/filters/network/source/router/router.h b/contrib/generic_proxy/filters/network/source/router/router.h index 3acee5fa862c..7bae2856615a 100644 --- a/contrib/generic_proxy/filters/network/source/router/router.h +++ b/contrib/generic_proxy/filters/network/source/router/router.h @@ -45,7 +45,7 @@ class RouterFilter; class UpstreamRequest : public UpstreamRequestCallbacks, public LinkedObject, - public EncodingCallbacks, + public EncodingContext, Logger::Loggable { public: UpstreamRequest(RouterFilter& parent, StreamFlags stream_flags, @@ -68,17 +68,16 @@ class UpstreamRequest : public UpstreamRequestCallbacks, void onDecodingSuccess(ResponseCommonFramePtr response_common_frame) override; void onDecodingFailure(absl::string_view reason) override; - // RequestEncoderCallback - void onEncodingSuccess(Buffer::Instance& buffer, bool end_stream) override; - void onEncodingFailure(absl::string_view reason) override; + // EncodingContext OptRef routeEntry() const override; void onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host); void onUpstreamConnectionReady(); void encodeBufferToUpstream(Buffer::Instance& buffer); - void sendRequestStartToUpstream(); - void sendRequestFrameToUpstream(); + void sendHeaderFrameToUpstream(); + void sendCommonFrameToUpstream(); + bool sendFrameToUpstream(const StreamFrame& frame, bool header_frame); void onUpstreamResponseComplete(bool drain_close); @@ -140,8 +139,8 @@ class RouterFilter : public DecoderFilter, } FilterStatus onStreamDecoded(StreamRequest& request) override; - void onResponseStart(ResponseHeaderFramePtr response_header_frame); - void onResponseFrame(ResponseCommonFramePtr response_common_frame); + void onResponseHeaderFrame(ResponseHeaderFramePtr response_header_frame); + void onResponseCommonFrame(ResponseCommonFramePtr response_common_frame); void completeDirectly(); void onUpstreamRequestReset(UpstreamRequest& upstream_request, StreamResetReason reason, diff --git a/contrib/generic_proxy/filters/network/test/codecs/dubbo/config_test.cc b/contrib/generic_proxy/filters/network/test/codecs/dubbo/config_test.cc index eb6f579e4d37..17792dc903ed 100644 --- a/contrib/generic_proxy/filters/network/test/codecs/dubbo/config_test.cc +++ b/contrib/generic_proxy/filters/network/test/codecs/dubbo/config_test.cc @@ -243,15 +243,15 @@ TEST(DubboServerCodecTest, DubboServerCodecTest) { // Encode response. { - MockEncodingCallbacks encoding_callbacks; + MockEncodingContext encoding_context; DubboRequest request(createDubboRequst(false)); DubboResponse response( createDubboResponse(request, ResponseStatus::Ok, RpcResponseType::ResponseWithValue)); EXPECT_CALL(*raw_serializer, serializeRpcResponse(_, _)); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, _)); + EXPECT_CALL(callbacks, writeToConnection(_)); - server_codec.encode(response, encoding_callbacks); + EXPECT_TRUE(server_codec.encode(response, encoding_context).ok()); } { @@ -368,26 +368,26 @@ TEST(DubboClientCodecTest, DubboClientCodecTest) { // Encode normal request. { - MockEncodingCallbacks encoding_callbacks; + MockEncodingContext encoding_context; DubboRequest request(createDubboRequst(false)); EXPECT_CALL(*raw_serializer, serializeRpcRequest(_, _)); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, _)); + EXPECT_CALL(callbacks, writeToConnection(_)); - client_codec.encode(request, encoding_callbacks); + EXPECT_TRUE(client_codec.encode(request, encoding_context).ok()); } // Encode one-way request. { - MockEncodingCallbacks encoding_callbacks; + MockEncodingContext encoding_context; DubboRequest request(createDubboRequst(true)); EXPECT_CALL(*raw_serializer, serializeRpcRequest(_, _)); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, _)); + EXPECT_CALL(callbacks, writeToConnection(_)); - client_codec.encode(request, encoding_callbacks); + EXPECT_TRUE(client_codec.encode(request, encoding_context).ok()); } } diff --git a/contrib/generic_proxy/filters/network/test/codecs/http1/config_test.cc b/contrib/generic_proxy/filters/network/test/codecs/http1/config_test.cc index 14ee1d70ed6d..ee5416ddb45a 100644 --- a/contrib/generic_proxy/filters/network/test/codecs/http1/config_test.cc +++ b/contrib/generic_proxy/filters/network/test/codecs/http1/config_test.cc @@ -524,27 +524,53 @@ TEST_F(Http1ServerCodecTest, RespondTest) { } TEST_F(Http1ServerCodecTest, HeaderOnlyResponseEncodingTest) { + // Mock request. + codec_->active_request_ = ActiveRequest{nullptr, true}; + // Create a response. auto headers = Http::ResponseHeaderMapImpl::create(); headers->setStatus(200); HttpResponseFrame response(std::move(headers), true); - NiceMock encoding_callbacks; + NiceMock encoding_context; // Encode the response. { - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "HTTP/1.1 200 OK\r\n" "\r\n"); + buffer.drain(buffer.length()); })); - codec_->encode(response, encoding_callbacks); + EXPECT_TRUE(codec_->encode(response, encoding_context).ok()); + } +} + +TEST_F(Http1ServerCodecTest, MissingRequiredHeadersEncodingTest) { + // Mock request. + codec_->active_request_ = ActiveRequest{nullptr, true}; + + // Create a request without method. + auto headers = Http::ResponseHeaderMapImpl::create(); + + HttpResponseFrame response(std::move(headers), true); + + NiceMock encoding_context; + + // Encode the request. + { + auto status_or = codec_->encode(response, encoding_context); + EXPECT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().message(), "missing required headers"); } } TEST_F(Http1ServerCodecTest, ResponseEncodingTest) { + // Mock request. + codec_->active_request_ = ActiveRequest{nullptr, true}; + // Create a response. auto headers = Http::ResponseHeaderMapImpl::create(); headers->setStatus(200); @@ -555,32 +581,34 @@ TEST_F(Http1ServerCodecTest, ResponseEncodingTest) { Buffer::OwnedImpl body_buffer("body"); HttpRawBodyFrame body(body_buffer, true); - NiceMock encoding_callbacks; + NiceMock encoding_context; // Encode the response. { - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, false)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "HTTP/1.1 200 OK\r\n" "content-length: 4\r\n" "\r\n"); buffer.drain(buffer.length()); })); - codec_->encode(response, encoding_callbacks); + EXPECT_TRUE(codec_->encode(response, encoding_context).ok()); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "body"); buffer.drain(buffer.length()); })); - - codec_->encode(body, encoding_callbacks); + EXPECT_TRUE(codec_->encode(body, encoding_context).ok()); } } TEST_F(Http1ServerCodecTest, ChunkedResponseEncodingTest) { + // Mock request. + codec_->active_request_ = ActiveRequest{nullptr, true}; + // Create a response. auto headers = Http::ResponseHeaderMapImpl::create(); headers->setStatus(200); @@ -591,22 +619,22 @@ TEST_F(Http1ServerCodecTest, ChunkedResponseEncodingTest) { Buffer::OwnedImpl body_buffer("body"); HttpRawBodyFrame body(body_buffer, true); - NiceMock encoding_callbacks; + NiceMock encoding_context; // Encode the response. { - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, false)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "HTTP/1.1 200 OK\r\n" "transfer-encoding: chunked\r\n" "\r\n"); buffer.drain(buffer.length()); })); - codec_->encode(response, encoding_callbacks); + EXPECT_TRUE(codec_->encode(response, encoding_context).ok()); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "4\r\n" // Chunk header. "body" // Chunk body. "\r\n" // Chunk footer. @@ -615,7 +643,7 @@ TEST_F(Http1ServerCodecTest, ChunkedResponseEncodingTest) { buffer.drain(buffer.length()); })); - codec_->encode(body, encoding_callbacks); + EXPECT_TRUE(codec_->encode(body, encoding_context).ok()); } } @@ -653,12 +681,12 @@ TEST_F(Http1ServerCodecTest, RequestAndResponseTest) { Buffer::OwnedImpl body_buffer("body"); HttpRawBodyFrame body(body_buffer, true); - NiceMock encoding_callbacks; - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, _)).Times(2); + NiceMock encoding_context; + EXPECT_CALL(codec_callbacks_, writeToConnection(_)).Times(2); // Encode the response. - codec_->encode(response, encoding_callbacks); - codec_->encode(body, encoding_callbacks); + EXPECT_TRUE(codec_->encode(response, encoding_context).ok()); + EXPECT_TRUE(codec_->encode(body, encoding_context).ok()); } } @@ -693,17 +721,17 @@ TEST_F(Http1ServerCodecTest, ResponseCompleteBeforeRequestCompleteTest) { Buffer::OwnedImpl body_buffer("body"); HttpRawBodyFrame body(body_buffer, true); - NiceMock encoding_callbacks; + NiceMock encoding_context; - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, false)); + EXPECT_CALL(codec_callbacks_, writeToConnection(_)); // Encode the response. - codec_->encode(response, encoding_callbacks); + EXPECT_TRUE(codec_->encode(response, encoding_context).ok()); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)); - // Response is complete, but request is not complete, so the codec should close the connection. - EXPECT_CALL(mock_connection, close(Network::ConnectionCloseType::FlushWrite)); + EXPECT_CALL(codec_callbacks_, writeToConnection(_)); - codec_->encode(body, encoding_callbacks); + auto status_or = codec_->encode(body, encoding_context); + EXPECT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().message(), "response complete before request complete"); } TEST_F(Http1ServerCodecTest, NewRequestBeforeFirstRequestCompleteTest) { @@ -888,19 +916,19 @@ TEST_F(Http1ServerCodecTest, SingleFrameModeResponseEncodingTest) { HttpResponseFrame response(std::move(headers), true); response.optionalBuffer().add("body"); - NiceMock encoding_callbacks; + NiceMock encoding_context; // Encode the response. { - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "HTTP/1.1 200 OK\r\n" "content-length: 4\r\n" "\r\n" "body"); buffer.drain(buffer.length()); })); - codec_->encode(response, encoding_callbacks); + EXPECT_TRUE(codec_->encode(response, encoding_context).ok()); } } @@ -929,12 +957,12 @@ class Http1ClientCodecTest : public testing::Test { Buffer::OwnedImpl body_buffer("body"); HttpRawBodyFrame body(body_buffer, true); - NiceMock encoding_callbacks; + NiceMock encoding_context; // Encode the request. { - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, false)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "GET /path HTTP/1.1\r\n" "host: host\r\n" "content-length: 4\r\n" @@ -942,15 +970,15 @@ class Http1ClientCodecTest : public testing::Test { buffer.drain(buffer.length()); })); - codec_->encode(request, encoding_callbacks); + EXPECT_TRUE(codec_->encode(request, encoding_context).ok()); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "body"); buffer.drain(buffer.length()); })); - codec_->encode(body, encoding_callbacks); + EXPECT_TRUE(codec_->encode(body, encoding_context).ok()); } } @@ -1171,19 +1199,41 @@ TEST_F(Http1ClientCodecTest, HeaderOnlyRequestEncodingTest) { HttpRequestFrame request(std::move(headers), true); - NiceMock encoding_callbacks; + NiceMock encoding_context; // Encode the request. { - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "GET /path HTTP/1.1\r\n" "host: host\r\n" "custom: value\r\n" "\r\n"); })); - codec_->encode(request, encoding_callbacks); + EXPECT_TRUE(codec_->encode(request, encoding_context).ok()); + } +} + +TEST_F(Http1ClientCodecTest, MissingRequiredHeadersEncodingTest) { + ON_CALL(codec_callbacks_, connection()) + .WillByDefault(testing::Return(makeOptRef(mock_connection))); + + // Create a request without method. + auto headers = Http::RequestHeaderMapImpl::create(); + headers->addCopy(Http::Headers::get().HostLegacy, "host"); + headers->addCopy(Http::Headers::get().Path, "/path"); + headers->addCopy(Http::LowerCaseString("custom"), "value"); + + HttpRequestFrame request(std::move(headers), true); + + NiceMock encoding_context; + + // Encode the request. + { + auto status_or = codec_->encode(request, encoding_context); + EXPECT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().message(), "missing required headers"); } } @@ -1205,12 +1255,12 @@ TEST_F(Http1ClientCodecTest, ChunkedRequestEncodingTest) { Buffer::OwnedImpl body_buffer("body"); HttpRawBodyFrame body(body_buffer, true); - NiceMock encoding_callbacks; + NiceMock encoding_context; // Encode the request. { - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, false)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "GET /path HTTP/1.1\r\n" "host: host\r\n" "transfer-encoding: chunked\r\n" @@ -1218,10 +1268,10 @@ TEST_F(Http1ClientCodecTest, ChunkedRequestEncodingTest) { buffer.drain(buffer.length()); })); - codec_->encode(request, encoding_callbacks); + EXPECT_TRUE(codec_->encode(request, encoding_context).ok()); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "4\r\n" // Chunk header. "body" // Chunk body. "\r\n" // Chunk footer. @@ -1230,7 +1280,7 @@ TEST_F(Http1ClientCodecTest, ChunkedRequestEncodingTest) { buffer.drain(buffer.length()); })); - codec_->encode(body, encoding_callbacks); + EXPECT_TRUE(codec_->encode(body, encoding_context).ok()); } } @@ -1252,12 +1302,12 @@ TEST_F(Http1ClientCodecTest, RequestAndResponseTest) { Buffer::OwnedImpl body_buffer("body"); HttpRawBodyFrame body(body_buffer, true); - NiceMock encoding_callbacks; - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, _)).Times(2); + NiceMock encoding_context; + EXPECT_CALL(codec_callbacks_, writeToConnection(_)).Times(2); // Encode the request. - codec_->encode(request, encoding_callbacks); - codec_->encode(body, encoding_callbacks); + EXPECT_TRUE(codec_->encode(request, encoding_context).ok()); + EXPECT_TRUE(codec_->encode(body, encoding_context).ok()); Buffer::OwnedImpl buffer; @@ -1292,11 +1342,11 @@ TEST_F(Http1ClientCodecTest, ResponseCompleteBeforeRequestCompleteTest) { HttpRequestFrame request(std::move(headers), false); - NiceMock encoding_callbacks; - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, _)); + NiceMock encoding_context; + EXPECT_CALL(codec_callbacks_, writeToConnection(_)); // Encode the request. Only the headers are encoded and the body is not encoded. - codec_->encode(request, encoding_callbacks); + EXPECT_TRUE(codec_->encode(request, encoding_context).ok()); Buffer::OwnedImpl buffer; @@ -1336,19 +1386,19 @@ TEST_F(Http1ClientCodecTest, SingleFrameModeRequestEncodingTest) { HttpRequestFrame request(std::move(headers), true); request.optionalBuffer().add("body"); - NiceMock encoding_callbacks; + NiceMock encoding_context; // Encode the request. { - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { + EXPECT_CALL(codec_callbacks_, writeToConnection(_)) + .WillOnce(Invoke([](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), "GET /path HTTP/1.1\r\n" "host: host\r\n" "content-length: 4\r\n" "\r\n" "body"); })); - codec_->encode(request, encoding_callbacks); + EXPECT_TRUE(codec_->encode(request, encoding_context).ok()); } } diff --git a/contrib/generic_proxy/filters/network/test/codecs/kafka/config_test.cc b/contrib/generic_proxy/filters/network/test/codecs/kafka/config_test.cc index c9ec702f133e..e2b9c80e53f0 100644 --- a/contrib/generic_proxy/filters/network/test/codecs/kafka/config_test.cc +++ b/contrib/generic_proxy/filters/network/test/codecs/kafka/config_test.cc @@ -144,7 +144,7 @@ TEST(KafkaCodecTest, KafkaServerCodecTest) { { // Test encode() method with non-response frame. - NiceMock encoding_callbacks; + NiceMock encoding_context; auto request = std::make_shared>( @@ -153,30 +153,27 @@ TEST(KafkaCodecTest, KafkaServerCodecTest) { NetworkFilters::Kafka::FetchRequest({}, {}, {}, {})); KafkaRequestFrame request_frame(request); - // Do nothing. - server_codec.encode(request_frame, encoding_callbacks); + auto status_or = server_codec.encode(request_frame, encoding_context); + EXPECT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().message(), "Invalid response frame type"); } { // Test encode() method without actual response. - NiceMock encoding_callbacks; - NiceMock mock_connection; + NiceMock encoding_context; KafkaResponseFrame response_frame(nullptr); - // Expect close connection. - EXPECT_CALL(callbacks, connection()) - .WillOnce(testing::Return(makeOptRef(mock_connection))); - EXPECT_CALL(mock_connection, close(Network::ConnectionCloseType::FlushWrite)); - - server_codec.encode(response_frame, encoding_callbacks); + auto status_or = server_codec.encode(response_frame, encoding_context); + EXPECT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().message(), "Invalid empty response frame"); } { // Test encode() method with response. - NiceMock encoding_callbacks; + NiceMock encoding_context; auto response = std::make_shared>( @@ -191,11 +188,12 @@ TEST(KafkaCodecTest, KafkaServerCodecTest) { dst_buffer.add(&size, sizeof(size)); // Encode data length. response->encode(dst_buffer); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(testing::Invoke([&](Buffer::Instance& buffer, bool) { + EXPECT_CALL(callbacks, writeToConnection(_)) + .WillOnce(testing::Invoke([&](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), dst_buffer.toString()); + buffer.drain(buffer.length()); })); - server_codec.encode(response_frame, encoding_callbacks); + EXPECT_TRUE(server_codec.encode(response_frame, encoding_context).ok()); } } @@ -233,7 +231,7 @@ TEST(KafkaCodecTest, KafkaClientCodecTest) { { // Test encode() method with non-request frame. - NiceMock encoding_callbacks; + NiceMock encoding_context; auto response = std::make_shared>( @@ -242,14 +240,15 @@ TEST(KafkaCodecTest, KafkaClientCodecTest) { NetworkFilters::Kafka::FetchResponse({}, {})); KafkaResponseFrame response_frame(response); - // Do nothing. - client_codec.encode(response_frame, encoding_callbacks); + auto status_or = client_codec.encode(response_frame, encoding_context); + EXPECT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().message(), "Invalid request frame type"); } { // Test encode() method with request. - NiceMock encoding_callbacks; + NiceMock encoding_context; auto request = std::make_shared>( @@ -264,12 +263,13 @@ TEST(KafkaCodecTest, KafkaClientCodecTest) { dst_buffer.add(&size, sizeof(size)); // Encode data length. request->encode(dst_buffer); - EXPECT_CALL(encoding_callbacks, onEncodingSuccess(_, true)) - .WillOnce(testing::Invoke([&](Buffer::Instance& buffer, bool) { + EXPECT_CALL(callbacks, writeToConnection(_)) + .WillOnce(testing::Invoke([&](Buffer::Instance& buffer) { EXPECT_EQ(buffer.toString(), dst_buffer.toString()); + buffer.drain(buffer.length()); })); - client_codec.encode(request_frame, encoding_callbacks); + EXPECT_TRUE(client_codec.encode(request_frame, encoding_context).ok()); } } diff --git a/contrib/generic_proxy/filters/network/test/fake_codec.h b/contrib/generic_proxy/filters/network/test/fake_codec.h index 0cad7217edb3..e25a525e6320 100644 --- a/contrib/generic_proxy/filters/network/test/fake_codec.h +++ b/contrib/generic_proxy/filters/network/test/fake_codec.h @@ -178,7 +178,7 @@ class FakeStreamCodecFactory : public CodecFactory { } } - void encode(const StreamFrame& response, EncodingCallbacks& callback) override { + EncodingResult encode(const StreamFrame& response, EncodingContext&) override { std::string buffer; buffer.reserve(512); buffer += "FAKE-RSP|"; @@ -212,7 +212,11 @@ class FakeStreamCodecFactory : public CodecFactory { encoding_buffer_.writeBEInt(buffer.size()); encoding_buffer_.add(buffer); - callback.onEncodingSuccess(encoding_buffer_, response.frameFlags().endStream()); + const uint64_t encoded_size = encoding_buffer_.length(); + + callback_->writeToConnection(encoding_buffer_); + + return encoded_size; } ResponsePtr respond(Status status, absl::string_view, const Request&) override { @@ -332,7 +336,7 @@ class FakeStreamCodecFactory : public CodecFactory { } } - void encode(const StreamFrame& request, EncodingCallbacks& callback) override { + EncodingResult encode(const StreamFrame& request, EncodingContext&) override { std::string buffer; buffer.reserve(512); buffer += "FAKE-REQ|"; @@ -366,7 +370,11 @@ class FakeStreamCodecFactory : public CodecFactory { encoding_buffer_.writeBEInt(buffer.size()); encoding_buffer_.add(buffer); - callback.onEncodingSuccess(encoding_buffer_, request.frameFlags().endStream()); + const uint64_t encoded_size = encoding_buffer_.length(); + + callback_->writeToConnection(encoding_buffer_); + + return encoded_size; } absl::optional message_size_; diff --git a/contrib/generic_proxy/filters/network/test/integration_test.cc b/contrib/generic_proxy/filters/network/test/integration_test.cc index 0f49099a3cc3..36645c576f84 100644 --- a/contrib/generic_proxy/filters/network/test/integration_test.cc +++ b/contrib/generic_proxy/filters/network/test/integration_test.cc @@ -34,6 +34,8 @@ class GenericProxyIntegrationTest : public BaseIntegrationTest { } }; +// The integration test class for generic proxy. The chain of calls is: +// [Test client] -> [Envoy with Generic proxy] -> [Fake upstream] class IntegrationTest : public testing::TestWithParam { public: struct ConnectionCallbacks : public Network::ConnectionCallbacks { @@ -69,30 +71,27 @@ class IntegrationTest : public testing::TestWithParam; - struct TestRequestEncoderCallback : public EncodingCallbacks { + struct TestEncodingContext : public EncodingContext { OptRef routeEntry() const override { return {}; } - void onEncodingSuccess(Buffer::Instance& buffer, bool) override { buffer_.move(buffer); } - void onEncodingFailure(absl::string_view) override {} - Buffer::OwnedImpl buffer_; }; - using TestRequestEncoderCallbackSharedPtr = std::shared_ptr; + using TestEncodingContextSharedPtr = std::shared_ptr; - struct TestResponseEncoderCallback : public EncodingCallbacks { - OptRef routeEntry() const override { return {}; } - void onEncodingSuccess(Buffer::Instance& buffer, bool) override { buffer_.move(buffer); } - void onEncodingFailure(absl::string_view) override {} - Buffer::OwnedImpl buffer_; + struct SingleResponse { + bool end_stream_{}; + ResponseHeaderFramePtr response_; + std::list response_frames_; }; - using TestResponseEncoderCallbackSharedPtr = std::shared_ptr; - struct TestResponseDecoderCallback : public ClientCodecCallbacks { - TestResponseDecoderCallback(IntegrationTest& parent) : parent_(parent) {} + struct SingleRequest { + bool end_stream_{}; + RequestHeaderFramePtr request_; + std::list request_frames_; + }; - struct SingleResponse { - bool end_stream_{}; - ResponseHeaderFramePtr response_; - std::list response_frames_; - }; + // The callbacks for test client codec. This will used to encode request to Envoy + // and decode response from Envoy. + struct TestClientCodecCallbacks : public ClientCodecCallbacks { + TestClientCodecCallbacks(IntegrationTest& parent) : parent_(parent) {} void onDecodingSuccess(ResponseHeaderFramePtr response_frame, absl::optional) override { @@ -119,26 +118,72 @@ class IntegrationTest : public testing::TestWithParamwrite(buffer, false); + } + } OptRef connection() override { - if (parent_.upstream_connection_ != nullptr) { - return parent_.upstream_connection_->connection(); + if (parent_.client_connection_ != nullptr) { + return *parent_.client_connection_; } return {}; } - OptRef upstreamCluster() const override { - auto result = parent_.integration_->clusterManager().clusters().getCluster("cluster_0"); - if (result.has_value()) { - return makeOptRefFromPtr(result.value().get().info().get()); + OptRef upstreamCluster() const override { return {}; } + + uint64_t waiting_for_stream_id_{}; + std::map responses_; + IntegrationTest& parent_; + }; + using TestClientCodecCallbacksSharedPtr = std::shared_ptr; + + // The callbacks for fake upstream codec. This will used to encode response to Envoy + // and decode request from Envoy. + struct TestServerCodecCallbacks : public ServerCodecCallbacks { + TestServerCodecCallbacks(IntegrationTest& parent) : parent_(parent) {} + + void onDecodingSuccess(RequestHeaderFramePtr request_frame, + absl::optional) override { + auto& request = requests_[request_frame->frameFlags().streamFlags().streamId()]; + ASSERT(!request.end_stream_); + request.end_stream_ = request_frame->frameFlags().endStream(); + request.request_ = std::move(request_frame); + + // Exit dispatcher if we have received all the expected request frames. + if (requests_[waiting_for_stream_id_].end_stream_) { + parent_.integration_->dispatcher_->exit(); + } + } + void onDecodingSuccess(RequestCommonFramePtr frame) override { + auto& request = requests_[frame->frameFlags().streamFlags().streamId()]; + ASSERT(!request.end_stream_); + request.end_stream_ = frame->frameFlags().endStream(); + request.request_frames_.push_back(std::move(frame)); + + // Exit dispatcher if we have received all the expected request frames. + if (requests_[waiting_for_stream_id_].end_stream_) { + parent_.integration_->dispatcher_->exit(); + } + } + + void onDecodingFailure(absl::string_view) override {} + void writeToConnection(Buffer::Instance& buffer) override { + if (parent_.upstream_connection_ != nullptr) { + parent_.upstream_connection_->connection().write(buffer, false); + } + } + OptRef connection() override { + if (parent_.upstream_connection_ != nullptr) { + return parent_.upstream_connection_->connection(); } return {}; } uint64_t waiting_for_stream_id_{}; - std::map responses_; + std::map requests_; IntegrationTest& parent_; }; - using TestResponseDecoderCallbackSharedPtr = std::shared_ptr; + using TestServerCodecCallbacksSharedPtr = std::shared_ptr; void initialize(const std::string& config_yaml, CodecFactoryPtr codec_factory) { integration_ = std::make_unique(config_yaml); @@ -147,14 +192,15 @@ class IntegrationTest : public testing::TestWithParamcreateClientCodec(); + server_codec_ = codec_factory_->createServerCodec(); - request_encoder_callback_ = std::make_shared(); - response_decoder_callback_ = std::make_shared(*this); - client_codec_->setCodecCallbacks(*response_decoder_callback_); + test_encoding_context_ = std::make_shared(); - // Helper codec for upstream server to encode response. - server_codec_ = codec_factory_->createServerCodec(); - response_encoder_callback_ = std::make_shared(); + client_codec_callabcks_ = std::make_shared(*this); + client_codec_->setCodecCallbacks(*client_codec_callabcks_); + + server_codec_callbacks_ = std::make_shared(*this); + server_codec_->setCodecCallbacks(*server_codec_callbacks_); } std::string defaultConfig(bool bind_upstream_connection = false) { @@ -284,11 +330,10 @@ class IntegrationTest : public testing::TestWithParamencode(request, *request_encoder_callback_); - client_connection_->write(request_encoder_callback_->buffer_, false); + // Encode request and write data to client connection. + auto status_or = client_codec_->encode(request, *test_encoding_context_); + ASSERT(status_or.ok()); client_connection_->dispatcher().run(Envoy::Event::Dispatcher::RunType::NonBlock); - // Clear buffer for next encoding. - request_encoder_callback_->buffer_.drain(request_encoder_callback_->buffer_.length()); } // Waiting upstream connection to be created. @@ -307,27 +352,23 @@ class IntegrationTest : public testing::TestWithParamencode(response, *response_encoder_callback_); - - auto result = - upstream_connection_->write(response_encoder_callback_->buffer_.toString(), false); - // Clear buffer for next encoding. - response_encoder_callback_->buffer_.drain(response_encoder_callback_->buffer_.length()); - RELEASE_ASSERT(result, result.failure_message()); + // Encode response and write data to upstream connection. + auto status_or = server_codec_->encode(response, *test_encoding_context_); + ASSERT(status_or.ok()); } // Waiting for downstream response. AssertionResult waitDownstreamResponseForTest(std::chrono::milliseconds timeout, uint64_t stream_id) { bool timer_fired = false; - if (!response_decoder_callback_->responses_[stream_id].end_stream_) { + if (!client_codec_callabcks_->responses_[stream_id].end_stream_) { Envoy::Event::TimerPtr timer( integration_->dispatcher_->createTimer([this, &timer_fired]() -> void { timer_fired = true; integration_->dispatcher_->exit(); })); timer->enableTimer(timeout); - response_decoder_callback_->waiting_for_stream_id_ = stream_id; + client_codec_callabcks_->waiting_for_stream_id_ = stream_id; integration_->dispatcher_->run(Envoy::Event::Dispatcher::RunType::Block); if (timer_fired) { return AssertionFailure() << "Timed out waiting for response"; @@ -336,7 +377,7 @@ class IntegrationTest : public testing::TestWithParamdisableTimer(); } } - if (!response_decoder_callback_->responses_[stream_id].end_stream_) { + if (!client_codec_callabcks_->responses_[stream_id].end_stream_) { return AssertionFailure() << "No response or response not complete"; } return AssertionSuccess(); @@ -360,9 +401,9 @@ class IntegrationTest : public testing::TestWithParam integration_; @@ -406,8 +447,8 @@ TEST_P(IntegrationTest, RequestRouteNotFound) { RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 0), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->responses_[0].response_, nullptr); - EXPECT_EQ(response_decoder_callback_->responses_[0].response_->status().code(), + EXPECT_NE(client_codec_callabcks_->responses_[0].response_, nullptr); + EXPECT_EQ(client_codec_callabcks_->responses_[0].response_->status().code(), static_cast(absl::StatusCode::kNotFound)); cleanup(); @@ -445,9 +486,9 @@ TEST_P(IntegrationTest, RequestAndResponse) { RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 0), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->responses_[0].response_, nullptr); - EXPECT_EQ(response_decoder_callback_->responses_[0].response_->status().code(), 0); - EXPECT_EQ(response_decoder_callback_->responses_[0].response_->get("zzzz"), "xxxx"); + EXPECT_NE(client_codec_callabcks_->responses_[0].response_, nullptr); + EXPECT_EQ(client_codec_callabcks_->responses_[0].response_->status().code(), 0); + EXPECT_EQ(client_codec_callabcks_->responses_[0].response_->get("zzzz"), "xxxx"); cleanup(); } @@ -479,8 +520,8 @@ TEST_P(IntegrationTest, RequestTimeout) { RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 0), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->responses_[0].response_, nullptr); - EXPECT_EQ(response_decoder_callback_->responses_[0].response_->status().code(), 4); + EXPECT_NE(client_codec_callabcks_->responses_[0].response_, nullptr); + EXPECT_EQ(client_codec_callabcks_->responses_[0].response_->status().code(), 4); cleanup(); } @@ -560,7 +601,7 @@ TEST_P(IntegrationTest, MultipleRequests) { request_2.data_ = {{"version", "v1"}, {"stream_id", "2"}, {"frame", "2_header"}}; // Reset request encoder callback. - request_encoder_callback_ = std::make_shared(); + test_encoding_context_ = std::make_shared(); // Send the second request with the different stream id and expect the connection to be alive. sendRequestForTest(request_2); @@ -581,12 +622,11 @@ TEST_P(IntegrationTest, MultipleRequests) { RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 2), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->responses_[2].response_, nullptr); - EXPECT_EQ(response_decoder_callback_->responses_[2].response_->status().code(), 0); - EXPECT_EQ(response_decoder_callback_->responses_[2].response_->get("zzzz"), "xxxx"); - EXPECT_EQ( - response_decoder_callback_->responses_[2].response_->frameFlags().streamFlags().streamId(), - 2); + EXPECT_NE(client_codec_callabcks_->responses_[2].response_, nullptr); + EXPECT_EQ(client_codec_callabcks_->responses_[2].response_->status().code(), 0); + EXPECT_EQ(client_codec_callabcks_->responses_[2].response_->get("zzzz"), "xxxx"); + EXPECT_EQ(client_codec_callabcks_->responses_[2].response_->frameFlags().streamFlags().streamId(), + 2); FakeStreamCodecFactory::FakeResponse response_1; response_1.protocol_ = "fake_fake_fake"; @@ -599,12 +639,11 @@ TEST_P(IntegrationTest, MultipleRequests) { RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 1), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->responses_[1].response_, nullptr); - EXPECT_EQ(response_decoder_callback_->responses_[1].response_->status().code(), 0); - EXPECT_EQ(response_decoder_callback_->responses_[1].response_->get("zzzz"), "yyyy"); - EXPECT_EQ( - response_decoder_callback_->responses_[1].response_->frameFlags().streamFlags().streamId(), - 1); + EXPECT_NE(client_codec_callabcks_->responses_[1].response_, nullptr); + EXPECT_EQ(client_codec_callabcks_->responses_[1].response_->status().code(), 0); + EXPECT_EQ(client_codec_callabcks_->responses_[1].response_->get("zzzz"), "yyyy"); + EXPECT_EQ(client_codec_callabcks_->responses_[1].response_->frameFlags().streamFlags().streamId(), + 1); cleanup(); } @@ -716,12 +755,11 @@ TEST_P(IntegrationTest, MultipleRequestsWithMultipleFrames) { RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 2), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->responses_[2].response_, nullptr); - EXPECT_EQ(response_decoder_callback_->responses_[2].response_->status().code(), 0); - EXPECT_EQ(response_decoder_callback_->responses_[2].response_->get("zzzz"), "xxxx"); - EXPECT_EQ( - response_decoder_callback_->responses_[2].response_->frameFlags().streamFlags().streamId(), - 2); + EXPECT_NE(client_codec_callabcks_->responses_[2].response_, nullptr); + EXPECT_EQ(client_codec_callabcks_->responses_[2].response_->status().code(), 0); + EXPECT_EQ(client_codec_callabcks_->responses_[2].response_->get("zzzz"), "xxxx"); + EXPECT_EQ(client_codec_callabcks_->responses_[2].response_->frameFlags().streamFlags().streamId(), + 2); FakeStreamCodecFactory::FakeResponse response_1; response_1.protocol_ = "fake_fake_fake"; @@ -740,12 +778,11 @@ TEST_P(IntegrationTest, MultipleRequestsWithMultipleFrames) { RELEASE_ASSERT(waitDownstreamResponseForTest(TestUtility::DefaultTimeout, 1), "unexpected timeout"); - EXPECT_NE(response_decoder_callback_->responses_[1].response_, nullptr); - EXPECT_EQ(response_decoder_callback_->responses_[1].response_->status().code(), 0); - EXPECT_EQ(response_decoder_callback_->responses_[1].response_->get("zzzz"), "yyyy"); - EXPECT_EQ( - response_decoder_callback_->responses_[1].response_->frameFlags().streamFlags().streamId(), - 1); + EXPECT_NE(client_codec_callabcks_->responses_[1].response_, nullptr); + EXPECT_EQ(client_codec_callabcks_->responses_[1].response_->status().code(), 0); + EXPECT_EQ(client_codec_callabcks_->responses_[1].response_->get("zzzz"), "yyyy"); + EXPECT_EQ(client_codec_callabcks_->responses_[1].response_->frameFlags().streamFlags().streamId(), + 1); cleanup(); } diff --git a/contrib/generic_proxy/filters/network/test/mocks/codec.h b/contrib/generic_proxy/filters/network/test/mocks/codec.h index a1e8da1ccef4..baff48a13746 100644 --- a/contrib/generic_proxy/filters/network/test/mocks/codec.h +++ b/contrib/generic_proxy/filters/network/test/mocks/codec.h @@ -8,8 +8,16 @@ namespace Extensions { namespace NetworkFilters { namespace GenericProxy { +using testing::_; + class MockServerCodecCallbacks : public ServerCodecCallbacks { public: + MockServerCodecCallbacks() { + ON_CALL(*this, writeToConnection(_)) + .WillByDefault( + testing::Invoke([](Buffer::Instance& buffer) { buffer.drain(buffer.length()); })); + } + MOCK_METHOD(void, onDecodingSuccess, (RequestHeaderFramePtr, absl::optional)); MOCK_METHOD(void, onDecodingSuccess, (RequestCommonFramePtr)); MOCK_METHOD(void, onDecodingFailure, (absl::string_view)); @@ -20,6 +28,12 @@ class MockServerCodecCallbacks : public ServerCodecCallbacks { class MockClientCodecCallbacks : public ClientCodecCallbacks { public: + MockClientCodecCallbacks() { + ON_CALL(*this, writeToConnection(_)) + .WillByDefault( + testing::Invoke([](Buffer::Instance& buffer) { buffer.drain(buffer.length()); })); + } + MOCK_METHOD(void, onDecodingSuccess, (ResponseHeaderFramePtr, absl::optional)); MOCK_METHOD(void, onDecodingSuccess, (ResponseCommonFramePtr)); MOCK_METHOD(void, onDecodingFailure, (absl::string_view)); @@ -28,26 +42,33 @@ class MockClientCodecCallbacks : public ClientCodecCallbacks { MOCK_METHOD(OptRef, upstreamCluster, (), (const)); }; -class MockEncodingCallbacks : public EncodingCallbacks { +class MockEncodingContext : public EncodingContext { public: - MOCK_METHOD(void, onEncodingSuccess, (Buffer::Instance & buffer, bool end_stream)); - MOCK_METHOD(void, onEncodingFailure, (absl::string_view)); MOCK_METHOD(OptRef, routeEntry, (), (const)); }; class MockServerCodec : public ServerCodec { public: + MockServerCodec() { + ON_CALL(*this, encode(_, _)).WillByDefault(testing::Return(EncodingResult{0})); + } + MOCK_METHOD(void, setCodecCallbacks, (ServerCodecCallbacks & callbacks)); MOCK_METHOD(void, decode, (Buffer::Instance & buffer, bool end_stream)); - MOCK_METHOD(void, encode, (const StreamFrame&, EncodingCallbacks& callbacks)); - MOCK_METHOD(ResponsePtr, respond, (Status status, absl::string_view, const Request&)); + MOCK_METHOD(EncodingResult, encode, (const StreamFrame&, EncodingContext& ctx)); + MOCK_METHOD(ResponseHeaderFramePtr, respond, + (Status status, absl::string_view, const RequestHeaderFrame&)); }; class MockClientCodec : public ClientCodec { public: + MockClientCodec() { + ON_CALL(*this, encode(_, _)).WillByDefault(testing::Return(EncodingResult{0})); + } + MOCK_METHOD(void, setCodecCallbacks, (ClientCodecCallbacks & callbacks)); MOCK_METHOD(void, decode, (Buffer::Instance & buffer, bool end_stream)); - MOCK_METHOD(void, encode, (const StreamFrame&, EncodingCallbacks& callbacks)); + MOCK_METHOD(EncodingResult, encode, (const StreamFrame&, EncodingContext& ctx)); }; class MockCodecFactory : public CodecFactory { diff --git a/contrib/generic_proxy/filters/network/test/mocks/filter.h b/contrib/generic_proxy/filters/network/test/mocks/filter.h index 6017bf868445..7a92048bf1ae 100644 --- a/contrib/generic_proxy/filters/network/test/mocks/filter.h +++ b/contrib/generic_proxy/filters/network/test/mocks/filter.h @@ -108,8 +108,8 @@ class MockDecoderFilterCallback : public MockStreamFilterCallbacks(filter_config_, factory_context_); - EXPECT_EQ(filter_.get(), decoder_callback_); + EXPECT_EQ(filter_.get(), server_codec_callbacks_); filter_->initializeReadFilterCallbacks(filter_callbacks_); } std::shared_ptr filter_; - ServerCodecCallbacks* decoder_callback_{}; + ServerCodecCallbacks* server_codec_callbacks_{}; NiceMock* server_codec_{}; @@ -232,7 +232,7 @@ TEST_F(FilterTest, OnDecodingFailureWithoutActiveStreams) { filter_->onData(fake_empty_buffer, false); EXPECT_CALL(filter_callbacks_.connection_, close(_)); - decoder_callback_->onDecodingFailure(); + server_codec_callbacks_->onDecodingFailure(); EXPECT_EQ(filter_config_->stats().downstream_rq_decoding_error_.value(), 1); EXPECT_EQ(filter_config_->stats().downstream_rq_total_.value(), 0); @@ -258,7 +258,7 @@ TEST_F(FilterTest, OnDecodingSuccessWithNormalRequest) { // Three mock factories was added. EXPECT_CALL(*mock_stream_filter, onStreamDecoded(_)).Times(3); - decoder_callback_->onDecodingSuccess(std::move(request)); + server_codec_callbacks_->onDecodingSuccess(std::move(request)); EXPECT_EQ(filter_config_->stats().downstream_rq_total_.value(), 1); EXPECT_EQ(filter_config_->stats().downstream_rq_active_.value(), 1); @@ -283,32 +283,6 @@ TEST_F(FilterTest, OnConnectionClosedEvent) { EXPECT_EQ(Network::FilterStatus::StopIteration, filter_->onData(fake_empty_buffer, false)); } -TEST_F(FilterTest, SendReplyDownstream) { - initializeFilter(); - - NiceMock encoder_callback; - - auto response = std::make_unique(); - - Buffer::OwnedImpl response_buffer; - - EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); - - EXPECT_CALL(encoder_callback, onEncodingSuccess(_, _)) - .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) { - filter_callbacks_.connection_.write(buffer, false); - })); - - EXPECT_CALL(*server_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { - Buffer::OwnedImpl buffer; - buffer.add("test"); - callback.onEncodingSuccess(buffer, true); - })); - - filter_->sendFrameToDownstream(*response, encoder_callback); -} - TEST_F(FilterTest, GetConnection) { initializeFilter(); @@ -406,7 +380,7 @@ TEST_F(FilterTest, OnDecodingFailureWithActiveStreams) { filter_->onData(fake_empty_buffer, false); EXPECT_CALL(filter_callbacks_.connection_, close(_)); - decoder_callback_->onDecodingFailure(); + server_codec_callbacks_->onDecodingFailure(); EXPECT_EQ(0, filter_->activeStreamsForTest().size()); @@ -436,8 +410,10 @@ TEST_F(FilterTest, OnEncodingFailureWithActiveStreams) { EXPECT_EQ(filter_config_->stats().downstream_rq_total_.value(), 2); EXPECT_EQ(filter_config_->stats().downstream_rq_active_.value(), 2); - // One of stream encoding failed. - filter_->activeStreamsForTest().begin()->get()->onEncodingFailure(); + EXPECT_CALL(*server_codec_, encode(_, _)) + .WillOnce(Return(EncodingResult{absl::InvalidArgumentError("encoding-error")})); + auto response_0 = std::make_unique(); + filter_->activeStreamsForTest().begin()->get()->onResponseHeaderFrame(std::move(response_0)); EXPECT_EQ(1, filter_->activeStreamsForTest().size()); @@ -674,8 +650,8 @@ TEST_F(FilterTest, ActiveStreamFiltersContinueEncoding) { EXPECT_EQ(0, active_stream->nextEncoderFilterIndexForTest()); auto response = std::make_unique(); - // `continueEncoding` will be called in the `onResponseStart`. - active_stream->onResponseStart(std::move(response)); + // `continueEncoding` will be called in the `onResponseHeaderFrame`. + active_stream->onResponseHeaderFrame(std::move(response)); // Encoding will be stopped when `onStreamEncoded` of `mock_stream_filter_1` is called. EXPECT_EQ(2, active_stream->nextEncoderFilterIndexForTest()); @@ -683,10 +659,14 @@ TEST_F(FilterTest, ActiveStreamFiltersContinueEncoding) { EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); EXPECT_CALL(*server_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { + .WillOnce(Invoke([&](const StreamFrame&, EncodingContext&) { Buffer::OwnedImpl buffer; buffer.add("test"); - callback.onEncodingSuccess(buffer, true); + + server_codec_callbacks_->writeToConnection(buffer); + buffer.drain(buffer.length()); + + return EncodingResult{4}; })); active_stream->encoderFiltersForTest()[1]->continueEncoding(); @@ -729,12 +709,16 @@ TEST_F(FilterTest, ActiveStreamSendLocalReply) { EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); EXPECT_CALL(*server_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame& response, EncodingCallbacks& callback) { + .WillOnce(Invoke([&](const StreamFrame& response, EncodingContext&) { Buffer::OwnedImpl buffer; EXPECT_EQ(dynamic_cast(&response)->status().code(), static_cast(StatusCode::kUnknown)); buffer.add("test"); - callback.onEncodingSuccess(buffer, true); + + server_codec_callbacks_->writeToConnection(buffer); + buffer.drain(buffer.length()); + + return EncodingResult{4}; })); active_stream->sendLocalReply( @@ -817,10 +801,14 @@ TEST_F(FilterTest, NewStreamAndReplyNormally) { EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); EXPECT_CALL(*server_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { + .WillOnce(Invoke([&](const StreamFrame&, EncodingContext&) { Buffer::OwnedImpl buffer; buffer.add("test"); - callback.onEncodingSuccess(buffer, true); + + server_codec_callbacks_->writeToConnection(buffer); + buffer.drain(buffer.length()); + + return EncodingResult{4}; })); EXPECT_CALL(*factory_context_.server_factory_context_.access_log_manager_.file_, @@ -834,7 +822,7 @@ TEST_F(FilterTest, NewStreamAndReplyNormally) { response->status_ = {0, true}; response->data_["response-key"] = "response-value"; - active_stream->onResponseStart(std::move(response)); + active_stream->onResponseHeaderFrame(std::move(response)); EXPECT_EQ(filter_config_->stats().downstream_rq_total_.value(), 1); EXPECT_EQ(filter_config_->stats().downstream_rq_active_.value(), 0); @@ -904,10 +892,14 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithMultipleFrames) { EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)).Times(2); EXPECT_CALL(*server_codec_, encode(_, _)) .Times(2) - .WillRepeatedly(Invoke([&](const StreamFrame& frame, EncodingCallbacks& callback) { + .WillRepeatedly(Invoke([&](const StreamFrame&, EncodingContext&) { Buffer::OwnedImpl buffer; buffer.add("test"); - callback.onEncodingSuccess(buffer, frame.frameFlags().endStream()); + + server_codec_callbacks_->writeToConnection(buffer); + buffer.drain(buffer.length()); + + return EncodingResult{4}; })); auto response = std::make_unique(); @@ -915,12 +907,12 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithMultipleFrames) { response->stream_frame_flags_ = FrameFlags(StreamFlags(), false); response->status_ = {123, false}; // Response non-OK. - active_stream->onResponseStart(std::move(response)); + active_stream->onResponseHeaderFrame(std::move(response)); auto response_frame_1 = std::make_unique(); response_frame_1->stream_frame_flags_ = FrameFlags(StreamFlags(), true); - active_stream->onResponseFrame(std::move(response_frame_1)); + active_stream->onResponseCommonFrame(std::move(response_frame_1)); EXPECT_EQ(filter_config_->stats().downstream_rq_total_.value(), 1); EXPECT_EQ(filter_config_->stats().downstream_rq_active_.value(), 0); @@ -950,10 +942,14 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithDrainClose) { EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); EXPECT_CALL(*server_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { + .WillOnce(Invoke([&](const StreamFrame&, EncodingContext&) { Buffer::OwnedImpl buffer; buffer.add("test"); - callback.onEncodingSuccess(buffer, true); + + server_codec_callbacks_->writeToConnection(buffer); + buffer.drain(buffer.length()); + + return EncodingResult{4}; })); EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(true)); @@ -963,7 +959,7 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithDrainClose) { auto response = std::make_unique(); response->status_ = {234, false}; // Response non-OK. active_stream->streamInfo().setResponseFlag(StreamInfo::CoreResponseFlag::UpstreamProtocolError); - active_stream->onResponseStart(std::move(response)); + active_stream->onResponseHeaderFrame(std::move(response)); EXPECT_EQ(filter_config_->stats().downstream_rq_total_.value(), 1); EXPECT_EQ(filter_config_->stats().downstream_rq_active_.value(), 0); @@ -992,10 +988,14 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithStreamDrainClose) { EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); EXPECT_CALL(*server_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { + .WillOnce(Invoke([&](const StreamFrame&, EncodingContext&) { Buffer::OwnedImpl buffer; buffer.add("test"); - callback.onEncodingSuccess(buffer, true); + + server_codec_callbacks_->writeToConnection(buffer); + buffer.drain(buffer.length()); + + return EncodingResult{4}; })); // The drain close of factory_context_.drain_manager_ is false, but the drain close of @@ -1006,7 +1006,7 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithStreamDrainClose) { auto response = std::make_unique(); response->stream_frame_flags_ = FrameFlags(StreamFlags(0, false, true, false), true); - active_stream->onResponseStart(std::move(response)); + active_stream->onResponseHeaderFrame(std::move(response)); } TEST_F(FilterTest, NewStreamAndReplyNormallyWithTracing) { @@ -1037,17 +1037,21 @@ TEST_F(FilterTest, NewStreamAndReplyNormallyWithTracing) { EXPECT_CALL(filter_callbacks_.connection_, write(BufferStringEqual("test"), false)); EXPECT_CALL(*server_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) { + .WillOnce(Invoke([&](const StreamFrame&, EncodingContext&) { Buffer::OwnedImpl buffer; buffer.add("test"); - callback.onEncodingSuccess(buffer, true); + + server_codec_callbacks_->writeToConnection(buffer); + buffer.drain(buffer.length()); + + return EncodingResult{4}; })); EXPECT_CALL(factory_context_.drain_manager_, drainClose()).WillOnce(Return(false)); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)); auto response = std::make_unique(); - active_stream->onResponseStart(std::move(response)); + active_stream->onResponseHeaderFrame(std::move(response)); } } // namespace diff --git a/contrib/generic_proxy/filters/network/test/router/router_test.cc b/contrib/generic_proxy/filters/network/test/router/router_test.cc index 42c27df6b5e3..6dfbb9bb2f91 100644 --- a/contrib/generic_proxy/filters/network/test/router/router_test.cc +++ b/contrib/generic_proxy/filters/network/test/router/router_test.cc @@ -258,7 +258,6 @@ class RouterFilterTest : public testing::Test { NiceMock tracing_config_; NiceMock active_span_; NiceMock* child_span_{}; - bool with_tracing_{}; uint32_t creating_connection_{}; }; @@ -524,21 +523,10 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndExpectNoResponse) { EXPECT_EQ(0, filter_->upstreamRequestsSize()); })); - EXPECT_CALL(*mock_generic_upstream_, upstreamConnection()); - EXPECT_CALL(mock_generic_upstream_->mock_upstream_connection_, write(_, _)) - .WillOnce(Invoke( - [](Buffer::Instance& buffer, bool) -> void { EXPECT_EQ(buffer.toString(), "hello"); })); - EXPECT_CALL(*mock_generic_upstream_, removeUpstreamRequest(_)); EXPECT_CALL(*mock_generic_upstream_, cleanUp(false)); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect no response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); expectInjectContextToUpstreamRequest(); expectFinalizeUpstreamSpanAny(); @@ -555,13 +543,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyButConnectionErrorBeforeRespons setup(); kickOffNewUpstreamRequest(); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); notifyUpstreamSuccess(); @@ -585,13 +567,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyButConnectionTerminationBeforeR setup(); kickOffNewUpstreamRequest(); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); notifyUpstreamSuccess(); @@ -615,13 +591,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyButStreamDestroyBeforeResponse) setup(); kickOffNewUpstreamRequest(); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); notifyUpstreamSuccess(); @@ -640,19 +610,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponse) { setup(); kickOffNewUpstreamRequest(true); - EXPECT_CALL(*mock_generic_upstream_, upstreamConnection()); - EXPECT_CALL(mock_generic_upstream_->mock_upstream_connection_, write(_, _)) - .WillOnce(Invoke([](Buffer::Instance& buffer, bool) -> void { - EXPECT_EQ(buffer.toString(), "helloxxxxxx"); - })); - - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("helloxxxxxx"); - // Expect response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); expectInjectContextToUpstreamRequest(); @@ -662,7 +620,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponse) { EXPECT_CALL(*mock_generic_upstream_, removeUpstreamRequest(_)); EXPECT_CALL(*mock_generic_upstream_, cleanUp(false)); - EXPECT_CALL(mock_filter_callback_, onResponseStart(_)).WillOnce(Invoke([this](ResponsePtr) { + EXPECT_CALL(mock_filter_callback_, onResponseHeaderFrame(_)).WillOnce(Invoke([this](ResponsePtr) { // When the response is sent to callback, the upstream request should be removed. EXPECT_EQ(0, filter_->upstreamRequestsSize()); })); @@ -678,13 +636,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponseWithStartTime) { setup(); kickOffNewUpstreamRequest(true); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); expectInjectContextToUpstreamRequest(); @@ -694,7 +646,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponseWithStartTime) { EXPECT_CALL(*mock_generic_upstream_, removeUpstreamRequest(_)); EXPECT_CALL(*mock_generic_upstream_, cleanUp(false)); - EXPECT_CALL(mock_filter_callback_, onResponseStart(_)).WillOnce(Invoke([this](ResponsePtr) { + EXPECT_CALL(mock_filter_callback_, onResponseHeaderFrame(_)).WillOnce(Invoke([this](ResponsePtr) { // When the response is sent to callback, the upstream request should be removed. EXPECT_EQ(0, filter_->upstreamRequestsSize()); })); @@ -724,13 +676,8 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponseAndTimeout) { kickOffNewUpstreamRequest(true); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); + expectInjectContextToUpstreamRequest(); notifyUpstreamSuccess(); @@ -739,7 +686,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponseAndTimeout) { EXPECT_CALL(*mock_generic_upstream_, removeUpstreamRequest(_)); EXPECT_CALL(*mock_generic_upstream_, cleanUp(false)); - EXPECT_CALL(mock_filter_callback_, onResponseStart(_)).WillOnce(Invoke([this](ResponsePtr) { + EXPECT_CALL(mock_filter_callback_, onResponseHeaderFrame(_)).WillOnce(Invoke([this](ResponsePtr) { // When the response is sent to callback, the upstream request should be removed. EXPECT_EQ(0, filter_->upstreamRequestsSize()); })); @@ -762,26 +709,14 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponseWithMultipleFrames) // This only store the frame and does nothing else because the pool is not ready yet. filter_->onRequestCommonFrame(std::move(frame_1)); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .Times(2) - .WillRepeatedly(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect response. - callback.onEncodingSuccess(buffer, false); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)).Times(2); + expectInjectContextToUpstreamRequest(); // This will trigger two frames to be sent. notifyUpstreamSuccess(); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); // End stream is set to true by default. auto frame_2 = std::make_unique(); @@ -793,8 +728,8 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponseWithMultipleFrames) EXPECT_CALL(*mock_generic_upstream_, removeUpstreamRequest(_)); EXPECT_CALL(*mock_generic_upstream_, cleanUp(false)); - EXPECT_CALL(mock_filter_callback_, onResponseStart(_)); - EXPECT_CALL(mock_filter_callback_, onResponseFrame(_)) + EXPECT_CALL(mock_filter_callback_, onResponseHeaderFrame(_)); + EXPECT_CALL(mock_filter_callback_, onResponseCommonFrame(_)) .Times(2) .WillRepeatedly(Invoke([this](ResponseCommonFramePtr frame) { // When the entire response is sent to callback, the upstream request should be removed. @@ -826,20 +761,14 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponseWithDrainCloseSetInR setup(); kickOffNewUpstreamRequest(); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); notifyUpstreamSuccess(); EXPECT_CALL(*mock_generic_upstream_, removeUpstreamRequest(_)); EXPECT_CALL(*mock_generic_upstream_, cleanUp(true)); - EXPECT_CALL(mock_filter_callback_, onResponseStart(_)).WillOnce(Invoke([this](ResponsePtr) { + EXPECT_CALL(mock_filter_callback_, onResponseHeaderFrame(_)).WillOnce(Invoke([this](ResponsePtr) { // When the response is sent to callback, the upstream request should be removed. EXPECT_EQ(0, filter_->upstreamRequestsSize()); })); @@ -856,13 +785,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndResponseDecodingFailure) { setup(); kickOffNewUpstreamRequest(); - EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - Buffer::OwnedImpl buffer; - buffer.add("hello"); - // Expect response. - callback.onEncodingSuccess(buffer, true); - })); + EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)); notifyUpstreamSuccess(); @@ -887,9 +810,7 @@ TEST_F(RouterFilterTest, UpstreamRequestPoolReadyAndRequestEncodingFailure) { kickOffNewUpstreamRequest(); EXPECT_CALL(mock_generic_upstream_->mock_client_codec_, encode(_, _)) - .WillOnce(Invoke([&](const StreamFrame&, EncodingCallbacks& callback) -> void { - callback.onEncodingFailure("encoding-failure"); - })); + .WillOnce(Return(EncodingResult(absl::InvalidArgumentError("encoding-failure")))); EXPECT_CALL(mock_filter_callback_, sendLocalReply(_, _, _)) .WillOnce(Invoke([this](Status status, absl::string_view data, ResponseUpdateFunction) { From c292f796736be6e3c46cbca655cd2a653e129671 Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Tue, 4 Jun 2024 21:19:33 -0500 Subject: [PATCH 04/61] mobile: Enable CI for C++ tests with xDS and full protos enabled (#34516) Risk Level: low Testing: CI Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a Signed-off-by: Fredy Wijaya --- .github/workflows/mobile-compile_time_options.yml | 12 ++++++++++++ mobile/.bazelrc | 10 +++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mobile-compile_time_options.yml b/.github/workflows/mobile-compile_time_options.yml index 20522bfc21fb..a94335618cc0 100644 --- a/.github/workflows/mobile-compile_time_options.yml +++ b/.github/workflows/mobile-compile_time_options.yml @@ -56,6 +56,18 @@ jobs: build --config=mobile-remote-ci-cc-no-exceptions //test/performance:test_binary_size //library/cc/... + - name: Running C++ tests with xDS enabled + target: cc-tests-xds-enabled + args: >- + test + --config=mobile-remote-ci-cc-xds-enabled + //test/common/integration/... + - name: Running C++ tests with full protos enabled + target: cc-tests-full-protos-enabled + args: >- + test + --config=mobile-remote-ci-cc-full-protos-enabled + //test/common/... //test/cc/... build: permissions: diff --git a/mobile/.bazelrc b/mobile/.bazelrc index 861a52bec7ae..e8efba686abb 100644 --- a/mobile/.bazelrc +++ b/mobile/.bazelrc @@ -254,9 +254,13 @@ build:mobile-remote-ci-cc-no-exceptions --config=mobile-remote-ci-cc build:mobile-remote-ci-cc-no-exceptions --define envoy_exceptions=disabled build:mobile-remote-ci-cc-no-exceptions --copt=-fno-exceptions -build:mobile-remote-ci-cc-test --config=mobile-remote-ci-cc -test:mobile-remote-ci-cc-test --test_output=all -test:mobile-remote-ci-cc-test --config=mobile-remote-ci-cc +build:mobile-remote-ci-cc-xds-enabled --config=mobile-remote-ci-cc +test:mobile-remote-ci-cc-xds-enabled --config=mobile-remote-ci-cc +test:mobile-remote-ci-cc-xds-enabled --define=envoy_mobile_xds=enabled + +build:mobile-remote-ci-cc-full-protos-enabled --config=mobile-remote-ci-cc +test:mobile-remote-ci-cc-full-protos-enabled --config=mobile-remote-ci-cc +test:mobile-remote-ci-cc-full-protos-enabled --define=envoy_full_protos=enabled build:mobile-remote-ci-macos-kotlin --config=mobile-remote-ci-macos build:mobile-remote-ci-macos-kotlin --fat_apk_cpu=x86_64 From ea91935163bd573ec5a0ec46b661b4e4dd65343b Mon Sep 17 00:00:00 2001 From: Pandurang Khandeparker Date: Wed, 5 Jun 2024 08:03:16 +0530 Subject: [PATCH 05/61] Fix zipkin_test on big-endian systems (#34448) The subtest "ZipkinSpanBufferTest.SerializeSpan" was failing due to discrepancies in the id field of the expected JSON string and the serialized JSON representation. The expected id was AQAAAAAAAAA= but we were getting AAAAAAAAAAE= on s390x. This was only occurring when serializing the buffer with HTTP_PROTO whereas HTTP_JSON was working fine. On checking more into the serialized MessageToJsonString, I found that the hexadecimal representation of id was id: "\000\000\000\000\000\000\000\001" which is id:"AAAAAAAAAAE=" on s390x and that on Intel was id: "\001\000\000\000\000\000\000\000" which is id:"AQAAAAAAAAA=" . Further found that this is occurring due to endianness, and the value of id on both Intel and s390x is same in respective endianness and because the expected content is hard coded in little-endian format, it was failing on big-endian system. Signed-off-by: Pandurang Khandeparker --- .../tracers/zipkin/span_buffer_test.cc | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/extensions/tracers/zipkin/span_buffer_test.cc b/test/extensions/tracers/zipkin/span_buffer_test.cc index 85ae80edd300..cd5cd5f5ac85 100644 --- a/test/extensions/tracers/zipkin/span_buffer_test.cc +++ b/test/extensions/tracers/zipkin/span_buffer_test.cc @@ -327,7 +327,11 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { EXPECT_EQ(withDefaultTimestampAndDuration("{" R"("spans":[{)" R"("traceId":"AAAAAAAAAAE=",)" +#ifdef ABSL_IS_BIG_ENDIAN + R"("id":"AAAAAAAAAAE=",)" +#else R"("id":"AQAAAAAAAAA=",)" +#endif R"("kind":"CLIENT",)" R"("timestamp":"ANNOTATION_TEST_TIMESTAMP",)" R"("duration":"DEFAULT_TEST_DURATION",)" @@ -346,7 +350,11 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { "{" R"("spans":[{)" R"("traceId":"AAAAAAAAAAE=",)" +#ifdef ABSL_IS_BIG_ENDIAN + R"("id":"AAAAAAAAAAE=",)" +#else R"("id":"AQAAAAAAAAA=",)" +#endif R"("kind":"CLIENT",)" R"("timestamp":"ANNOTATION_TEST_TIMESTAMP",)" R"("duration":"DEFAULT_TEST_DURATION",)" @@ -365,7 +373,11 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { EXPECT_EQ(withDefaultTimestampAndDuration("{" R"("spans":[{)" R"("traceId":"AAAAAAAAAAE=",)" +#ifdef ABSL_IS_BIG_ENDIAN + R"("id":"AAAAAAAAAAE=",)" +#else R"("id":"AQAAAAAAAAA=",)" +#endif R"("kind":"CLIENT",)" R"("timestamp":"ANNOTATION_TEST_TIMESTAMP",)" R"("duration":"DEFAULT_TEST_DURATION",)" @@ -377,7 +389,11 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { R"("response_size":"DEFAULT_TEST_DURATION"}},)" R"({)" R"("traceId":"AAAAAAAAAAE=",)" +#ifdef ABSL_IS_BIG_ENDIAN + R"("id":"AAAAAAAAAAE=",)" +#else R"("id":"AQAAAAAAAAA=",)" +#endif R"("kind":"SERVER",)" R"("timestamp":"ANNOTATION_TEST_TIMESTAMP",)" R"("duration":"DEFAULT_TEST_DURATION",)" @@ -396,7 +412,11 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { EXPECT_EQ(withDefaultTimestampAndDuration("{" R"("spans":[{)" R"("traceId":"AAAAAAAAAAE=",)" +#ifdef ABSL_IS_BIG_ENDIAN + R"("id":"AAAAAAAAAAE=",)" +#else R"("id":"AQAAAAAAAAA=",)" +#endif R"("kind":"CLIENT",)" R"("timestamp":"ANNOTATION_TEST_TIMESTAMP",)" R"("duration":"DEFAULT_TEST_DURATION",)" @@ -408,7 +428,11 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { R"("response_size":"DEFAULT_TEST_DURATION"}},)" R"({)" R"("traceId":"AAAAAAAAAAE=",)" +#ifdef ABSL_IS_BIG_ENDIAN + R"("id":"AAAAAAAAAAE=",)" +#else R"("id":"AQAAAAAAAAA=",)" +#endif R"("kind":"SERVER",)" R"("timestamp":"ANNOTATION_TEST_TIMESTAMP",)" R"("duration":"DEFAULT_TEST_DURATION",)" From 94aa1fc58902702aef49c68ba7d85e79cb2be6ff Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Tue, 4 Jun 2024 23:33:24 -0500 Subject: [PATCH 06/61] mobile: Temporarily disable RtdsIntegrationTest (#34538) See #34537 Risk Level: low Testing: unit tests Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a Signed-off-by: Fredy Wijaya --- mobile/test/common/integration/rtds_integration_test.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mobile/test/common/integration/rtds_integration_test.cc b/mobile/test/common/integration/rtds_integration_test.cc index 6cda02b8bc5e..60c72c66cc93 100644 --- a/mobile/test/common/integration/rtds_integration_test.cc +++ b/mobile/test/common/integration/rtds_integration_test.cc @@ -100,12 +100,14 @@ INSTANTIATE_TEST_SUITE_P( // Envoy Mobile's xDS APIs only support state-of-the-world, not delta. testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::UnifiedSotw))); -TEST_P(RtdsIntegrationTest, RtdsReloadWithDfpMixedScheme) { +// https://github.com/envoyproxy/envoy/issues/34537 +TEST_P(RtdsIntegrationTest, DISABLED_RtdsReloadWithDfpMixedScheme) { TestScopedStaticReloadableFeaturesRuntime scoped_runtime({{"dfp_mixed_scheme", true}}); runReloadTest(); } -TEST_P(RtdsIntegrationTest, RtdsReloadWithoutDfpMixedScheme) { +// https://github.com/envoyproxy/envoy/issues/34537 +TEST_P(RtdsIntegrationTest, DISABLED_RtdsReloadWithoutDfpMixedScheme) { TestScopedStaticReloadableFeaturesRuntime scoped_runtime({{"dfp_mixed_scheme", false}}); runReloadTest(); } From 65bb971a4626bd33bbcd5ff6f12e623780b0b8ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:03:23 +0100 Subject: [PATCH 07/61] build(deps): bump github/codeql-action from 3.25.7 to 3.25.8 (#34541) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.25.7 to 3.25.8. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/f079b8493333aace61c81488f8bd40919487bd9f...2e230e8fe0ad3a14a340ad0815ddb96d599d2aff) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-daily.yml | 4 ++-- .github/workflows/codeql-push.yml | 4 ++-- .github/workflows/scorecard.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 6b889ae351cc..0d8347258ed3 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -37,7 +37,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@f079b8493333aace61c81488f8bd40919487bd9f # codeql-bundle-v3.25.7 + uses: github/codeql-action/init@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # codeql-bundle-v3.25.8 # Override language selection by uncommenting this and choosing your languages with: languages: cpp @@ -71,4 +71,4 @@ jobs: git clean -xdf - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@f079b8493333aace61c81488f8bd40919487bd9f # codeql-bundle-v3.25.7 + uses: github/codeql-action/analyze@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # codeql-bundle-v3.25.8 diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index c0572bef4caf..4a82d699b216 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -68,7 +68,7 @@ jobs: - name: Initialize CodeQL if: ${{ env.BUILD_TARGETS != '' }} - uses: github/codeql-action/init@f079b8493333aace61c81488f8bd40919487bd9f # codeql-bundle-v3.25.7 + uses: github/codeql-action/init@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # codeql-bundle-v3.25.8 with: languages: cpp @@ -112,4 +112,4 @@ jobs: - name: Perform CodeQL Analysis if: ${{ env.BUILD_TARGETS != '' }} - uses: github/codeql-action/analyze@f079b8493333aace61c81488f8bd40919487bd9f # codeql-bundle-v3.25.7 + uses: github/codeql-action/analyze@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # codeql-bundle-v3.25.8 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index c6afce282318..fac066899006 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -40,6 +40,6 @@ jobs: retention-days: 5 - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@f079b8493333aace61c81488f8bd40919487bd9f # v3.25.7 + uses: github/codeql-action/upload-sarif@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8 with: sarif_file: results.sarif From 51c4a7414db8b9c4c1321fcbae4466bacf3777b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:03:32 +0100 Subject: [PATCH 08/61] build(deps): bump cryptography from 42.0.7 to 42.0.8 in /tools/base (#34542) Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.7 to 42.0.8. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.7...42.0.8) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 66 ++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 412d81692d14..d36bef92a33d 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -420,39 +420,39 @@ coloredlogs==15.0.1 \ crcmod==1.7 \ --hash=sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e # via gsutil -cryptography==42.0.7 \ - --hash=sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55 \ - --hash=sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785 \ - --hash=sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b \ - --hash=sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886 \ - --hash=sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82 \ - --hash=sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1 \ - --hash=sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda \ - --hash=sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f \ - --hash=sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68 \ - --hash=sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60 \ - --hash=sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7 \ - --hash=sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd \ - --hash=sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582 \ - --hash=sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc \ - --hash=sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858 \ - --hash=sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b \ - --hash=sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2 \ - --hash=sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678 \ - --hash=sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13 \ - --hash=sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4 \ - --hash=sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8 \ - --hash=sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604 \ - --hash=sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477 \ - --hash=sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e \ - --hash=sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a \ - --hash=sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9 \ - --hash=sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14 \ - --hash=sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda \ - --hash=sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da \ - --hash=sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562 \ - --hash=sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2 \ - --hash=sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9 +cryptography==42.0.8 \ + --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ + --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ + --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ + --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ + --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ + --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ + --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ + --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ + --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ + --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ + --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ + --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ + --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ + --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ + --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ + --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ + --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ + --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ + --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ + --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ + --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ + --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ + --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ + --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ + --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ + --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ + --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ + --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ + --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ + --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ + --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ + --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e # via # -r requirements.in # aioquic From 46407a8622776005e00ec2922452dc4f606f5444 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:03:42 +0100 Subject: [PATCH 09/61] build(deps): bump golang.org/x/net from 0.25.0 to 0.26.0 in /examples/grpc-bridge/server in the examples-grpc-bridge group (#34543) build(deps): bump golang.org/x/net Bumps the examples-grpc-bridge group in /examples/grpc-bridge/server with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.25.0 to 0.26.0 - [Commits](https://github.com/golang/net/compare/v0.25.0...v0.26.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor dependency-group: examples-grpc-bridge ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/grpc-bridge/server/go.mod | 2 +- examples/grpc-bridge/server/go.sum | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/examples/grpc-bridge/server/go.mod b/examples/grpc-bridge/server/go.mod index 784d578fb1f6..f5db949d588d 100644 --- a/examples/grpc-bridge/server/go.mod +++ b/examples/grpc-bridge/server/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/golang/protobuf v1.5.4 - golang.org/x/net v0.25.0 + golang.org/x/net v0.26.0 google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/grpc v1.64.0 ) diff --git a/examples/grpc-bridge/server/go.sum b/examples/grpc-bridge/server/go.sum index 75d93af29f2c..ea3adc7978c1 100644 --- a/examples/grpc-bridge/server/go.sum +++ b/examples/grpc-bridge/server/go.sum @@ -1729,6 +1729,7 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1793,6 +1794,8 @@ golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1864,8 +1867,9 @@ golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1925,6 +1929,7 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2017,8 +2022,10 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2040,6 +2047,7 @@ golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2061,8 +2069,9 @@ golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2137,6 +2146,7 @@ golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 41ef3f086bd17a196845b74e12944fdf3591437f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:03:55 +0100 Subject: [PATCH 10/61] build(deps): bump golang from 1.22.3-bookworm to 1.22.4-bookworm in /examples/shared/golang (#34544) build(deps): bump golang in /examples/shared/golang Bumps golang from 1.22.3-bookworm to 1.22.4-bookworm. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/shared/golang/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared/golang/Dockerfile b/examples/shared/golang/Dockerfile index aaea2fd502a0..25473d6681e6 100644 --- a/examples/shared/golang/Dockerfile +++ b/examples/shared/golang/Dockerfile @@ -3,7 +3,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean \ && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee /etc/apt/apt.conf.d/keep-cache -FROM golang:1.22.3-bookworm@sha256:5c56bd47228dd572d8a82971cf1f946cd8bb1862a8ec6dc9f3d387cc94136976 as golang-base +FROM golang:1.22.4-bookworm@sha256:aec47843e52fee4436bdd3ce931417fa980e9055658b5142140925eea3044bea as golang-base FROM golang-base as golang-control-plane-builder From 9e456480a71c607c021ff4511a6c1c5e3b9bc909 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:04:03 +0100 Subject: [PATCH 11/61] build(deps): bump otel/opentelemetry-collector from `fe77148` to `1cffbc3` in /examples/opentelemetry (#34546) build(deps): bump otel/opentelemetry-collector Bumps otel/opentelemetry-collector from `fe77148` to `1cffbc3`. --- updated-dependencies: - dependency-name: otel/opentelemetry-collector dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/opentelemetry/Dockerfile-opentelemetry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry index e21ca2af794a..e27962f834f3 100644 --- a/examples/opentelemetry/Dockerfile-opentelemetry +++ b/examples/opentelemetry/Dockerfile-opentelemetry @@ -1,7 +1,7 @@ FROM alpine:3.20@sha256:77726ef6b57ddf65bb551896826ec38bc3e53f75cdde31354fbffb4f25238ebd as otelc_curl RUN apk --update add curl -FROM otel/opentelemetry-collector:latest@sha256:fe7714849adfd251129be75e39c5c4fa7031d87146645afa5d391ab957845c18 +FROM otel/opentelemetry-collector:latest@sha256:1cffbc33556826c5185cebc1b1dbe1e056b3417bd422cdd4a03747dc6d19388f COPY --from=otelc_curl / / From 162ce267812e32d828c2c9acfbfd1fba6eadc738 Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Wed, 5 Jun 2024 07:37:50 -0500 Subject: [PATCH 12/61] mobile: Remove Mobile/Core CI (#34522) This PR removes Mobile/Core since everything has been covered by Mobile/CC. This should help to free up some runners. Risk Level: low Testing: CI Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a Signed-off-by: Fredy Wijaya --- .github/workflows/mobile-core.yml | 67 ------------------------------- 1 file changed, 67 deletions(-) delete mode 100644 .github/workflows/mobile-core.yml diff --git a/.github/workflows/mobile-core.yml b/.github/workflows/mobile-core.yml deleted file mode 100644 index 70de7033c9a0..000000000000 --- a/.github/workflows/mobile-core.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Mobile/Core - -permissions: - contents: read - -on: - workflow_run: - workflows: - - Request - types: - - completed - -concurrency: - group: ${{ github.head_ref || github.run_id }}-${{ github.workflow }} - cancel-in-progress: true - - -jobs: - load: - secrets: - app-key: ${{ secrets.ENVOY_CI_APP_KEY }} - app-id: ${{ secrets.ENVOY_CI_APP_ID }} - lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} - lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} - permissions: - actions: read - contents: read - packages: read - pull-requests: read - if: ${{ github.event.workflow_run.conclusion == 'success' }} - uses: ./.github/workflows/_load.yml - with: - check-name: mobile-core - - unit-tests: - permissions: - contents: read - packages: read - if: ${{ fromJSON(needs.load.outputs.request).run.mobile-core }} - needs: load - uses: ./.github/workflows/_mobile_container_ci.yml - with: - args: >- - test - --config=mobile-remote-ci-core - //test/common/... - request: ${{ needs.load.outputs.request }} - target: unit-tests - timeout-minutes: 120 - - request: - secrets: - app-id: ${{ secrets.ENVOY_CI_APP_ID }} - app-key: ${{ secrets.ENVOY_CI_APP_KEY }} - permissions: - actions: read - contents: read - if: >- - ${{ always() - && github.event.workflow_run.conclusion == 'success' - && fromJSON(needs.load.outputs.request).run.mobile-core }} - needs: - - load - - unit-tests - uses: ./.github/workflows/_finish.yml - with: - needs: ${{ toJSON(needs) }} From 0d4f6de27bdadfb1b815f02932754f53f43d33ef Mon Sep 17 00:00:00 2001 From: Namrata Bhave Date: Wed, 5 Jun 2024 18:21:09 +0530 Subject: [PATCH 13/61] test: Fix //test/common/network:multi_connection_base_impl_test on big endian (#34442) Signed-off-by: namrata-ibm --- test/common/network/multi_connection_base_impl_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/common/network/multi_connection_base_impl_test.cc b/test/common/network/multi_connection_base_impl_test.cc index a5cf8cccab50..673eafe91831 100644 --- a/test/common/network/multi_connection_base_impl_test.cc +++ b/test/common/network/multi_connection_base_impl_test.cc @@ -297,7 +297,8 @@ TEST_F(MultiConnectionBaseImplTest, HashKey) { startConnect(); std::vector hash_key = {'A', 'B', 'C'}; - uint8_t* id_array = reinterpret_cast(&id); + uint8_t id_array[sizeof(id)]; + absl::little_endian::Store64(id_array, id); impl_->hashKey(hash_key); EXPECT_EQ(3 + sizeof(id), hash_key.size()); EXPECT_EQ('A', hash_key[0]); From 21fba6209fdeee43c86a917b6a26823fd718d941 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 5 Jun 2024 09:19:48 -0400 Subject: [PATCH 14/61] tls: removing exceptions from certs (#34460) Risk Level: low Testing: updated tests Docs Changes: n/a Release Notes: n/a envoyproxy/envoy-mobile#176 Signed-off-by: Alyssa Wilk --- source/common/ssl/tls_certificate_config_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/ssl/tls_certificate_config_impl.cc b/source/common/ssl/tls_certificate_config_impl.cc index d206870a85e6..24393f420a75 100644 --- a/source/common/ssl/tls_certificate_config_impl.cc +++ b/source/common/ssl/tls_certificate_config_impl.cc @@ -17,7 +17,7 @@ namespace { // string. std::string maybeSet(absl::StatusOr to_set, absl::Status error) { if (to_set.status().ok()) { - return to_set.value(); + return std::move(to_set.value()); } error = to_set.status(); return ""; From fd598803a7608a5c5e0f3822f5acce1f71773cc8 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 5 Jun 2024 14:42:57 +0100 Subject: [PATCH 15/61] ci/coverage: Adjust `extensions/tracers` (#34553) Signed-off-by: Ryan Northey --- test/per_file_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 2d2b536f3949..f48c76c288a0 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -44,7 +44,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/rate_limit_descriptors/expr:95.0" "source/extensions/stat_sinks/graphite_statsd:82.8" # Death tests don't report LCOV "source/extensions/stat_sinks/statsd:85.2" # Death tests don't report LCOV -"source/extensions/tracers:96.5" +"source/extensions/tracers:96.4" "source/extensions/tracers/common:74.8" "source/extensions/tracers/common/ot:72.9" "source/extensions/tracers/opencensus:93.9" From f4930b1ffd7c7bffec32e73f4b7c8482a72dc23a Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 5 Jun 2024 14:56:27 +0100 Subject: [PATCH 16/61] docs/examples: Shift rst files into example folders (#34429) Signed-off-by: Ryan Northey --- docs/BUILD | 9 +- docs/root/start/sandboxes/index.rst | 42 +-- .../start/sandboxes/win32_front_proxy.rst | 354 ------------------ examples/BUILD | 52 ++- .../brotli.rst => examples/brotli/example.rst | 0 .../cache.rst => examples/cache/example.rst | 0 .../cors.rst => examples/cors/example.rst | 0 .../csrf.rst => examples/csrf/example.rst | 0 .../double-proxy/example.rst | 0 ...sponse-config-active-clusters-updated.json | 0 .../response-config-active-clusters.json | 0 .../_include}/response-config-cluster.json | 0 .../dynamic-config-cp/example.rst | 12 +- ...sponse-config-active-clusters-updated.json | 0 .../response-config-active-clusters.json | 0 .../dynamic-config-fs/example.rst | 10 +- .../ext_authz/example.rst | 0 .../fault-injection/example.rst | 0 .../front-proxy/example.rst | 0 .../golang-http/example.rst | 0 .../golang-network/example.rst | 0 .../grpc-bridge/example.rst | 0 .../gzip.rst => examples/gzip/example.rst | 0 .../jaeger-tracing/example.rst | 0 .../kafka.rst => examples/kafka/example.rst | 0 .../load-reporting-service/example.rst | 0 .../local_ratelimit/example.rst | 0 .../locality-load-balancing/example.rst | 0 .../lua-cluster-specifier/example.rst | 0 .../lua.rst => examples/lua/example.rst | 0 .../mysql.rst => examples/mysql/example.rst | 0 .../opentelemetry/example.rst | 0 .../postgres/example.rst | 0 .../rbac.rst => examples/rbac/example.rst | 0 .../redis.rst => examples/redis/example.rst | 0 .../route-mirror/example.rst | 0 .../single-page-app}/_static/spa-cookies.png | Bin .../_static/spa-github-oauth.png | Bin .../_static/spa-login-github.png | Bin .../single-page-app}/_static/spa-login.png | Bin .../_static/spa-resources.png | Bin .../single-page-app/example.rst | 10 +- .../_static/skywalking-services.png | Bin .../_static/skywalking-topology.png | Bin .../skywalking}/_static/skywalking-trace.png | Bin .../skywalking/example.rst | 6 +- .../tls-inspector/example.rst | 0 .../tls-sni/example.rst | 0 .../tls.rst => examples/tls/example.rst | 0 .../udp.rst => examples/udp/example.rst | 0 examples/wasm-cc/BUILD | 4 +- .../wasm-cc/example.rst | 0 .../websocket/example.rst | 0 .../zipkin}/_static/zipkin-ui-dependency.png | Bin .../zipkin}/_static/zipkin-ui.png | Bin .../zipkin.rst => examples/zipkin/example.rst | 4 +- .../zstd.rst => examples/zstd/example.rst | 0 57 files changed, 77 insertions(+), 426 deletions(-) delete mode 100644 docs/root/start/sandboxes/win32_front_proxy.rst rename docs/root/start/sandboxes/brotli.rst => examples/brotli/example.rst (100%) rename docs/root/start/sandboxes/cache.rst => examples/cache/example.rst (100%) rename docs/root/start/sandboxes/cors.rst => examples/cors/example.rst (100%) rename docs/root/start/sandboxes/csrf.rst => examples/csrf/example.rst (100%) rename docs/root/start/sandboxes/double-proxy.rst => examples/double-proxy/example.rst (100%) rename {docs/root/start/sandboxes/_include/dynamic-config-cp => examples/dynamic-config-cp/_include}/response-config-active-clusters-updated.json (100%) rename {docs/root/start/sandboxes/_include/dynamic-config-cp => examples/dynamic-config-cp/_include}/response-config-active-clusters.json (100%) rename {docs/root/start/sandboxes/_include/dynamic-config-cp => examples/dynamic-config-cp/_include}/response-config-cluster.json (100%) rename docs/root/start/sandboxes/dynamic-configuration-control-plane.rst => examples/dynamic-config-cp/example.rst (92%) rename {docs/root/start/sandboxes/_include/dynamic-config-fs => examples/dynamic-config-fs/_include}/response-config-active-clusters-updated.json (100%) rename {docs/root/start/sandboxes/_include/dynamic-config-fs => examples/dynamic-config-fs/_include}/response-config-active-clusters.json (100%) rename docs/root/start/sandboxes/dynamic-configuration-filesystem.rst => examples/dynamic-config-fs/example.rst (88%) rename docs/root/start/sandboxes/ext_authz.rst => examples/ext_authz/example.rst (100%) rename docs/root/start/sandboxes/fault_injection.rst => examples/fault-injection/example.rst (100%) rename docs/root/start/sandboxes/front_proxy.rst => examples/front-proxy/example.rst (100%) rename docs/root/start/sandboxes/golang-http.rst => examples/golang-http/example.rst (100%) rename docs/root/start/sandboxes/golang-network.rst => examples/golang-network/example.rst (100%) rename docs/root/start/sandboxes/grpc_bridge.rst => examples/grpc-bridge/example.rst (100%) rename docs/root/start/sandboxes/gzip.rst => examples/gzip/example.rst (100%) rename docs/root/start/sandboxes/jaeger_tracing.rst => examples/jaeger-tracing/example.rst (100%) rename docs/root/start/sandboxes/kafka.rst => examples/kafka/example.rst (100%) rename docs/root/start/sandboxes/load_reporting_service.rst => examples/load-reporting-service/example.rst (100%) rename docs/root/start/sandboxes/local_ratelimit.rst => examples/local_ratelimit/example.rst (100%) rename docs/root/start/sandboxes/locality_load_balancing.rst => examples/locality-load-balancing/example.rst (100%) rename docs/root/start/sandboxes/lua-cluster-specifier.rst => examples/lua-cluster-specifier/example.rst (100%) rename docs/root/start/sandboxes/lua.rst => examples/lua/example.rst (100%) rename docs/root/start/sandboxes/mysql.rst => examples/mysql/example.rst (100%) rename docs/root/start/sandboxes/opentelemetry.rst => examples/opentelemetry/example.rst (100%) rename docs/root/start/sandboxes/postgres.rst => examples/postgres/example.rst (100%) rename docs/root/start/sandboxes/rbac.rst => examples/rbac/example.rst (100%) rename docs/root/start/sandboxes/redis.rst => examples/redis/example.rst (100%) rename docs/root/start/sandboxes/route-mirror.rst => examples/route-mirror/example.rst (100%) rename {docs/root/start/sandboxes => examples/single-page-app}/_static/spa-cookies.png (100%) rename {docs/root/start/sandboxes => examples/single-page-app}/_static/spa-github-oauth.png (100%) rename {docs/root/start/sandboxes => examples/single-page-app}/_static/spa-login-github.png (100%) rename {docs/root/start/sandboxes => examples/single-page-app}/_static/spa-login.png (100%) rename {docs/root/start/sandboxes => examples/single-page-app}/_static/spa-resources.png (100%) rename docs/root/start/sandboxes/single-page-app.rst => examples/single-page-app/example.rst (98%) rename {docs/root/start/sandboxes => examples/skywalking}/_static/skywalking-services.png (100%) rename {docs/root/start/sandboxes => examples/skywalking}/_static/skywalking-topology.png (100%) rename {docs/root/start/sandboxes => examples/skywalking}/_static/skywalking-trace.png (100%) rename docs/root/start/sandboxes/skywalking.rst => examples/skywalking/example.rst (95%) rename docs/root/start/sandboxes/tls-inspector.rst => examples/tls-inspector/example.rst (100%) rename docs/root/start/sandboxes/tls-sni.rst => examples/tls-sni/example.rst (100%) rename docs/root/start/sandboxes/tls.rst => examples/tls/example.rst (100%) rename docs/root/start/sandboxes/udp.rst => examples/udp/example.rst (100%) rename docs/root/start/sandboxes/wasm-cc.rst => examples/wasm-cc/example.rst (100%) rename docs/root/start/sandboxes/websocket.rst => examples/websocket/example.rst (100%) rename {docs/root/start/sandboxes => examples/zipkin}/_static/zipkin-ui-dependency.png (100%) rename {docs/root/start/sandboxes => examples/zipkin}/_static/zipkin-ui.png (100%) rename docs/root/start/sandboxes/zipkin.rst => examples/zipkin/example.rst (96%) rename docs/root/start/sandboxes/zstd.rst => examples/zstd/example.rst (100%) diff --git a/docs/BUILD b/docs/BUILD index 5859a9dcbe91..46df13e77e19 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -209,17 +209,9 @@ pkg_files( strip_prefix = "/configs", ) -pkg_files( - name = "examples_rst", - srcs = ["//examples:files"], - prefix = "start/sandboxes/_include", - strip_prefix = "/examples", -) - pkg_filegroup( name = "rst_files", srcs = [ - ":examples_rst", ":repo_configs", ":sphinx_base", ":sphinx_inventories", @@ -244,6 +236,7 @@ pkg_tar( ":extensions_security_rst", ":external_deps_rst", ":version_history_rst", + "//examples:docs", ], ) diff --git a/docs/root/start/sandboxes/index.rst b/docs/root/start/sandboxes/index.rst index 450fd5176fd2..e84bd196d74c 100644 --- a/docs/root/start/sandboxes/index.rst +++ b/docs/root/start/sandboxes/index.rst @@ -42,44 +42,4 @@ Before you begin you will need to install the sandbox environment. The following sandboxes are available: -.. toctree:: - :maxdepth: 1 - - brotli - cache - cors - csrf - double-proxy - dynamic-configuration-filesystem - dynamic-configuration-control-plane - ext_authz - fault_injection - front_proxy - golang-http - golang-network - grpc_bridge - gzip - jaeger_tracing - kafka - load_reporting_service - locality_load_balancing - local_ratelimit - lua-cluster-specifier - lua - mysql - opentelemetry - postgres - rbac - redis - route-mirror - single-page-app - skywalking - tls-inspector - tls-sni - tls - udp - wasm-cc - websocket - win32_front_proxy - zipkin - zstd +.. include:: toctree.rst diff --git a/docs/root/start/sandboxes/win32_front_proxy.rst b/docs/root/start/sandboxes/win32_front_proxy.rst deleted file mode 100644 index 2342bacc7eb3..000000000000 --- a/docs/root/start/sandboxes/win32_front_proxy.rst +++ /dev/null @@ -1,354 +0,0 @@ -Windows based Front proxy -========================= - -.. include:: ../../_include/windows_support_ended.rst - -.. sidebar:: Requirements - - .. include:: _include/docker-env-setup-link.rst - -To get a flavor of what Envoy has to offer on Windows, we are releasing a -`docker compose `_ sandbox that deploys a front Envoy and a -couple of services (simple Flask apps) colocated with a running service Envoy. - -The three containers will be deployed inside a virtual network called ``envoymesh``. - -Below you can see a graphic showing the docker compose deployment: - -.. image:: /_static/docker_compose_front_proxy.svg - :width: 100% - -All incoming requests are routed via the front Envoy, which is acting as a reverse proxy sitting on -the edge of the ``envoymesh`` network. Port ``8080``, ``8443``, and ``8001`` are exposed by docker -compose (see :download:`docker-compose.yaml <_include/front-proxy/docker-compose.yaml>`) to handle -``HTTP``, ``HTTPS`` calls to the services and requests to ``/admin`` respectively. - -Moreover, notice that all traffic routed by the front Envoy to the service containers is actually -routed to the service Envoys (routes setup in :download:`envoy.yaml <_include/front-proxy/envoy.yaml>`). - -In turn the service Envoys route the request to the Flask app via the loopback -address (routes setup in :download:`service-envoy.yaml <_include/front-proxy/service-envoy.yaml>`). This -setup illustrates the advantage of running service Envoys collocated with your services: all -requests are handled by the service Envoy, and efficiently routed to your services. - -Step 1: Start all of our containers -*********************************** - -Change to the ``examples/front-proxy`` directory. - -.. code-block:: console - - PS> $PWD - D:\envoy\examples\win32-front-proxy - PS> docker-compose build --pull - PS> docker-compose up -d - PS> docker-compose ps - Name Command State Ports - ------------------------------------------------------------------------------------------------------------------------------------------------------------ - envoy-front-proxy_front-envoy_1 powershell.exe ./start_env ... Up 10000/tcp, 0.0.0.0:8003->8003/tcp, 0.0.0.0:8080->8080/tcp, 0.0.0.0:8443->8443/tcp - envoy-front-proxy_service1_1 powershell.exe ./start_ser ... Up 10000/tcp - envoy-front-proxy_service2_1 powershell.exe ./start_ser ... Up 10000/tcp - -Step 2: Test Envoy's routing capabilities -***************************************** - -You can now send a request to both services via the ``front-envoy``. - -For ``service1``: - -.. code-block:: console - - PS> curl -v localhost:8080/service/1 - * Trying ::1... - * TCP_NODELAY set - * Trying 127.0.0.1... - * TCP_NODELAY set - * Connected to localhost (127.0.0.1) port 8080 (#0) - > GET /service/1 HTTP/1.1 - > Host: localhost:8080 - > User-Agent: curl/7.55.1 - > Accept: */* - > - < HTTP/1.1 200 OK - < content-type: text/html; charset=utf-8 - < content-length: 92 - < server: envoy - < date: Wed, 05 May 2021 05:55:55 GMT - < x-envoy-upstream-service-time: 18 - < - Hello from behind Envoy (service 1)! hostname: 8a45bba91d83 resolvedhostname: 172.30.97.237 - * Connection #0 to host localhost left intact - -For ``service2``: - -.. code-block:: console - - PS> curl -v localhost:8080/service/2 - * Trying ::1... - * TCP_NODELAY set - * Trying 127.0.0.1... - * TCP_NODELAY set - * Connected to localhost (127.0.0.1) port 8080 (#0) - > GET /service/2 HTTP/1.1 - > Host: localhost:8080 - > User-Agent: curl/7.55.1 - > Accept: */* - > - < HTTP/1.1 200 OK - < content-type: text/html; charset=utf-8 - < content-length: 93 - < server: envoy - < date: Wed, 05 May 2021 05:57:03 GMT - < x-envoy-upstream-service-time: 14 - < - Hello from behind Envoy (service 2)! hostname: 51e28eb3c8b8 resolvedhostname: 172.30.109.113 - * Connection #0 to host localhost left intact - -Notice that each request, while sent to the front Envoy, was correctly routed to the respective -application. - -We can also use ``HTTPS`` to call services behind the front Envoy. For example, calling ``service1``: - -.. code-block:: console - - PS> curl https://localhost:8443/service/1 -k -v - * Trying ::1... - * TCP_NODELAY set - * Trying 127.0.0.1... - * TCP_NODELAY set - * Connected to localhost (127.0.0.1) port 8443 (#0) - * schannel: SSL/TLS connection with localhost port 8443 (step 1/3) - * schannel: disabled server certificate revocation checks - * schannel: verifyhost setting prevents Schannel from comparing the supplied target name with the subject names in server certificates. - * schannel: sending initial handshake data: sending 171 bytes... - * schannel: sent initial handshake data: sent 171 bytes - * schannel: SSL/TLS connection with localhost port 8443 (step 2/3) - * schannel: failed to receive handshake, need more data - * schannel: SSL/TLS connection with localhost port 8443 (step 2/3) - * schannel: encrypted data got 1081 - * schannel: encrypted data buffer: offset 1081 length 4096 - * schannel: sending next handshake data: sending 93 bytes... - * schannel: SSL/TLS connection with localhost port 8443 (step 2/3) - * schannel: encrypted data got 258 - * schannel: encrypted data buffer: offset 258 length 4096 - * schannel: SSL/TLS handshake complete - * schannel: SSL/TLS connection with localhost port 8443 (step 3/3) - * schannel: stored credential handle in session cache - > GET /service/1 HTTP/1.1 - > Host: localhost:8443 - > User-Agent: curl/7.55.1 - > Accept: */* - > - * schannel: client wants to read 102400 bytes - * schannel: encdata_buffer resized 103424 - * schannel: encrypted data buffer: offset 0 length 103424 - * schannel: encrypted data got 286 - * schannel: encrypted data buffer: offset 286 length 103424 - * schannel: decrypted data length: 257 - * schannel: decrypted data added: 257 - * schannel: decrypted data cached: offset 257 length 102400 - * schannel: encrypted data buffer: offset 0 length 103424 - * schannel: decrypted data buffer: offset 257 length 102400 - * schannel: schannel_recv cleanup - * schannel: decrypted data returned 257 - * schannel: decrypted data buffer: offset 0 length 102400 - < HTTP/1.1 200 OK - < content-type: text/html; charset=utf-8 - < content-length: 92 - < server: envoy - < date: Wed, 05 May 2021 05:57:45 GMT - < x-envoy-upstream-service-time: 3 - < - Hello from behind Envoy (service 1)! hostname: 8a45bba91d83 resolvedhostname: 172.30.97.237 - * Connection #0 to host localhost left intact - -Step 3: Test Envoy's load balancing capabilities -************************************************ - -Now let's scale up our ``service1`` nodes to demonstrate the load balancing abilities of Envoy: - -.. code-block:: console - - PS> docker-compose scale service1=3 - Creating and starting example_service1_2 ... done - Creating and starting example_service1_3 ... done - -Now if we send a request to ``service1`` multiple times, the front Envoy will load balance the -requests by doing a round robin of the three ``service1`` machines: - -.. code-block:: console - - PS> curl -v localhost:8080/service/1 - * Trying ::1... - * TCP_NODELAY set - * Trying 127.0.0.1... - * TCP_NODELAY set - * Connected to localhost (127.0.0.1) port 8080 (#0) - > GET /service/1 HTTP/1.1 - > Host: localhost:8080 - > User-Agent: curl/7.55.1 - > Accept: */* - > - < HTTP/1.1 200 OK - < content-type: text/html; charset=utf-8 - < content-length: 93 - < server: envoy - < date: Wed, 05 May 2021 05:58:40 GMT - < x-envoy-upstream-service-time: 22 - < - Hello from behind Envoy (service 1)! hostname: 8d2359ee21a8 resolvedhostname: 172.30.101.143 - * Connection #0 to host localhost left intact - PS> curl -v localhost:8080/service/1 - * Trying ::1... - * TCP_NODELAY set - * Trying 127.0.0.1... - * TCP_NODELAY set - * Connected to localhost (127.0.0.1) port 8080 (#0) - > GET /service/1 HTTP/1.1 - > Host: localhost:8080 - > User-Agent: curl/7.55.1 - > Accept: */* - > - < HTTP/1.1 200 OK - < content-type: text/html; charset=utf-8 - < content-length: 91 - < server: envoy - < date: Wed, 05 May 2021 05:58:43 GMT - < x-envoy-upstream-service-time: 11 - < - Hello from behind Envoy (service 1)! hostname: 41e1141eebf4 resolvedhostname: 172.30.96.11 - * Connection #0 to host localhost left intact - PS> curl -v localhost:8080/service/1 - * Trying ::1... - * TCP_NODELAY set - * Trying 127.0.0.1... - * TCP_NODELAY set - * Connected to localhost (127.0.0.1) port 8080 (#0) - > GET /service/1 HTTP/1.1 - > Host: localhost:8080 - > User-Agent: curl/7.55.1 - > Accept: */* - > - < HTTP/1.1 200 OK - < content-type: text/html; charset=utf-8 - < content-length: 92 - < server: envoy - < date: Wed, 05 May 2021 05:58:44 GMT - < x-envoy-upstream-service-time: 7 - < - Hello from behind Envoy (service 1)! hostname: 8a45bba91d83 resolvedhostname: 172.30.97.237 - * Connection #0 to host localhost left intact - -Step 4: Enter containers and curl services -****************************************** - -In addition of using ``curl`` from your host machine, you can also enter the -containers themselves and ``curl`` from inside them. To enter a container you -can use ``docker-compose exec /bin/bash``. For example we can -enter the ``front-envoy`` container, and ``curl`` for services locally: - -.. code-block:: console - - PS> docker-compose exec front-envoy powershell - PS C:\> (curl -UseBasicParsing http://localhost:8080/service/1).Content - Hello from behind Envoy (service 1)! hostname: 41e1141eebf4 resolvedhostname: 172.30.96.11 - - PS C:\> (curl -UseBasicParsing http://localhost:8080/service/1).Content - Hello from behind Envoy (service 1)! hostname: 8a45bba91d83 resolvedhostname: 172.30.97.237 - - PS C:\> (curl -UseBasicParsing http://localhost:8080/service/1).Content - Hello from behind Envoy (service 1)! hostname: 8d2359ee21a8 resolvedhostname: 172.30.101.143 - - -Step 5: Enter container and curl admin interface -************************************************ - -When Envoy runs it also attaches an ``admin`` to your desired port. - -In the example configs the admin listener is bound to port ``8001``. - -We can ``curl`` it to gain useful information: - -- :ref:`/server_info ` provides information about the Envoy version you are running. -- :ref:`/stats ` provides statistics about the Envoy server. - -In the example we can enter the ``front-envoy`` container to query admin: - -.. code-block:: console - - PS> docker-compose exec front-envoy powershell - PS C:\> (curl http://localhost:8003/server_info -UseBasicParsing).Content - -.. code-block:: json - - { - "version": "093e2ffe046313242144d0431f1bb5cf18d82544/1.15.0-dev/Clean/RELEASE/BoringSSL", - "state": "LIVE", - "hot_restart_version": "11.104", - "command_line_options": { - "base_id": "0", - "use_dynamic_base_id": false, - "base_id_path": "", - "concurrency": 8, - "config_path": "/etc/front-envoy.yaml", - "config_yaml": "", - "allow_unknown_static_fields": false, - "reject_unknown_dynamic_fields": false, - "ignore_unknown_dynamic_fields": false, - "admin_address_path": "", - "local_address_ip_version": "v4", - "log_level": "info", - "component_log_level": "", - "log_format": "[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v", - "log_format_escaped": false, - "log_path": "", - "service_cluster": "front-proxy", - "service_node": "", - "service_zone": "", - "drain_strategy": "Gradual", - "mode": "Serve", - "disable_hot_restart": false, - "enable_mutex_tracing": false, - "restart_epoch": 0, - "cpuset_threads": false, - "disabled_extensions": [], - "bootstrap_version": 0, - "hidden_envoy_deprecated_max_stats": "0", - "hidden_envoy_deprecated_max_obj_name_len": "0", - "file_flush_interval": "10s", - "drain_time": "600s", - "parent_shutdown_time": "900s" - }, - "uptime_current_epoch": "188s", - "uptime_all_epochs": "188s" - } - -.. code-block:: console - - PS C:\> (curl http://localhost:8003/stats -UseBasicParsing).Content - cluster.service1.external.upstream_rq_200: 7 - ... - cluster.service1.membership_change: 2 - cluster.service1.membership_total: 3 - ... - cluster.service1.upstream_cx_http2_total: 3 - ... - cluster.service1.upstream_rq_total: 7 - ... - cluster.service2.external.upstream_rq_200: 2 - ... - cluster.service2.membership_change: 1 - cluster.service2.membership_total: 1 - ... - cluster.service2.upstream_cx_http2_total: 1 - ... - cluster.service2.upstream_rq_total: 2 - ... - -Notice that we can get the number of members of upstream clusters, number of requests fulfilled by -them, information about http ingress, and a plethora of other useful stats. - -.. seealso:: - - :ref:`Envoy admin quick start guide ` - Quick start guide to the Envoy admin interface. diff --git a/examples/BUILD b/examples/BUILD index 61024d861c46..bcf5f121b8ef 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -1,3 +1,5 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_files") +load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") load( "//bazel:envoy_build_system.bzl", "envoy_package", @@ -51,6 +53,38 @@ filegroup( srcs = glob(["_extra_certs/*.pem"]), ) +filegroup( + name = "docs_rst", + srcs = glob(["**/example.rst"]) + ["//examples/wasm-cc:example.rst"], +) + +pkg_files( + name = "examples_files", + srcs = [":files"], + prefix = "_include", + strip_prefix = "/examples", +) + +genrule( + name = "examples_docs", + srcs = [":docs_rst"], + outs = ["examples_docs.tar.gz"], + cmd = """ + TEMP=$$(mktemp -d) + for location in $(locations :docs_rst); do + example=$$(echo $$location | cut -d/ -f2) + cp -a $$location "$${TEMP}/$${example}.rst" + echo " $${example}" >> "$${TEMP}/_toctree.rst" + done + echo ".. toctree::" > "$${TEMP}/toctree.rst" + echo " :maxdepth: 1" >> "$${TEMP}/toctree.rst" + echo "" >> "$${TEMP}/toctree.rst" + cat "$${TEMP}/_toctree.rst" | sort >> "$${TEMP}/toctree.rst" + rm "$${TEMP}/_toctree.rst" + tar czf $@ -C $${TEMP} . + """, +) + filegroup( name = "lua", srcs = glob(["**/*.lua"]), @@ -58,7 +92,23 @@ filegroup( filegroup( name = "files", - srcs = glob(["**/*"]) + [ + srcs = glob( + [ + "**/*", + ], + exclude = [ + "**/node_modules/**", + "**/*.rst", + ], + ) + [ "//examples/wasm-cc:files", ], ) + +pkg_tar( + name = "docs", + srcs = [":examples_files"], + extension = "tar.gz", + package_dir = "start/sandboxes", + deps = [":examples_docs"], +) diff --git a/docs/root/start/sandboxes/brotli.rst b/examples/brotli/example.rst similarity index 100% rename from docs/root/start/sandboxes/brotli.rst rename to examples/brotli/example.rst diff --git a/docs/root/start/sandboxes/cache.rst b/examples/cache/example.rst similarity index 100% rename from docs/root/start/sandboxes/cache.rst rename to examples/cache/example.rst diff --git a/docs/root/start/sandboxes/cors.rst b/examples/cors/example.rst similarity index 100% rename from docs/root/start/sandboxes/cors.rst rename to examples/cors/example.rst diff --git a/docs/root/start/sandboxes/csrf.rst b/examples/csrf/example.rst similarity index 100% rename from docs/root/start/sandboxes/csrf.rst rename to examples/csrf/example.rst diff --git a/docs/root/start/sandboxes/double-proxy.rst b/examples/double-proxy/example.rst similarity index 100% rename from docs/root/start/sandboxes/double-proxy.rst rename to examples/double-proxy/example.rst diff --git a/docs/root/start/sandboxes/_include/dynamic-config-cp/response-config-active-clusters-updated.json b/examples/dynamic-config-cp/_include/response-config-active-clusters-updated.json similarity index 100% rename from docs/root/start/sandboxes/_include/dynamic-config-cp/response-config-active-clusters-updated.json rename to examples/dynamic-config-cp/_include/response-config-active-clusters-updated.json diff --git a/docs/root/start/sandboxes/_include/dynamic-config-cp/response-config-active-clusters.json b/examples/dynamic-config-cp/_include/response-config-active-clusters.json similarity index 100% rename from docs/root/start/sandboxes/_include/dynamic-config-cp/response-config-active-clusters.json rename to examples/dynamic-config-cp/_include/response-config-active-clusters.json diff --git a/docs/root/start/sandboxes/_include/dynamic-config-cp/response-config-cluster.json b/examples/dynamic-config-cp/_include/response-config-cluster.json similarity index 100% rename from docs/root/start/sandboxes/_include/dynamic-config-cp/response-config-cluster.json rename to examples/dynamic-config-cp/_include/response-config-cluster.json diff --git a/docs/root/start/sandboxes/dynamic-configuration-control-plane.rst b/examples/dynamic-config-cp/example.rst similarity index 92% rename from docs/root/start/sandboxes/dynamic-configuration-control-plane.rst rename to examples/dynamic-config-cp/example.rst index d0c0c908102d..53968bee3c50 100644 --- a/docs/root/start/sandboxes/dynamic-configuration-control-plane.rst +++ b/examples/dynamic-config-cp/example.rst @@ -61,7 +61,7 @@ configuration and you should see the cluster named ``xds_cluster`` configured fo $ curl -s http://localhost:19000/config_dump | jq '.configs[1].static_clusters' -.. literalinclude:: _include/dynamic-config-cp/response-config-cluster.json +.. literalinclude:: /start/sandboxes/_include/dynamic-config-cp/_include/response-config-cluster.json :language: json :emphasize-lines: 10, 18-19 @@ -123,7 +123,7 @@ configuration, you should see it is configured with the ``example_proxy_cluster` $ curl -s http://localhost:19000/config_dump | jq '.configs[1].dynamic_active_clusters' -.. literalinclude:: _include/dynamic-config-cp/response-config-active-clusters.json +.. literalinclude:: /start/sandboxes/_include/dynamic-config-cp/_include/response-config-active-clusters.json :language: json :emphasize-lines: 3, 11, 19-20 @@ -147,7 +147,7 @@ Step 7: Edit ``go`` file and restart the control plane ****************************************************** The example setup starts the ``go-control-plane`` -service with a custom :download:`resource.go <_include/dynamic-config-cp/resource.go>` file which +service with a custom :download:`resource.go ` file which specifies the configuration provided to Envoy. Update this to have Envoy proxy instead to ``service2``. @@ -155,7 +155,7 @@ Update this to have Envoy proxy instead to ``service2``. Edit ``resource.go`` in the dynamic configuration example folder and change the ``UpstreamHost`` from ``service1`` to ``service2``: -.. literalinclude:: _include/dynamic-config-cp/resource.go +.. literalinclude:: /start/sandboxes/_include/dynamic-config-cp/resource.go :language: go :lines: 34-43 :lineno-start: 35 @@ -165,7 +165,7 @@ from ``service1`` to ``service2``: Further down in this file you must also change the configuration snapshot version number from ``"1"`` to ``"2"`` to ensure Envoy sees the configuration as newer: -.. literalinclude:: _include/dynamic-config-cp/resource.go +.. literalinclude:: /start/sandboxes/_include/dynamic-config-cp/resource.go :language: go :lineno-start: 175 :lines: 174-186 @@ -198,7 +198,7 @@ is configured to proxy to ``service2``: $ curl -s http://localhost:19000/config_dump | jq '.configs[1].dynamic_active_clusters' -.. literalinclude:: _include/dynamic-config-cp/response-config-active-clusters-updated.json +.. literalinclude:: /start/sandboxes/_include/dynamic-config-cp/_include/response-config-active-clusters-updated.json :language: json :emphasize-lines: 3, 11, 19-20 diff --git a/docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters-updated.json b/examples/dynamic-config-fs/_include/response-config-active-clusters-updated.json similarity index 100% rename from docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters-updated.json rename to examples/dynamic-config-fs/_include/response-config-active-clusters-updated.json diff --git a/docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters.json b/examples/dynamic-config-fs/_include/response-config-active-clusters.json similarity index 100% rename from docs/root/start/sandboxes/_include/dynamic-config-fs/response-config-active-clusters.json rename to examples/dynamic-config-fs/_include/response-config-active-clusters.json diff --git a/docs/root/start/sandboxes/dynamic-configuration-filesystem.rst b/examples/dynamic-config-fs/example.rst similarity index 88% rename from docs/root/start/sandboxes/dynamic-configuration-filesystem.rst rename to examples/dynamic-config-fs/example.rst index 73da55505409..4935cd5bbe6e 100644 --- a/docs/root/start/sandboxes/dynamic-configuration-filesystem.rst +++ b/examples/dynamic-config-fs/example.rst @@ -70,7 +70,7 @@ configuration, you should see it is configured with the ``example_proxy_cluster` $ curl -s http://localhost:19000/config_dump | jq -r '.configs[1].dynamic_active_clusters' -.. literalinclude:: _include/dynamic-config-fs/response-config-active-clusters.json +.. literalinclude:: /start/sandboxes/_include/dynamic-config-fs/_include/response-config-active-clusters.json :language: json :emphasize-lines: 10, 18-19 @@ -79,15 +79,15 @@ Step 4: Replace ``cds.yaml`` inside the container to update upstream cluster The example setup provides Envoy with two dynamic configuration files: -- :download:`configs/cds.yaml <_include/dynamic-config-fs/configs/cds.yaml>` to provide a :ref:`Cluster +- :download:`configs/cds.yaml ` to provide a :ref:`Cluster discovery service (CDS) `. -- :download:`configs/lds.yaml <_include/dynamic-config-fs/configs/lds.yaml>` to provide a :ref:`Listener +- :download:`configs/lds.yaml ` to provide a :ref:`Listener discovery service (LDS) `. Edit ``cds.yaml`` inside the container and change the cluster address from ``service1`` to ``service2``: -.. literalinclude:: _include/dynamic-config-fs/configs/cds.yaml +.. literalinclude:: /start/sandboxes/_include/dynamic-config-fs/configs/cds.yaml :language: yaml :linenos: :lines: 6-13 @@ -121,7 +121,7 @@ the ``example_proxy_cluster`` should now be configured to proxy to ``service2``: $ curl -s http://localhost:19000/config_dump | jq -r '.configs[1].dynamic_active_clusters' -.. literalinclude:: _include/dynamic-config-fs/response-config-active-clusters-updated.json +.. literalinclude:: /start/sandboxes/_include/dynamic-config-fs/_include/response-config-active-clusters-updated.json :language: json :emphasize-lines: 10, 18-19 diff --git a/docs/root/start/sandboxes/ext_authz.rst b/examples/ext_authz/example.rst similarity index 100% rename from docs/root/start/sandboxes/ext_authz.rst rename to examples/ext_authz/example.rst diff --git a/docs/root/start/sandboxes/fault_injection.rst b/examples/fault-injection/example.rst similarity index 100% rename from docs/root/start/sandboxes/fault_injection.rst rename to examples/fault-injection/example.rst diff --git a/docs/root/start/sandboxes/front_proxy.rst b/examples/front-proxy/example.rst similarity index 100% rename from docs/root/start/sandboxes/front_proxy.rst rename to examples/front-proxy/example.rst diff --git a/docs/root/start/sandboxes/golang-http.rst b/examples/golang-http/example.rst similarity index 100% rename from docs/root/start/sandboxes/golang-http.rst rename to examples/golang-http/example.rst diff --git a/docs/root/start/sandboxes/golang-network.rst b/examples/golang-network/example.rst similarity index 100% rename from docs/root/start/sandboxes/golang-network.rst rename to examples/golang-network/example.rst diff --git a/docs/root/start/sandboxes/grpc_bridge.rst b/examples/grpc-bridge/example.rst similarity index 100% rename from docs/root/start/sandboxes/grpc_bridge.rst rename to examples/grpc-bridge/example.rst diff --git a/docs/root/start/sandboxes/gzip.rst b/examples/gzip/example.rst similarity index 100% rename from docs/root/start/sandboxes/gzip.rst rename to examples/gzip/example.rst diff --git a/docs/root/start/sandboxes/jaeger_tracing.rst b/examples/jaeger-tracing/example.rst similarity index 100% rename from docs/root/start/sandboxes/jaeger_tracing.rst rename to examples/jaeger-tracing/example.rst diff --git a/docs/root/start/sandboxes/kafka.rst b/examples/kafka/example.rst similarity index 100% rename from docs/root/start/sandboxes/kafka.rst rename to examples/kafka/example.rst diff --git a/docs/root/start/sandboxes/load_reporting_service.rst b/examples/load-reporting-service/example.rst similarity index 100% rename from docs/root/start/sandboxes/load_reporting_service.rst rename to examples/load-reporting-service/example.rst diff --git a/docs/root/start/sandboxes/local_ratelimit.rst b/examples/local_ratelimit/example.rst similarity index 100% rename from docs/root/start/sandboxes/local_ratelimit.rst rename to examples/local_ratelimit/example.rst diff --git a/docs/root/start/sandboxes/locality_load_balancing.rst b/examples/locality-load-balancing/example.rst similarity index 100% rename from docs/root/start/sandboxes/locality_load_balancing.rst rename to examples/locality-load-balancing/example.rst diff --git a/docs/root/start/sandboxes/lua-cluster-specifier.rst b/examples/lua-cluster-specifier/example.rst similarity index 100% rename from docs/root/start/sandboxes/lua-cluster-specifier.rst rename to examples/lua-cluster-specifier/example.rst diff --git a/docs/root/start/sandboxes/lua.rst b/examples/lua/example.rst similarity index 100% rename from docs/root/start/sandboxes/lua.rst rename to examples/lua/example.rst diff --git a/docs/root/start/sandboxes/mysql.rst b/examples/mysql/example.rst similarity index 100% rename from docs/root/start/sandboxes/mysql.rst rename to examples/mysql/example.rst diff --git a/docs/root/start/sandboxes/opentelemetry.rst b/examples/opentelemetry/example.rst similarity index 100% rename from docs/root/start/sandboxes/opentelemetry.rst rename to examples/opentelemetry/example.rst diff --git a/docs/root/start/sandboxes/postgres.rst b/examples/postgres/example.rst similarity index 100% rename from docs/root/start/sandboxes/postgres.rst rename to examples/postgres/example.rst diff --git a/docs/root/start/sandboxes/rbac.rst b/examples/rbac/example.rst similarity index 100% rename from docs/root/start/sandboxes/rbac.rst rename to examples/rbac/example.rst diff --git a/docs/root/start/sandboxes/redis.rst b/examples/redis/example.rst similarity index 100% rename from docs/root/start/sandboxes/redis.rst rename to examples/redis/example.rst diff --git a/docs/root/start/sandboxes/route-mirror.rst b/examples/route-mirror/example.rst similarity index 100% rename from docs/root/start/sandboxes/route-mirror.rst rename to examples/route-mirror/example.rst diff --git a/docs/root/start/sandboxes/_static/spa-cookies.png b/examples/single-page-app/_static/spa-cookies.png similarity index 100% rename from docs/root/start/sandboxes/_static/spa-cookies.png rename to examples/single-page-app/_static/spa-cookies.png diff --git a/docs/root/start/sandboxes/_static/spa-github-oauth.png b/examples/single-page-app/_static/spa-github-oauth.png similarity index 100% rename from docs/root/start/sandboxes/_static/spa-github-oauth.png rename to examples/single-page-app/_static/spa-github-oauth.png diff --git a/docs/root/start/sandboxes/_static/spa-login-github.png b/examples/single-page-app/_static/spa-login-github.png similarity index 100% rename from docs/root/start/sandboxes/_static/spa-login-github.png rename to examples/single-page-app/_static/spa-login-github.png diff --git a/docs/root/start/sandboxes/_static/spa-login.png b/examples/single-page-app/_static/spa-login.png similarity index 100% rename from docs/root/start/sandboxes/_static/spa-login.png rename to examples/single-page-app/_static/spa-login.png diff --git a/docs/root/start/sandboxes/_static/spa-resources.png b/examples/single-page-app/_static/spa-resources.png similarity index 100% rename from docs/root/start/sandboxes/_static/spa-resources.png rename to examples/single-page-app/_static/spa-resources.png diff --git a/docs/root/start/sandboxes/single-page-app.rst b/examples/single-page-app/example.rst similarity index 98% rename from docs/root/start/sandboxes/single-page-app.rst rename to examples/single-page-app/example.rst index d408748155a7..ea702bda6f5f 100644 --- a/docs/root/start/sandboxes/single-page-app.rst +++ b/examples/single-page-app/example.rst @@ -145,7 +145,7 @@ Step 4: Browse to the dev app and login The development app should now be available at http://localhost:10001 and provide a login button: -.. image:: /start/sandboxes/_static/spa-login.png +.. image:: /start/sandboxes/_include/single-page-app/_static/spa-login.png :align: center .. note:: @@ -212,7 +212,7 @@ Envoy then uses this authorization code with its client secret to confirm author Once logged in, you should be able to make queries to the API using the OAuth credentials: -.. image:: /start/sandboxes/_static/spa-resources.png +.. image:: /start/sandboxes/_include/single-page-app/_static/spa-resources.png :align: center .. warning:: @@ -252,7 +252,7 @@ For the sandbox app, :ref:`forward_bearer_token ` is set, and so Envoy also passes the acquired access token back to the user as a cookie: -.. image:: /start/sandboxes/_static/spa-cookies.png +.. image:: /start/sandboxes/_include/single-page-app/_static/spa-cookies.png :align: center This cookie is then passed through Envoy in any subsequent requests to the proxied Myhub API: @@ -476,7 +476,7 @@ more control and is generally preferable. This can be done either at the `user `_ or organization levels: -.. image:: /start/sandboxes/_static/spa-github-oauth.png +.. image:: /start/sandboxes/_include/single-page-app/_static/spa-github-oauth.png :align: center .. note:: @@ -605,7 +605,7 @@ Browse to the production server https://localhost:10000 You can now log in and use the `Github APIs `__.: -.. image:: /start/sandboxes/_static/spa-login-github.png +.. image:: /start/sandboxes/_include/single-page-app/_static/spa-login-github.png :align: center .. seealso:: diff --git a/docs/root/start/sandboxes/_static/skywalking-services.png b/examples/skywalking/_static/skywalking-services.png similarity index 100% rename from docs/root/start/sandboxes/_static/skywalking-services.png rename to examples/skywalking/_static/skywalking-services.png diff --git a/docs/root/start/sandboxes/_static/skywalking-topology.png b/examples/skywalking/_static/skywalking-topology.png similarity index 100% rename from docs/root/start/sandboxes/_static/skywalking-topology.png rename to examples/skywalking/_static/skywalking-topology.png diff --git a/docs/root/start/sandboxes/_static/skywalking-trace.png b/examples/skywalking/_static/skywalking-trace.png similarity index 100% rename from docs/root/start/sandboxes/_static/skywalking-trace.png rename to examples/skywalking/_static/skywalking-trace.png diff --git a/docs/root/start/sandboxes/skywalking.rst b/examples/skywalking/example.rst similarity index 95% rename from docs/root/start/sandboxes/skywalking.rst rename to examples/skywalking/example.rst index 3b0ffb77151a..d3c8baddc3d7 100644 --- a/docs/root/start/sandboxes/skywalking.rst +++ b/examples/skywalking/example.rst @@ -109,15 +109,15 @@ You should see the Skywalking dashboard. You may need to wait a moment for the traces to be added, but clicking on ``General Service > Services``, you should see the Envoy services listed. -.. image:: /start/sandboxes/_static/skywalking-services.png +.. image:: /start/sandboxes/_include/skywalking/_static/skywalking-services.png From here you can explore the metrics and views that skywalking offers, such as the ``Topology``: -.. image:: /start/sandboxes/_static/skywalking-topology.png +.. image:: /start/sandboxes/_include/skywalking/_static/skywalking-topology.png You can also view tracing information for the requests that you made: -.. image:: /start/sandboxes/_static/skywalking-trace.png +.. image:: /start/sandboxes/_include/skywalking/_static/skywalking-trace.png .. seealso:: diff --git a/docs/root/start/sandboxes/tls-inspector.rst b/examples/tls-inspector/example.rst similarity index 100% rename from docs/root/start/sandboxes/tls-inspector.rst rename to examples/tls-inspector/example.rst diff --git a/docs/root/start/sandboxes/tls-sni.rst b/examples/tls-sni/example.rst similarity index 100% rename from docs/root/start/sandboxes/tls-sni.rst rename to examples/tls-sni/example.rst diff --git a/docs/root/start/sandboxes/tls.rst b/examples/tls/example.rst similarity index 100% rename from docs/root/start/sandboxes/tls.rst rename to examples/tls/example.rst diff --git a/docs/root/start/sandboxes/udp.rst b/examples/udp/example.rst similarity index 100% rename from docs/root/start/sandboxes/udp.rst rename to examples/udp/example.rst diff --git a/examples/wasm-cc/BUILD b/examples/wasm-cc/BUILD index d8a2c77e2c9f..2ea0b7d13906 100644 --- a/examples/wasm-cc/BUILD +++ b/examples/wasm-cc/BUILD @@ -9,6 +9,8 @@ licenses(["notice"]) # Apache 2 envoy_package() +exports_files(["example.rst"]) + selects.config_setting_group( name = "include_wasm_config", match_all = [ @@ -46,5 +48,5 @@ envoy_wasm_cc_binary( filegroup( name = "files", - srcs = glob(["**/*"]), + srcs = glob(["**/*"], exclude = ["example.rst"]), ) diff --git a/docs/root/start/sandboxes/wasm-cc.rst b/examples/wasm-cc/example.rst similarity index 100% rename from docs/root/start/sandboxes/wasm-cc.rst rename to examples/wasm-cc/example.rst diff --git a/docs/root/start/sandboxes/websocket.rst b/examples/websocket/example.rst similarity index 100% rename from docs/root/start/sandboxes/websocket.rst rename to examples/websocket/example.rst diff --git a/docs/root/start/sandboxes/_static/zipkin-ui-dependency.png b/examples/zipkin/_static/zipkin-ui-dependency.png similarity index 100% rename from docs/root/start/sandboxes/_static/zipkin-ui-dependency.png rename to examples/zipkin/_static/zipkin-ui-dependency.png diff --git a/docs/root/start/sandboxes/_static/zipkin-ui.png b/examples/zipkin/_static/zipkin-ui.png similarity index 100% rename from docs/root/start/sandboxes/_static/zipkin-ui.png rename to examples/zipkin/_static/zipkin-ui.png diff --git a/docs/root/start/sandboxes/zipkin.rst b/examples/zipkin/example.rst similarity index 96% rename from docs/root/start/sandboxes/zipkin.rst rename to examples/zipkin/example.rst index 8d402711ae67..34c0f9c82e9e 100644 --- a/docs/root/start/sandboxes/zipkin.rst +++ b/examples/zipkin/example.rst @@ -112,11 +112,11 @@ and other contextual information. Note that Zipkin identifies the Envoy proxies by the name provided in the bootstrap ``node/cluster`` configuration. -.. image:: /start/sandboxes/_static/zipkin-ui.png +.. image:: /start/sandboxes/_include/zipkin/_static/zipkin-ui.png You can also explore the Zipkin dependency UI to view relationships between nodes and the path of traces. -.. image:: /start/sandboxes/_static/zipkin-ui-dependency.png +.. image:: /start/sandboxes/_include/zipkin/_static/zipkin-ui-dependency.png .. seealso:: diff --git a/docs/root/start/sandboxes/zstd.rst b/examples/zstd/example.rst similarity index 100% rename from docs/root/start/sandboxes/zstd.rst rename to examples/zstd/example.rst From afd019b08de39d4ab864b626af2c839c74bc6c48 Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 5 Jun 2024 16:25:33 +0100 Subject: [PATCH 17/61] mobile/ci: Remove config for `Mobile/Core` (#34563) Signed-off-by: Ryan Northey --- .github/config.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/config.yml b/.github/config.yml index 846771e06a19..189bbc77eae1 100644 --- a/.github/config.yml +++ b/.github/config.yml @@ -53,11 +53,6 @@ checks: on-run: - mobile-compile-time-cc - mobile-compile-time-options - mobile-core: - name: Mobile/Core - required: true - on-run: - - mobile-core mobile-coverage: name: Mobile/Coverage required: true @@ -258,22 +253,6 @@ run: - mobile/.bazelrc - mobile/**/* - tools/code_format/check_format.py - mobile-core: - paths: - - .bazelrc - - .bazelversion - - .github/config.yml - - api/**/* - - bazel/external/quiche.BUILD - - bazel/repository_locations.bzl - - envoy/**/* - - mobile/.bazelrc - - mobile/**/* - - source/**/* - - test/config/**/* - - test/integration/* - - test/mocks/**/* - - test/test_common/**/* mobile-format: paths: - .bazelrc From bdbf0de7ee1070eeb04638d6290cdccde7051b6b Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Wed, 5 Jun 2024 11:46:25 -0500 Subject: [PATCH 18/61] mobile: Add response flags in the error message (#34533) This PR adds response flags into the error message for more debugging info. This PR also updates the logic in Cronvoy to perform an error check based on the response flags first and then do the HTTP/3 protocol error check last. Risk Level: low Testing: unit tests Docs Changes: n/a Release Notes: n/a Platform Specific Features: mobile Signed-off-by: Fredy Wijaya --- mobile/library/common/http/client.cc | 8 ++++- .../java/org/chromium/net/impl/Errors.java | 32 ++++++++++--------- mobile/test/common/http/client_test.cc | 8 +++-- .../chromium/net/CronetUrlRequestTest.java | 22 ++++++------- .../org/chromium/net/impl/ErrorsTest.java | 11 ++++++- 5 files changed, 51 insertions(+), 30 deletions(-) diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index a1522e4982c3..f44c09b08066 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -450,14 +450,20 @@ void Client::DirectStreamCallbacks::latchError() { } error_msg_details.push_back(absl::StrCat("ERROR_CODE: ", error_->error_code)); + std::vector response_flags(info.responseFlags().size()); + std::transform(info.responseFlags().begin(), info.responseFlags().end(), response_flags.begin(), + [](StreamInfo::ResponseFlag flag) { return std::to_string(flag.value()); }); + error_msg_details.push_back(absl::StrCat("RESPONSE_FLAGS: ", absl::StrJoin(response_flags, ","))); if (std::string resp_code_details = info.responseCodeDetails().value_or(""); !resp_code_details.empty()) { error_msg_details.push_back(absl::StrCat("DETAILS: ", std::move(resp_code_details))); } // The format of the error message propogated to callbacks is: - // RESPONSE_CODE: {RESPONSE_CODE}|ERROR_CODE: {ERROR_CODE}|DETAILS: {DETAILS} + // RESPONSE_CODE: {value}|ERROR_CODE: {value}|RESPONSE_FLAGS: {value}|DETAILS: {value} + // // Where RESPONSE_CODE is the HTTP response code from StreamInfo::responseCode(). // ERROR_CODE is of the envoy_error_code_t enum type, and gets mapped from RESPONSE_CODE. + // RESPONSE_FLAGS comes from StreamInfo::responseFlags(). // DETAILS is the contents of StreamInfo::responseCodeDetails(). error_->message = absl::StrJoin(std::move(error_msg_details), "|"); error_->attempt_count = info.attemptCount().value_or(0); diff --git a/mobile/library/java/org/chromium/net/impl/Errors.java b/mobile/library/java/org/chromium/net/impl/Errors.java index 66e686c7e419..7c6aa9132b7e 100644 --- a/mobile/library/java/org/chromium/net/impl/Errors.java +++ b/mobile/library/java/org/chromium/net/impl/Errors.java @@ -21,8 +21,10 @@ public class Errors { public static final int QUIC_INTERNAL_ERROR = 1; private static final Map ENVOYMOBILE_ERROR_TO_NET_ERROR = buildErrorMap(); - /**Subset of errors defined in - * https://github.com/envoyproxy/envoy/blob/main/envoy/stream_info/stream_info.h */ + /** + * Subset of errors defined in + * + */ @LongDef(flag = true, value = {EnvoyMobileError.DNS_RESOLUTION_FAILED, EnvoyMobileError.DURATION_TIMEOUT, EnvoyMobileError.STREAM_IDLE_TIMEOUT, @@ -31,12 +33,12 @@ public class Errors { EnvoyMobileError.UPSTREAM_REMOTE_RESET}) @Retention(RetentionPolicy.SOURCE) public @interface EnvoyMobileError { - long DNS_RESOLUTION_FAILED = 0x4000000; - long DURATION_TIMEOUT = 0x400000; - long STREAM_IDLE_TIMEOUT = 0x10000; - long UPSTREAM_CONNECTION_FAILURE = 0x20; - long UPSTREAM_CONNECTION_TERMINATION = 0x40; - long UPSTREAM_REMOTE_RESET = 0x10; + long DNS_RESOLUTION_FAILED = 1 << 26; // 0x4000000; + long DURATION_TIMEOUT = 1 << 22; // 0x400000; + long STREAM_IDLE_TIMEOUT = 1 << 16; // 0x10000 + long UPSTREAM_CONNECTION_FAILURE = 1 << 5; // 0x20 + long UPSTREAM_CONNECTION_TERMINATION = 1 << 6; // 0x40 + long UPSTREAM_REMOTE_RESET = 1 << 4; // 0x10 } /** Subset of errors defined in chromium/src/net/base/net_error_list.h */ @@ -81,19 +83,19 @@ public static NetError mapEnvoyMobileErrorToNetError(EnvoyFinalStreamIntel final return NetError.ERR_INTERNET_DISCONNECTED; } - // Check if negotiated_protocol is quic - // TODO(abeyad): Assigning all errors that happen when using HTTP/3 to QUIC_PROTOCOL_ERROR - // can mask the real error (DNS, upstream connection, etc). Revisit the error conversion logic. - if (finalStreamIntel.getUpstreamProtocol() == UpstreamHttpProtocol.HTTP3) { - return NetError.ERR_QUIC_PROTOCOL_ERROR; - } - // This will only map the first matched error to a NetError code. for (Map.Entry entry : ENVOYMOBILE_ERROR_TO_NET_ERROR.entrySet()) { if ((responseFlag & entry.getKey()) != 0) { return entry.getValue(); } } + + // Check if negotiated_protocol is quic + // TODO(abeyad): Assigning all errors that happen when using HTTP/3 to QUIC_PROTOCOL_ERROR + // can mask the real error (DNS, upstream connection, etc). Revisit the error conversion logic. + if (finalStreamIntel.getUpstreamProtocol() == UpstreamHttpProtocol.HTTP3) { + return NetError.ERR_QUIC_PROTOCOL_ERROR; + } return NetError.ERR_OTHER; } diff --git a/mobile/test/common/http/client_test.cc b/mobile/test/common/http/client_test.cc index e672e3a1dc41..2aebc92598c0 100644 --- a/mobile/test/common/http/client_test.cc +++ b/mobile/test/common/http/client_test.cc @@ -22,6 +22,7 @@ using testing::_; using testing::AnyNumber; using testing::ContainsRegex; +using testing::Eq; using testing::NiceMock; using testing::Return; using testing::ReturnPointee; @@ -476,8 +477,9 @@ TEST_P(ClientTest, EnvoyLocalError) { stream_callbacks.on_error_ = [&](EnvoyError error, envoy_stream_intel, envoy_final_stream_intel) -> void { EXPECT_EQ(error.error_code, ENVOY_CONNECTION_FAILURE); - EXPECT_THAT(error.message, - ContainsRegex("RESPONSE_CODE: 503|ERROR_CODE: 2|DETAILS: failed miserably")); + EXPECT_THAT( + error.message, + Eq("RESPONSE_CODE: 503|ERROR_CODE: 2|RESPONSE_FLAGS: 4,26|DETAILS: failed miserably")); EXPECT_EQ(error.attempt_count, 123); callbacks_called.on_error_calls_++; }; @@ -498,6 +500,8 @@ TEST_P(ClientTest, EnvoyLocalError) { EXPECT_CALL(dispatcher_, deferredDelete_(_)); stream_info_.setResponseCode(503); stream_info_.setResponseCodeDetails("failed miserably"); + stream_info_.setResponseFlag(StreamInfo::ResponseFlag(StreamInfo::UpstreamRemoteReset)); + stream_info_.setResponseFlag(StreamInfo::ResponseFlag(StreamInfo::DnsResolutionFailed)); stream_info_.setAttemptCount(123); response_encoder_->getStream().resetStream(Http::StreamResetReason::LocalConnectionFailure); ASSERT_EQ(callbacks_called.on_headers_calls_, 0); diff --git a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java index b6acfda3f8d9..a442014b3315 100644 --- a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java +++ b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java @@ -2057,23 +2057,23 @@ public void testDestroyUploadDataStreamAdapterOnSucceededCallback() throws Excep public void testErrorCodes() throws Exception { checkSpecificErrorCode(EnvoyMobileError.DNS_RESOLUTION_FAILED, NetError.ERR_NAME_NOT_RESOLVED, NetworkException.ERROR_HOSTNAME_NOT_RESOLVED, "NAME_NOT_RESOLVED", false, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0"); + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 26"); checkSpecificErrorCode(EnvoyMobileError.UPSTREAM_CONNECTION_TERMINATION, NetError.ERR_CONNECTION_CLOSED, NetworkException.ERROR_CONNECTION_CLOSED, "CONNECTION_CLOSED", true, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0"); + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 6"); checkSpecificErrorCode(EnvoyMobileError.UPSTREAM_CONNECTION_FAILURE, NetError.ERR_CONNECTION_REFUSED, NetworkException.ERROR_CONNECTION_REFUSED, "CONNECTION_REFUSED", false, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0"); + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 5"); checkSpecificErrorCode(EnvoyMobileError.UPSTREAM_REMOTE_RESET, NetError.ERR_CONNECTION_RESET, NetworkException.ERROR_CONNECTION_RESET, "CONNECTION_RESET", true, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0"); + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 4"); checkSpecificErrorCode(EnvoyMobileError.STREAM_IDLE_TIMEOUT, NetError.ERR_TIMED_OUT, NetworkException.ERROR_TIMED_OUT, "TIMED_OUT", true, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0"); + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 16"); checkSpecificErrorCode(0x2000, NetError.ERR_OTHER, NetworkException.ERROR_OTHER, "OTHER", false, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0"); + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 13"); } /* @@ -2095,10 +2095,10 @@ public void testInternetDisconnectedError() throws Exception { androidNetworkMonitor.onReceive(getContext(), intent); // send request and confirm errorcode - checkSpecificErrorCode(EnvoyMobileError.DNS_RESOLUTION_FAILED, - NetError.ERR_INTERNET_DISCONNECTED, - NetworkException.ERROR_INTERNET_DISCONNECTED, "INTERNET_DISCONNECTED", - false, /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0"); + checkSpecificErrorCode( + EnvoyMobileError.DNS_RESOLUTION_FAILED, NetError.ERR_INTERNET_DISCONNECTED, + NetworkException.ERROR_INTERNET_DISCONNECTED, "INTERNET_DISCONNECTED", false, + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 26"); // bring back online since the AndroidNetworkMonitor class is a singleton connectivityManager.setActiveNetworkInfo(networkInfo); @@ -2319,7 +2319,7 @@ public void test404Head() throws Exception { checkSpecificErrorCode(EnvoyMobileError.UPSTREAM_CONNECTION_FAILURE, NetError.ERR_CONNECTION_REFUSED, NetworkException.ERROR_CONNECTION_REFUSED, "CONNECTION_REFUSED", false, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0"); + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 5"); } @Test diff --git a/mobile/test/java/org/chromium/net/impl/ErrorsTest.java b/mobile/test/java/org/chromium/net/impl/ErrorsTest.java index 6536beda875c..dd1db3b8ee0d 100644 --- a/mobile/test/java/org/chromium/net/impl/ErrorsTest.java +++ b/mobile/test/java/org/chromium/net/impl/ErrorsTest.java @@ -13,7 +13,6 @@ @RunWith(RobolectricTestRunner.class) public class ErrorsTest { - @Test public void testMapEnvoyMobileErrorToNetErrorHttp3() throws Exception { // 8 corresponds to NoRouteFound in StreamInfo::CoreResponseFlag: @@ -35,6 +34,16 @@ public void testMapEnvoyMobileErrorToNetErrorFoundInMap() throws Exception { assertEquals(NetError.ERR_CONNECTION_RESET, error); } + @Test + public void testMapEnvoyMobileErrorToNetErrorFoundInMapOnHttp3() throws Exception { + // 4 corresponds to UpstreamRemoteReset in StreamInfo::CoreResponseFlag: + // https://github.com/envoyproxy/envoy/blob/410e9a77bd6b74abb3e1545b4fd077e734d0fce3/envoy/stream_info/stream_info.h#L39 + long responseFlags = 1L << 4; + EnvoyFinalStreamIntel intel = constructStreamIntel(responseFlags, UpstreamHttpProtocol.HTTP3); + NetError error = mapEnvoyMobileErrorToNetError(intel); + assertEquals(NetError.ERR_CONNECTION_RESET, error); + } + @Test public void testMapEnvoyMobileErrorToNetErrorMultipleResponseFlags() throws Exception { // 4 corresponds to UpstreamRemoteReset and 16 corresponds to StreamIdleTimeout in From dc38617159d9eb2dd5902e7a6b5bddf4db1fb838 Mon Sep 17 00:00:00 2001 From: "Antonio V. Leonti" <53806445+antoniovleonti@users.noreply.github.com> Date: Wed, 5 Jun 2024 12:56:00 -0400 Subject: [PATCH 19/61] add rbac http fuzzer (#34552) --------- Signed-off-by: antoniovleonti --- .../filters/http/rbac/rbac_filter.h | 2 +- test/extensions/filters/http/rbac/BUILD | 27 ++++++ .../http/rbac/rbac_filter_corpus/example | 7 ++ .../no_registered_implementation | 14 +++ .../rbac_filter_corpus/track_per_rule_stats | 5 ++ .../filters/http/rbac/rbac_filter_fuzz.proto | 13 +++ .../http/rbac/rbac_filter_fuzz_test.cc | 88 +++++++++++++++++++ 7 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 test/extensions/filters/http/rbac/rbac_filter_corpus/example create mode 100644 test/extensions/filters/http/rbac/rbac_filter_corpus/no_registered_implementation create mode 100644 test/extensions/filters/http/rbac/rbac_filter_corpus/track_per_rule_stats create mode 100644 test/extensions/filters/http/rbac/rbac_filter_fuzz.proto create mode 100644 test/extensions/filters/http/rbac/rbac_filter_fuzz_test.cc diff --git a/source/extensions/filters/http/rbac/rbac_filter.h b/source/extensions/filters/http/rbac/rbac_filter.h index e20568432fcd..6de163c11369 100644 --- a/source/extensions/filters/http/rbac/rbac_filter.h +++ b/source/extensions/filters/http/rbac/rbac_filter.h @@ -98,7 +98,7 @@ class RoleBasedAccessControlFilter : public Http::StreamDecoderFilter, public Logger::Loggable { public: RoleBasedAccessControlFilter(RoleBasedAccessControlFilterConfigSharedPtr config) - : config_(config) {} + : config_(std::move(config)) {} // Http::StreamDecoderFilter Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, diff --git a/test/extensions/filters/http/rbac/BUILD b/test/extensions/filters/http/rbac/BUILD index 4c397d1e0389..92e1d8ab3bbc 100644 --- a/test/extensions/filters/http/rbac/BUILD +++ b/test/extensions/filters/http/rbac/BUILD @@ -1,6 +1,8 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_fuzz_test", "envoy_package", + "envoy_proto_library", ) load( "//test/extensions:extensions_build_system.bzl", @@ -89,3 +91,28 @@ envoy_extension_cc_mock( "@envoy_api//envoy/extensions/filters/http/rbac/v3:pkg_cc_proto", ], ) + +envoy_proto_library( + name = "rbac_filter_fuzz_proto", + srcs = ["rbac_filter_fuzz.proto"], + deps = [ + "//test/fuzz:common_proto", + "@envoy_api//envoy/extensions/filters/http/rbac/v3:pkg", + ], +) + +envoy_cc_fuzz_test( + name = "rbac_filter_fuzz_test", + srcs = ["rbac_filter_fuzz_test.cc"], + corpus = "rbac_filter_corpus", + deps = [ + ":rbac_filter_fuzz_proto_cc_proto", + "//source/common/buffer:buffer_lib", + "//source/extensions/filters/http/rbac:rbac_filter_lib", + "//test/extensions/filters/http/common/fuzz:http_filter_fuzzer_lib", + "//test/mocks/http:http_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:server_factory_context_mocks", + "@envoy_api//envoy/extensions/filters/http/rbac/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/filters/http/rbac/rbac_filter_corpus/example b/test/extensions/filters/http/rbac/rbac_filter_corpus/example new file mode 100644 index 000000000000..7da000f76a2f --- /dev/null +++ b/test/extensions/filters/http/rbac/rbac_filter_corpus/example @@ -0,0 +1,7 @@ +config { +} +request_data { + http_body { + data: "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" + } +} diff --git a/test/extensions/filters/http/rbac/rbac_filter_corpus/no_registered_implementation b/test/extensions/filters/http/rbac/rbac_filter_corpus/no_registered_implementation new file mode 100644 index 000000000000..fc6e9dc504cc --- /dev/null +++ b/test/extensions/filters/http/rbac/rbac_filter_corpus/no_registered_implementation @@ -0,0 +1,14 @@ +config { + matcher { + on_no_match { + action { + name: "8888888888888888888888888888888888888888888888" + typed_config { + value: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377" + } + } + } + } +} +request_data { +} diff --git a/test/extensions/filters/http/rbac/rbac_filter_corpus/track_per_rule_stats b/test/extensions/filters/http/rbac/rbac_filter_corpus/track_per_rule_stats new file mode 100644 index 000000000000..7146655025b9 --- /dev/null +++ b/test/extensions/filters/http/rbac/rbac_filter_corpus/track_per_rule_stats @@ -0,0 +1,5 @@ +config { + track_per_rule_stats: true +} +request_data { +} diff --git a/test/extensions/filters/http/rbac/rbac_filter_fuzz.proto b/test/extensions/filters/http/rbac/rbac_filter_fuzz.proto new file mode 100644 index 000000000000..0a87f76dd07b --- /dev/null +++ b/test/extensions/filters/http/rbac/rbac_filter_fuzz.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; +package envoy.extensions.filters.http.rbac; + +import "envoy/extensions/filters/http/rbac/v3/rbac.proto"; +import "test/fuzz/common.proto"; +import "validate/validate.proto"; + +// We only fuzz a single request per iteration. +message RbacTestCase { + envoy.extensions.filters.http.rbac.v3.RBAC config = 1 + [(validate.rules).message = {required: true}]; + test.fuzz.HttpData request_data = 2 [(validate.rules).message = {required: true}]; +} diff --git a/test/extensions/filters/http/rbac/rbac_filter_fuzz_test.cc b/test/extensions/filters/http/rbac/rbac_filter_fuzz_test.cc new file mode 100644 index 000000000000..c8e7660278ee --- /dev/null +++ b/test/extensions/filters/http/rbac/rbac_filter_fuzz_test.cc @@ -0,0 +1,88 @@ +#include "envoy/extensions/filters/http/rbac/v3/rbac.pb.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/common/network/address_impl.h" +#include "source/extensions/filters/http/rbac/rbac_filter.h" + +#include "test/extensions/filters/http/common/fuzz/http_filter_fuzzer.h" +#include "test/extensions/filters/http/rbac/rbac_filter_fuzz.pb.validate.h" +#include "test/fuzz/fuzz_runner.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/server/server_factory_context.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using Envoy::Extensions::HttpFilters::RBACFilter::RoleBasedAccessControlFilter; +using Envoy::Extensions::HttpFilters::RBACFilter::RoleBasedAccessControlFilterConfig; +using Envoy::Extensions::HttpFilters::RBACFilter::RoleBasedAccessControlFilterConfigSharedPtr; +using testing::WithArgs; + +namespace Envoy { +namespace Extensions { +namespace Http { +namespace Rbac { + +class ReusableFilterFactory { +public: + ReusableFilterFactory() + : addr_(std::make_shared("/test/test.sock")) { + connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress(addr_); + connection_.stream_info_.downstream_connection_info_provider_->setLocalAddress(addr_); + + ON_CALL(callbacks_, connection()) + .WillByDefault(testing::Return(OptRef{connection_})); + } + + absl::StatusOr> + newFilter(const envoy::extensions::filters::http::rbac::v3::RBAC& proto_config) { + RoleBasedAccessControlFilterConfigSharedPtr config; + try { + config = make_shared( + proto_config, "stats_prefix", *stats_store_.rootScope(), context_, + ProtobufMessage::getStrictValidationVisitor()); + } catch (const EnvoyException& e) { + return absl::InvalidArgumentError( + absl::StrCat("EnvoyException during validation: ", e.what())); + } + auto filter = std::make_unique(std::move(config)); + filter->setDecoderFilterCallbacks(callbacks_); + return filter; + } + +private: + Network::Address::InstanceConstSharedPtr addr_; + NiceMock callbacks_; + NiceMock connection_; + NiceMock context_; + Stats::TestUtil::TestStore stats_store_; +}; + +DEFINE_PROTO_FUZZER(const envoy::extensions::filters::http::rbac::RbacTestCase& input) { + try { + TestUtility::validate(input); + } catch (const EnvoyException& e) { + ENVOY_LOG_MISC(debug, "EnvoyException during validation: {}", e.what()); + return; + } + + // This is static to avoid recreating all the mocks between each fuzz test. The class is + // stateless; it just stores mocks and uses them when you call newFilter(). + static ReusableFilterFactory filter_factory; + absl::StatusOr> filter = + filter_factory.newFilter(input.config()); + if (!filter.ok()) { + ENVOY_LOG_MISC(debug, "Failed to create filter: {}", filter.status()); + return; + } + + static Envoy::Extensions::HttpFilters::HttpFilterFuzzer fuzzer; + fuzzer.runData(static_cast(filter->get()), + input.request_data()); +} + +} // namespace Rbac +} // namespace Http +} // namespace Extensions +} // namespace Envoy From fe61ffe55e50a2c3e05388d53a0b8cf7b90c9e7a Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Wed, 5 Jun 2024 12:08:12 -0500 Subject: [PATCH 20/61] mobile: Increase the engine running timeout to 30 seconds (#34566) Risk Level: low Testing: unit tests Docs Changes: n/a Release Notes: n/a Platform Specific Features: mobile Signed-off-by: Fredy Wijaya --- mobile/library/common/internal_engine.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/library/common/internal_engine.cc b/mobile/library/common/internal_engine.cc index 46d6a09e2e61..19c25085c0b8 100644 --- a/mobile/library/common/internal_engine.cc +++ b/mobile/library/common/internal_engine.cc @@ -12,7 +12,7 @@ namespace Envoy { namespace { -constexpr absl::Duration ENGINE_RUNNING_TIMEOUT = absl::Seconds(3); +constexpr absl::Duration ENGINE_RUNNING_TIMEOUT = absl::Seconds(30); } // namespace static std::atomic current_stream_handle_{0}; From 1bfbdacaae6862d17620f92f293d0b529921c8ff Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Wed, 5 Jun 2024 12:50:35 -0500 Subject: [PATCH 21/61] mobile: Pass EnvoyError by const ref (#34562) This PR updates the on_error_ callback to pass EnvoyError by const ref instead of value. This PR also appends _ in the struct field members of EnvoyError to conform with Envoy naming convention. Risk Level: low Testing: unit tests Docs Changes: n/a Release Notes: n/a Platform Specific Features: mobile Signed-off-by: Fredy Wijaya --- .../examples/cc/fetch_client/fetch_client.cc | 4 ++-- mobile/library/common/bridge/utility.cc | 8 ++++---- mobile/library/common/engine_types.h | 10 +++++----- mobile/library/common/http/client.cc | 14 +++++++------- mobile/library/jni/jni_impl.cc | 8 ++++---- .../library/objective-c/EnvoyHTTPStreamImpl.mm | 3 ++- mobile/test/cc/engine_test.cc | 15 ++++++--------- mobile/test/cc/integration/lifetimes_test.cc | 5 ++--- .../test/cc/integration/receive_data_test.cc | 5 ++--- .../cc/integration/receive_headers_test.cc | 5 ++--- .../cc/integration/receive_trailers_test.cc | 5 ++--- mobile/test/cc/integration/send_data_test.cc | 5 ++--- .../test/cc/integration/send_headers_test.cc | 5 ++--- .../test/cc/integration/send_trailers_test.cc | 5 ++--- mobile/test/common/http/client_test.cc | 18 +++++++++--------- .../integration/client_integration_test.cc | 5 ++--- 16 files changed, 55 insertions(+), 65 deletions(-) diff --git a/mobile/examples/cc/fetch_client/fetch_client.cc b/mobile/examples/cc/fetch_client/fetch_client.cc index d731dff7cd89..f7123e94fadf 100644 --- a/mobile/examples/cc/fetch_client/fetch_client.cc +++ b/mobile/examples/cc/fetch_client/fetch_client.cc @@ -79,10 +79,10 @@ void Fetch::sendRequest(absl::string_view url_string) { << final_intel.stream_end_ms - final_intel.stream_start_ms << "ms\n"; request_finished.Notify(); }; - stream_callbacks.on_error_ = [&request_finished](EnvoyError error, envoy_stream_intel, + stream_callbacks.on_error_ = [&request_finished](const EnvoyError& error, envoy_stream_intel, envoy_final_stream_intel final_intel) { std::cerr << "Request failed after " << final_intel.stream_end_ms - final_intel.stream_start_ms - << "ms with error message: " << error.message << "\n"; + << "ms with error message: " << error.message_ << "\n"; request_finished.Notify(); }; stream_callbacks.on_cancel_ = [&request_finished](envoy_stream_intel, diff --git a/mobile/library/common/bridge/utility.cc b/mobile/library/common/bridge/utility.cc index 25d7f6629c35..f63b6a35f3dd 100644 --- a/mobile/library/common/bridge/utility.cc +++ b/mobile/library/common/bridge/utility.cc @@ -32,10 +32,10 @@ envoy_map makeEnvoyMap(std::initializer_list envoy_error toBridgeError(const EnvoyError& error) { envoy_error error_bridge{}; - error_bridge.message = copyToBridgeData(error.message); - error_bridge.error_code = error.error_code; - if (error.attempt_count.has_value()) { - error_bridge.attempt_count = *error.attempt_count; + error_bridge.message = copyToBridgeData(error.message_); + error_bridge.error_code = error.error_code_; + if (error.attempt_count_.has_value()) { + error_bridge.attempt_count = *error.attempt_count_; } return error_bridge; } diff --git a/mobile/library/common/engine_types.h b/mobile/library/common/engine_types.h index a923751d69c0..479bc344b863 100644 --- a/mobile/library/common/engine_types.h +++ b/mobile/library/common/engine_types.h @@ -38,9 +38,9 @@ struct EnvoyEventTracker { /** The Envoy error passed into `EnvoyStreamCallbacks::on_error_` callback. */ struct EnvoyError { - envoy_error_code_t error_code; - std::string message; - absl::optional attempt_count = absl::nullopt; + envoy_error_code_t error_code_; + std::string message_; + absl::optional attempt_count_ = absl::nullopt; }; /** The callbacks for the stream. */ @@ -106,8 +106,8 @@ struct EnvoyStreamCallbacks { * - stream_intel: contains internal stream metrics. * - final_stream_intel: contains final internal stream metrics. */ - absl::AnyInvocable on_error_ = - [](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) {}; + absl::AnyInvocable + on_error_ = [](const EnvoyError&, envoy_stream_intel, envoy_final_stream_intel) {}; /** * The callback for when the stream is cancelled. diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index f44c09b08066..9db7e5db94f5 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -433,23 +433,23 @@ void Client::DirectStreamCallbacks::latchError() { OptRef request_decoder = direct_stream_.requestDecoder(); if (!request_decoder) { - error_->message = ""; + error_->message_ = ""; return; } const auto& info = request_decoder->streamInfo(); std::vector error_msg_details; if (info.responseCode().has_value()) { - error_->error_code = Bridge::Utility::errorCodeFromLocalStatus( + error_->error_code_ = Bridge::Utility::errorCodeFromLocalStatus( static_cast(info.responseCode().value())); error_msg_details.push_back(absl::StrCat("RESPONSE_CODE: ", info.responseCode().value())); } else if (StreamInfo::isStreamIdleTimeout(info)) { - error_->error_code = ENVOY_REQUEST_TIMEOUT; + error_->error_code_ = ENVOY_REQUEST_TIMEOUT; } else { - error_->error_code = ENVOY_STREAM_RESET; + error_->error_code_ = ENVOY_STREAM_RESET; } - error_msg_details.push_back(absl::StrCat("ERROR_CODE: ", error_->error_code)); + error_msg_details.push_back(absl::StrCat("ERROR_CODE: ", error_->error_code_)); std::vector response_flags(info.responseFlags().size()); std::transform(info.responseFlags().begin(), info.responseFlags().end(), response_flags.begin(), [](StreamInfo::ResponseFlag flag) { return std::to_string(flag.value()); }); @@ -465,8 +465,8 @@ void Client::DirectStreamCallbacks::latchError() { // ERROR_CODE is of the envoy_error_code_t enum type, and gets mapped from RESPONSE_CODE. // RESPONSE_FLAGS comes from StreamInfo::responseFlags(). // DETAILS is the contents of StreamInfo::responseCodeDetails(). - error_->message = absl::StrJoin(std::move(error_msg_details), "|"); - error_->attempt_count = info.attemptCount().value_or(0); + error_->message_ = absl::StrJoin(std::move(error_msg_details), "|"); + error_->attempt_count_ = info.attemptCount().value_or(0); } Client::DirectStream::DirectStream(envoy_stream_t stream_handle, Client& http_client) diff --git a/mobile/library/jni/jni_impl.cc b/mobile/library/jni/jni_impl.cc index 0857f7671e9a..1cd9b8923e4e 100644 --- a/mobile/library/jni/jni_impl.cc +++ b/mobile/library/jni/jni_impl.cc @@ -841,7 +841,7 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibra jni_helper.getEnv()->DeleteGlobalRef(java_stream_callbacks_global_ref); }; stream_callbacks.on_error_ = [java_stream_callbacks_global_ref]( - Envoy::EnvoyError error, envoy_stream_intel stream_intel, + const Envoy::EnvoyError& error, envoy_stream_intel stream_intel, envoy_final_stream_intel final_stream_intel) { Envoy::JNI::JniHelper jni_helper(Envoy::JNI::JniHelper::getThreadLocalEnv()); auto java_stream_intel = Envoy::JNI::cppStreamIntelToJavaStreamIntel(jni_helper, stream_intel); @@ -852,10 +852,10 @@ extern "C" JNIEXPORT jint JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibra java_stream_callbacks_class.get(), "onError", "(ILjava/lang/String;ILio/envoyproxy/envoymobile/engine/types/EnvoyStreamIntel;" "Lio/envoyproxy/envoymobile/engine/types/EnvoyFinalStreamIntel;)V"); - auto java_error_message = Envoy::JNI::cppStringToJavaString(jni_helper, error.message); + auto java_error_message = Envoy::JNI::cppStringToJavaString(jni_helper, error.message_); jni_helper.callVoidMethod(java_stream_callbacks_global_ref, java_on_error_method_id, - static_cast(error.error_code), java_error_message.get(), - error.attempt_count.value_or(-1), java_stream_intel.get(), + static_cast(error.error_code_), java_error_message.get(), + error.attempt_count_.value_or(-1), java_stream_intel.get(), java_final_stream_intel.get()); // on_error_ is a terminal callback, delete the java_stream_callbacks_global_ref. jni_helper.getEnv()->DeleteGlobalRef(java_stream_callbacks_global_ref); diff --git a/mobile/library/objective-c/EnvoyHTTPStreamImpl.mm b/mobile/library/objective-c/EnvoyHTTPStreamImpl.mm index 430e91cd9f78..16d2d2745e7f 100644 --- a/mobile/library/objective-c/EnvoyHTTPStreamImpl.mm +++ b/mobile/library/objective-c/EnvoyHTTPStreamImpl.mm @@ -183,7 +183,8 @@ - (instancetype)initWithHandle:(envoy_stream_t)handle envoy_final_stream_intel final_stream_intel) { ios_on_complete(stream_intel, final_stream_intel, context); }; - streamCallbacks.on_error_ = [context](Envoy::EnvoyError error, envoy_stream_intel stream_intel, + streamCallbacks.on_error_ = [context](const Envoy::EnvoyError &error, + envoy_stream_intel stream_intel, envoy_final_stream_intel final_stream_intel) { envoy_error bridge_error = Envoy::Bridge::Utility::toBridgeError(error); ios_on_error(bridge_error, stream_intel, final_stream_intel, context); diff --git a/mobile/test/cc/engine_test.cc b/mobile/test/cc/engine_test.cc index b8d111bce024..5beba1d965c0 100644 --- a/mobile/test/cc/engine_test.cc +++ b/mobile/test/cc/engine_test.cc @@ -41,9 +41,8 @@ TEST(EngineTest, SetLogger) { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; @@ -89,9 +88,8 @@ TEST(EngineTest, SetEngineCallbacks) { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; @@ -160,9 +158,8 @@ TEST(EngineTest, DontWaitForOnEngineRunning) { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; diff --git a/mobile/test/cc/integration/lifetimes_test.cc b/mobile/test/cc/integration/lifetimes_test.cc index fc057d7396f3..8a7837e780f5 100644 --- a/mobile/test/cc/integration/lifetimes_test.cc +++ b/mobile/test/cc/integration/lifetimes_test.cc @@ -33,9 +33,8 @@ void sendRequest() { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; diff --git a/mobile/test/cc/integration/receive_data_test.cc b/mobile/test/cc/integration/receive_data_test.cc index e5d9499e94d6..56dd70d3d05d 100644 --- a/mobile/test/cc/integration/receive_data_test.cc +++ b/mobile/test/cc/integration/receive_data_test.cc @@ -40,9 +40,8 @@ TEST(ReceiveDataTest, Success) { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; diff --git a/mobile/test/cc/integration/receive_headers_test.cc b/mobile/test/cc/integration/receive_headers_test.cc index 6c635da9f956..eabe8efa3981 100644 --- a/mobile/test/cc/integration/receive_headers_test.cc +++ b/mobile/test/cc/integration/receive_headers_test.cc @@ -36,9 +36,8 @@ TEST(ReceiveHeadersTest, Success) { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; diff --git a/mobile/test/cc/integration/receive_trailers_test.cc b/mobile/test/cc/integration/receive_trailers_test.cc index 1519cc53d748..9e92d2a1ef2d 100644 --- a/mobile/test/cc/integration/receive_trailers_test.cc +++ b/mobile/test/cc/integration/receive_trailers_test.cc @@ -36,9 +36,8 @@ TEST(ReceiveTrailersTest, Success) { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; diff --git a/mobile/test/cc/integration/send_data_test.cc b/mobile/test/cc/integration/send_data_test.cc index 13a013e6138e..5f4ba1ab2887 100644 --- a/mobile/test/cc/integration/send_data_test.cc +++ b/mobile/test/cc/integration/send_data_test.cc @@ -47,9 +47,8 @@ TEST(SendDataTest, Success) { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; diff --git a/mobile/test/cc/integration/send_headers_test.cc b/mobile/test/cc/integration/send_headers_test.cc index 2b6f017c0b8a..a7bafda0dd44 100644 --- a/mobile/test/cc/integration/send_headers_test.cc +++ b/mobile/test/cc/integration/send_headers_test.cc @@ -47,9 +47,8 @@ TEST(SendHeadersTest, Success) { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; diff --git a/mobile/test/cc/integration/send_trailers_test.cc b/mobile/test/cc/integration/send_trailers_test.cc index 263c817abd46..a773d316e1ce 100644 --- a/mobile/test/cc/integration/send_trailers_test.cc +++ b/mobile/test/cc/integration/send_trailers_test.cc @@ -49,9 +49,8 @@ TEST(SendTrailersTest, Success) { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - stream_complete.Notify(); - }; + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { stream_complete.Notify(); }; stream_callbacks.on_cancel_ = [&](envoy_stream_intel, envoy_final_stream_intel) { stream_complete.Notify(); }; diff --git a/mobile/test/common/http/client_test.cc b/mobile/test/common/http/client_test.cc index 2aebc92598c0..a65e38333642 100644 --- a/mobile/test/common/http/client_test.cc +++ b/mobile/test/common/http/client_test.cc @@ -122,7 +122,7 @@ class ClientTest : public testing::TestWithParam { stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) -> void { callbacks_called.on_complete_calls_++; }; - stream_callbacks.on_error_ = [&](EnvoyError, envoy_stream_intel, + stream_callbacks.on_error_ = [&](const EnvoyError&, envoy_stream_intel, envoy_final_stream_intel) -> void { callbacks_called.on_error_calls_++; }; @@ -474,13 +474,13 @@ TEST_P(ClientTest, EnvoyLocalError) { // Override the on_error default with some custom checks. StreamCallbacksCalled callbacks_called; auto stream_callbacks = createDefaultStreamCallbacks(callbacks_called); - stream_callbacks.on_error_ = [&](EnvoyError error, envoy_stream_intel, + stream_callbacks.on_error_ = [&](const EnvoyError& error, envoy_stream_intel, envoy_final_stream_intel) -> void { - EXPECT_EQ(error.error_code, ENVOY_CONNECTION_FAILURE); + EXPECT_EQ(error.error_code_, ENVOY_CONNECTION_FAILURE); EXPECT_THAT( - error.message, + error.message_, Eq("RESPONSE_CODE: 503|ERROR_CODE: 2|RESPONSE_FLAGS: 4,26|DETAILS: failed miserably")); - EXPECT_EQ(error.attempt_count, 123); + EXPECT_EQ(error.attempt_count_, 123); callbacks_called.on_error_calls_++; }; @@ -553,11 +553,11 @@ TEST_P(ClientTest, RemoteResetAfterStreamStart) { callbacks_called.end_stream_with_headers_ = false; auto stream_callbacks = createDefaultStreamCallbacks(callbacks_called); - stream_callbacks.on_error_ = [&](EnvoyError error, envoy_stream_intel, + stream_callbacks.on_error_ = [&](const EnvoyError& error, envoy_stream_intel, envoy_final_stream_intel) -> void { - EXPECT_EQ(error.error_code, ENVOY_STREAM_RESET); - EXPECT_THAT(error.message, ContainsRegex("ERROR_CODE: 1")); - EXPECT_EQ(error.attempt_count, 0); + EXPECT_EQ(error.error_code_, ENVOY_STREAM_RESET); + EXPECT_THAT(error.message_, ContainsRegex("ERROR_CODE: 1")); + EXPECT_EQ(error.attempt_count_, 0); callbacks_called.on_error_calls_++; }; diff --git a/mobile/test/common/integration/client_integration_test.cc b/mobile/test/common/integration/client_integration_test.cc index e9d57dc0886b..37a3e94636cb 100644 --- a/mobile/test/common/integration/client_integration_test.cc +++ b/mobile/test/common/integration/client_integration_test.cc @@ -471,9 +471,8 @@ void ClientIntegrationTest::explicitFlowControlWithCancels(const uint32_t body_s // Allow reading up to 100 bytes. streams[i]->readData(100); }; - stream_callbacks.on_error_ = [](EnvoyError, envoy_stream_intel, envoy_final_stream_intel) { - RELEASE_ASSERT(0, "unexpected"); - }; + stream_callbacks.on_error_ = [](const EnvoyError&, envoy_stream_intel, + envoy_final_stream_intel) { RELEASE_ASSERT(0, "unexpected"); }; auto stream = createNewStream(std::move(stream_callbacks)); stream->sendHeaders(std::make_unique(default_request_headers_), From 9bfb2c9e8350fb2cf746b9772968b6db276de368 Mon Sep 17 00:00:00 2001 From: Tianyu <72890320+tyxia@users.noreply.github.com> Date: Wed, 5 Jun 2024 14:30:12 -0400 Subject: [PATCH 22/61] async_client: Fix yoda condition (#34517) Signed-off-by: tyxia --- source/common/grpc/async_client_impl.cc | 2 +- source/common/http/async_client_impl.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/grpc/async_client_impl.cc b/source/common/grpc/async_client_impl.cc index ce6fefb67e0e..dd89bc7bec97 100644 --- a/source/common/grpc/async_client_impl.cc +++ b/source/common/grpc/async_client_impl.cc @@ -88,7 +88,7 @@ AsyncStreamImpl::AsyncStreamImpl(AsyncClientImpl& parent, absl::string_view serv // Configure the maximum frame length decoder_.setMaxFrameLength(parent_.max_recv_message_length_); - if (nullptr != options.parent_span_) { + if (options.parent_span_ != nullptr) { const std::string child_span_name = options.child_span_name_.empty() ? absl::StrCat("async ", service_full_name, ".", method_name, " egress") diff --git a/source/common/http/async_client_impl.cc b/source/common/http/async_client_impl.cc index e51d701326d6..d761b76dcd6f 100644 --- a/source/common/http/async_client_impl.cc +++ b/source/common/http/async_client_impl.cc @@ -306,7 +306,7 @@ AsyncRequestSharedImpl::AsyncRequestSharedImpl(AsyncClientImpl& parent, if (!creation_status.ok()) { return; } - if (nullptr != options.parent_span_) { + if (options.parent_span_ != nullptr) { const std::string child_span_name = options.child_span_name_.empty() ? absl::StrCat("async ", parent.cluster_->name(), " egress") From 1c0ffc8ea0491a751d659fbc90c80601dd8356db Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 5 Jun 2024 14:54:11 -0400 Subject: [PATCH 23/61] cidr: removing exceptions (#34485) Risk Level: low Testing: updated tests Docs Changes: n/a Release Notes: n/a envoyproxy/envoy-mobile#176 Signed-off-by: Alyssa Wilk --- .../filter_chain_manager_impl.cc | 16 +- source/common/network/cidr_range.cc | 43 ++++-- source/common/network/cidr_range.h | 30 ++-- source/common/router/router_ratelimit.cc | 4 +- .../extensions/common/matcher/trie_matcher.h | 4 +- .../extensions/filters/common/rbac/matchers.h | 4 +- .../common/rbac/matchers/upstream_ip_port.cc | 3 +- .../http/ip_tagging/ip_tagging_filter.cc | 8 +- .../matching/input_matchers/ip/config.cc | 9 +- test/common/network/cidr_range_test.cc | 137 ++++++------------ test/common/network/lc_trie_speed_test.cc | 6 +- test/common/network/lc_trie_test.cc | 6 +- .../input_matchers/ip/matcher_test.cc | 14 +- 13 files changed, 127 insertions(+), 157 deletions(-) diff --git a/source/common/listener_manager/filter_chain_manager_impl.cc b/source/common/listener_manager/filter_chain_manager_impl.cc index 9c3331ab6f45..ea25462bc6b8 100644 --- a/source/common/listener_manager/filter_chain_manager_impl.cc +++ b/source/common/listener_manager/filter_chain_manager_impl.cc @@ -194,7 +194,8 @@ void FilterChainManagerImpl::addFilterChains( std::vector ips; ips.reserve(prefix_ranges.size()); for (const auto& ip : prefix_ranges) { - const auto& cidr_range = Network::Address::CidrRange::create(ip); + const auto& cidr_range = THROW_OR_RETURN_VALUE(Network::Address::CidrRange::create(ip), + Network::Address::CidrRange); ips.push_back(cidr_range.asString()); } return ips; @@ -457,15 +458,18 @@ std::pair> makeCidrListEntry(const s std::vector subnets; if (cidr == EMPTY_STRING) { if (Network::SocketInterfaceSingleton::get().ipFamilySupported(AF_INET)) { - subnets.push_back( - Network::Address::CidrRange::create(Network::Utility::getIpv4CidrCatchAllAddress())); + subnets.push_back(THROW_OR_RETURN_VALUE( + Network::Address::CidrRange::create(Network::Utility::getIpv4CidrCatchAllAddress()), + Network::Address::CidrRange)); } if (Network::SocketInterfaceSingleton::get().ipFamilySupported(AF_INET6)) { - subnets.push_back( - Network::Address::CidrRange::create(Network::Utility::getIpv6CidrCatchAllAddress())); + subnets.push_back(THROW_OR_RETURN_VALUE( + Network::Address::CidrRange::create(Network::Utility::getIpv6CidrCatchAllAddress()), + Network::Address::CidrRange)); } } else { - subnets.push_back(Network::Address::CidrRange::create(cidr)); + subnets.push_back(THROW_OR_RETURN_VALUE(Network::Address::CidrRange::create(cidr), + Network::Address::CidrRange)); } return std::make_pair>(T(data), std::move(subnets)); } diff --git a/source/common/network/cidr_range.cc b/source/common/network/cidr_range.cc index 7fc622644d52..8101552d3bbc 100644 --- a/source/common/network/cidr_range.cc +++ b/source/common/network/cidr_range.cc @@ -99,30 +99,42 @@ std::string CidrRange::asString() const { } // static -CidrRange CidrRange::create(InstanceConstSharedPtr address, int length) { +absl::StatusOr +CidrRange::create(InstanceConstSharedPtr address, int length, + absl::optional original_address_str) { InstanceConstSharedPtr ptr = truncateIpAddressAndLength(std::move(address), &length); - return {std::move(ptr), length}; + if (!ptr) { + return absl::InvalidArgumentError(absl::StrCat( + "malformed IP address: ", original_address_str.has_value() ? *original_address_str : "")); + } + CidrRange ret = CidrRange(std::move(ptr), length); + if (ret.isValid()) { + return ret; + } + return absl::InvalidArgumentError("Invalid CIDR range"); } // static -CidrRange CidrRange::create(const std::string& address, int length) { - return create(Utility::parseInternetAddress(address), length); +absl::StatusOr CidrRange::create(const std::string& address, int length) { + return create(Utility::parseInternetAddressNoThrow(address), length, address); } -CidrRange CidrRange::create(const envoy::config::core::v3::CidrRange& cidr) { - return create(Utility::parseInternetAddress(cidr.address_prefix()), cidr.prefix_len().value()); +absl::StatusOr CidrRange::create(const envoy::config::core::v3::CidrRange& cidr) { + return create(Utility::parseInternetAddressNoThrow(cidr.address_prefix()), + cidr.prefix_len().value(), cidr.address_prefix()); } -CidrRange CidrRange::create(const xds::core::v3::CidrRange& cidr) { - return create(Utility::parseInternetAddress(cidr.address_prefix()), cidr.prefix_len().value()); +absl::StatusOr CidrRange::create(const xds::core::v3::CidrRange& cidr) { + return create(Utility::parseInternetAddressNoThrow(cidr.address_prefix()), + cidr.prefix_len().value(), cidr.address_prefix()); } // static -CidrRange CidrRange::create(const std::string& range) { +absl::StatusOr CidrRange::create(const std::string& range) { const auto parts = StringUtil::splitToken(range, "/"); if (parts.size() == 2) { - InstanceConstSharedPtr ptr = Utility::parseInternetAddress(std::string{parts[0]}); - if (ptr->type() == Type::Ip) { + InstanceConstSharedPtr ptr = Utility::parseInternetAddressNoThrow(std::string{parts[0]}); + if (ptr && ptr->type() == Type::Ip) { uint64_t length64; if (absl::SimpleAtoi(parts[1], &length64)) { if ((ptr->ip()->version() == IpVersion::v6 && length64 <= 128) || @@ -132,7 +144,7 @@ CidrRange CidrRange::create(const std::string& range) { } } } - return {nullptr, -1}; + return absl::InvalidArgumentError("Invalid CIDR range"); } // static @@ -202,12 +214,13 @@ IpList::create(const Protobuf::RepeatedPtrField& cidrs) { ip_list_.reserve(cidrs.size()); for (const envoy::config::core::v3::CidrRange& entry : cidrs) { - CidrRange list_entry = CidrRange::create(entry); - if (list_entry.isValid()) { - ip_list_.push_back(std::move(list_entry)); + absl::StatusOr range_or_error = CidrRange::create(entry); + if (range_or_error.status().ok()) { + ip_list_.push_back(std::move(range_or_error.value())); } else { error_ = fmt::format("invalid ip/mask combo '{}/{}' (format is /<# mask bits>)", entry.address_prefix(), entry.prefix_len().value()); diff --git a/source/common/network/cidr_range.h b/source/common/network/cidr_range.h index b78555dc41f6..33f3af9ebf92 100644 --- a/source/common/network/cidr_range.h +++ b/source/common/network/cidr_range.h @@ -68,43 +68,34 @@ class CidrRange { std::string asString() const; /** - * @return true if this instance is valid; address != nullptr && length is appropriate for the - * IP version (these are checked during construction, and reduced down to a check of - * the length). - */ - bool isValid() const { return length_ >= 0; } - - /** - * TODO(ccaraman): Update CidrRange::create to support only constructing valid ranges. * @return a CidrRange instance with the specified address and length, modified so that the only * bits that might be non-zero are in the high-order length bits, and so that length is * in the appropriate range (0 to 32 for IPv4, 0 to 128 for IPv6). If the address or * length is invalid, then the range will be invalid (i.e. length == -1). */ - static CidrRange create(InstanceConstSharedPtr address, int length); - static CidrRange create(const std::string& address, int length); + static absl::StatusOr + create(InstanceConstSharedPtr address, int length, + absl::optional original_address_str = ""); + static absl::StatusOr create(const std::string& address, int length); /** * Constructs an CidrRange from a string with this format (same as returned * by CidrRange::asString above): *
/ e.g. "10.240.0.0/16" or "1234:5678::/64" - * TODO(ccaraman): Update CidrRange::create to support only constructing valid ranges. * @return a CidrRange instance with the specified address and length if parsed successfully, * else with no address and a length of -1. */ - static CidrRange create(const std::string& range); + static absl::StatusOr create(const std::string& range); /** * Constructs a CidrRange from envoy::config::core::v3::CidrRange. - * TODO(ccaraman): Update CidrRange::create to support only constructing valid ranges. */ - static CidrRange create(const envoy::config::core::v3::CidrRange& cidr); + static absl::StatusOr create(const envoy::config::core::v3::CidrRange& cidr); /** * Constructs a CidrRange from xds::core::v3::CidrRange. - * TODO(ccaraman): Update CidrRange::create to support only constructing valid ranges. */ - static CidrRange create(const xds::core::v3::CidrRange& cidr); + static absl::StatusOr create(const xds::core::v3::CidrRange& cidr); /** * Given an IP address and a length of high order bits to keep, returns an address @@ -120,6 +111,13 @@ class CidrRange { int* length_io); private: + /** + * @return true if this instance is valid; address != nullptr && length is appropriate for the + * IP version (these are checked during construction, and reduced down to a check of + * the length). + */ + bool isValid() const { return length_ >= 0; } + CidrRange(InstanceConstSharedPtr address, int length); InstanceConstSharedPtr address_; diff --git a/source/common/router/router_ratelimit.cc b/source/common/router/router_ratelimit.cc index e48b93d791a9..db2dcc45e298 100644 --- a/source/common/router/router_ratelimit.cc +++ b/source/common/router/router_ratelimit.cc @@ -170,8 +170,10 @@ bool MaskedRemoteAddressAction::populateDescriptor(RateLimit::DescriptorEntry& d } // TODO: increase the efficiency, avoid string transform back and forth + // Note: we don't do validity checking for CIDR range here because we know + // from addressAsString this is a valid address. Network::Address::CidrRange cidr_entry = - Network::Address::CidrRange::create(remote_address->ip()->addressAsString(), mask_len); + *Network::Address::CidrRange::create(remote_address->ip()->addressAsString(), mask_len); descriptor_entry = {"masked_remote_address", cidr_entry.asString()}; return true; diff --git a/source/extensions/common/matcher/trie_matcher.h b/source/extensions/common/matcher/trie_matcher.h index b50ae214eac2..6084d9c82449 100644 --- a/source/extensions/common/matcher/trie_matcher.h +++ b/source/extensions/common/matcher/trie_matcher.h @@ -144,7 +144,9 @@ class TrieMatcherFactoryBase : public ::Envoy::Matcher::CustomMatcherFactory node = {i, range.prefix_len().value(), range_matcher.exclusive(), on_match}; - data.push_back({node, {Network::Address::CidrRange::create(range)}}); + data.push_back({node, + {THROW_OR_RETURN_VALUE(Network::Address::CidrRange::create(range), + Network::Address::CidrRange)}}); } } auto lc_trie = std::make_shared>>(data); diff --git a/source/extensions/filters/common/rbac/matchers.h b/source/extensions/filters/common/rbac/matchers.h index 362d2c51c979..1fb0a05e6483 100644 --- a/source/extensions/filters/common/rbac/matchers.h +++ b/source/extensions/filters/common/rbac/matchers.h @@ -159,7 +159,9 @@ class IPMatcher : public Matcher { enum Type { ConnectionRemote = 0, DownstreamLocal, DownstreamDirectRemote, DownstreamRemote }; IPMatcher(const envoy::config::core::v3::CidrRange& range, Type type) - : range_(Network::Address::CidrRange::create(range)), type_(type) {} + : range_(THROW_OR_RETURN_VALUE(Network::Address::CidrRange::create(range), + Network::Address::CidrRange)), + type_(type) {} bool matches(const Network::Connection& connection, const Envoy::Http::RequestHeaderMap& headers, const StreamInfo::StreamInfo& info) const override; diff --git a/source/extensions/filters/common/rbac/matchers/upstream_ip_port.cc b/source/extensions/filters/common/rbac/matchers/upstream_ip_port.cc index 89ffe42eb9ca..a0446d54c85b 100644 --- a/source/extensions/filters/common/rbac/matchers/upstream_ip_port.cc +++ b/source/extensions/filters/common/rbac/matchers/upstream_ip_port.cc @@ -23,7 +23,8 @@ UpstreamIpPortMatcher::UpstreamIpPortMatcher( } if (proto.has_upstream_ip()) { - cidr_ = Network::Address::CidrRange::create(proto.upstream_ip()); + cidr_ = THROW_OR_RETURN_VALUE(Network::Address::CidrRange::create(proto.upstream_ip()), + Network::Address::CidrRange); } if (proto.has_upstream_port_range()) { port_ = proto.upstream_port_range(); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 52a6ad00c87e..a8a8d3b8bcbc 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -38,10 +38,10 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( cidr_set.reserve(ip_tag.ip_list().size()); for (const envoy::config::core::v3::CidrRange& entry : ip_tag.ip_list()) { - // Currently, CidrRange::create doesn't guarantee that the CidrRanges are valid. - Network::Address::CidrRange cidr_entry = Network::Address::CidrRange::create(entry); - if (cidr_entry.isValid()) { - cidr_set.emplace_back(std::move(cidr_entry)); + absl::StatusOr cidr_or_error = + Network::Address::CidrRange::create(entry); + if (cidr_or_error.status().ok()) { + cidr_set.emplace_back(std::move(cidr_or_error.value())); } else { throw EnvoyException( fmt::format("invalid ip/mask combo '{}/{}' (format is /<# mask bits>)", diff --git a/source/extensions/matching/input_matchers/ip/config.cc b/source/extensions/matching/input_matchers/ip/config.cc index 811a279ae5e9..6ae3ea720033 100644 --- a/source/extensions/matching/input_matchers/ip/config.cc +++ b/source/extensions/matching/input_matchers/ip/config.cc @@ -19,13 +19,8 @@ Config::createInputMatcherFactoryCb(const Protobuf::Message& config, for (const auto& cidr_range : cidr_ranges) { const std::string& address = cidr_range.address_prefix(); const uint32_t prefix_len = cidr_range.prefix_len().value(); - const auto range = Network::Address::CidrRange::create(address, prefix_len); - // We only assert that the range is valid because: - // * if "address" can't be parsed, it will throw an EnvoyException - // * prefix_len can't be < 0 as per the protobuf definition as an uint32_t - // * if prefix_len is too big, CidrRange::create clamps it to a valid value - // => it is thus not possible to create an invalid range. - ASSERT(range.isValid(), "address range should be valid!"); + const auto range = THROW_OR_RETURN_VALUE( + Network::Address::CidrRange::create(address, prefix_len), Network::Address::CidrRange); ranges.emplace_back(std::move(range)); } diff --git a/test/common/network/cidr_range_test.cc b/test/common/network/cidr_range_test.cc index 2938159c38ee..b8060738c605 100644 --- a/test/common/network/cidr_range_test.cc +++ b/test/common/network/cidr_range_test.cc @@ -85,15 +85,10 @@ TEST(TruncateIpAddressAndLength, Various) { } TEST(IsInRange, Various) { - { - CidrRange rng = CidrRange::create("foo"); - EXPECT_FALSE(rng.isValid()); - EXPECT_FALSE(rng.isInRange(Ipv4Instance("0.0.0.0"))); - } + { EXPECT_FALSE(CidrRange::create("foo").status().ok()); } { - CidrRange rng = CidrRange::create("10.255.255.255/0"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create("10.255.255.255/0"); EXPECT_EQ(rng.asString(), "0.0.0.0/0"); EXPECT_EQ(rng.length(), 0); EXPECT_EQ(rng.ip()->version(), IpVersion::v4); @@ -105,8 +100,7 @@ TEST(IsInRange, Various) { } { - CidrRange rng = CidrRange::create("10.255.255.255/10"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create("10.255.255.255/10"); EXPECT_EQ(rng.asString(), "10.192.0.0/10"); EXPECT_EQ(rng.length(), 10); EXPECT_EQ(rng.ip()->version(), IpVersion::v4); @@ -117,8 +111,7 @@ TEST(IsInRange, Various) { } { - CidrRange rng = CidrRange::create("::/0"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create("::/0"); EXPECT_EQ(rng.asString(), "::/0"); EXPECT_EQ(rng.length(), 0); EXPECT_EQ(rng.ip()->version(), IpVersion::v6); @@ -130,8 +123,7 @@ TEST(IsInRange, Various) { } { - CidrRange rng = CidrRange::create("::1/128"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create("::1/128"); EXPECT_EQ(rng.asString(), "::1/128"); EXPECT_EQ(rng.length(), 128); EXPECT_EQ(rng.ip()->version(), IpVersion::v6); @@ -142,8 +134,7 @@ TEST(IsInRange, Various) { } { - CidrRange rng = CidrRange::create("2001:abcd:ef01:2345:6789:abcd:ef01:234/64"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create("2001:abcd:ef01:2345:6789:abcd:ef01:234/64"); EXPECT_EQ(rng.asString(), "2001:abcd:ef01:2345::/64"); EXPECT_EQ(rng.length(), 64); EXPECT_EQ(rng.ip()->version(), IpVersion::v6); @@ -156,8 +147,7 @@ TEST(IsInRange, Various) { } { - CidrRange rng = CidrRange::create("2001:abcd:ef01:2345:6789:abcd:ef01:234/60"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create("2001:abcd:ef01:2345:6789:abcd:ef01:234/60"); EXPECT_EQ(rng.asString(), "2001:abcd:ef01:2340::/60"); EXPECT_EQ(rng.length(), 60); EXPECT_EQ(rng.ip()->version(), IpVersion::v6); @@ -173,57 +163,47 @@ TEST(IsInRange, Various) { TEST(CidrRangeTest, OperatorIsEqual) { { - CidrRange rng1 = CidrRange::create("192.0.0.0/8"); - CidrRange rng2 = CidrRange::create("192.168.0.0/16"); + CidrRange rng1 = *CidrRange::create("192.0.0.0/8"); + CidrRange rng2 = *CidrRange::create("192.168.0.0/16"); EXPECT_FALSE(rng1 == rng2); } { - CidrRange rng1 = CidrRange::create("192.0.0.0/8"); - CidrRange rng2 = CidrRange::create("192.168.0.0/8"); + CidrRange rng1 = *CidrRange::create("192.0.0.0/8"); + CidrRange rng2 = *CidrRange::create("192.168.0.0/8"); EXPECT_TRUE(rng1 == rng2); } { - CidrRange rng1 = CidrRange::create("192.0.0.0/8"); - CidrRange rng2 = CidrRange::create("2001::/8"); + CidrRange rng1 = *CidrRange::create("192.0.0.0/8"); + CidrRange rng2 = *CidrRange::create("2001::/8"); EXPECT_FALSE(rng1 == rng2); } { - CidrRange rng1 = CidrRange::create("2002::/16"); - CidrRange rng2 = CidrRange::create("2001::/16"); + CidrRange rng1 = *CidrRange::create("2002::/16"); + CidrRange rng2 = *CidrRange::create("2001::/16"); EXPECT_FALSE(rng1 == rng2); } { - CidrRange rng1 = CidrRange::create("2002::/16"); - CidrRange rng2 = CidrRange::create("192.168.0.1/16"); + CidrRange rng1 = *CidrRange::create("2002::/16"); + CidrRange rng2 = *CidrRange::create("192.168.0.1/16"); EXPECT_FALSE(rng1 == rng2); } { - CidrRange rng1 = CidrRange::create("2002::/16"); - CidrRange rng2 = CidrRange::create("2002::1/16"); + CidrRange rng1 = *CidrRange::create("2002::/16"); + CidrRange rng2 = *CidrRange::create("2002::1/16"); EXPECT_TRUE(rng1 == rng2); } } -TEST(CidrRangeTest, InvalidCidrRange) { - CidrRange rng1 = CidrRange::create("foo"); - EXPECT_EQ(nullptr, rng1.ip()); - EXPECT_EQ("/-1", rng1.asString()); - // Not equal due to invalid CidrRange. - EXPECT_FALSE(rng1 == rng1); - - CidrRange rng2 = CidrRange::create("192.0.0.0/8"); - EXPECT_FALSE(rng1 == rng2); -} +TEST(CidrRangeTest, InvalidCidrRange) { EXPECT_FALSE(CidrRange::create("foo").status().ok()); } TEST(Ipv4CidrRangeTest, InstanceConstSharedPtrAndLengthCtor) { InstanceConstSharedPtr ptr = Utility::parseInternetAddress("1.2.3.5"); - CidrRange rng(CidrRange::create(ptr, 31)); // Copy ctor. - EXPECT_TRUE(rng.isValid()); + CidrRange rng(*CidrRange::create(ptr, 31)); // Copy ctor. EXPECT_EQ(rng.length(), 31); EXPECT_EQ(rng.ip()->version(), IpVersion::v4); EXPECT_EQ(rng.asString(), "1.2.3.4/31"); @@ -232,18 +212,15 @@ TEST(Ipv4CidrRangeTest, InstanceConstSharedPtrAndLengthCtor) { EXPECT_TRUE(rng.isInRange(Ipv4Instance("1.2.3.5"))); EXPECT_FALSE(rng.isInRange(Ipv4Instance("1.2.3.6"))); - CidrRange rng2(CidrRange::create(ptr, -1)); // Invalid length. - EXPECT_FALSE(rng2.isValid()); + EXPECT_FALSE(CidrRange::create(ptr, -1).status().ok()); // Invalid length. ptr.reset(); - CidrRange rng3(CidrRange::create(ptr, 10)); // Invalid address. - EXPECT_FALSE(rng3.isValid()); + EXPECT_FALSE(CidrRange::create(ptr, 10).status().ok()); // Invalid address. } TEST(Ipv4CidrRangeTest, StringAndLengthCtor) { CidrRange rng; - rng = CidrRange::create("1.2.3.4", 31); // Assignment operator. - EXPECT_TRUE(rng.isValid()); + rng = *CidrRange::create("1.2.3.4", 31); // Assignment operator. EXPECT_EQ(rng.asString(), "1.2.3.4/31"); EXPECT_EQ(rng.length(), 31); EXPECT_EQ(rng.ip()->version(), IpVersion::v4); @@ -252,15 +229,13 @@ TEST(Ipv4CidrRangeTest, StringAndLengthCtor) { EXPECT_TRUE(rng.isInRange(Ipv4Instance("1.2.3.5"))); EXPECT_FALSE(rng.isInRange(Ipv4Instance("1.2.3.6"))); - rng = CidrRange::create("1.2.3.4", -10); // Invalid length. - EXPECT_FALSE(rng.isValid()); + EXPECT_FALSE(CidrRange::create("1.2.3.4", -10).status().ok()); // Invalid length. - EXPECT_THROW(CidrRange::create("bogus", 31), EnvoyException); // Invalid address. + EXPECT_FALSE(CidrRange::create("bogus", 31).status().ok()); // Invalid address. } TEST(Ipv4CidrRangeTest, StringCtor) { - CidrRange rng = CidrRange::create("1.2.3.4/31"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create("1.2.3.4/31"); EXPECT_EQ(rng.asString(), "1.2.3.4/31"); EXPECT_EQ(rng.length(), 31); EXPECT_EQ(rng.ip()->version(), IpVersion::v4); @@ -269,21 +244,14 @@ TEST(Ipv4CidrRangeTest, StringCtor) { EXPECT_TRUE(rng.isInRange(Ipv4Instance("1.2.3.5"))); EXPECT_FALSE(rng.isInRange(Ipv4Instance("1.2.3.6"))); - CidrRange rng2 = CidrRange::create("1.2.3.4/-10"); // Invalid length. - EXPECT_FALSE(rng2.isValid()); - - EXPECT_THROW(CidrRange::create("bogus/31"), EnvoyException); // Invalid address. - - CidrRange rng4 = CidrRange::create("/31"); // Missing address. - EXPECT_FALSE(rng4.isValid()); - - CidrRange rng5 = CidrRange::create("1.2.3.4/"); // Missing length. - EXPECT_FALSE(rng5.isValid()); + EXPECT_FALSE(CidrRange::create("1.2.3.4/-10").status().ok()); // Invalid length. + EXPECT_FALSE(CidrRange::create("bogus/31").status().ok()); // Invalid address. + EXPECT_FALSE(CidrRange::create("/31").status().ok()); // Missing address. + EXPECT_FALSE(CidrRange::create("1.2.3.4/").status().ok()); // Missing length. } TEST(Ipv4CidrRangeTest, BigRange) { - CidrRange rng = CidrRange::create("10.255.255.255/8"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create("10.255.255.255/8"); EXPECT_EQ(rng.asString(), "10.0.0.0/8"); EXPECT_EQ(rng.length(), 8); EXPECT_EQ(rng.ip()->version(), IpVersion::v4); @@ -300,8 +268,7 @@ TEST(Ipv4CidrRangeTest, BigRange) { TEST(Ipv6CidrRange, InstanceConstSharedPtrAndLengthCtor) { InstanceConstSharedPtr ptr = Utility::parseInternetAddress("abcd::0345"); - CidrRange rng(CidrRange::create(ptr, 127)); // Copy ctor. - EXPECT_TRUE(rng.isValid()); + CidrRange rng(*CidrRange::create(ptr, 127)); // Copy ctor. EXPECT_EQ(rng.length(), 127); EXPECT_EQ(rng.ip()->version(), IpVersion::v6); EXPECT_EQ(rng.asString(), "abcd::344/127"); @@ -310,18 +277,14 @@ TEST(Ipv6CidrRange, InstanceConstSharedPtrAndLengthCtor) { EXPECT_TRUE(rng.isInRange(Ipv6Instance("abcd::345"))); EXPECT_FALSE(rng.isInRange(Ipv6Instance("abcd::346"))); - CidrRange rng2(CidrRange::create(ptr, -1)); // Invalid length. - EXPECT_FALSE(rng2.isValid()); - + EXPECT_FALSE(CidrRange::create(ptr, -1).status().ok()); // Invalid length. ptr.reset(); - CidrRange rng3(CidrRange::create(ptr, 127)); // Invalid address. - EXPECT_FALSE(rng3.isValid()); + EXPECT_FALSE(CidrRange::create(ptr, 127).status().ok()); // Invalid address. } TEST(Ipv6CidrRange, StringAndLengthCtor) { CidrRange rng; - rng = CidrRange::create("ff::ffff", 122); // Assignment operator. - EXPECT_TRUE(rng.isValid()); + rng = *CidrRange::create("ff::ffff", 122); // Assignment operator. EXPECT_EQ(rng.asString(), "ff::ffc0/122"); EXPECT_EQ(rng.length(), 122); EXPECT_EQ(rng.ip()->version(), IpVersion::v6); @@ -330,15 +293,12 @@ TEST(Ipv6CidrRange, StringAndLengthCtor) { EXPECT_TRUE(rng.isInRange(Ipv6Instance("ff::ffff"))); EXPECT_FALSE(rng.isInRange(Ipv6Instance("::1:0"))); - rng = CidrRange::create("::ffff", -2); // Invalid length. - EXPECT_FALSE(rng.isValid()); - - EXPECT_THROW(CidrRange::create("bogus", 122), EnvoyException); // Invalid address. + EXPECT_FALSE(CidrRange::create("::ffff", -2).status().ok()); // Invalid length. + EXPECT_FALSE(CidrRange::create("bogus", 122).status().ok()); // Invalid address. } TEST(Ipv6CidrRange, StringCtor) { - CidrRange rng = CidrRange::create("ff::fc1f/118"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create("ff::fc1f/118"); EXPECT_EQ(rng.asString(), "ff::fc00/118"); EXPECT_EQ(rng.length(), 118); EXPECT_EQ(rng.ip()->version(), IpVersion::v6); @@ -347,22 +307,15 @@ TEST(Ipv6CidrRange, StringCtor) { EXPECT_TRUE(rng.isInRange(Ipv6Instance("ff::ffff"))); EXPECT_FALSE(rng.isInRange(Ipv6Instance("::1:00"))); - CidrRange rng2 = CidrRange::create("::fc1f/-10"); // Invalid length. - EXPECT_FALSE(rng2.isValid()); - - EXPECT_THROW(CidrRange::create("::fc1f00/118"), EnvoyException); // Invalid address. - - CidrRange rng4 = CidrRange::create("/118"); // Missing address. - EXPECT_FALSE(rng4.isValid()); - - CidrRange rng5 = CidrRange::create("::fc1f/"); // Missing length. - EXPECT_FALSE(rng5.isValid()); + EXPECT_FALSE(CidrRange::create("::fc1f/-10").status().ok()); // Invalid length. + EXPECT_FALSE(CidrRange::create("::fc1f00/118").status().ok()); // Invalid address. + EXPECT_FALSE(CidrRange::create("/118").status().ok()); // Missing address. + EXPECT_FALSE(CidrRange::create("::fc1f/").status().ok()); // Missing length. } TEST(Ipv6CidrRange, BigRange) { std::string prefix = "2001:0db8:85a3:0000"; - CidrRange rng = CidrRange::create(prefix + "::/64"); - EXPECT_TRUE(rng.isValid()); + CidrRange rng = *CidrRange::create(prefix + "::/64"); EXPECT_EQ(rng.asString(), "2001:db8:85a3::/64"); EXPECT_EQ(rng.length(), 64); EXPECT_EQ(rng.ip()->version(), IpVersion::v6); @@ -389,7 +342,7 @@ makeCidrRangeList(const std::vector>& ranges) { } TEST(IpListTest, Errors) { - { EXPECT_THROW(IpList::create(makeCidrRangeList({{"foo", 0}})).IgnoreError(), EnvoyException); } + { EXPECT_FALSE(IpList::create(makeCidrRangeList({{"foo", 0}})).status().ok()); } } TEST(IpListTest, SpecificAddressAllowed) { diff --git a/test/common/network/lc_trie_speed_test.cc b/test/common/network/lc_trie_speed_test.cc index 99eb7e6aede1..3a59d87cc820 100644 --- a/test/common/network/lc_trie_speed_test.cc +++ b/test/common/network/lc_trie_speed_test.cc @@ -30,17 +30,17 @@ struct CidrInputs { tag_data_.emplace_back( std::pair>( {"tag_1", - {Envoy::Network::Address::CidrRange::create( + {*Envoy::Network::Address::CidrRange::create( fmt::format("192.0.{}.{}/32", i, j))}})); } } tag_data_nested_prefixes_ = tag_data_; tag_data_nested_prefixes_.emplace_back( std::pair>( - {"tag_0", {Envoy::Network::Address::CidrRange::create("0.0.0.0/0")}})); + {"tag_0", {*Envoy::Network::Address::CidrRange::create("0.0.0.0/0")}})); tag_data_minimal_.emplace_back( std::pair>( - {"tag_1", {Envoy::Network::Address::CidrRange::create("0.0.0.0/0")}})); + {"tag_1", {*Envoy::Network::Address::CidrRange::create("0.0.0.0/0")}})); } std::vector>> tag_data_; diff --git a/test/common/network/lc_trie_test.cc b/test/common/network/lc_trie_test.cc index c2da7417c96d..965d58141685 100644 --- a/test/common/network/lc_trie_test.cc +++ b/test/common/network/lc_trie_test.cc @@ -22,7 +22,7 @@ class LcTrieTest : public testing::Test { std::pair> ip_tags; ip_tags.first = fmt::format("tag_{0}", i); for (const auto& j : cidr_range_strings[i]) { - ip_tags.second.push_back(Address::CidrRange::create(j)); + ip_tags.second.push_back(*Address::CidrRange::create(j)); } output.push_back(ip_tags); } @@ -329,7 +329,7 @@ TEST_F(LcTrieTest, ExclusiveNestedPrefixesWithCatchAll) { // when using the default fill factor. TEST_F(LcTrieTest, MaximumEntriesExceptionDefault) { static const size_t num_prefixes = 1 << 19; - Address::CidrRange address = Address::CidrRange::create("10.0.0.1/8"); + Address::CidrRange address = *Address::CidrRange::create("10.0.0.1/8"); std::vector prefixes; prefixes.reserve(num_prefixes); for (size_t i = 0; i < num_prefixes; i++) { @@ -355,7 +355,7 @@ TEST_F(LcTrieTest, MaximumEntriesExceptionOverride) { for (size_t i = 0; i < 16; i++) { for (size_t j = 0; j < 16; j++) { for (size_t k = 0; k < 32; k++) { - prefixes.emplace_back(Address::CidrRange::create(fmt::format("10.{}.{}.{}/8", i, j, k))); + prefixes.emplace_back(*Address::CidrRange::create(fmt::format("10.{}.{}.{}/8", i, j, k))); } } } diff --git a/test/extensions/matching/input_matchers/ip/matcher_test.cc b/test/extensions/matching/input_matchers/ip/matcher_test.cc index 68842784e819..287343adedd1 100644 --- a/test/extensions/matching/input_matchers/ip/matcher_test.cc +++ b/test/extensions/matching/input_matchers/ip/matcher_test.cc @@ -27,8 +27,8 @@ class MatcherTest : public testing::Test { TEST_F(MatcherTest, TestV4) { std::vector ranges; - ranges.emplace_back(Network::Address::CidrRange::create("192.0.2.0", 24)); - ranges.emplace_back(Network::Address::CidrRange::create("10.0.0.0", 24)); + ranges.emplace_back(*Network::Address::CidrRange::create("192.0.2.0", 24)); + ranges.emplace_back(*Network::Address::CidrRange::create("10.0.0.0", 24)); initialize(std::move(ranges)); EXPECT_FALSE(m_->match("192.0.1.255")); EXPECT_TRUE(m_->match("192.0.2.0")); @@ -42,9 +42,9 @@ TEST_F(MatcherTest, TestV4) { TEST_F(MatcherTest, TestV6) { std::vector ranges; - ranges.emplace_back(Network::Address::CidrRange::create("::1/128")); - ranges.emplace_back(Network::Address::CidrRange::create("2001::/16")); - ranges.emplace_back(Network::Address::CidrRange::create("2002::/16")); + ranges.emplace_back(*Network::Address::CidrRange::create("::1/128")); + ranges.emplace_back(*Network::Address::CidrRange::create("2001::/16")); + ranges.emplace_back(*Network::Address::CidrRange::create("2002::/16")); initialize(std::move(ranges)); EXPECT_FALSE(m_->match("::")); @@ -66,7 +66,7 @@ TEST_F(MatcherTest, EmptyRanges) { TEST_F(MatcherTest, EmptyIP) { std::vector ranges; - ranges.emplace_back(Network::Address::CidrRange::create("192.0.2.0", 24)); + ranges.emplace_back(*Network::Address::CidrRange::create("192.0.2.0", 24)); initialize(std::move(ranges)); EXPECT_FALSE(m_->match("")); EXPECT_FALSE(m_->match(absl::monostate())); @@ -74,7 +74,7 @@ TEST_F(MatcherTest, EmptyIP) { TEST_F(MatcherTest, InvalidIP) { std::vector ranges; - ranges.emplace_back(Network::Address::CidrRange::create("192.0.2.0", 24)); + ranges.emplace_back(*Network::Address::CidrRange::create("192.0.2.0", 24)); initialize(std::move(ranges)); EXPECT_EQ(m_->stats()->ip_parsing_failed_.value(), 0); EXPECT_FALSE(m_->match("foo")); From 32830d6e2e6f402fbca855e1552c39dd8464fd6c Mon Sep 17 00:00:00 2001 From: wbpcode Date: Wed, 24 Jan 2024 10:28:23 +0000 Subject: [PATCH 24/61] fix brotli decompression endless loop Signed-off-by: wbpcode Signed-off-by: Ryan Northey --- changelogs/current.yaml | 4 ++ .../decompressor/brotli_decompressor_impl.cc | 51 +++++++++++++------ .../decompressor/brotli_decompressor_impl.h | 5 +- .../brotli_decompressor_impl_test.cc | 45 ++++++++++++++++ 4 files changed, 89 insertions(+), 16 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 75038cddb542..6a848a4fa7c6 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -136,6 +136,10 @@ bug_fixes: Set the SNI value from the requested server name if it isn't available on the connection/socket. This applies when ``include_tls_session`` is true. The requested server name is set on a connection when filters such as the TLS inspector are used. +- area: decompression + change: | + Fixed a bug where Envoy will go into an endless loop when using the brotli decompressor. If the input stream has + redundant data, the decompressor will loop forever. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc index eb1bb144baa5..d096179d1717 100644 --- a/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc +++ b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc @@ -60,26 +60,47 @@ void BrotliDecompressorImpl::decompress(const Buffer::Instance& input_buffer, } bool BrotliDecompressorImpl::process(Common::BrotliContext& ctx, Buffer::Instance& output_buffer) { - BrotliDecoderResult result; - result = BrotliDecoderDecompressStream(state_.get(), &ctx.avail_in_, &ctx.next_in_, - &ctx.avail_out_, &ctx.next_out_, nullptr); - if (result == BROTLI_DECODER_RESULT_ERROR) { - // TODO(rojkov): currently the Brotli library doesn't specify possible errors in its API. Add - // more detailed stats when they are documented. - stats_.brotli_error_.inc(); - return false; - } + BrotliDecoderResult result = BrotliDecoderDecompressStream( + state_.get(), &ctx.avail_in_, &ctx.next_in_, &ctx.avail_out_, &ctx.next_out_, nullptr); + + switch (result) { + case BROTLI_DECODER_RESULT_SUCCESS: + // The decompression is done successfully but there is still some input left. + // We treat this as an error and stop the decompression directly to avoid + // possible endless loop. + if (ctx.avail_in_ > 0) { + stats_.brotli_error_.inc(); + stats_.brotli_redundant_input_.inc(); + return false; + } + // The decompression is done successfully and fall through to the next case + // to check if the output buffer is full and flush chunk to the output buffer. + FALLTHRU; + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + ASSERT(ctx.avail_in_ == 0); + FALLTHRU; + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + // Check if the output buffer is full first. If it is full then we treat it + // as an error and stop the decompression directly to avoid possible decompression + // bomb. + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.enable_compression_bomb_protection") && + (output_buffer.length() > ctx.max_output_size_)) { + stats_.brotli_error_.inc(); + stats_.brotli_output_overflow_.inc(); + return false; + } - if (Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.enable_compression_bomb_protection") && - (output_buffer.length() > ctx.max_output_size_)) { + // If current chunk is full then flush it to the output buffer and reset + // the chunk or do nothing. + ctx.updateOutput(output_buffer); + return true; + case BROTLI_DECODER_RESULT_ERROR: stats_.brotli_error_.inc(); return false; } - ctx.updateOutput(output_buffer); - - return true; + PANIC("Unexpected BrotliDecoderResult"); } } // namespace Decompressor diff --git a/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.h b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.h index f55f6a7c545a..1f3328807fb3 100644 --- a/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.h +++ b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.h @@ -17,7 +17,10 @@ namespace Decompressor { /** * All brotli decompressor stats. @see stats_macros.h */ -#define ALL_BROTLI_DECOMPRESSOR_STATS(COUNTER) COUNTER(brotli_error) +#define ALL_BROTLI_DECOMPRESSOR_STATS(COUNTER) \ + COUNTER(brotli_error) /*Decompression error of all.*/ \ + COUNTER(brotli_output_overflow) /*Decompression error because of the overflow output.*/ \ + COUNTER(brotli_redundant_input) /*Decompression error because of the redundant input.*/ /** * Struct definition for brotli decompressor stats. @see stats_macros.h diff --git a/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc b/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc index 514a5e0fd2b5..4b2451c04d49 100644 --- a/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc +++ b/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc @@ -106,6 +106,51 @@ TEST_F(BrotliDecompressorImplTest, CompressAndDecompress) { EXPECT_EQ(original_text, decompressed_text); } +TEST_F(BrotliDecompressorImplTest, CompressAndDecompressWithRedundantInput) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; + + Brotli::Compressor::BrotliCompressorImpl compressor{ + default_quality, + default_window_bits, + default_input_block_bits, + false, + Brotli::Compressor::BrotliCompressorImpl::EncoderMode::Default, + 4096}; + + std::string original_text{}; + for (uint64_t i = 0; i < 20; ++i) { + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + original_text.append(buffer.toString()); + compressor.compress(buffer, Envoy::Compression::Compressor::State::Flush); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + } + + ASSERT_EQ(0, buffer.length()); + + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); + ASSERT_GE(10, buffer.length()); + + accumulation_buffer.add(buffer); + accumulation_buffer.add("redundant_input_here"); // Add some redundant input. + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + + Stats::IsolatedStoreImpl stats_store{}; + BrotliDecompressorImpl decompressor{*stats_store.rootScope(), "test.", 16, false}; + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{buffer.toString()}; + ASSERT_EQ(original_text.length(), decompressed_text.length()); + EXPECT_EQ(original_text, decompressed_text); + + // Although we finally get the original text, we still have some redundant input and + // the decompression is considered as failed. + EXPECT_EQ(1, stats_store.counterFromString("test.brotli_error").value()); + EXPECT_EQ(1, stats_store.counterFromString("test.brotli_redundant_input").value()); +} + // Exercises decompression with a very small output buffer. TEST_F(BrotliDecompressorImplTest, DecompressWithSmallOutputBuffer) { Buffer::OwnedImpl buffer; From 1a0b2a22333ede255188885c548b4800463a7be0 Mon Sep 17 00:00:00 2001 From: Dan Zhang Date: Thu, 18 Apr 2024 21:29:11 +0000 Subject: [PATCH 25/61] quic: add 2 quiche patches Signed-off-by: Dan Zhang Signed-off-by: Ryan Northey --- bazel/external/quiche_sequencer_fix.patch | 16 +++++++ bazel/external/quiche_stream_fix.patch | 51 +++++++++++++++++++++++ bazel/repositories.bzl | 5 +++ changelogs/current.yaml | 3 ++ 4 files changed, 75 insertions(+) create mode 100644 bazel/external/quiche_sequencer_fix.patch create mode 100644 bazel/external/quiche_stream_fix.patch diff --git a/bazel/external/quiche_sequencer_fix.patch b/bazel/external/quiche_sequencer_fix.patch new file mode 100644 index 000000000000..b4203e92b6e3 --- /dev/null +++ b/bazel/external/quiche_sequencer_fix.patch @@ -0,0 +1,16 @@ +# Fix https://github.com/envoyproxy/envoy-setec/issues/1496#issue-2251291349 + +diff --git a/quiche/quic/core/quic_stream_sequencer_buffer.cc b/quiche/quic/core/quic_stream_sequencer_buffer.cc +index d364d61bc..0966af4b0 100644 +--- a/quiche/quic/core/quic_stream_sequencer_buffer.cc ++++ b/quiche/quic/core/quic_stream_sequencer_buffer.cc +@@ -388,7 +388,8 @@ bool QuicStreamSequencerBuffer::PeekRegion(QuicStreamOffset offset, + + // Determine if entire block has been received. + size_t end_block_idx = GetBlockIndex(FirstMissingByte()); +- if (block_idx == end_block_idx) { ++ if (block_idx == end_block_idx && ++ block_offset < GetInBlockOffset(FirstMissingByte())) { + // Only read part of block before FirstMissingByte(). + iov->iov_len = GetInBlockOffset(FirstMissingByte()) - block_offset; + } else { diff --git a/bazel/external/quiche_stream_fix.patch b/bazel/external/quiche_stream_fix.patch new file mode 100644 index 000000000000..898a54843f95 --- /dev/null +++ b/bazel/external/quiche_stream_fix.patch @@ -0,0 +1,51 @@ +# Fix https://github.com/envoyproxy/envoy-setec/issues/1496#issuecomment-2064844217 + +diff --git a/quiche/quic/core/http/quic_spdy_stream.cc b/quiche/quic/core/http/quic_spdy_stream.cc +index 4a5c2ede2..d69895055 100644 +--- a/quiche/quic/core/http/quic_spdy_stream.cc ++++ b/quiche/quic/core/http/quic_spdy_stream.cc +@@ -1865,6 +1865,18 @@ bool QuicSpdyStream::AreHeaderFieldValuesValid( + return true; + } + ++void QuicSpdyStream::StopReading() { ++ QuicStream::StopReading(); ++ if (GetQuicReloadableFlag( ++ quic_stop_reading_also_stops_header_decompression) && ++ VersionUsesHttp3(transport_version()) && !fin_received() && ++ spdy_session_->qpack_decoder()) { ++ // Clean up Qpack decoding states. ++ spdy_session_->qpack_decoder()->OnStreamReset(id()); ++ qpack_decoded_headers_accumulator_.reset(); ++ } ++} ++ + void QuicSpdyStream::OnInvalidHeaders() { Reset(QUIC_BAD_APPLICATION_PAYLOAD); } + + void QuicSpdyStream::CloseReadSide() { +diff --git a/quiche/quic/core/http/quic_spdy_stream.h b/quiche/quic/core/http/quic_spdy_stream.h +index 10c34b10f..5c0cb0128 100644 +--- a/quiche/quic/core/http/quic_spdy_stream.h ++++ b/quiche/quic/core/http/quic_spdy_stream.h +@@ -117,6 +117,7 @@ class QUICHE_EXPORT QuicSpdyStream + + // QuicStream implementation + void OnClose() override; ++ void StopReading() override; + + // Override to maybe close the write side after writing. + void OnCanWrite() override; +diff --git a/quiche/quic/core/quic_flags_list.h b/quiche/quic/core/quic_flags_list.h +index 61fa4f9e6..6737823a5 100644 +--- a/quiche/quic/core/quic_flags_list.h ++++ b/quiche/quic/core/quic_flags_list.h +@@ -6,7 +6,8 @@ + + #ifdef QUIC_FLAG + +-QUIC_FLAG(quic_reloadable_flag_quic_stop_reading_also_stops_header_decompression, false) ++// If true, QUIC stream will not continue decompressing buffer headers after StopReading() called. ++QUIC_FLAG(quic_reloadable_flag_quic_stop_reading_also_stops_header_decompression, true) + // A testonly reloadable flag that will always default to false. + QUIC_FLAG(quic_reloadable_flag_quic_testonly_default_false, false) + // A testonly reloadable flag that will always default to true. diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index bcaa502620bf..a54efde0664b 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1141,6 +1141,11 @@ def _com_github_google_quiche(): external_http_archive( name = "com_github_google_quiche", patch_cmds = ["find quiche/ -type f -name \"*.bazel\" -delete"], + patches = [ + "@envoy//bazel/external:quiche_sequencer_fix.patch", + "@envoy//bazel/external:quiche_stream_fix.patch", + ], + patch_args = ["-p1"], build_file = "@envoy//bazel/external:quiche.BUILD", ) native.bind( diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 6a848a4fa7c6..bfce830ea055 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -96,6 +96,9 @@ bug_fixes: Fixed :ref:`successful_active_health_check_uneject_host `. Before, a failed health check could uneject the host if the ``FAILED_ACTIVE_HC`` health flag had not been set. +- area: quic + change: | + Applied 2 QUICHE patches for crash bugs in ``QuicSpdyStream`` ``OnDataAvailable()`` and ``OnInitialHeaderComplete()``. - area: tls change: | Fix a RELEASE_ASSERT when using :ref:`auto_sni ` From 1b3c6e3ab83ef62b7f0582b4c458432cc1e93342 Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Mon, 29 Apr 2024 10:37:04 -0400 Subject: [PATCH 26/61] Fix CVE from uncaught nlohmann json exception. Signed-off-by: Kevin Baichoo Signed-off-by: Ryan Northey --- source/common/json/json_internal.cc | 4 ++- test/integration/BUILD | 1 + .../access_log_integration_test.cc | 32 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/source/common/json/json_internal.cc b/source/common/json/json_internal.cc index f7e624ed7881..f5c8cc5c21d5 100644 --- a/source/common/json/json_internal.cc +++ b/source/common/json/json_internal.cc @@ -570,7 +570,9 @@ std::vector Field::asObjectArray() const { std::string Field::asJsonString() const { nlohmann::json j = asJsonDocument(); - return j.dump(); + // Call with defaults except in the case of UTF-8 errors which we replace + // invalid UTF-8 characters instead of throwing an exception. + return j.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace); } bool Field::empty() const { diff --git a/test/integration/BUILD b/test/integration/BUILD index 56fb0148d4f4..763531da0ec5 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -2482,6 +2482,7 @@ envoy_cc_test( deps = [ ":http_integration_lib", "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/access_loggers/file/v3:pkg_cc_proto", ], ) diff --git a/test/integration/access_log_integration_test.cc b/test/integration/access_log_integration_test.cc index 6a181c95df76..c62d220b189f 100644 --- a/test/integration/access_log_integration_test.cc +++ b/test/integration/access_log_integration_test.cc @@ -1,3 +1,7 @@ +#include "envoy/extensions/access_loggers/file/v3/file.pb.h" + +#include "source/common/protobuf/protobuf.h" + #include "test/integration/http_integration.h" #include "test/test_common/registry.h" #include "test/test_common/utility.h" @@ -24,4 +28,32 @@ TEST_P(AccessLogIntegrationTest, DownstreamDisconnectBeforeHeadersResponseCode) std::string log = waitForAccessLog(access_log_name_); EXPECT_THAT(log, HasSubstr("RESPONSE_CODE=0")); } + +TEST_P(AccessLogIntegrationTest, ShouldReplaceInvalidUtf8) { + // Add incomplete UTF-8 strings. + default_request_headers_.setForwardedFor("\xec"); + + // Update access logs to use json format sorted. + access_log_name_ = TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); + config_helper_.addConfigModifier( + [this]( + envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { + envoy::extensions::access_loggers::file::v3::FileAccessLog access_log_config; + auto* access_log_config_to_clobber = hcm.add_access_log(); + access_log_config.set_path(access_log_name_); + + auto* log_format = access_log_config.mutable_log_format(); + auto* json = log_format->mutable_json_format(); + Envoy::ProtobufWkt::Value v; + v.set_string_value("%REQ(X-FORWARDED-FOR)%"); + auto fields = json->mutable_fields(); + (*fields)["x_forwarded_for"] = v; + log_format->mutable_json_format_options()->set_sort_properties(true); + access_log_config_to_clobber->mutable_typed_config()->PackFrom(access_log_config); + }); + testRouterDownstreamDisconnectBeforeRequestComplete(); + const std::string log = waitForAccessLog(access_log_name_); + EXPECT_THAT(log, HasSubstr("x_forwarded_for\":\"\xEF\xBF\xBD")); +} } // namespace Envoy From 1d083a49c44df4dd3702cb8dda265c18173981f7 Mon Sep 17 00:00:00 2001 From: Dan Zhang Date: Mon, 18 Mar 2024 15:30:07 +0000 Subject: [PATCH 27/61] quic: fix crash from `EnvoyQuicServerSession::OnConnectionClosed()` Signed-off-by: Dan Zhang Signed-off-by: Ryan Northey --- changelogs/current.yaml | 3 +++ .../common/quic/envoy_quic_server_stream.cc | 5 +++- .../quic/envoy_quic_server_session_test.cc | 4 +-- .../integration/quic_http_integration_test.cc | 26 +++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index bfce830ea055..32c3ad5d049c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -99,6 +99,9 @@ bug_fixes: - area: quic change: | Applied 2 QUICHE patches for crash bugs in ``QuicSpdyStream`` ``OnDataAvailable()`` and ``OnInitialHeaderComplete()``. +- area: quic + change: | + Fixed crash bug when QUIC downstream stream was read closed and then timed out. - area: tls change: | Fix a RELEASE_ASSERT when using :ref:`auto_sni ` diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc index 94a82cce10d7..5bdf06608546 100644 --- a/source/common/quic/envoy_quic_server_stream.cc +++ b/source/common/quic/envoy_quic_server_stream.cc @@ -112,10 +112,13 @@ void EnvoyQuicServerStream::resetStream(Http::StreamResetReason reason) { // of propagating original reset reason. In QUICHE if a stream stops reading // before FIN or RESET received, it resets the steam with QUIC_STREAM_NO_ERROR. StopReading(); - runResetCallbacks(Http::StreamResetReason::LocalReset); } else { Reset(envoyResetReasonToQuicRstError(reason)); } + // Run reset callbacks once because HCM calls resetStream() without tearing + // down its own ActiveStream. It might be no-op if it has been called already + // in ResetWithError(). + runResetCallbacks(Http::StreamResetReason::LocalReset); } void EnvoyQuicServerStream::switchStreamBlockState() { diff --git a/test/common/quic/envoy_quic_server_session_test.cc b/test/common/quic/envoy_quic_server_session_test.cc index 11f43607b402..4fba3ac27b03 100644 --- a/test/common/quic/envoy_quic_server_session_test.cc +++ b/test/common/quic/envoy_quic_server_session_test.cc @@ -1019,11 +1019,11 @@ TEST_F(EnvoyQuicServerSessionTest, SendBufferWatermark) { EXPECT_TRUE(stream2->IsFlowControlBlocked()); // Resetting stream3 should lower the buffered bytes, but callbacks will not - // be triggered because end stream is already encoded. + // be triggered because end stream is already decoded and encoded. EXPECT_CALL(stream_callbacks3, onResetStream(Http::StreamResetReason::LocalReset, "")).Times(0); // Connection buffered data book keeping should also be updated. EXPECT_CALL(network_connection_callbacks_, onBelowWriteBufferLowWatermark()); - stream3->resetStream(Http::StreamResetReason::LocalReset); + stream3->Reset(quic::QUIC_STREAM_CANCELLED); // Update flow control window for stream1. quic::QuicWindowUpdateFrame window_update3(quic::kInvalidControlFrameId, stream1->id(), diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index b196ee0771c0..33a28f2dda61 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -2359,5 +2359,31 @@ TEST_P(QuicHttpIntegrationTest, ConnectionDebugVisitor) { }); } +TEST_P(QuicHttpIntegrationTest, StreamTimeoutWithHalfClose) { + // Tighten the stream idle timeout to 400ms. + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { + hcm.mutable_stream_idle_timeout()->set_seconds(0); + hcm.mutable_stream_idle_timeout()->set_nanos(400 * 1000 * 1000); + }); + initialize(); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + IntegrationStreamDecoderPtr response = + codec_client_->makeRequestWithBody(default_request_headers_, "partial body", false); + EnvoyQuicClientSession* quic_session = + static_cast(codec_client_->connection()); + quic::QuicStream* stream = quic_session->GetActiveStream(0); + // Only send RESET_STREAM to close write side of this stream. + stream->ResetWriteSide(quic::QuicResetStreamError::FromInternal(quic::QUIC_STREAM_NO_ERROR)); + + // Wait for the server to timeout this request and the local reply. + EXPECT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_rq_idle_timeout")->value()); + codec_client_->close(); +} + } // namespace Quic } // namespace Envoy From 76b5c920cf30297e118dc0d25f647796623407d5 Mon Sep 17 00:00:00 2001 From: Boteng Yao Date: Fri, 16 Feb 2024 14:26:44 +0000 Subject: [PATCH 28/61] websocket handshake check 101 protocol Signed-off-by: Boteng Yao Signed-off-by: Ryan Northey --- changelogs/current.yaml | 5 + envoy/http/filter.h | 5 + envoy/stream_info/stream_info.h | 2 + source/common/router/upstream_codec_filter.cc | 30 ++++ source/common/router/upstream_request.cc | 13 +- source/common/router/upstream_request.h | 9 + source/common/runtime/runtime_features.cc | 1 + test/integration/BUILD | 1 + .../integration/websocket_integration_test.cc | 157 +++++++++++++++++- test/integration/websocket_integration_test.h | 3 +- 10 files changed, 221 insertions(+), 5 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 32c3ad5d049c..c3c1026acca4 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -146,6 +146,11 @@ bug_fixes: change: | Fixed a bug where Envoy will go into an endless loop when using the brotli decompressor. If the input stream has redundant data, the decompressor will loop forever. +- area: websocket + change: | + Only 101 is considered a successful response for websocket handshake for HTTP/1.1, and Envoy as a proxy will proxy the response + header from upstream to downstream and then close the request if other status is received. This behavior can be + reverted by ``envoy_reloadable_features_check_switch_protocol_websocket_handshake``. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/envoy/http/filter.h b/envoy/http/filter.h index 9b627f1c26d7..8260924e34df 100644 --- a/envoy/http/filter.h +++ b/envoy/http/filter.h @@ -241,6 +241,11 @@ class UpstreamStreamFilterCallbacks { virtual bool pausedForConnect() const PURE; virtual void setPausedForConnect(bool value) PURE; + // Setters and getters to determine if sending body payload is paused on + // confirmation of a WebSocket upgrade. These should only be used by the upstream codec filter. + virtual bool pausedForWebsocketUpgrade() const PURE; + virtual void setPausedForWebsocketUpgrade(bool value) PURE; + // Return the upstreamStreamOptions for this stream. virtual const Http::ConnectionPool::Instance::StreamOptions& upstreamStreamOptions() const PURE; diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h index 8b2eb2a18b1e..f3b8232967d6 100644 --- a/envoy/stream_info/stream_info.h +++ b/envoy/stream_info/stream_info.h @@ -180,6 +180,8 @@ struct ResponseCodeDetailValues { const std::string PathNormalizationFailed = "path_normalization_failed"; // The request was rejected because it attempted an unsupported upgrade. const std::string UpgradeFailed = "upgrade_failed"; + // The websocket handshake is unsuccessful and only SwitchingProtocols is considering successful. + const std::string WebsocketHandshakeUnsuccessful = "websocket_handshake_unsuccessful"; // The request was rejected by the HCM because there was no route configuration found. const std::string RouteConfigurationNotFound = "route_configuration_not_found"; diff --git a/source/common/router/upstream_codec_filter.cc b/source/common/router/upstream_codec_filter.cc index 9882fda66eed..56ac395dd40b 100644 --- a/source/common/router/upstream_codec_filter.cc +++ b/source/common/router/upstream_codec_filter.cc @@ -83,6 +83,8 @@ Http::FilterHeadersStatus UpstreamCodecFilter::decodeHeaders(Http::RequestHeader } if (callbacks_->upstreamCallbacks()->pausedForConnect()) { return Http::FilterHeadersStatus::StopAllIterationAndWatermark; + } else if (callbacks_->upstreamCallbacks()->pausedForWebsocketUpgrade()) { + return Http::FilterHeadersStatus::StopAllIterationAndWatermark; } return Http::FilterHeadersStatus::Continue; } @@ -154,6 +156,34 @@ void UpstreamCodecFilter::CodecBridge::decodeHeaders(Http::ResponseHeaderMapPtr& filter_.callbacks_->continueDecoding(); } + if (filter_.callbacks_->upstreamCallbacks()->pausedForWebsocketUpgrade()) { + const uint64_t status = Http::Utility::getResponseStatus(*headers); + const auto protocol = filter_.callbacks_->upstreamCallbacks()->upstreamStreamInfo().protocol(); + if (status == static_cast(Http::Code::SwitchingProtocols) || + (protocol.has_value() && protocol.value() != Envoy::Http::Protocol::Http11)) { + // handshake is finished and continue the data processing. + filter_.callbacks_->upstreamCallbacks()->setPausedForWebsocketUpgrade(false); + filter_.callbacks_->continueDecoding(); + } else { + // Other status, e.g., 426 or 200, indicate a failed handshake, Envoy as a proxy will proxy + // back the response header to downstream and then close the request, since WebSocket + // just needs headers for handshake per RFC-6455. Note: HTTP/2 200 will be normalized to + // 101 before this point in codec and this patch will skip this scenario from the above + // proto check. + filter_.callbacks_->sendLocalReply( + static_cast(status), "", + [&headers](Http::ResponseHeaderMap& local_headers) { + headers->iterate([&local_headers](const Envoy::Http::HeaderEntry& header) { + local_headers.addCopy(Http::LowerCaseString(header.key().getStringView()), + header.value().getStringView()); + return Envoy::Http::HeaderMap::Iterate::Continue; + }); + }, + std::nullopt, StreamInfo::ResponseCodeDetails::get().WebsocketHandshakeUnsuccessful); + return; + } + } + maybeEndDecode(end_stream); filter_.callbacks_->encodeHeaders(std::move(headers), end_stream, StreamInfo::ResponseCodeDetails::get().ViaUpstream); diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index 41a42290d3c0..2d8c83b5b775 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -90,7 +90,7 @@ UpstreamRequest::UpstreamRequest(RouterFilterInterface& parent, encode_trailers_(false), retried_(false), awaiting_headers_(true), outlier_detection_timeout_recorded_(false), create_per_try_timeout_on_request_complete_(false), paused_for_connect_(false), - reset_stream_(false), + paused_for_websocket_(false), reset_stream_(false), record_timeout_budget_(parent_.cluster()->timeoutBudgetStats().has_value()), cleaned_up_(false), had_upstream_(false), stream_options_({can_send_early_data, can_use_http3}), grpc_rq_success_deferred_(false), @@ -389,6 +389,13 @@ void UpstreamRequest::acceptHeadersFromRouter(bool end_stream) { auto* headers = parent_.downstreamHeaders(); if (headers->getMethodValue() == Http::Headers::get().MethodValues.Connect) { paused_for_connect_ = true; + // If this is a websocket upgrade request, pause the request until the upstream sends + // the 101 Switching Protocols response code. Using the else logic here to obey CONNECT + // method which is expecting 2xx response. + } else if ((Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.check_switch_protocol_websocket_handshake")) && + Http::Utility::isWebSocketUpgradeRequest(*headers)) { + paused_for_websocket_ = true; } // Kick off creation of the upstream connection immediately upon receiving headers. @@ -602,8 +609,10 @@ void UpstreamRequest::onPoolReady(std::unique_ptr&& upstream, if (protocol) { stream_info_.protocol(protocol.value()); } else { - // We only pause for CONNECT for HTTP upstreams. If this is a TCP upstream, unpause. + // We only pause for CONNECT and WebSocket for HTTP upstreams. If this is a TCP upstream, + // unpause. paused_for_connect_ = false; + paused_for_websocket_ = false; } StreamInfo::UpstreamInfo& upstream_info = *stream_info_.upstreamInfo(); diff --git a/source/common/router/upstream_request.h b/source/common/router/upstream_request.h index c6e7a4ec4c94..eb32a8ec8431 100644 --- a/source/common/router/upstream_request.h +++ b/source/common/router/upstream_request.h @@ -245,6 +245,7 @@ class UpstreamRequest : public Logger::Loggable, // True if the CONNECT headers have been sent but proxying payload is paused // waiting for response headers. bool paused_for_connect_ : 1; + bool paused_for_websocket_ : 1; bool reset_stream_ : 1; // Sentinel to indicate if timeout budget tracking is configured for the cluster, @@ -361,6 +362,14 @@ class UpstreamRequestFilterManagerCallbacks : public Http::FilterManagerCallback } bool pausedForConnect() const override { return upstream_request_.paused_for_connect_; } void setPausedForConnect(bool value) override { upstream_request_.paused_for_connect_ = value; } + + bool pausedForWebsocketUpgrade() const override { + return upstream_request_.paused_for_websocket_; + } + void setPausedForWebsocketUpgrade(bool value) override { + upstream_request_.paused_for_websocket_ = value; + } + const Http::ConnectionPool::Instance::StreamOptions& upstreamStreamOptions() const override { return upstream_request_.upstreamStreamOptions(); } diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 91ecb569ee5f..708f029cdac2 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -32,6 +32,7 @@ RUNTIME_GUARD(envoy_reloadable_features_abort_filter_chain_on_stream_reset); RUNTIME_GUARD(envoy_reloadable_features_avoid_zombie_streams); RUNTIME_GUARD(envoy_reloadable_features_check_mep_on_first_eject); +RUNTIME_GUARD(envoy_reloadable_features_check_switch_protocol_websocket_handshake); RUNTIME_GUARD(envoy_reloadable_features_conn_pool_delete_when_idle); RUNTIME_GUARD(envoy_reloadable_features_consistent_header_validation); RUNTIME_GUARD(envoy_reloadable_features_defer_processing_backedup_streams); diff --git a/test/integration/BUILD b/test/integration/BUILD index 763531da0ec5..1635e574593e 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1407,6 +1407,7 @@ envoy_cc_test( "//source/common/http:header_map_lib", "//source/extensions/access_loggers/file:config", "//source/extensions/filters/http/buffer:config", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", diff --git a/test/integration/websocket_integration_test.cc b/test/integration/websocket_integration_test.cc index 66818889e2db..5853814e3e71 100644 --- a/test/integration/websocket_integration_test.cc +++ b/test/integration/websocket_integration_test.cc @@ -11,6 +11,7 @@ #include "test/integration/utility.h" #include "test/test_common/network_utility.h" #include "test/test_common/printers.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "absl/strings/str_cat.h" @@ -158,7 +159,7 @@ void WebsocketIntegrationTest::initialize() { void WebsocketIntegrationTest::performUpgrade( const Http::TestRequestHeaderMapImpl& upgrade_request_headers, - const Http::TestResponseHeaderMapImpl& upgrade_response_headers) { + const Http::TestResponseHeaderMapImpl& upgrade_response_headers, bool upgrade_should_fail) { // Establish the initial connection. codec_client_ = makeHttpConnection(lookupPort("http")); @@ -180,7 +181,9 @@ void WebsocketIntegrationTest::performUpgrade( // Verify the upgrade response was received downstream. response_->waitForHeaders(); - validateUpgradeResponseHeaders(response_->headers(), upgrade_response_headers); + if (!upgrade_should_fail) { + validateUpgradeResponseHeaders(response_->headers(), upgrade_response_headers); + } } void WebsocketIntegrationTest::sendBidirectionalData() { @@ -242,6 +245,10 @@ TEST_P(WebsocketIntegrationTest, EarlyData) { upstreamProtocol() != Http::CodecType::HTTP1) { return; } + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.check_switch_protocol_websocket_handshake", "false"}}); + config_helper_.addConfigModifier(setRouteUsingWebsocket()); initialize(); @@ -630,4 +637,150 @@ TEST_P(WebsocketIntegrationTest, BidirectionalConnectNoContentLengthNoTransferEn ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); } +// Test Websocket Upgrade in HTTP1 with 200 response code. +TEST_P(WebsocketIntegrationTest, Http1UpgradeStatusCodeOK) { + if (downstreamProtocol() != Http::CodecType::HTTP1 || + upstreamProtocol() != Http::CodecType::HTTP1) { + return; + } + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.check_switch_protocol_websocket_handshake", "true"}}); + + config_helper_.addConfigModifier(setRouteUsingWebsocket()); + initialize(); + + auto in_correct_status_response_headers = upgradeResponseHeaders(); + in_correct_status_response_headers.setStatus(200); + + // The upgrade should be paused, but the response header is proxied back to downstream. + performUpgrade(upgradeRequestHeaders(), in_correct_status_response_headers, true); + EXPECT_EQ("200", response_->headers().Status()->value().getStringView()); + EXPECT_EQ("upgrade", response_->headers().Connection()->value().getStringView()); + EXPECT_EQ("websocket", response_->headers().Upgrade()->value().getStringView()); + + test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_destroy", 1); + test_server_->waitForGaugeEq("http.config_test.downstream_cx_upgrades_active", 0); + ASSERT_TRUE(codec_client_->waitForDisconnect()); + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); +} + +// Test Websocket Upgrade with 200 response code from no HTTP1 upstream and downstream. +TEST_P(WebsocketIntegrationTest, NonHttp1UpgradeStatusCodeOK) { + if (upstreamProtocol() == Http::CodecType::HTTP1 || + downstreamProtocol() == Http::CodecType::HTTP1) { + return; + } + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.check_switch_protocol_websocket_handshake", "true"}}); + + config_helper_.addConfigModifier(setRouteUsingWebsocket()); + initialize(); + + auto correct_status_response_headers = upgradeResponseHeaders(); + correct_status_response_headers.setStatus(200); + performUpgrade(upgradeRequestHeaders(), correct_status_response_headers, true); + + // HTTP2 upstream response 200 is converted to 101. + EXPECT_EQ("101", response_->headers().Status()->value().getStringView()); + test_server_->waitForGaugeEq("http.config_test.downstream_cx_upgrades_active", 1); + codec_client_->close(); +} + +// Test Websocket Upgrade with 201 response code from no HTTP1 upstream and downstream. +// This patch will not impact no H/1 behaviors. +TEST_P(WebsocketIntegrationTest, NoHttp1UpstreamUpgradeStatus201) { + if (upstreamProtocol() == Http::CodecType::HTTP1 || + downstreamProtocol() == Http::CodecType::HTTP1) { + return; + } + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.check_switch_protocol_websocket_handshake", "true"}}); + + config_helper_.addConfigModifier(setRouteUsingWebsocket()); + initialize(); + + auto correct_status_response_headers = upgradeResponseHeaders(); + correct_status_response_headers.setStatus(201); + performUpgrade(upgradeRequestHeaders(), correct_status_response_headers, true); + + EXPECT_EQ("201", response_->headers().Status()->value().getStringView()); + test_server_->waitForGaugeEq("http.config_test.downstream_cx_upgrades_active", 1); + codec_client_->close(); +} + +// Test Websocket Upgrade in HTTP1 with 426 response code. +// Upgrade is a HTTP1 header. +TEST_P(WebsocketIntegrationTest, Http1UpgradeStatusCodeUpgradeRequired) { + if (downstreamProtocol() != Http::CodecType::HTTP1 || + upstreamProtocol() != Http::CodecType::HTTP1) { + return; + } + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.check_switch_protocol_websocket_handshake", "true"}}); + + useAccessLog("%RESPONSE_CODE_DETAILS%"); + config_helper_.addConfigModifier(setRouteUsingWebsocket()); + initialize(); + + auto in_correct_status_response_headers = upgradeResponseHeaders(); + in_correct_status_response_headers.setStatus(426); + + // The upgrade should be paused, but the response header is proxied back to downstream. + performUpgrade(upgradeRequestHeaders(), in_correct_status_response_headers, true); + EXPECT_EQ("426", response_->headers().Status()->value().getStringView()); + EXPECT_EQ("upgrade", response_->headers().Connection()->value().getStringView()); + EXPECT_EQ("websocket", response_->headers().Upgrade()->value().getStringView()); + + test_server_->waitForCounterEq("cluster.cluster_0.upstream_cx_destroy", 1); + test_server_->waitForGaugeEq("http.config_test.downstream_cx_upgrades_active", 0); + ASSERT_TRUE(codec_client_->waitForDisconnect()); + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); +} + +// Test data flow when websocket handshake failed. +TEST_P(WebsocketIntegrationTest, BidirectionalUpgradeFailedWithPrePayload) { + if (downstreamProtocol() != Http::CodecType::HTTP1 || + upstreamProtocol() != Http::CodecType::HTTP1) { + return; + } + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.check_switch_protocol_websocket_handshake", "true"}}); + + config_helper_.addConfigModifier(setRouteUsingWebsocket()); + initialize(); + + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("http")); + + // Send upgrade request with additional data. + ASSERT_TRUE(tcp_client->write( + "GET / HTTP/1.1\r\nHost: host\r\nconnection: upgrade\r\nupgrade: websocket\r\n\r\nfoo boo", + false, false)); + + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT(fake_upstream_connection != nullptr); + std::string received_data; + ASSERT_TRUE(fake_upstream_connection->waitForData( + FakeRawConnection::waitForInexactMatch("\r\n\r\n"), &received_data)); + // Make sure Envoy did not add TE or CL headers + ASSERT_FALSE(absl::StrContains(received_data, "content-length")); + ASSERT_FALSE(absl::StrContains(received_data, "transfer-encoding")); + ASSERT_TRUE(fake_upstream_connection->write( + "HTTP/1.1 426 Upgrade Required\r\nconnection: upgrade\r\nupgrade: websocket\r\n\r\n", false)); + + tcp_client->waitForData("\r\n\r\n", false); + + // Should not receive any data before handshake is finished. + std::string received_data_prepayload; + ASSERT_FALSE(fake_upstream_connection->waitForData( + FakeRawConnection::waitForInexactMatch("foo boo"), nullptr, std::chrono::milliseconds(10))); + + tcp_client->waitForDisconnect(); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); +} } // namespace Envoy diff --git a/test/integration/websocket_integration_test.h b/test/integration/websocket_integration_test.h index 4e03ca44bd87..066c8b3ba123 100644 --- a/test/integration/websocket_integration_test.h +++ b/test/integration/websocket_integration_test.h @@ -20,7 +20,8 @@ class WebsocketIntegrationTest : public HttpProtocolIntegrationTest { protected: void performUpgrade(const Http::TestRequestHeaderMapImpl& upgrade_request_headers, - const Http::TestResponseHeaderMapImpl& upgrade_response_headers); + const Http::TestResponseHeaderMapImpl& upgrade_response_headers, + bool upgrade_should_fail = false); void sendBidirectionalData(); void validateUpgradeRequestHeaders(const Http::RequestHeaderMap& proxied_request_headers, From 3da6d33e3d25f5ca69d2c8ac70e0b0235cf1119f Mon Sep 17 00:00:00 2001 From: Boteng Yao Date: Wed, 24 Jan 2024 10:28:23 +0000 Subject: [PATCH 29/61] async http: set buffer limit for response and do not buffer for mirror Signed-off-by: Boteng Yao Signed-off-by: Yan Avlasov Signed-off-by: Ryan Northey --- changelogs/current.yaml | 4 + envoy/http/async_client.h | 10 +- source/common/http/async_client_impl.cc | 37 +++++- source/common/http/async_client_impl.h | 9 +- source/common/router/router.cc | 3 +- source/extensions/common/wasm/context.cc | 5 +- .../rest/rest_api_fetcher.cc | 4 +- .../common/ext_authz/ext_authz_http_impl.cc | 4 +- .../filters/http/gcp_authn/gcp_authn_impl.cc | 5 +- .../tracers/datadog/agent_http_client.cc | 3 + test/common/http/BUILD | 1 + test/common/http/async_client_impl_test.cc | 123 ++++++++++++++++++ .../ext_authz/ext_authz_integration_test.cc | 26 ++++ 13 files changed, 218 insertions(+), 16 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index c3c1026acca4..8cec28a7b245 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -151,6 +151,10 @@ bug_fixes: Only 101 is considered a successful response for websocket handshake for HTTP/1.1, and Envoy as a proxy will proxy the response header from upstream to downstream and then close the request if other status is received. This behavior can be reverted by ``envoy_reloadable_features_check_switch_protocol_websocket_handshake``. +- area: async http client + change: | + Added one option to disable the response body buffering for mirror request. Also introduced a 32MB cap for the response + buffer, which can be changed by the runtime flag ``http.async_response_buffer_limit`` based on the product needs. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/envoy/http/async_client.h b/envoy/http/async_client.h index 3b495a795aae..5110246d902f 100644 --- a/envoy/http/async_client.h +++ b/envoy/http/async_client.h @@ -47,7 +47,9 @@ class AsyncClient { */ enum class FailureReason { // The stream has been reset. - Reset + Reset, + // The stream exceeds the response buffer limit. + ExceedResponseBufferLimit }; /** @@ -316,6 +318,11 @@ class AsyncClient { return *this; } + StreamOptions& setDiscardResponseBody(bool discard) { + discard_response_body = discard; + return *this; + } + StreamOptions& setIsShadowSuffixDisabled(bool d) { is_shadow_suffixed_disabled = d; return *this; @@ -380,6 +387,7 @@ class AsyncClient { bool is_shadow{false}; bool is_shadow_suffixed_disabled{false}; + bool discard_response_body{false}; // The parent span that child spans are created under to trace egress requests/responses. // If not set, requests will not be traced. diff --git a/source/common/http/async_client_impl.cc b/source/common/http/async_client_impl.cc index d761b76dcd6f..a0bec05e0d5a 100644 --- a/source/common/http/async_client_impl.cc +++ b/source/common/http/async_client_impl.cc @@ -16,6 +16,9 @@ namespace Envoy { namespace Http { + +const absl::string_view AsyncClientImpl::ResponseBufferLimit = "http.async_response_buffer_limit"; + AsyncClientImpl::AsyncClientImpl(Upstream::ClusterInfoConstSharedPtr cluster, Stats::Store& stats_store, Event::Dispatcher& dispatcher, Upstream::ClusterManager& cm, @@ -29,7 +32,7 @@ AsyncClientImpl::AsyncClientImpl(Upstream::ClusterInfoConstSharedPtr cluster, factory_context.api().randomGenerator(), std::move(shadow_writer), true, false, false, false, false, false, Protobuf::RepeatedPtrField{}, dispatcher.timeSource(), http_context, router_context)), - dispatcher_(dispatcher) {} + dispatcher_(dispatcher), runtime_(factory_context.runtime()) {} AsyncClientImpl::~AsyncClientImpl() { while (!active_streams_.empty()) { @@ -108,7 +111,8 @@ createRetryPolicy(AsyncClientImpl& parent, const AsyncClient::StreamOptions& opt AsyncStreamImpl::AsyncStreamImpl(AsyncClientImpl& parent, AsyncClient::StreamCallbacks& callbacks, const AsyncClient::StreamOptions& options, absl::Status& creation_status) - : parent_(parent), stream_callbacks_(callbacks), stream_id_(parent.config_->random_.random()), + : parent_(parent), discard_response_body_(options.discard_response_body), + stream_callbacks_(callbacks), stream_id_(parent.config_->random_.random()), router_(options.filter_config_ ? options.filter_config_ : parent.config_, parent.config_->async_stats_), stream_info_(Protocol::Http11, parent.dispatcher().timeSource(), nullptr, @@ -302,7 +306,9 @@ AsyncRequestSharedImpl::AsyncRequestSharedImpl(AsyncClientImpl& parent, AsyncClient::Callbacks& callbacks, const AsyncClient::RequestOptions& options, absl::Status& creation_status) - : AsyncStreamImpl(parent, *this, options, creation_status), callbacks_(callbacks) { + : AsyncStreamImpl(parent, *this, options, creation_status), callbacks_(callbacks), + response_buffer_limit_(parent.runtime_.snapshot().getInteger( + AsyncClientImpl::ResponseBufferLimit, kBufferLimitForResponse)) { if (!creation_status.ok()) { return; } @@ -367,7 +373,22 @@ void AsyncRequestSharedImpl::onHeaders(ResponseHeaderMapPtr&& headers, bool) { response_ = std::make_unique(std::move(headers)); } -void AsyncRequestSharedImpl::onData(Buffer::Instance& data, bool) { response_->body().move(data); } +void AsyncRequestSharedImpl::onData(Buffer::Instance& data, bool) { + if (discard_response_body_) { + data.drain(data.length()); + return; + } + + if (response_->body().length() + data.length() > response_buffer_limit_) { + ENVOY_LOG_EVERY_POW_2(warn, "the buffer size limit for async client response body " + "has been exceeded, draining data"); + data.drain(data.length()); + response_buffer_overlimit_ = true; + reset(); + } else { + response_->body().move(data); + } +} void AsyncRequestSharedImpl::onTrailers(ResponseTrailerMapPtr&& trailers) { response_->trailers(std::move(trailers)); @@ -393,8 +414,12 @@ void AsyncRequestSharedImpl::onReset() { Tracing::EgressConfig::get()); if (!cancelled_) { - // In this case we don't have a valid response so we do need to raise a failure. - callbacks_.onFailure(*this, AsyncClient::FailureReason::Reset); + if (response_buffer_overlimit_) { + callbacks_.onFailure(*this, AsyncClient::FailureReason::ExceedResponseBufferLimit); + } else { + // In this case we don't have a valid response so we do need to raise a failure. + callbacks_.onFailure(*this, AsyncClient::FailureReason::Reset); + } } } diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index 544c66ae7a18..af6422203293 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -51,6 +51,8 @@ namespace { // Limit the size of buffer for data used for retries. // This is currently fixed to 64KB. constexpr uint64_t kBufferLimitForRetry = 1 << 16; +// Response buffer limit 32MB. +constexpr uint64_t kBufferLimitForResponse = 32 * 1024 * 1024; } // namespace class AsyncStreamImpl; @@ -74,12 +76,14 @@ class AsyncClientImpl final : public AsyncClient { Server::Configuration::CommonFactoryContext& factory_context_; Upstream::ClusterInfoConstSharedPtr cluster_; Event::Dispatcher& dispatcher() override { return dispatcher_; } + static const absl::string_view ResponseBufferLimit; private: template T* internalStartRequest(T* async_request); const Router::FilterConfigSharedPtr config_; Event::Dispatcher& dispatcher_; std::list> active_streams_; + Runtime::Loader& runtime_; friend class AsyncStreamImpl; friend class AsyncRequestSharedImpl; @@ -92,7 +96,7 @@ class AsyncClientImpl final : public AsyncClient { class AsyncStreamImpl : public virtual AsyncClient::Stream, public StreamDecoderFilterCallbacks, public Event::DeferredDeletable, - Logger::Loggable, + public Logger::Loggable, public LinkedObject, public ScopeTrackedObject { public: @@ -161,6 +165,7 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, // Callback to listen for low/high/overflow watermark events. absl::optional> watermark_callbacks_; bool complete_{}; + const bool discard_response_body_; private: void cleanup(); @@ -325,6 +330,8 @@ class AsyncRequestSharedImpl : public virtual AsyncClient::Request, Tracing::SpanPtr child_span_; std::unique_ptr response_; bool cancelled_{}; + bool response_buffer_overlimit_{}; + const uint64_t response_buffer_limit_; }; class AsyncOngoingRequestImpl final : public AsyncClient::OngoingRequest, diff --git a/source/common/router/router.cc b/source/common/router/router.cc index e29c26cbd711..e37c0f08d2a0 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -789,7 +789,8 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, .setBufferAccount(callbacks_->account()) // A buffer limit of 1 is set in the case that retry_shadow_buffer_limit_ == 0, // because a buffer limit of zero on async clients is interpreted as no buffer limit. - .setBufferLimit(1 > retry_shadow_buffer_limit_ ? 1 : retry_shadow_buffer_limit_); + .setBufferLimit(1 > retry_shadow_buffer_limit_ ? 1 : retry_shadow_buffer_limit_) + .setDiscardResponseBody(true); options.setFilterConfig(config_); if (end_stream) { // This is a header-only request, and can be dispatched immediately to the shadow diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index 9164670de7b2..abf93e7fab73 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -1879,8 +1879,9 @@ void Context::onHttpCallFailure(uint32_t token, Http::AsyncClient::FailureReason return; } status_code_ = static_cast(WasmResult::BrokenConnection); - // This is the only value currently. - ASSERT(reason == Http::AsyncClient::FailureReason::Reset); + // TODO(botengyao): handle different failure reasons. + ASSERT(reason == Http::AsyncClient::FailureReason::Reset || + reason == Http::AsyncClient::FailureReason::ExceedResponseBufferLimit); status_message_ = "reset"; // Deferred "after VM call" actions are going to be executed upon returning from // ContextBase::*, which might include deleting Context object via proxy_done(). diff --git a/source/extensions/config_subscription/rest/rest_api_fetcher.cc b/source/extensions/config_subscription/rest/rest_api_fetcher.cc index 92c06f023d19..6b0d63fe73ef 100644 --- a/source/extensions/config_subscription/rest/rest_api_fetcher.cc +++ b/source/extensions/config_subscription/rest/rest_api_fetcher.cc @@ -50,8 +50,8 @@ void RestApiFetcher::onSuccess(const Http::AsyncClient::Request& request, void RestApiFetcher::onFailure(const Http::AsyncClient::Request&, Http::AsyncClient::FailureReason reason) { - // Currently Http::AsyncClient::FailureReason only has one value: "Reset". - ASSERT(reason == Http::AsyncClient::FailureReason::Reset); + ASSERT(reason == Http::AsyncClient::FailureReason::Reset || + reason == Http::AsyncClient::FailureReason::ExceedResponseBufferLimit); onFetchFailure(Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); requestComplete(); } diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc index 4b14c5fb5bb3..d35ef0840dcb 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc @@ -292,7 +292,9 @@ void RawHttpClientImpl::onSuccess(const Http::AsyncClient::Request&, void RawHttpClientImpl::onFailure(const Http::AsyncClient::Request&, Http::AsyncClient::FailureReason reason) { - ASSERT(reason == Http::AsyncClient::FailureReason::Reset); + // TODO(botengyao): handle different failure reasons. + ASSERT(reason == Http::AsyncClient::FailureReason::Reset || + reason == Http::AsyncClient::FailureReason::ExceedResponseBufferLimit); callbacks_->onComplete(std::make_unique(errorResponse())); callbacks_ = nullptr; } diff --git a/source/extensions/filters/http/gcp_authn/gcp_authn_impl.cc b/source/extensions/filters/http/gcp_authn/gcp_authn_impl.cc index ed16283f308c..05f108f48781 100644 --- a/source/extensions/filters/http/gcp_authn/gcp_authn_impl.cc +++ b/source/extensions/filters/http/gcp_authn/gcp_authn_impl.cc @@ -91,8 +91,9 @@ void GcpAuthnClient::onSuccess(const Http::AsyncClient::Request&, void GcpAuthnClient::onFailure(const Http::AsyncClient::Request&, Http::AsyncClient::FailureReason reason) { - // Http::AsyncClient::FailureReason only has one value: "Reset". - ASSERT(reason == Http::AsyncClient::FailureReason::Reset); + // TODO(botengyao): handle different failure reasons. + ASSERT(reason == Http::AsyncClient::FailureReason::Reset || + reason == Http::AsyncClient::FailureReason::ExceedResponseBufferLimit); ENVOY_LOG(error, "Request failed: stream has been reset"); active_request_ = nullptr; onError(); diff --git a/source/extensions/tracers/datadog/agent_http_client.cc b/source/extensions/tracers/datadog/agent_http_client.cc index 670dc0b79243..b002bfb0fb74 100644 --- a/source/extensions/tracers/datadog/agent_http_client.cc +++ b/source/extensions/tracers/datadog/agent_http_client.cc @@ -139,6 +139,9 @@ void AgentHTTPClient::onFailure(const Http::AsyncClient::Request& request, case Http::AsyncClient::FailureReason::Reset: message += "The stream has been reset."; break; + case Http::AsyncClient::FailureReason::ExceedResponseBufferLimit: + message += "The stream exceeds the response buffer limit."; + break; default: message += "Unknown error."; } diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 20782fb16392..acde87d4daa2 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -33,6 +33,7 @@ envoy_cc_test( "//test/mocks/server:server_factory_context_mocks", "//test/mocks/stats:stats_mocks", "//test/mocks/upstream:cluster_manager_mocks", + "//test/test_common:test_runtime_lib", "//test/test_common:test_time_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", diff --git a/test/common/http/async_client_impl_test.cc b/test/common/http/async_client_impl_test.cc index ae074cd026e2..28272c2a31ae 100644 --- a/test/common/http/async_client_impl_test.cc +++ b/test/common/http/async_client_impl_test.cc @@ -269,6 +269,129 @@ TEST_F(AsyncClientImplTest, Basic) { .value()); } +TEST_F(AsyncClientImplTest, NoResponseBodyBuffering) { + message_->body().add("test body"); + Buffer::Instance& data = message_->body(); + + EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) + .WillOnce(Invoke( + [&](ResponseDecoder& decoder, ConnectionPool::Callbacks& callbacks, + const ConnectionPool::Instance::StreamOptions&) -> ConnectionPool::Cancellable* { + callbacks.onPoolReady(stream_encoder_, cm_.thread_local_cluster_.conn_pool_.host_, + stream_info_, {}); + response_decoder_ = &decoder; + return nullptr; + })); + + TestRequestHeaderMapImpl copy(message_->headers()); + copy.addCopy("x-envoy-internal", "true"); + copy.addCopy("x-forwarded-for", "127.0.0.1"); + copy.addCopy(":scheme", "http"); + + EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(©), false)); + EXPECT_CALL(stream_encoder_, encodeData(BufferEqual(&data), true)); + + auto* request = client_.send(std::move(message_), callbacks_, + AsyncClient::RequestOptions().setDiscardResponseBody(true)); + EXPECT_NE(request, nullptr); + + EXPECT_CALL(callbacks_, onBeforeFinalizeUpstreamSpan(_, _)); + EXPECT_CALL(callbacks_, onSuccess_(_, _)) + .WillOnce(Invoke([](const AsyncClient::Request&, ResponseMessage* response) -> void { + // Verify that there is zero response body. + EXPECT_EQ(response->body().length(), 0); + })); + ResponseHeaderMapPtr response_headers(new TestResponseHeaderMapImpl{{":status", "200"}}); + response_decoder_->decodeHeaders(std::move(response_headers), false); + response_decoder_->decodeData(data, true); + + EXPECT_EQ( + 1UL, + cm_.thread_local_cluster_.cluster_.info_->stats_store_.counter("upstream_rq_200").value()); + EXPECT_EQ(1UL, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("internal.upstream_rq_200") + .value()); +} + +TEST_F(AsyncClientImplTest, LargeResponseBody) { + message_->body().add("test body"); + Buffer::Instance& data = message_->body(); + + EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) + .WillOnce(Invoke( + [&](ResponseDecoder& decoder, ConnectionPool::Callbacks& callbacks, + const ConnectionPool::Instance::StreamOptions&) -> ConnectionPool::Cancellable* { + callbacks.onPoolReady(stream_encoder_, cm_.thread_local_cluster_.conn_pool_.host_, + stream_info_, {}); + response_decoder_ = &decoder; + return nullptr; + })); + + TestRequestHeaderMapImpl copy(message_->headers()); + copy.addCopy("x-envoy-internal", "true"); + copy.addCopy("x-forwarded-for", "127.0.0.1"); + copy.addCopy(":scheme", "http"); + + EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(©), false)); + EXPECT_CALL(stream_encoder_, encodeData(BufferEqual(&data), true)); + ON_CALL(factory_context_.runtime_loader_.snapshot_, + getInteger(AsyncClientImpl::ResponseBufferLimit, kBufferLimitForResponse)) + .WillByDefault(Return(100)); + + auto* request = client_.send(std::move(message_), callbacks_, AsyncClient::RequestOptions()); + EXPECT_NE(request, nullptr); + + EXPECT_CALL(callbacks_, onBeforeFinalizeUpstreamSpan(_, _)); + EXPECT_CALL(callbacks_, onFailure(_, AsyncClient::FailureReason::ExceedResponseBufferLimit)); + + Buffer::InstancePtr large_body{new Buffer::OwnedImpl(std::string(100 + 1, 'a'))}; + ResponseHeaderMapPtr response_headers(new TestResponseHeaderMapImpl{{":status", "200"}}); + response_decoder_->decodeHeaders(std::move(response_headers), false); + response_decoder_->decodeData(*large_body, true); + EXPECT_EQ(large_body->length(), 0); +} + +TEST_F(AsyncClientImplTest, LargeResponseBodyMultipleRead) { + message_->body().add("test body"); + Buffer::Instance& data = message_->body(); + + EXPECT_CALL(cm_.thread_local_cluster_.conn_pool_, newStream(_, _, _)) + .WillOnce(Invoke( + [&](ResponseDecoder& decoder, ConnectionPool::Callbacks& callbacks, + const ConnectionPool::Instance::StreamOptions&) -> ConnectionPool::Cancellable* { + callbacks.onPoolReady(stream_encoder_, cm_.thread_local_cluster_.conn_pool_.host_, + stream_info_, {}); + response_decoder_ = &decoder; + return nullptr; + })); + + TestRequestHeaderMapImpl copy(message_->headers()); + copy.addCopy("x-envoy-internal", "true"); + copy.addCopy("x-forwarded-for", "127.0.0.1"); + copy.addCopy(":scheme", "http"); + + EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(©), false)); + EXPECT_CALL(stream_encoder_, encodeData(BufferEqual(&data), true)); + ON_CALL(factory_context_.runtime_loader_.snapshot_, + getInteger(AsyncClientImpl::ResponseBufferLimit, kBufferLimitForResponse)) + .WillByDefault(Return(100)); + + auto* request = client_.send(std::move(message_), callbacks_, AsyncClient::RequestOptions()); + EXPECT_NE(request, nullptr); + + EXPECT_CALL(callbacks_, onBeforeFinalizeUpstreamSpan(_, _)); + EXPECT_CALL(callbacks_, onFailure(_, AsyncClient::FailureReason::ExceedResponseBufferLimit)); + + Buffer::InstancePtr large_body{new Buffer::OwnedImpl(std::string(50, 'a'))}; + Buffer::InstancePtr large_body_second{new Buffer::OwnedImpl(std::string(50, 'a'))}; + Buffer::InstancePtr large_body_third{new Buffer::OwnedImpl(std::string(2, 'a'))}; + ResponseHeaderMapPtr response_headers(new TestResponseHeaderMapImpl{{":status", "200"}}); + response_decoder_->decodeHeaders(std::move(response_headers), false); + response_decoder_->decodeData(*large_body, false); + response_decoder_->decodeData(*large_body_second, false); + response_decoder_->decodeData(*large_body_third, true); +} + TEST_F(AsyncClientImplTest, BasicOngoingRequest) { auto headers = std::make_unique(); HttpTestUtility::addDefaultHeaders(*headers); diff --git a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc index e4fa04e707b8..0495698266bf 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc @@ -1321,6 +1321,32 @@ TEST_P(ExtAuthzHttpIntegrationTest, DirectReponse) { EXPECT_EQ("204", response_->headers().Status()->value().getStringView()); } +// Test exceeding the async client buffer limit. +TEST_P(ExtAuthzHttpIntegrationTest, ErrorReponseWithDefultBufferLimit) { + initializeConfig(false, /*failure_mode_allow=*/false); + config_helper_.addRuntimeOverride("http.async_response_buffer_limit", "1024"); + + HttpIntegrationTest::initialize(); + initiateClientConnection(); + waitForExtAuthzRequest(); + + Http::TestResponseHeaderMapImpl response_headers{ + {":status", "200"}, + {"baz", "baz"}, + {"bat", "bar"}, + {"x-append-bat", "append-foo"}, + {"x-append-bat", "append-bar"}, + {"x-envoy-auth-headers-to-remove", "remove-me"}, + }; + ext_authz_request_->encodeHeaders(response_headers, false); + ext_authz_request_->encodeData(2048, true); + + ASSERT_TRUE(response_->waitForEndStream()); + EXPECT_TRUE(response_->complete()); + // A forbidden response since the onFailure is called due to the async client buffer limit. + EXPECT_EQ("403", response_->headers().Status()->value().getStringView()); +} + // (uses new config for allowed_headers). TEST_P(ExtAuthzHttpIntegrationTest, RedirectResponse) { config_helper_.addConfigModifier( From f3303a5189ca04cde2a9adda4d8ac0465fdc0346 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 5 Jun 2024 15:11:09 -0400 Subject: [PATCH 30/61] dns: hopefully fixing a test flake (#34561) Signed-off-by: Alyssa Wilk --- .../network/dns_resolver/getaddrinfo/getaddrinfo_test.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc index 2968fb828953..4c260d16501a 100644 --- a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc +++ b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc @@ -216,7 +216,6 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainThenCancel) { EXPECT_CALL(os_sys_calls_, getaddrinfo(_, _, _, _)) .Times(testing::AnyNumber()) .WillOnce(Invoke([&](const char*, const char*, const addrinfo*, addrinfo**) { - query.load()->cancel(ActiveDnsQuery::CancelReason::QueryAbandoned); dispatcher_->exit(); return Api::SysCallIntResult{EAI_AGAIN, 0}; })); From ad0a10013c977a0f6a1e9de876d0a9eb22f1a187 Mon Sep 17 00:00:00 2001 From: ashish b <108897222+ashishb-solo@users.noreply.github.com> Date: Wed, 5 Jun 2024 15:55:22 -0400 Subject: [PATCH 31/61] Fix: OpenTelemetry access logs: Missing span ID (#33909) --------- Signed-off-by: Ashish Banerjee --- changelogs/current.yaml | 3 +++ envoy/tracing/trace_driver.h | 6 ++++++ source/common/tracing/null_span_impl.h | 1 + .../access_loggers/open_telemetry/access_log_impl.cc | 4 ++++ .../tracers/common/ot/opentracing_driver_impl.h | 7 ++++--- source/extensions/tracers/datadog/span.cc | 9 +++++++-- source/extensions/tracers/datadog/span.h | 1 + .../tracers/opencensus/opencensus_tracer_impl.cc | 3 +++ source/extensions/tracers/opentelemetry/tracer.h | 2 ++ source/extensions/tracers/skywalking/tracer.h | 1 + source/extensions/tracers/xray/tracer.h | 4 +++- source/extensions/tracers/zipkin/zipkin_tracer_impl.h | 3 +++ test/common/tracing/tracer_impl_test.cc | 1 + .../open_telemetry/access_log_impl_test.cc | 2 ++ .../filters/http/ext_proc/tracer_test_filter.cc | 4 ++++ .../tracers/common/ot/opentracing_driver_impl_test.cc | 2 ++ test/extensions/tracers/datadog/span_test.cc | 2 ++ test/extensions/tracers/opencensus/tracer_test.cc | 5 +++++ test/extensions/tracers/skywalking/tracer_test.cc | 2 ++ test/extensions/tracers/xray/tracer_test.cc | 3 +++ .../extensions/tracers/zipkin/zipkin_tracer_impl_test.cc | 2 ++ test/mocks/tracing/mocks.h | 1 + 22 files changed, 62 insertions(+), 6 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 8cec28a7b245..19bbdf4287c9 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -106,6 +106,9 @@ bug_fixes: change: | Fix a RELEASE_ASSERT when using :ref:`auto_sni ` if the downstream request ``:authority`` was longer than 255 characters. +- area: tracing + change: | + Fix an issue where span id is missing from opentelemetry access log entries. - area: ext_authz change: | Added field diff --git a/envoy/tracing/trace_driver.h b/envoy/tracing/trace_driver.h index 60e25ec84a68..94a7ca27050e 100644 --- a/envoy/tracing/trace_driver.h +++ b/envoy/tracing/trace_driver.h @@ -136,6 +136,12 @@ class Span { * @return trace ID */ virtual std::string getTraceId() const PURE; + + /** + * Retrieve the span's identifier. + * @return span ID as a hex string + */ + virtual std::string getSpanId() const PURE; }; /** diff --git a/source/common/tracing/null_span_impl.h b/source/common/tracing/null_span_impl.h index d573318e5751..cba151272db4 100644 --- a/source/common/tracing/null_span_impl.h +++ b/source/common/tracing/null_span_impl.h @@ -25,6 +25,7 @@ class NullSpan : public Span { void injectContext(Tracing::TraceContext&, const UpstreamContext&) override {} void setBaggage(absl::string_view, absl::string_view) override {} std::string getBaggage(absl::string_view) override { return EMPTY_STRING; } + std::string getSpanId() const override { return EMPTY_STRING; } std::string getTraceId() const override { return EMPTY_STRING; } SpanPtr spawnChild(const Config&, const std::string&, SystemTime) override { return SpanPtr{new NullSpan()}; diff --git a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc index e3866dfc98e7..b630c9c0cf53 100644 --- a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc @@ -102,6 +102,10 @@ void AccessLog::emitLog(const Formatter::HttpFormatterContext& log_context, auto trace_id = absl::StrCat(Hex::uint64ToHex(0), trace_id_hex); *log_entry.mutable_trace_id() = absl::HexStringToBytes(trace_id); } + std::string span_id_hex = log_context.activeSpan().getSpanId(); + if (!span_id_hex.empty()) { + *log_entry.mutable_span_id() = absl::HexStringToBytes(span_id_hex); + } tls_slot_->getTyped().logger_->log(std::move(log_entry)); } diff --git a/source/extensions/tracers/common/ot/opentracing_driver_impl.h b/source/extensions/tracers/common/ot/opentracing_driver_impl.h index 5fd46bfe96a4..b06323043f8d 100644 --- a/source/extensions/tracers/common/ot/opentracing_driver_impl.h +++ b/source/extensions/tracers/common/ot/opentracing_driver_impl.h @@ -45,10 +45,11 @@ class OpenTracingSpan : public Tracing::Span, Logger::Loggableid())); } +std::string Span::getSpanId() const { + // TODO(#34412): This method is not yet implemented for Datadog. + return EMPTY_STRING; +} + } // namespace Datadog } // namespace Tracers } // namespace Extensions diff --git a/source/extensions/tracers/datadog/span.h b/source/extensions/tracers/datadog/span.h index b4c14245cc04..0e41272ef49b 100644 --- a/source/extensions/tracers/datadog/span.h +++ b/source/extensions/tracers/datadog/span.h @@ -48,6 +48,7 @@ class Span : public Tracing::Span { std::string getBaggage(absl::string_view key) override; void setBaggage(absl::string_view key, absl::string_view value) override; std::string getTraceId() const override; + std::string getSpanId() const override; private: datadog::tracing::Optional span_; diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 026f9e70f4cf..476c3681d2c6 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -81,6 +81,7 @@ class Span : public Tracing::Span { std::string getBaggage(absl::string_view) override { return EMPTY_STRING; }; std::string getTraceId() const override; + std::string getSpanId() const override; private: ::opencensus::trace::Span span_; @@ -236,6 +237,8 @@ void Span::injectContext(Tracing::TraceContext& trace_context, const Tracing::Up } } +std::string Span::getSpanId() const { return EMPTY_STRING; } + std::string Span::getTraceId() const { const auto& ctx = span_.context(); return ctx.trace_id().ToHex(); diff --git a/source/extensions/tracers/opentelemetry/tracer.h b/source/extensions/tracers/opentelemetry/tracer.h index 057814365b9a..392bf7825420 100644 --- a/source/extensions/tracers/opentelemetry/tracer.h +++ b/source/extensions/tracers/opentelemetry/tracer.h @@ -119,6 +119,8 @@ class Span : Logger::Loggable, public Tracing::Span { std::string getTraceId() const override { return absl::BytesToHexString(span_.trace_id()); }; + std::string getSpanId() const override { return absl::BytesToHexString(span_.span_id()); }; + OTelSpanKind spankind() const { return span_.kind(); } /** diff --git a/source/extensions/tracers/skywalking/tracer.h b/source/extensions/tracers/skywalking/tracer.h index d5bb1f65f329..70f03f9e50b5 100644 --- a/source/extensions/tracers/skywalking/tracer.h +++ b/source/extensions/tracers/skywalking/tracer.h @@ -89,6 +89,7 @@ class Span : public Tracing::Span { std::string getBaggage(absl::string_view) override { return EMPTY_STRING; } void setBaggage(absl::string_view, absl::string_view) override {} std::string getTraceId() const override { return tracing_context_->traceId(); } + std::string getSpanId() const override { return EMPTY_STRING; } const TracingContextPtr tracingContext() { return tracing_context_; } const TracingSpanPtr spanEntity() { return span_entity_; } diff --git a/source/extensions/tracers/xray/tracer.h b/source/extensions/tracers/xray/tracer.h index bf9d696c0836..6d21b5999a94 100644 --- a/source/extensions/tracers/xray/tracer.h +++ b/source/extensions/tracers/xray/tracer.h @@ -230,9 +230,11 @@ class Span : public Tracing::Span, Logger::Loggable { void setBaggage(absl::string_view, absl::string_view) override {} std::string getBaggage(absl::string_view) override { return EMPTY_STRING; } - // TODO: This method is unimplemented for X-Ray. std::string getTraceId() const override { return trace_id_; }; + // TODO(#34412): This method is unimplemented for X-Ray. + std::string getSpanId() const override { return EMPTY_STRING; }; + /** * Creates a child span. * In X-Ray terms this creates a sub-segment and sets its parent ID to the current span's ID. diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h index cfa3c95a4fc3..2739bc61010b 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h @@ -85,6 +85,9 @@ class ZipkinSpan : public Tracing::Span { std::string getTraceId() const override { return span_.traceIdAsHexString(); }; + // TODO(#34412): This method is unimplemented for Zipkin. + std::string getSpanId() const override { return EMPTY_STRING; }; + /** * @return a reference to the Zipkin::Span object. */ diff --git a/test/common/tracing/tracer_impl_test.cc b/test/common/tracing/tracer_impl_test.cc index 851795817631..a6c70556b0a4 100644 --- a/test/common/tracing/tracer_impl_test.cc +++ b/test/common/tracing/tracer_impl_test.cc @@ -265,6 +265,7 @@ TEST(NullTracerTest, BasicFunctionality) { span_ptr->setBaggage("key", "value"); ASSERT_EQ("", span_ptr->getBaggage("baggage_key")); ASSERT_EQ(span_ptr->getTraceId(), ""); + ASSERT_EQ(span_ptr->getSpanId(), ""); span_ptr->injectContext(trace_context, upstream_context); span_ptr->log(SystemTime(), "fake_event"); diff --git a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc index b16d47d135c2..14c19f5bd185 100644 --- a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc @@ -176,7 +176,9 @@ TEST_F(AccessLogTest, TraceId) { NiceMock active_span; EXPECT_CALL(active_span, getTraceId()).WillOnce(Return("404142434445464748494a4b4c4d4e4f")); + EXPECT_CALL(active_span, getSpanId()).WillOnce(Return("4041424344454647")); expectLog(R"EOF( + span_id: "QEFCQ0RFRkc=" trace_id: "QEFCQ0RFRkdISUpLTE1OTw==" time_unix_nano: 3600000000000 )EOF"); diff --git a/test/extensions/filters/http/ext_proc/tracer_test_filter.cc b/test/extensions/filters/http/ext_proc/tracer_test_filter.cc index d58c97372160..15c7b1b348ea 100644 --- a/test/extensions/filters/http/ext_proc/tracer_test_filter.cc +++ b/test/extensions/filters/http/ext_proc/tracer_test_filter.cc @@ -73,6 +73,10 @@ class Span : public Tracing::Span { /* not implemented */ return EMPTY_STRING; }; + std::string getSpanId() const { + /* not implemented */ + return EMPTY_STRING; + }; Tracing::SpanPtr spawnChild(const Tracing::Config&, const std::string& operation_name, SystemTime) { diff --git a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc index 99c2ef75712e..352dfd8f2c14 100644 --- a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc +++ b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc @@ -353,6 +353,8 @@ TEST_F(OpenTracingDriverTest, GetTraceId) { // This method is unimplemented and a noop. ASSERT_EQ(first_span->getTraceId(), ""); + // This method is unimplemented and a noop. + ASSERT_EQ(first_span->getSpanId(), ""); } TEST_F(OpenTracingDriverTest, ExtractUsingForeach) { diff --git a/test/extensions/tracers/datadog/span_test.cc b/test/extensions/tracers/datadog/span_test.cc index 2dccafbea424..6104989bd7c9 100644 --- a/test/extensions/tracers/datadog/span_test.cc +++ b/test/extensions/tracers/datadog/span_test.cc @@ -389,6 +389,7 @@ TEST_F(DatadogTracerSpanTest, Baggage) { TEST_F(DatadogTracerSpanTest, GetTraceId) { Span span{std::move(span_)}; EXPECT_EQ("cafebabe", span.getTraceId()); + EXPECT_EQ("", span.getSpanId()); } TEST_F(DatadogTracerSpanTest, NoOpMode) { @@ -430,6 +431,7 @@ TEST_F(DatadogTracerSpanTest, NoOpMode) { EXPECT_EQ("", span.getBaggage("foo")); span.setBaggage("foo", "bar"); EXPECT_EQ("", span.getTraceId()); + EXPECT_EQ("", span.getSpanId()); } } // namespace diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index 65a1632596f6..cd4adebe4af7 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -130,6 +130,9 @@ TEST(OpenCensusTracerTest, Span) { // Trace id is automatically created when no parent context exists. ASSERT_NE(span->getTraceId(), ""); + + // Span id should be empty since this is not yet supported. + ASSERT_EQ(span->getSpanId(), ""); } // Retrieve SpanData from the OpenCensus trace exporter. @@ -222,6 +225,8 @@ void testIncomingHeaders( // Check contents via public API. // Trace id is set via context propagation headers. EXPECT_EQ(span->getTraceId(), "404142434445464748494a4b4c4d4e4f"); + // TODO(#34412) This method is unimplemented. + EXPECT_EQ(span->getSpanId(), ""); } // Retrieve SpanData from the OpenCensus trace exporter. diff --git a/test/extensions/tracers/skywalking/tracer_test.cc b/test/extensions/tracers/skywalking/tracer_test.cc index cfa2564a5816..60a12a42ed15 100644 --- a/test/extensions/tracers/skywalking/tracer_test.cc +++ b/test/extensions/tracers/skywalking/tracer_test.cc @@ -90,6 +90,8 @@ TEST_F(TracerTest, TracerTestCreateNewSpanWithNoPropagationHeaders) { span->setOperation("FakeStringAndNothingToDo"); span->setBaggage("FakeStringAndNothingToDo", "FakeStringAndNothingToDo"); ASSERT_EQ(span->getTraceId(), segment_context->traceId()); + // This method is unimplemented and a noop. + ASSERT_EQ(span->getSpanId(), ""); // Test whether the basic functions of Span are normal. EXPECT_FALSE(span->spanEntity()->skipAnalysis()); span->setSampled(false); diff --git a/test/extensions/tracers/xray/tracer_test.cc b/test/extensions/tracers/xray/tracer_test.cc index 12b7c50238f9..c451fb993f1f 100644 --- a/test/extensions/tracers/xray/tracer_test.cc +++ b/test/extensions/tracers/xray/tracer_test.cc @@ -388,6 +388,9 @@ TEST_F(XRayTracerTest, GetTraceId) { // Trace ID is always generated EXPECT_NE(span->getTraceId(), ""); + + // This method is unimplemented and a noop. + EXPECT_EQ(span->getSpanId(), ""); } TEST_F(XRayTracerTest, ChildSpanHasParentInfo) { diff --git a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc index 384d53bee290..e4292aa90452 100644 --- a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc @@ -726,6 +726,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanTest) { Tracing::SpanPtr span6 = driver_->startSpan(config_, request_headers_, stream_info_, operation_name_, {Tracing::Reason::Sampling, true}); EXPECT_EQ(span6->getTraceId(), "0000000000000000"); + EXPECT_EQ(span6->getSpanId(), ""); } TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersTest) { @@ -799,6 +800,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3Headers128TraceIdTest) { EXPECT_EQ(parent_id, zipkin_span->span().parentIdAsHexString()); EXPECT_TRUE(zipkin_span->span().sampled()); EXPECT_EQ(trace_id, zipkin_span->getTraceId()); + EXPECT_EQ("", zipkin_span->getSpanId()); } TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidTraceIdB3HeadersTest) { diff --git a/test/mocks/tracing/mocks.h b/test/mocks/tracing/mocks.h index 063902b3aadf..51b0fbc6c83e 100644 --- a/test/mocks/tracing/mocks.h +++ b/test/mocks/tracing/mocks.h @@ -44,6 +44,7 @@ class MockSpan : public Span { MOCK_METHOD(void, setBaggage, (absl::string_view key, absl::string_view value)); MOCK_METHOD(std::string, getBaggage, (absl::string_view key)); MOCK_METHOD(std::string, getTraceId, (), (const)); + MOCK_METHOD(std::string, getSpanId, (), (const)); SpanPtr spawnChild(const Config& config, const std::string& name, SystemTime start_time) override { From b6b585b2baf468f5876349658c5a7d3424702810 Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Wed, 5 Jun 2024 20:03:29 -0400 Subject: [PATCH 32/61] xds-failover: add unified gRPC mux support (#34534) This PR introduces xDS-Failover support for unified gRPC mux implementations. This is similar to #34437, and another part of #34417. Risk Level: medium if the runtime flag is set to true and unified-gRPC-mux is used. Testing: Updated the test cases (parameterized). Signed-off-by: Adi Suissa-Peleg --- .../config_subscription/grpc/xds_mux/BUILD | 1 + .../grpc/xds_mux/grpc_mux_impl.cc | 49 +++++++++---- .../grpc/xds_mux/grpc_mux_impl.h | 30 ++++++-- .../grpc/xds_grpc_mux_impl_test.cc | 70 +++++++++++-------- 4 files changed, 100 insertions(+), 50 deletions(-) diff --git a/source/extensions/config_subscription/grpc/xds_mux/BUILD b/source/extensions/config_subscription/grpc/xds_mux/BUILD index a9e7f75fd70a..a0fa33be7558 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/BUILD +++ b/source/extensions/config_subscription/grpc/xds_mux/BUILD @@ -70,6 +70,7 @@ envoy_cc_extension( "//source/common/memory:utils_lib", "//source/extensions/config_subscription/grpc:eds_resources_cache_lib", "//source/extensions/config_subscription/grpc:grpc_mux_context_lib", + "//source/extensions/config_subscription/grpc:grpc_mux_failover_lib", "//source/extensions/config_subscription/grpc:grpc_stream_lib", "//source/extensions/config_subscription/grpc:pausable_ack_queue_lib", "//source/extensions/config_subscription/grpc:watch_map_lib", diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc index 544a813e7a26..b564b0b929db 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc @@ -39,28 +39,49 @@ using AllMuxes = ThreadSafeSingleton; template GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory, - GrpcMuxContext& grpc_mux_content, bool skip_subsequent_node) - : grpc_stream_(this, std::move(grpc_mux_content.async_client_), - grpc_mux_content.service_method_, grpc_mux_content.dispatcher_, - grpc_mux_content.scope_, std::move(grpc_mux_content.backoff_strategy_), - grpc_mux_content.rate_limit_settings_), + GrpcMuxContext& grpc_mux_context, bool skip_subsequent_node) + : grpc_stream_(createGrpcStreamObject(grpc_mux_context)), subscription_state_factory_(std::move(subscription_state_factory)), - skip_subsequent_node_(skip_subsequent_node), local_info_(grpc_mux_content.local_info_), + skip_subsequent_node_(skip_subsequent_node), local_info_(grpc_mux_context.local_info_), dynamic_update_callback_handle_( - grpc_mux_content.local_info_.contextProvider().addDynamicContextUpdateCallback( + grpc_mux_context.local_info_.contextProvider().addDynamicContextUpdateCallback( [this](absl::string_view resource_type_url) { onDynamicContextUpdate(resource_type_url); return absl::OkStatus(); })), - config_validators_(std::move(grpc_mux_content.config_validators_)), - xds_config_tracker_(grpc_mux_content.xds_config_tracker_), - xds_resources_delegate_(grpc_mux_content.xds_resources_delegate_), - eds_resources_cache_(std::move(grpc_mux_content.eds_resources_cache_)), - target_xds_authority_(grpc_mux_content.target_xds_authority_) { - THROW_IF_NOT_OK(Config::Utility::checkLocalInfo("ads", grpc_mux_content.local_info_)); + config_validators_(std::move(grpc_mux_context.config_validators_)), + xds_config_tracker_(grpc_mux_context.xds_config_tracker_), + xds_resources_delegate_(grpc_mux_context.xds_resources_delegate_), + eds_resources_cache_(std::move(grpc_mux_context.eds_resources_cache_)), + target_xds_authority_(grpc_mux_context.target_xds_authority_) { + THROW_IF_NOT_OK(Config::Utility::checkLocalInfo("ads", grpc_mux_context.local_info_)); AllMuxes::get().insert(this); } +template +std::unique_ptr> +GrpcMuxImpl::createGrpcStreamObject(GrpcMuxContext& grpc_mux_context) { + if (Runtime::runtimeFeatureEnabled("envoy.restart_features.xds_failover_support")) { + return std::make_unique>( + /*primary_stream_creator=*/ + [&grpc_mux_context](GrpcStreamCallbacks* callbacks) -> GrpcStreamInterfacePtr { + return std::make_unique>( + callbacks, std::move(grpc_mux_context.async_client_), + grpc_mux_context.service_method_, grpc_mux_context.dispatcher_, + grpc_mux_context.scope_, std::move(grpc_mux_context.backoff_strategy_), + grpc_mux_context.rate_limit_settings_); + }, + /*failover_stream_creator=*/ + // TODO(adisuissa): implement when failover is fully plumbed. + absl::nullopt, + /*grpc_mux_callbacks=*/*this, + /*dispatch=*/grpc_mux_context.dispatcher_); + } + return std::make_unique>( + this, std::move(grpc_mux_context.async_client_), grpc_mux_context.service_method_, + grpc_mux_context.dispatcher_, grpc_mux_context.scope_, + std::move(grpc_mux_context.backoff_strategy_), grpc_mux_context.rate_limit_settings_); +} template GrpcMuxImpl::~GrpcMuxImpl() { AllMuxes::get().erase(this); } @@ -231,7 +252,7 @@ template void GrpcMuxImpl:: } started_ = true; ENVOY_LOG(debug, "GrpcMuxImpl now trying to establish a stream"); - grpc_stream_.establishNewStream(); + grpc_stream_->establishNewStream(); } template diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h index 8af1675ba5cb..2e0ce7407862 100644 --- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h @@ -22,7 +22,7 @@ #include "source/common/config/api_version.h" #include "source/common/grpc/common.h" #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" -#include "source/extensions/config_subscription/grpc/grpc_stream.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" #include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" #include "source/extensions/config_subscription/grpc/watch_map.h" #include "source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h" @@ -107,7 +107,14 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, return makeOptRefFromPtr(eds_resources_cache_.get()); } - GrpcStream& grpcStreamForTest() { return grpc_stream_; } + GrpcStreamInterface& grpcStreamForTest() { + // TODO(adisuissa): Once envoy.restart_features.xds_failover_support is deprecated, + // return grpc_stream_.currentStreamForTest() directly (defined in GrpcMuxFailover). + if (Runtime::runtimeFeatureEnabled("envoy.restart_features.xds_failover_support")) { + return dynamic_cast*>(grpc_stream_.get())->currentStreamForTest(); + } + return *grpc_stream_.get(); + } protected: class WatchImpl : public Envoy::Config::GrpcMuxWatch { @@ -137,10 +144,10 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, }; void sendGrpcMessage(RQ& msg_proto, S& sub_state); - void maybeUpdateQueueSizeStat(uint64_t size) { grpc_stream_.maybeUpdateQueueSizeStat(size); } - bool grpcStreamAvailable() { return grpc_stream_.grpcStreamAvailable(); } - bool rateLimitAllowsDrain() { return grpc_stream_.checkRateLimitAllowsDrain(); } - void sendMessage(RQ& msg_proto) { grpc_stream_.sendMessage(msg_proto); } + void maybeUpdateQueueSizeStat(uint64_t size) { grpc_stream_->maybeUpdateQueueSizeStat(size); } + bool grpcStreamAvailable() { return grpc_stream_->grpcStreamAvailable(); } + bool rateLimitAllowsDrain() { return grpc_stream_->checkRateLimitAllowsDrain(); } + void sendMessage(RQ& msg_proto) { grpc_stream_->sendMessage(msg_proto); } S& subscriptionStateFor(const std::string& type_url); WatchMap& watchMapFor(const std::string& type_url); @@ -157,6 +164,12 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, const LocalInfo::LocalInfo& localInfo() const { return local_info_; } private: + // Helper function to create the grpc_stream_ object. + // TODO(adisuissa): this should be removed when envoy.restart_features.xds_failover_support + // is deprecated. + std::unique_ptr> + createGrpcStreamObject(GrpcMuxContext& grpc_mux_context); + // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a (Delta)DiscoveryRequest). bool canSendDiscoveryRequest(const std::string& type_url); @@ -172,7 +185,10 @@ class GrpcMuxImpl : public GrpcStreamCallbacks, // Invoked when dynamic context parameters change for a resource type. void onDynamicContextUpdate(absl::string_view resource_type_url); - GrpcStream grpc_stream_; + // Multiplexes the stream to the primary and failover sources. + // TODO(adisuissa): Once envoy.restart_features.xds_failover_support is deprecated, + // convert from unique_ptr to GrpcMuxFailover directly. + std::unique_ptr> grpc_stream_; // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All // of our different resource types' ACKs are mixed together in this queue. See class for diff --git a/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc b/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc index 2e9ea564397c..123ffd52a56d 100644 --- a/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc +++ b/test/extensions/config_subscription/grpc/xds_grpc_mux_impl_test.cc @@ -26,6 +26,7 @@ #include "test/test_common/logging.h" #include "test/test_common/resources.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/test_time.h" #include "test/test_common/utility.h" @@ -48,7 +49,7 @@ namespace { // We test some mux specific stuff below, other unit test coverage for singleton use of GrpcMuxImpl // is provided in [grpc_]subscription_impl_test.cc. -class GrpcMuxImplTestBase : public testing::Test { +class GrpcMuxImplTestBase : public testing::TestWithParam { public: GrpcMuxImplTestBase() : async_client_(new Grpc::MockAsyncClient()), @@ -56,8 +57,13 @@ class GrpcMuxImplTestBase : public testing::Test { control_plane_stats_(Utility::generateControlPlaneStats(*stats_.rootScope())), control_plane_connected_state_( stats_.gauge("control_plane.connected_state", Stats::Gauge::ImportMode::NeverImport)), - control_plane_pending_requests_(stats_.gauge("control_plane.pending_requests", - Stats::Gauge::ImportMode::NeverImport)) {} + control_plane_pending_requests_( + stats_.gauge("control_plane.pending_requests", Stats::Gauge::ImportMode::NeverImport)) { + // Once "envoy.restart_features.xds_failover_support" is deprecated, the + // test should no longer be parameterized. + scoped_runtime_.mergeValues( + {{"envoy.restart_features.xds_failover_support", GetParam() ? "true" : "false"}}); + } void setup() { setup(rate_limit_settings_); } @@ -123,6 +129,7 @@ class GrpcMuxImplTestBase : public testing::Test { return grpc_mux_->addWatch(type_url, resources, callbacks, resource_decoder, {}); } + TestScopedRuntime scoped_runtime_; NiceMock dispatcher_; NiceMock random_; Grpc::MockAsyncClient* async_client_; @@ -147,8 +154,10 @@ class GrpcMuxImplTest : public GrpcMuxImplTestBase { Event::SimulatedTimeSystem time_system_; }; +INSTANTIATE_TEST_SUITE_P(GrpcMuxImpl, GrpcMuxImplTest, ::testing::Bool()); + // Validate behavior when multiple type URL watches are maintained, watches are created/destroyed. -TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { +TEST_P(GrpcMuxImplTest, MultipleTypeUrlStreams) { setup(); InSequence s; @@ -169,7 +178,7 @@ TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { } // Validate behavior when multiple type URL watches are maintained and the stream is reset. -TEST_F(GrpcMuxImplTest, ResetStream) { +TEST_P(GrpcMuxImplTest, ResetStream) { InSequence s; auto* timer = new Event::MockTimer(&dispatcher_); @@ -213,7 +222,7 @@ TEST_F(GrpcMuxImplTest, ResetStream) { } // Validate pause-resume behavior. -TEST_F(GrpcMuxImplTest, PauseResume) { +TEST_P(GrpcMuxImplTest, PauseResume) { setup(); InSequence s; GrpcMuxWatchPtr foo1; @@ -248,7 +257,7 @@ TEST_F(GrpcMuxImplTest, PauseResume) { } // Validate behavior when type URL mismatches occur. -TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { +TEST_P(GrpcMuxImplTest, TypeUrlMismatch) { setup(); auto invalid_response = std::make_unique(); @@ -289,7 +298,7 @@ TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { expectSendMessage("type_url_foo", {}, ""); } -TEST_F(GrpcMuxImplTest, RpcErrorMessageTruncated) { +TEST_P(GrpcMuxImplTest, RpcErrorMessageTruncated) { setup(); auto invalid_response = std::make_unique(); InSequence s; @@ -351,7 +360,7 @@ resourceWithEmptyTtl(envoy::config::endpoint::v3::ClusterLoadAssignment& cla) { return resource; } // Validates the behavior when the TTL timer expires. -TEST_F(GrpcMuxImplTest, ResourceTTL) { +TEST_P(GrpcMuxImplTest, ResourceTTL) { setup(); time_system_.setSystemTime(std::chrono::seconds(0)); @@ -487,7 +496,7 @@ TEST_F(GrpcMuxImplTest, ResourceTTL) { } // Checks that the control plane identifier is logged -TEST_F(GrpcMuxImplTest, LogsControlPlaneIndentifier) { +TEST_P(GrpcMuxImplTest, LogsControlPlaneIndentifier) { setup(); std::string type_url = "foo"; @@ -521,7 +530,7 @@ TEST_F(GrpcMuxImplTest, LogsControlPlaneIndentifier) { } // Validate behavior when watches has an unknown resource name. -TEST_F(GrpcMuxImplTest, WildcardWatch) { +TEST_P(GrpcMuxImplTest, WildcardWatch) { setup(); const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; @@ -553,7 +562,7 @@ TEST_F(GrpcMuxImplTest, WildcardWatch) { } // Validate behavior when watches specify resources (potentially overlapping). -TEST_F(GrpcMuxImplTest, WatchDemux) { +TEST_P(GrpcMuxImplTest, WatchDemux) { setup(); // We will not require InSequence here: an update that causes multiple onConfigUpdates // causes them in an indeterminate order, based on the whims of the hash map. @@ -643,7 +652,7 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { } // Validate behavior when we have multiple watchers that send empty updates. -TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { +TEST_P(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { setup(); InSequence s; const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; @@ -666,7 +675,7 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { } // Validate behavior when we have Single Watcher that sends Empty updates. -TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { +TEST_P(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { setup(); const std::string& type_url = Config::TypeUrl::get().Cluster; NiceMock foo_callbacks; @@ -696,8 +705,11 @@ class GrpcMuxImplTestWithMockTimeSystem : public GrpcMuxImplTestBase { Event::DelegatingTestTimeSystem mock_time_system_; }; +INSTANTIATE_TEST_SUITE_P(GrpcMuxImplTestWithMockTimeSystem, GrpcMuxImplTestWithMockTimeSystem, + ::testing::Bool()); + // Verifies that rate limiting is not enforced with defaults. -TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { +TEST_P(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { auto ttl_timer = new Event::MockTimer(&dispatcher_); // Retry timer, @@ -735,7 +747,7 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { } // Verifies that default rate limiting is enforced with empty RateLimitSettings. -TEST_F(GrpcMuxImplTest, TooManyRequestsWithEmptyRateLimitSettings) { +TEST_P(GrpcMuxImplTest, TooManyRequestsWithEmptyRateLimitSettings) { // Validate that request drain timer is created. auto ttl_timer = new Event::MockTimer(&dispatcher_); @@ -792,7 +804,7 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithEmptyRateLimitSettings) { } // Verifies that rate limiting is enforced with custom RateLimitSettings. -TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { +TEST_P(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { // Validate that request drain timer is created. // TTL timer. @@ -845,7 +857,7 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { } // Verifies that a message with no resources is accepted. -TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { +TEST_P(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { setup(); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); @@ -885,7 +897,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { // are accompanied by interesting ones. // Note: this was previously "rejects", not "accepts". See // https://github.com/envoyproxy/envoy/pull/8350#discussion_r328218220 for discussion. -TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsResources) { +TEST_P(GrpcMuxImplTest, UnwatchedTypeAcceptsResources) { setup(); EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; @@ -908,7 +920,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsResources) { grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); } -TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { +TEST_P(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { EXPECT_CALL(local_info_, clusterName()).WillOnce(ReturnRef(EMPTY_STRING)); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), @@ -934,7 +946,7 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { "--service-node and --service-cluster options."); } -TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { +TEST_P(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { EXPECT_CALL(local_info_, nodeName()).WillOnce(ReturnRef(EMPTY_STRING)); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(async_client_), @@ -961,7 +973,7 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { } // Validate that a valid resource decoder is used after removing a subscription. -TEST_F(GrpcMuxImplTest, ValidResourceDecoderAfterRemoval) { +TEST_P(GrpcMuxImplTest, ValidResourceDecoderAfterRemoval) { setup(); const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; @@ -1039,7 +1051,7 @@ TEST_F(GrpcMuxImplTest, ValidResourceDecoderAfterRemoval) { } // Validate behavior when dynamic context parameters are updated. -TEST_F(GrpcMuxImplTest, DynamicContextParameters) { +TEST_P(GrpcMuxImplTest, DynamicContextParameters) { setup(); InSequence s; auto foo = grpc_mux_->addWatch("foo", {"x", "y"}, callbacks_, resource_decoder_, {}); @@ -1061,7 +1073,7 @@ TEST_F(GrpcMuxImplTest, DynamicContextParameters) { expectSendMessage("foo", {}, "", false); } -TEST_F(GrpcMuxImplTest, AllMuxesStateTest) { +TEST_P(GrpcMuxImplTest, AllMuxesStateTest) { setup(); GrpcMuxContext grpc_mux_context{ /*async_client_=*/std::unique_ptr(), @@ -1089,20 +1101,20 @@ TEST_F(GrpcMuxImplTest, AllMuxesStateTest) { } // Validates that the EDS cache getter returns the cache. -TEST_F(GrpcMuxImplTest, EdsResourcesCacheForEds) { +TEST_P(GrpcMuxImplTest, EdsResourcesCacheForEds) { eds_resources_cache_ = new NiceMock(); setup(); EXPECT_TRUE(grpc_mux_->edsResourcesCache().has_value()); } // Validates that the EDS cache getter returns empty if there is no cache. -TEST_F(GrpcMuxImplTest, EdsResourcesCacheForEdsNoCache) { +TEST_P(GrpcMuxImplTest, EdsResourcesCacheForEdsNoCache) { setup(); EXPECT_FALSE(grpc_mux_->edsResourcesCache().has_value()); } // Validate that an EDS resource is cached if there's a cache. -TEST_F(GrpcMuxImplTest, CacheEdsResource) { +TEST_P(GrpcMuxImplTest, CacheEdsResource) { // Create the cache that will also be passed to the GrpcMux object via setup(). eds_resources_cache_ = new NiceMock(); setup(); @@ -1144,7 +1156,7 @@ TEST_F(GrpcMuxImplTest, CacheEdsResource) { // Validate that an update to an EDS resource watcher is reflected in the cache, // if there's a cache. -TEST_F(GrpcMuxImplTest, UpdateCacheEdsResource) { +TEST_P(GrpcMuxImplTest, UpdateCacheEdsResource) { // Create the cache that will also be passed to the GrpcMux object via setup(). eds_resources_cache_ = new NiceMock(); setup(); @@ -1191,7 +1203,7 @@ TEST_F(GrpcMuxImplTest, UpdateCacheEdsResource) { // Validate that adding and removing watchers reflects on the cache changes, // if there's a cache. -TEST_F(GrpcMuxImplTest, AddRemoveSubscriptions) { +TEST_P(GrpcMuxImplTest, AddRemoveSubscriptions) { // Create the cache that will also be passed to the GrpcMux object via setup(). eds_resources_cache_ = new NiceMock(); setup(); From 964ea7dfddde433eca763880dca2774074e01441 Mon Sep 17 00:00:00 2001 From: Ted Poole Date: Thu, 6 Jun 2024 12:13:44 +0100 Subject: [PATCH 33/61] Fixed bazel configuration ambiguity for non HTTP3 builds (#34483) To avoid ambiguity when building with //bazel:http3=False (on PPC and Windows) this commit adds 2 new config_setting_groups and updates the relevant select() statements. Signed-off-by: Ted Poole --- bazel/BUILD | 58 ++++++++++++++++++++++++++++++++++++++++++ source/exe/BUILD | 7 ++--- test/config_test/BUILD | 7 ++--- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/bazel/BUILD b/bazel/BUILD index c4749d3140d3..b6c11d2f029c 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -351,6 +351,64 @@ selects.config_setting_group( ], ) +selects.config_setting_group( + name = "disable_http3_on_linux_ppc", + match_all = [ + ":disable_http3", + ":linux_ppc", + ], +) + +selects.config_setting_group( + name = "disable_http3_on_windows_x86_64", + match_all = [ + ":disable_http3", + ":windows_x86_64", + ], +) + +bool_flag( + name = "enabled", + build_setting_default = True, + visibility = ["//visibility:private"], +) + +bool_flag( + name = "disabled", + build_setting_default = False, + visibility = ["//visibility:private"], +) + +# Alias equal to "not(":disable_http3")" (if "not()" existed). +alias( + name = "enable_http3_setting", + actual = select({ + ":disable_http3": ":disabled", + "//conditions:default": ":enabled", + }), +) + +config_setting( + name = "enable_http3", + flag_values = {":enable_http3_setting": "True"}, +) + +selects.config_setting_group( + name = "enable_http3_on_linux_ppc", + match_all = [ + ":enable_http3", + ":linux_ppc", + ], +) + +selects.config_setting_group( + name = "enable_http3_on_windows_x86_64", + match_all = [ + ":enable_http3", + ":windows_x86_64", + ], +) + config_setting( name = "disable_admin_html", values = {"define": "admin_html=disabled"}, diff --git a/source/exe/BUILD b/source/exe/BUILD index 905a09ff4627..59246022ad08 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -45,9 +45,10 @@ envoy_cc_library( "//source/server:options_base", "//source/server:server_base_lib", ] + select({ - "//bazel:windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS), - "//bazel:linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS), - "//bazel:disable_http3": envoy_all_extensions(NO_HTTP3_SKIP_TARGETS), + "//bazel:enable_http3_on_windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS), + "//bazel:enable_http3_on_linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS), + "//bazel:disable_http3_on_windows_x86_64": envoy_all_extensions(NO_HTTP3_SKIP_TARGETS + WINDOWS_SKIP_TARGETS), + "//bazel:disable_http3_on_linux_ppc": envoy_all_extensions(NO_HTTP3_SKIP_TARGETS + PPC_SKIP_TARGETS), "//conditions:default": envoy_all_extensions(), }), ) diff --git a/test/config_test/BUILD b/test/config_test/BUILD index b0b664f2bece..43561cd54f84 100644 --- a/test/config_test/BUILD +++ b/test/config_test/BUILD @@ -63,9 +63,10 @@ envoy_cc_test_library( "//test/test_common:simulated_time_system_lib", "//test/test_common:threadsafe_singleton_injector_lib", ] + select({ - "//bazel:windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS), - "//bazel:linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS), - "//bazel:disable_http3": envoy_all_extensions(NO_HTTP3_SKIP_TARGETS), + "//bazel:enable_http3_on_windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS), + "//bazel:enable_http3_on_linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS), + "//bazel:disable_http3_on_windows_x86_64": envoy_all_extensions(NO_HTTP3_SKIP_TARGETS + WINDOWS_SKIP_TARGETS), + "//bazel:disable_http3_on_linux_ppc": envoy_all_extensions(NO_HTTP3_SKIP_TARGETS + PPC_SKIP_TARGETS), "//conditions:default": envoy_all_extensions(), }), ) From de78171a00ae5c067b3159ba42f456ea123f46c7 Mon Sep 17 00:00:00 2001 From: "publish-envoy[bot]" <140627008+publish-envoy[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:33:34 +0000 Subject: [PATCH 34/61] repo: Sync version histories (#34559) Signed-off-by: publish-envoy[bot] Co-authored-by: publish-envoy[bot] <140627008+publish-envoy[bot]@users.noreply.github.com> --- changelogs/1.27.6.yaml | 33 +++++++++++++++++++++++++++++ changelogs/1.28.4.yaml | 26 +++++++++++++++++++++++ changelogs/1.29.5.yaml | 26 +++++++++++++++++++++++ changelogs/1.30.2.yaml | 23 ++++++++++++++++++++ docs/inventories/v1.27/objects.inv | Bin 160000 -> 160022 bytes docs/inventories/v1.28/objects.inv | Bin 164421 -> 164467 bytes docs/inventories/v1.29/objects.inv | Bin 168176 -> 168248 bytes docs/inventories/v1.30/objects.inv | Bin 171817 -> 171905 bytes docs/versions.yaml | 8 +++---- 9 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 changelogs/1.27.6.yaml create mode 100644 changelogs/1.28.4.yaml create mode 100644 changelogs/1.29.5.yaml create mode 100644 changelogs/1.30.2.yaml diff --git a/changelogs/1.27.6.yaml b/changelogs/1.27.6.yaml new file mode 100644 index 000000000000..cc73ba5da9a9 --- /dev/null +++ b/changelogs/1.27.6.yaml @@ -0,0 +1,33 @@ +date: June 4, 2024 + +bug_fixes: +- area: router + change: | + Fix a timing issue when upstream requests are empty when decoding data and send local reply when that happends. This is + controlled by ``envoy_reloadable_features_send_local_reply_when_no_buffer_and_upstream_request``. +- area: quic + change: | + Applied 2 QUICHE patches for crash bugs in ``QuicSpdyStream`` ``OnDataAvailable()`` and ``OnInitialHeaderComplete()``. +- area: quic + change: | + Fixed crash bug when QUIC downstream stream was read closed and then timed out. +- area: decompression + change: | + Fixed a bug where Envoy will go into an endless loop when using the brotli decompressor. If the input stream has + redundant data, the decompressor will loop forever. +- area: websocket + change: | + Only 101 is considered a successful response for websocket handshake for HTTP/1.1, and Envoy as a proxy will proxy the response + header from upstream to downstream and then close the request if other status is received. This behavior can be + reverted by ``envoy_reloadable_features_check_switch_protocol_websocket_handshake``. +- area: async http client + change: | + Added one option to disable the response body buffering for mirror request. Also introduced a 32MB cap for the response + buffer, which can be changed by the runtime flag ``http.async_response_buffer_limit`` based on the product needs. + +removed_config_or_runtime: +# *Normally occurs at the end of the* :ref:`deprecation period ` + +new_features: + +deprecated: diff --git a/changelogs/1.28.4.yaml b/changelogs/1.28.4.yaml new file mode 100644 index 000000000000..d4f34e447097 --- /dev/null +++ b/changelogs/1.28.4.yaml @@ -0,0 +1,26 @@ +date: June 5, 2024 + +bug_fixes: +- area: router + change: | + Fix a timing issue when upstream requests are empty when decoding data and send local reply when that happends. This is + controlled by ``envoy_reloadable_features_send_local_reply_when_no_buffer_and_upstream_request``. +- area: quic + change: | + Applied 2 QUICHE patches for crash bugs in ``QuicSpdyStream`` ``OnDataAvailable()`` and ``OnInitialHeaderComplete()``. +- area: quic + change: | + Fixed crash bug when QUIC downstream stream was read closed and then timed out. +- area: decompression + change: | + Fixed a bug where Envoy will go into an endless loop when using the brotli decompressor. If the input stream has + redundant data, the decompressor will loop forever. +- area: websocket + change: | + Only 101 is considered a successful response for websocket handshake for HTTP/1.1, and Envoy as a proxy will proxy the response + header from upstream to downstream and then close the request if other status is received. This behavior can be + reverted by ``envoy_reloadable_features_check_switch_protocol_websocket_handshake``. +- area: async http client + change: | + Added one option to disable the response body buffering for mirror request. Also introduced a 32MB cap for the response + buffer, which can be changed by the runtime flag ``http.async_response_buffer_limit`` based on the product needs. diff --git a/changelogs/1.29.5.yaml b/changelogs/1.29.5.yaml new file mode 100644 index 000000000000..d4f34e447097 --- /dev/null +++ b/changelogs/1.29.5.yaml @@ -0,0 +1,26 @@ +date: June 5, 2024 + +bug_fixes: +- area: router + change: | + Fix a timing issue when upstream requests are empty when decoding data and send local reply when that happends. This is + controlled by ``envoy_reloadable_features_send_local_reply_when_no_buffer_and_upstream_request``. +- area: quic + change: | + Applied 2 QUICHE patches for crash bugs in ``QuicSpdyStream`` ``OnDataAvailable()`` and ``OnInitialHeaderComplete()``. +- area: quic + change: | + Fixed crash bug when QUIC downstream stream was read closed and then timed out. +- area: decompression + change: | + Fixed a bug where Envoy will go into an endless loop when using the brotli decompressor. If the input stream has + redundant data, the decompressor will loop forever. +- area: websocket + change: | + Only 101 is considered a successful response for websocket handshake for HTTP/1.1, and Envoy as a proxy will proxy the response + header from upstream to downstream and then close the request if other status is received. This behavior can be + reverted by ``envoy_reloadable_features_check_switch_protocol_websocket_handshake``. +- area: async http client + change: | + Added one option to disable the response body buffering for mirror request. Also introduced a 32MB cap for the response + buffer, which can be changed by the runtime flag ``http.async_response_buffer_limit`` based on the product needs. diff --git a/changelogs/1.30.2.yaml b/changelogs/1.30.2.yaml new file mode 100644 index 000000000000..2e4316c5184c --- /dev/null +++ b/changelogs/1.30.2.yaml @@ -0,0 +1,23 @@ +date: June 5, 2024 + +bug_fixes: +# *Changes expected to improve the state of the world and are unlikely to have negative effects* +- area: quic + change: | + Applied 2 QUICHE patches for crash bugs in ``QuicSpdyStream`` ``OnDataAvailable()`` and ``OnInitialHeaderComplete()``. +- area: quic + change: | + Fixed crash bug when QUIC downstream stream was read closed and then timed out. +- area: decompression + change: | + Fixed a bug where Envoy will go into an endless loop when using the brotli decompressor. If the input stream has + redundant data, the decompressor will loop forever. +- area: websocket + change: | + Only 101 is considered a successful response for websocket handshake for HTTP/1.1, and Envoy as a proxy will proxy the response + header from upstream to downstream and then close the request if other status is received. This behavior can be + reverted by ``envoy_reloadable_features_check_switch_protocol_websocket_handshake``. +- area: async http client + change: | + Added one option to disable the response body buffering for mirror request. Also introduced a 32MB cap for the response + buffer, which can be changed by the runtime flag ``http.async_response_buffer_limit`` based on the product needs. diff --git a/docs/inventories/v1.27/objects.inv b/docs/inventories/v1.27/objects.inv index 01563bfb768291679df12aaaeea1be817960262f..f875b6a4258958214fe4a9a7d4980f1bc18d54b8 100644 GIT binary patch delta 3582 zcmXYxc_0)18^@bthPg&;&dE)fE4epECP!N)DR<>6N3Q1F!DQuEQ=uFg5lu*lkiM=& z#$wLqnIQT_bt```Qd<9R->=lMMEhqxe|zaUI=030DXQXB!zj(h^=2r{q8 zgQpK<%j9=5F$>Wg zqbRIZ9Lo!p``dbnqNK;j%VoyF#0fIYvAg%I6QuOHZ9b;VFhJyxZ~V95N@Vw&+}sAO za^t6=^M@3*lVx-64SeQHOdh5=F0tBANBY919S#3|arMG~fyny98-;H{_taW-V`B_j zok?nrTdUE%+ehw7zXq)(gVl|*3YEk2J3RtUNDc>jeB{gc+GJqOawFJNMWEnpFFLE! zmm*Oko?jl^MVu25*>LsPp??n)J}v`BV`y|3AM^E$3^@BF?TH1MIS_FW405GUbv9*6 z6xDdTc@_sEBgnl>$*5zK^gv|lzLbixMdgtJ79x;V_I3P z9ZaKJ@G-;Bz`)t(X?F#?4`*{>;MXBEx`}}0gOA||cm*nWSZX=-S~rk?R9^iPpQ_<% zp#nHNlGf^)*ZhnYvV>|Evq=SX;NZ#CPn47UBqf2l~AnqA@$k} zkl$EAJ(ffhZTku2?^aOX1kVpwUsD95FVm{yGA*mZ;}P&GlJPEC-gepEZ$Q073@C5} zJq7Lxk-lRNrE%b-IDz7^=A%%_dH7|~%(Iwr3n*;~jtvDR92;8;L{?|?7OM?e_;h6e z8pLGY6};ODD&X2bd=x`Kz+a*kLO|w4^fz5ZxR7wM+FUV%-9>a1qMm`X$q#3{h+#rh z3x~TVPYZ?i(X^i#zL@XvuhzR({<@1OBSbA!W#=&7b`g*6OEq>eW2TGf`)}@I7m@gH z?o%QDTgvL|d|d{bVQC2X8xnc9ThVr_lF)WdI9Uk{O8B#5BFbR&71|R)vc6_mG6KGg z;v13N%3*x!B6bK-EuE|53O9w~?3JKTwZ`WGj7doWfCRAV2qrKfvgazUzA5aGBDgx5 z_5?;|YQBy|(ECw*E1;vL@cn-5g@k?Kb7bB9enR$3IaHez-WB54B`B*eD+UcyKPiJ( zlWBBOfi}2EI)c89A{$7p-qHqi6VC{VY^){h(79;{?5hmG4{?aOL+i9akOWZ120f<; zSiwD#0GisO*?eWPraTEC&JH~U>n@gBLPA*pRn!V7-a`fygoI@Z_`z9y0t1pjwhbmG z;C+wK#K`GugmoeQWEJ&@ER0E^Bq&1!ZmOO5c={sa2q>c*Zi+~5h4~m{0!kzyN1a(2FZaEjZoQ{06_=@PbC(_ zheLp-F27ga2=zDs|ehPpm4Y_h7WEhqrOfDB}tkK~VO z#z7^qs#rrQu7;N%1)wT(XQsfOhaE>MmD`K(k7}z&WyP4Zg@|MX>zE=Ed-q3_Ba-ob zUnjqw7PxQ_sNog1u6S-my7v?mz1>htH`NCJ`&#Lf;vhb=#hEmHK*gXoML{?_T zJA#KA4@8s`MfnHy|6o;!W}GqjjkPAMYX}}}XHBhQL!~nVpqNzEvbfA*>B>MTi>Qhc z6*zsg)&AqQ946+R&`9mmSP*x7AENvXBf)k5Srw$KhE0!S_ks59Km=fpLOpZYx79 z&e#-Mr{%cV^+?(LmbF-1vN5!wN_$b2A*un1L_xY{~ym%IvSEwKl*LO z4kfJywJYGoMehkWe$3#B!d`B@Dn%$Aff6(qM>=>~ZudoBmF`X+4~DPIQz7T)%*!bCxV8CQpfqqX^k>4SyJv%POb=^9?Y`Apzv7JARLIqK?NA6V zD$5fL_?H=>#gV;>rx9919^4F(eH6iW_ikHjn)zq&IheAx}$s|+Z}oAo>f z;rDS3*y^rH7}(U|&mxdyD-E}Y^f3ET51Eu)VYtoF)7cL)2qew}!|m=Gz`$c$wK5S! zkr&&8M~wB7(bel&_5t{2$$NP^>THBVnPFb?yu)fw zt?u)Zo_8816@Rf_*P=Wv1w?#FuvKD6dc)n zcC!6!;$*^Zap}!x+?n)b^QHQgHSTwO%A1Yh2g8;v?Nv=tp5gfsjkvZl;fj?w>3n34XaDErsCCFi^P|Lb zdQ&&eQzjHox3|?iJDC3*(GlHTrey~+SUMFRhfZkq4`y~kCS^|4PHX(S_j5{7d9bJL zOqGg+`{Y*YyE@%Tm$=)hHM2zyu>tiA9LJ`TnZDaAPGC0FOx1mya*yNG3TvgVv#l~W zJaw;BrCwnKb2%+1sGQqdHdM~%s>c3=H_@*g5{9D(KTXY=U2RKAZZ5bAU|UL=RI;gk zNvgk6rq9)KzX1A=UN-{ZQ!>np+?yfatY6x;IeftHO(<8}G3)^8eYT000+a_M6?$$) zrp26aB&ZZBcv)eQi)_*~^k~lFIo`Vl3*}P>Z*b54$0b7>njKl_HBf448u;E4tnBC{ zJtt_(iy`TdfjoFi%8d8QSzbqt_@|ZnBk2(=jM013clw6q>3B|D`JK_UfX%3zOObgU z_|HAN1=Pey?Dw|m{4Y214>WmM_<9NXb2S#w1TpbSYAB3wWBsO|tHTY&o_J- zLpOMBeoEXmF07a>_>X_a^;42DIF;aNNxW_{7MNtLbq!^|#rwc5{YT#tn4-xw*5}aLa0F*}YI z&vpAk5>uw?hc=oBugor~JvC$tAxvK{yifEouNIG72w zU%lS$ma+T3_i?3dD$NcTZ#Y*|LGF365=G=CJzwNKF3NJ`(MT8g^54rGmHH!G<6wZZ?;1J@alXp52_uPmZb?{ zd{w27Tx7F4sHgP%hDP6il)JCi`<@y&n1|rYW;+|x@;Lw#Ih9xH%&SZE5s5qA5Gmx6 z3X;fB9Zwx|vJnTsh9dCxA`Kq`2ha^g>~X3h__%NYFJ)E6)RUL!jJTBJ=M>>kvanu5 z+BYw+1gF>w>j~06qi{zYN{tLuht{LS6dX#2%;{I0Pb)E#utZ=1!6}PTI0ObEl+p7* zHYWU)G82BmO_=4vYsf8hl(1sJ2NzwO*1GtO$B(BV;1uVr^cOUh;TK$lA_B1QxL6t9 z8YZ>0(G$^W!^ZHFIf+(^z}+* zn1m&i5hSo zjKklADtj_5kisV@HUpgN5r;=A4x(r<-jTr!OQdiZwJQwDaD4u16Ajd{%HNgv0v30G z_H8#(u?I!7;APWP@kn7mipJn&dstc`6`%cA>$wLh9Q~hmB9Iir<@uprbbt-EK^!co zF#&tpWLrGFcadGW01X8Mp85YX7QhA26%e(gst$060;FgUWl95>N3*m>D!xY1=AdzI z9`6F(96|Q0Jp4Y=iY5njSHeO-;4?p^iWg)JD za#Qs1G3jiE_&F4$r;o;*?|c72AKsNNte27QoIi(#LoW;KEqHrxnPuQmu6EP?N{fk{ zP&rVL0eT4tz?PXB!n>{u%UpPk=nvUARJa|-A2kRQ56Lk>2qW|d1H0y;6Ztrl2gm(~ zoxy&wW|{pkmQJeYwf8ATJ9^{k)E*w>ak_Pat15v`o#1VL5g|vEAQer?UA!5*kjPPy z2Wc9k@s9oPr;XudgKaNXj}bFCaRVF$#<1K}Srt_D`9f853Ss zvBg!?Q;8GR*%kVuhGE%?Fp#`Cde%Vt5Pk3gPE^RT`$06_U#1Cn%PP9+ zrW&7lJVGvU;J2 z0VbCAFq<};YgdR-{J=3?ld^_jf<}ijmuxT`^a_ifm61L~rAToih&{(&Z_uUi5FC_( zF|NM*ewr&LUqqLyL-78TVR%8P=NEN|X8>hbt6&ySKtf^zDZN0tngbgN$)HmPHGnsj z41$q^DZ1o2Xsnz1Wje{`e7hfNT*i<2*Li@A#@6DBEzp3dgi^*~yaaMa9-U+W;jhL8 zB~mRzGW2xXSkZ8~Y~8^$jBy+53S5w*Kc0)()yArY3w%-I91O9I^$IT7YhQ0s2v-2D zTWndNT#D~C>E1j#*V1Im{oUGCL$j{(Br!HLf32GmEqag zUO{zWSKgsi*>drAl`m?u!++rd51;G+dCDk}`Fz4JRUj6&yt@zsW7wG@tBi`L0pfyT zG+T96g@y|{tlVMm+2$`eXwrMUNHNPN>&DY+j~7Xf*Y|vpE_N83-5XOU-jdJa*@c}x zV0&_2j%Xm6-_2a^&0hz{2?G-|TWo*Jt*c1Z8q)F;-X6|+sht+kwZ^yF;FXt4)+{ub zvlngYdcL@g@ke57hS~nBB}kdK*q>%(i@HPD<5Fkb#?YfgECW`5#SjF38?dGxe=Pcq z>rm@b;7^y|oZN*x{M6H(z4NfV1n-EDXoCYo!jZslqs$FewHv;KbbJ*SX}f&axO4o- z(IQ>bO$z2k%&c02Zu2+zW9-k=*SI<{s9e?t$(#-N#3slw+?mu(6l_#~t~IV)S0nAw zQYyrv)bpVDjqF?OFZq&%yxn4T%As|Ry8nc8{9%oUuvp!sf-wJkl+BG zbD_G?s50iS}CIF8jStI@NL_-fHDj!my+ov2+G0`|Valff@Z0=`bwb$tCvnCU!O@Q}k=a_Hw8SS{| z4HGuSaXAMcc;*xjnUACeEaqKlbWK)N6Vy^RzJ6csT;4}BWPS}h^11Otm6;W%?qobO z%F^cbDJVK3IG31bbtd@w-%S#LmaioF;mZfJt*=J>YVK6^^4}y?DA;`Ojxnh_m}4() zTR}vuX4aSdbD|IPec)T?>=IS``3UjXKjDq%=f>i_8r-sTW_&+QYF+tp&Z~HO(Ijeh z%qTuQ-LE6wwKF>VZM3WPo8Zz(-N=0R2Ik&(O&+Qqb1g{+XYzQTooDiFZ`xG)CR=EW z8Oa{kIu|Pm!xPUcn)ks)xq>W{kwfPFun)|+Xpv>3jjZ!)0a@D*?(Y{1pk2@Qo>)5ZJ2o0LVTSv=NBimW!8N0 z;GR$UUh*>ftkO!`+dSps+n;`%K9gX6vSOwqM`!ZUq3Vd#q0zqD+h?l5qB}ir#Ka|c ztdi4-7-QFcPu3VA%!MPGr)LKD#($Y|nPhf18`n-g3xDiJo=igTxVk~n|CXjYtJAbz zQu)05aQ4bX=NaSAt;}~Rj-Zm173WCQj*)`KJK6%3FtJGxNH7#HyYcN7U38~qedZd@ z_#ESAjK_D>y@2+AT@tN(Hs}o#e|r9W6#7$isbbqDo7z}D0--?iX;_!0R*YOf#m{3d zzhcn!?tEvsdr{V|WsH>$y>GH&?)}~R8*Ok`m^F=6qWJhBR2k2{3ah>1=GQC-V6!RS zXNW%BPSby*YKSSs|E$H75w8QF8LvpT`>u`sI=}pRo7YMdFcJ~LX!{s>>EzujV~2HK zLtnVr{2Z8KyAjo&J3IXP);1D$78CLh^Ff7rZsx~TG|5!!QB(6fs~xt5abIhry%lU5 z4&OJQ9U0so>KAoh!N`=-&@%SLqvc&@p}TgD5i#P@=%)|V0PlgRB1~8220A(_`9|{4 zsK=O-_rLo}^-rcP1>Akt=QWv<=$=2C*%;}z?~!4QJg}78Y8##det}NPNv^GRi(z(- z?KLtC4JtBy9FQ%WEb#6=;jtWYB|hLzJEO=|d$Ff~l7GjV|1*!E9nVg4aJh9(MfP5D zsDB|XNKfEcy#`=5RKz=XGus0$WscL6`)8usuKsBCHM50YC!Oy6b!5D5;E9Xu_BM;+K0(9bbm6|H!j-=A^4$IE2TgR!>cW6`C*EGp@L?ow5HmfUXk zz(e(L^UYuFjf>TGIMwug+ee?B_Aci6P+R}_U6tR#9@ol_Dic|+n&-=@>8{V~{Fwi} K{hcibB~Pq%xKH`Srg4JlAz!&-L7&`?^2(bw9n)sLg0pq7h7+Y0Nf;)f>a( z=!|{01SQYGG)L<~HKo!%ULNeQ^m)%7NNoJx!VwQ>PkR+)R+UtF3$1DnqoFQ}**pt7^~|(CU@Id+R>-8U&sU@@!hfzB{?93KNpl4Q)H}{oYvjVV{>c4SpeZ zxaZVcw(gf{*}d~QzNvS!#;y-*VVLR5P%GF3=hViWvI!XfJbFhy(K^LaY%V*{0JqPp z|3qhX=1KVxOQ}dR>q6xWuVzi|cuyqr<49lf>)X|pZPbEm>c8&f>DwRfwT+x}4l9Wp zx#bX`oUwkXaIg^e?dR5=u}wl!J31}W?4K6SJ*6CnUxsLKE($x+g%tHpNxz5E8c~ewO=W)Xqm?UYY@}@uAWmYv zv@I!|)SK}Fb<-Sbpg8TWJ6QHq^n*iceSl!u(AVrbu2NB5I z9H~&Xm=iC9L_Gnu3APo}_USfO&Y4D+TftLD8i<-3o~r zAWj!!+~95I_RMyDL_GO}By191vMW(j1Qm<{7}?WU3uK})`K*;dx@|#>2<)Q* zao8CMj;*#=u7K^-z)1O6-lJ0}j8KTUf!0Bm3?*%SzgLVY_7-H3NI z!;_xL$6c_4H(F8(n`RNVd1kjv}eh{hP;Ux8-Ph*H?$%BUKK8gE7^5YFv!=bTmo+F zKr7w^7G;XsDG<#Ze3@#_xN=tA_@t>)F^J1%vDDyICagjf%uAU#eG&*~IXdG>?t*N+ zKc=m!n^h3V3J%Q|HY#lu&jv&!v7hvz<7O9XL0mp2x}I(4R%KzGc!j5oKyo=j3BaW! zP7?(hMWMnmK#u28CnV@W&b1OKw#`yT!>+0l2b}?W%qxaJo@AqdypT}Zf>9xS8J>_pGkP{jeDh8dA zFKz;HO&COjf}>k!rLmGJara3V3$6y|2)aCA1J;t*WoL2z#kyNliq5C6%eF-A{CE<2V=Qx@N9P^Eu73865!AazMx^58pL4}AUFvpWCUHZ0LL9JU4ac;Ry7@F0}4{u z05OtU5Jm$L&@xRjx~AstW$%H!f1?W~T zrZkbm+KBlV9LPfu@;J-(M-C^7!WujxN*jF`gzQEG*N1WgVlEXv8@zDyow{ zM<5@G1*&DRU*sYEF77;t`#Y*TTPUbVq*)h%T;xE$uxV)r|6NvD?4mxTYZkBu;+ms= zuiojX9Q=GZt#|-f#Bg62L7IC}TZf_R@eJE#JgHQey{!ywqe?y)L6cUy%{jRn2|;J7 ztQ;H>{@+Se*#wDwQ39fKP2m~1S4JH3y`p;QO9XO9Ebu|@w;-}I55&zy!Lx-(MNYR5 zz0VT@9FTo31VVxUC089@j4nG2!tRlXj8ShOB>p9eK$1tYuSs{d z8kY)-;r2(vBP4(b_NX|4#8X5DB=DO^z1@`ihd`JYi8xAtTs-Yr$V6dEt`)pkJnIq& zW08r2{s8@^9fm-nApaYi2?`<(mHRt3?tyVY1|GXgfLc12_KM;3qr1-t1*L_XFbMF1 zI9gL0Y|^5@R19{NLZ-RWo(U?a26_d@Ai;f<-ycV%zA-clqQWqp+0*rLNDxo?U?tFG z+o~%Dt1%`{Is;)bqbdYa@$TTg&`>qX6hW2BrAfg?1+8Aj0o%E_LN(HCHx_M|)NG7E zVsih#o|ulx1X3OY$<`B@wo05ZLBvVsl8Ru%N)FHC0O^C+MO8@m@xp$wgH$Z5PCi}+ z^17pDZG#7AqB>Q6h6~!fkKJdG19BcdW6S_p2|4qk*nMZS!90cX84{LT zl*H2Op|0RG(I$%@9mJjUD|5zAHDlFzA& zPHCI*M?6)vy4brCzRG5sY}dYFS`uucP5qmhpy!r z`yRU5@w=AzBju3As%F9qCZ=>?dsOMJ`KsoHLFQj216zY7{<}b>k2zgDur*mi*#%3z zOv*0kR@!e~SD=^|zd~;dR*t}wC&&9){&4fqojVfm&|qBW8ByUO(lzj-#^W8M;=z6! zxquOe2c_j-x&5J58gpSU6YOiJMvqDLQ}`gbNc1hao$oQb;QMVX)WUG(!4CuH6RQJf z+6XPB8n3-QCj8)^##0Tv|FW)FGhg(L5UWZ<__4ASpLw&~1b$K~z4@N6YmR+%`I_7+ z6d$-4=uuy=G|+CYAQ)&C-P7yF&=EQxIsLg-C35HUk_Xp*%AlnqLO#T-^w*h9YI8tG z#kXU#4QqSFc~Eod+Qone8=Jk&&4%wMhl8c8&-JZ*)v9dCQXKHE**z4VXf)n+e&;<^ zi_Nf1`n59i^qW+&rTlin1OK)n=ZQ4em@vPuXHtg3mI<9M3PXS7B)6ESF&2WO_Q>eP z{Hi*h7-RT!EU=zS8FcP$u{sp|@677eOy$e-$x^GTUe9yA=HhKGe0{^1J)yl4@uq?s zveO>Oi`iJ=mAd&pKlH;i^H^Z0V`XVp+t5RwjvMlU^ktQr%hzlC@BBFWG3M^=GSy`{ zdgSqAC3DjR}@3^q*tAtqb9Q|JYdz!}<#JMNeFO z_JY1M^;+^(se9wueaT+8fU@kN2P>)Vcgq~gzeeouyQ^0(FuF2!3Q8T1&5`MEm<-YS ze#J$q>2W?f^HAy6g85mmVl#q4TOQOaHnfb0;D;N=Dl0 zn(u-3Wwnk!qs#Z|Xs1{~tr2$_{mDP$jjAx{bjIq^sWauZf6KpWT$>VFcj4Vvk@=IV zTxD1#UhGysc4`YU);g(x(qmE}hIzZ%I&HmTs5y|Ux&zAgq*=pTG3pd=h&Q8tTFSJ303vdeDBJ#$@7GE~SlGmHjsnJD+Qz!qyF>-ycW*UAr zooe5C+N;M?RPm28wr5C3%|2InMRYuaztd1Uk&}AOMa!}7(aX(OH;T_r*>ewGpjRCT z9NRC{X3!KbZ?y53~Gbr?G)O-)|V-ap}UBb=V{?yqouCUd3tDPMZfZ=`M5bZ{wl ztcjZH^8Am81;0D5-aKe_J^JO7;YM@%e~bdtL+WL}&Q_7S zzpUiE-;fQ&rs!W5T`SOY592npR}6Lb-A-27PH3B`OOH&PE2+)p>h&*B^$vbn2&v;+ zw$$JK_^lb!`>E##SLSo)6~myPZI;PQ?TqXvKHRs6#<~tVL2K@yl;O@R#^g(ayEixR zRNwh`xh~af)Us4|gyyR;q6|+!W6aN>2PHA`SF*EMu#LM1+Ry(jY>V zB}Ebni6~pbppeuMKiB*I{yEom|8dTB&i%cw&v!Wr^D+vPXas778#9f;YGYKaKZ3t; zbWuARcRM}#vh0$Sda+)P+f@(YL(sj^j@qVNERf%1c-r+~bFLZC0j)6Ze8mcC@WPgT zJW~@^8k>qq*J>>rA57o2evN%sY={0yyl|~s5BHgQqlAG?betf0XYwS@&aW_@! zGW@ly;o}-n+#}b&@d5F~9W$}7@`$y;T`89il2b)@H7ze)Z9O5^%v$qhz!@7xT=lP) z`L}nm>s_3WB+-@~x$-8VCFkM`*Qu3{4vFQ|2l2P@@?J@!CAbnh~b+|I+) z6+PXRgi(NKS6F0eu!q-tITEN4#ZGB5>da1ncyjs9Sg)f&wTXq}`7d{e!nm>U7FAR& zuf^7iCq%K+N`PK-aTttS4sXfi+gC{YD~gID(OHiHWLl^YO=V!w82=qjnT76AFis?b z>I5Z9?=HbZ?Mzl;8aPRCIvfRnV%P;&U{##Gm&!oX;DJ=ocdX0UTg;XpPnLq1_Rv1T zJSQ|r#S?o>fq)207*E!Nn7YEJ&66?|q1ZSU`8?PoYwj8agoNu2ZmLgQ@-cya>7)E6A>D%T@G=oA+H8`!67UKa%ZWbis#y0$?(M*k=j^M)Ygq z$*mAmPnc!yJF5(}XR!922Rmg?evbljB(QUeKs}94#*-f^p}pSRsbx6FS=^v&;K|P+ z-jfhnu1~*08WEw2DmRI+6982R#8FcqBpK&SWnA2vwhhR3;XQ_? zYyRJ$RsnEG`9FiO26*y3#QPM@P)pP#LeuwII4bpv-3=KGs6r(6DKbuuPM$#bnKXw-FP&^?RVjdp zcOrO1+-k_zi(poE1S%Wy*+sLLME$g)vwDGP1iQP6%4or00;GT}e7-HZ$B`1=#l!Ni zk`V+|_7Xpu06|H(ZYo1f5ECK`IF0itVVI; zYDqiCA{RXWp}S1LL<;LJ2B`K}s6w7qqdrD;s!odi4(Jr`<}p4k$J4y0&{lp(GkBcZ zB=PW-AULi@9ERs0HkZm67W_Agb#OJM1TEuBnAT+|3g(qL;W9kc! znoI2WK*J9xU^l8s>){b$aA`lWFBAw&=quYI5Z}K~Uw-5OMEl61r-6M0n@`ceiS5`~ zS71fFFvy6}DU4aR29_l)Hlu;!j4g25Odu#p!OFyK2cX!2|1+@~Pw(R6m+SCd)TB&D zlbPG)g7yuVMEnv4bG83l$m#{2%*W5GC(JOf;Oao=2^Rf4*e7eD5d##NB3~$B*!GRRAug_@xoU%N*{L z>hcYbXEKQf@~Eh-9uuItNo5Ykl|@psAo?zyxALggAo|^2UO8e+v@iu!UPuJ%iOelQD`fAd zR`;JUt}F82kHe`t2B_PCT~K1^H*skmVAH z#EQ)dgrBr95TR+TN=PF=@P~YNLFYXdbVgf86_3Ql$joHIxrFQqGFJY4W}b+~Hk~b4 z@V0@wPe#jl%el0AQ5mV3H+kq+&%%n@t?i$1KQJVx@RVoxLxswa`|f@nz~pP>dp7=% zk8+f|X_pTCvcx417S<5okM3-_Wj~td(S9o2Ho-Syq{75&4yk+-7~UVVSf>`kI#S4i zWM$aFT@5~#$6@D1uJq55b%{w*!^W=R&DI$bMW=~_bo@P@wILC-artj|?Phu}(qc$q zZ|bg+g1OG4!9JmX)rY~ZxH!X`L){ZAjRA-Am6g>rJlV{F&3t_)4UNTfCpZhwHk|7w zI+OMLE?Y<-m-@b59T%b=F=NpC;!m?z(-xX?S|MZOvU@+d>cipUaRM4KC&9@Tl5b4* zPURp8PxCdn@b?Vc5||6P)-4fS z;=d*IJXx7De$RY5Jj7K4r@iu)S=mE^qVziKJtwB6L%D-t)g#Hs@wKUI z$3NU!^Rs2QKbn0LIGA=IgQ5Gx^|-8a3a37qYwrxr5x@-GHvzwN+yKznb77G5v3l-i zZR%eoHpS)MBe7~WSyIsr^q`2A-t{{H44?to{1aQDI`1=EovPh{X= z@}nU(=dW9g~z59{QpeS7VfzIUs{ExwsQ zYZE-Uy4OV6@*J}Ed-3l}B?;4wakrD-JyL&tO&ycZ2(!&!maj*b+r;@tew%#s-c@e| z$J3T@hU6uZm(%SV=rCUb`=|bRbX!1vy|zBHpkq6ClBs{)&cDbmG`BAU3Hzn+NA%bg z=O-M`tj0_Cp66Vxf37lM|M>Ouq^V=~M*2%t&a?fC2fyzy{Uo;*KX)(ND`Vo?&DycE zJNF3Iw`Dqa&3twIoiZGJOSUoj-5%M-rL^;+oP-Y|-hq@0aZ4Y+9Pl0?J=gi+1rR=v zHFH|FEiGqmu2wUvO9s4UUggOkJAn&@%Wn1g-*=W;mxg^gv@~Hp4+LHblwc8l7 zR1H2QbS?7(@4}Dih=RBhgEg0^6a24VF^ouRd%1cC+JdZ^IdfKL^TN|$cM%?GDT`ymTZWR2|5{&JC0)xh?Ny6?elf`PS(_NcgEywMB){6}m0!tdg!`Psp~jstXTOZ$ z4)fW39*j}H@`lZKhQlLrEMEe5>C1y9C1f0a*ovCEzjoQumNU02Ze@P0hSHE~_!Sew zmyCHRItQuPdIwq>;7*uXAaO0`Ec(;u`OYLcA+tS46-X2O@y{K1HCU-7ew6SvhzmY! zi0kUs&#K%Us2UsEVH@^wRVTkV1@##q?zX+}9$3xYzi@`$un<8QE1n6q*iV~I&n)P! zdsoBW^j%EYczt9DccS0<6)x02?w<~8+t>$XB~J=nvu!#hy%&BqtROo)KL+>Dwhzh8 z9d=lW27OX49R5N|seh3={^g1F$}zt8=k(r2KN;Nd+0j*7;d7(g=_`wi2%lC)fO?hK zGIwmP^wzqQ&+2UQ*US{o7Pd2GuU+vsa@Nau5csQL!AAI5-K_?-g17YTZl}Cs40QJ< z?hC6gthsu4y9uHD-V6O1Bw=8rq>Npv<>NOod1S+deL%H9;aupf*Ys?HpWgc}bl7G2 zc~bAi`3JwBJu))f!j!mzIPPDz{7S)3X>cj)#@#UI~s?XoY#)64I*oeh%F5bC`de zY}ne}7Vk{IrFEs3?)TmO($_0)&X>yjb_WYTAr@|Mqu69>C)7yrBh2*Qn&%S=@F2PyrRcYR`X8rer q`T1JX<6MTwMltgEhS!jk+qY6;f=!23c)H+rF8_A!CYaZzb@zYRnRxFi5+(QL$&JIEys>M z;gRea&DgS*S~Fg;ZV>D$b~Q&;#WKn2R{Ltb#(KT=BtJ57V3GuZMD@Y5FVrfM$oxK# zAc*6a&rsUBC<787@`_QGFs@`fuz`oH0V_<2k|!7~HfBfzE!qK%9%i4|rHR%g69tEV zseIYtj!`{P&Rm02zD%-4rkwd@y+4ng%fV~p%-i@|)Z8C(D~_}(%g=?IKcutbPAZ+3 zG^DrcP1kXa@5ISVd4GOJZm(^~3d0U4Z*odwXpQ!_|L+FFN{;6|4ItLI-w)%bY zlD5!+T)q6+q)#EPv12;pD@=E$@vvr26>78!#WaJz7~_MXxztZN2*Un0uW7ZovMyy9$Km!#ryf z6_JL5^eMc>tD~Cr{p8oh79_sqmpzKMGQ*jyJc_&nuTSX*Oad(q%5}byADcQdv&ve4xytj&Ssv%% z%e(5Qy^&UL!0O>Q;ol(t(OEx#HP^q#J3NN$%jpSkF>G#)`EFwP@Tlxigg!cKjb(o# z!4XJ4?l&=nl|?k;E|81&C8l7rH}ThFxEq@&_I;o}5fiBWROZ-0+|Oyz>vqgX7+} z#N%$hJl>S$Mt-*P=`-+7TQFY^Uj7A>ef)kJ&dalWllZ#KaBGhueX#b)NFtea=c34-~v@z=6{jx9xAz0Ie{hQaga zWOTYekeSloCVk|rlx>@IG$ZABdlaTMp=ATJl(PJlyC&O=lh>(nt`buVnLNy1o8gqD z1?j`T!7HD#O$1XK9O7A6)94W6IiRs2VsfL#MxQ4m8f!ccYV0+J{uaV1X4~iCZ@|Vg z@S{o*lLPY#Jx>;YEZTS;%zHeN2|NE+W&4e%(t%DGqzDtWlYPpgK~YnDAK~0MmFr3^ zDbKct9CWK1FCm?VMQP~SVE$mQd{9@$>D^@Y&7s^Dt9vgmNG4P|7B}M7ApKRI`hcf6 zZ0LmO6EUN6pbnOF{*%R6$&M9=r=n~9JGfrrwRZ;((WW_ncvcu&elXgv7nrg5wQemf zd1`D)ut`(wo7wa+-%sMuV7!Am`EJ#GZJ8(U{h_kfXHmsO%#V9Bx0lT}v3 z8nmH30<6oG&wYe*C|_a)FGpG33(T$#p(3{BW;3RmrfzsIo{NpVLZ6@kNexJm8+G+BIk? z%@j{_#N}%6Q-aTAyv|SLObt=~H-7-{4ht`He1!6u6l)MP_7}ynBF|l@mNt_bZUiDQ z9!~nMU|XDf2lsh%S5}`@!<+==m0xMtacbLt?88ee{u&0`@m%UQENxUNz(F}R^@+ct z>Dl199GQ@v$gUO(GTpwQgVj(#murz^xAb$7-6~6636iI278e|?(@!ZcDD6*b4E<k0 zla~d)1GmLGf-+W>XsdY2A54BSW!l2|m{o_BSXDW0JaQ@(b`nWX@ z%zlkoFC~&&m0LbbgJni|xUpnBM%wB%K`&Gs;Jspfi${3%94Q0&qor54FUPVd0j=n)dV zucOQD%eo#eem6@N8MF+B85+@l7#vnuN*!BMxLFU1L7)?bM+9j=tBO z&iI>J*W2eME&(;^%gZ16gw0JRpB?o==CK`8R(UYDw}nd$TpA|1-q+B7dMUTJc@@*M z2Rn6)qj&Zwin26IEx3m)?xYxBlKUuCJ7B0sA7oPHcU95kPt3)jc-Z`UELOCb6>EWg z($Oe#jxk z_t&{cOeRELV8G%c6S6+ZxN%4RmYs|IsRlO?BQ+RaGPnvsjL1Gef7eSBl9h{}wtB&Z zY*7}WG6rn}XdDAI^P9;|FX4f8e&!~@ocE&E^rYQsdeQUP$%T-=A!p=m8D5GK%=nTs?8mz05XSTIoRIp6-mnUvQi7jVEPoJ$*pe>fxE2Ud3CKx?G0(q zaIvkl;V^)IgjUg-lHjeyl;6K?h*Q^x{cKB1yj}UoJPdDDW(ov?yk+7%o@TDq{_+Td zh!o_5-z}J~k0b>qSoF7b1ScGCN3uTH?@BWgOe~gN?R5|){?d~%8w1x8US>NxoU9p1233pkwRvl3` zH>Y)fEKltu)lO8`*2g(t*WuYe(`qrA=VD#dE4X?epab4bV9vxE8D;b2>+j!QelyNZ zzPtsr71#TxERlyA&V6bIw}ew33qkM2Vmvflhw98SE)VA{LG=#9x0(x!1d zPrbLuw0~!#xklX8U$f@)`>Kq)cUHATv|A==m70Hxf0GSD2i`{W&J|H!JC|rLPTVH2=U$}6qcg~-i-nOt*Ter2fd&rz- zPVA0fICS9fISuNCuL*x0MSyf~ZpS`?J znwvcN^7bMsR{W4UsXAVXWhc%jm^O`aqqKML6Kl5{O03_HwiVX zSy^cB{G(EC@9{}D7KRn;ds&I?9+eew`vXCUzG0=$(wnfVNBxnGpLe4kI_YtvzZHzj1KXi<%P{xya9z@sL^%qwj7vcXzee zX1l9(7MC4shWzajpYUCsYFq|Gvj!Bx(+_lm7*u=qhI8+96nV1>)KCh4kgkV5Bt}V1*9sAl~)Eq=gtFaDd=-cHG7_SRm4n%Yyb5la+tf@*?g|08zOrJ{LU)xNFNE#bp9qrGyKIKWqo>$(eL7gG~sNRIh5&C+&24=^R7H@_as$U3`kYy#5Ib8OasL=HzWVK9$>lRNM532!g^y#+;` zIA%f39v2-z2{DOU(-5I=@8ZK+zfk9m-smdEgQAUUclL}`)S&J;HeYF|Pbd{n4)c?m zH`LHDj&9ZfTW*u4{`HvEadWfE1li#)J}-q8V)Xkcwz~NG1Y2Fqzr?k~I>{O4$$Oq4ud4dyqbA_!Cfe<7?O43bxoY9KObxQH?cR80=4MNn85f)-?NbY%{=b zP}>{?_ZIr-Q{zq>ZC%F^4{SjkXC2Y19I`%tv1K`A?QdPu4cTAr?dr8?-d%G&rA5v} zaaR&lmJ5S@v#lTSqIRsY9;PRvsDsuM=Pk7=zS{yLJ=EK%MOtP_YtLNmuI-%Su(uk& zXo|wbft6bh5#p#y#`5pk>mi!Xxu_hp%CVtZ#!3_`efQXv5he#c^SCi4t5mGi-Wkh( zA}s6U)m0g{%D^&iO>w%uJ&d9ZX2I12hHk#PDPwtp4bI!k|6`aT7K0MT$J~`yg9;4jK?meun*+q)f6@dj0F{jW@okjbA(@VQ|?4z;?Rzo z?Y4YRo$$6aLv!AS-~TU4`ss!Hb9ViIITKe^`8hsM)E0;0$#D#Ov`aMyF6CWb-<(Dm zEN#hYPsKAv3Sqt{_A3nOM;fFz)cM^C!+%%qHz$5Ni79T+ZVzK9qZzwG+1=N_Zcn7! znj+oyFp4givFRSly4lEv%xR3f*0xl4PsflqmTdkfm+J=%SsSu3r8s8b+ER>v9J{u> z{KMCl5{l`Z^xTPQE6Wr8u+fbFq*459_v?v_JJxCpo}P)~tu5KqPaT=J`r3~)IBzl# zF+OjzU_NUi$AK^=+-5Hc)M&S+B*nxOMEie{uDE(H- z)^vWr2sJ0%$euZ>19eZuaJHtwJ1T3L6E~~3)tsH>Rka;|su!xb;7qR$?0xzYL1$UD zb2N{*bWj)KsKtXOY~-P47!8-{&{2zDCPPtQ!PKj^a$j8Qfp@2HuLd&ERWPvJA|cbaxI}M|5e--&5f)%#IMIZEEV4}8r*Yx*MCPix7rZ%g=w6svD|6_G59!Gz6dPQ0@__vHe3;@(PDSU* z*avc>;&AqNK1Ob=R00YvPY>K?sZj&gUT0e83kJk1Ak`@CrUnY za9V^{5!&!bZOv=xrsz`j^4F=R8<}JI^R_6D^~t&IInZwjJ@wFB8h0n=u+FaK`&yL{ zdQmnx!T|<{c^tUFxv~DqhvpL;rsy!$M29(rkhQme+4h)|E>tK#)`oPNcuQCzpe0o_G^u|FJ}8kX7@Rk??t5AthM71>pIgv`?H zxH@yWQV}NQW#0|Ac~SmSkSpPJC zrInor*~o7B__nL{v$4vz0!Eb;;u9AeO;uv1fT@ldB-gT;AUoW6ufuABPkEOKlhi3U z=%bDinf^`4BNqmJ)laF*W3|eE?|ZMhB8m(1?(b!!ufAN8fh>{AFuu;Ke@lh=hHxkNj4_f<-bKU+_H&(X@noE z;`eH%Na#>A#dg=AukbwmOWPxZt^%W9*;17E30D&~IOfEVJ{{{?4n66?=-ye@&2IB! z?Nh^#ldXu#xt9$krS8{JAJX*K!tI~SJ$ZJl0^OPD)! zL)0Kx{&Kwy>ODJDL$vjow6?>45%RZ>xI-&Mdr5)AIBDqE$QDYum9HBPMdVz<)bb21 zan=uhh>lkVZRq2e^r>ROq`bCglyRzgt(u{<=JgII&kM z5ccq4nV`IeSvh3Qje246E7Rn=!xnZ_(gSy;KFk*QQt$8?HODpI19!fEe1IeMuHyH{ z7BRnw8O%`DbUIJwEVp6qemxeem#gaj(3C61!3OnrdUzUjbbfu=8#%zeX`bDh&LuS4S79g zqo7B)>R#!g%CGaP%GUy_i@b(`r!V2zZX`VBLM;!LUc^`m#%cr1l`dn}KGpWAB00<& zQ4L<_Dtt18Ov}F> zhnZ?^+e!|O;>e?2iVQe}i#-f_z`{FyDW3Phg_@G?os7$cF}IhTnQ5wfaZ}cvRc$3y zvhtnC^fgB5l5Wl*>Dj;*MM(Q6{W+1&QbNZEN$xhles(&>Hb0%S zj4!4qUwG29Rmhz&yA(3^8FdXit13@?I(8YQ|Ir+(qLeM@f|75~@p7Qk-=8@=bh6T( zW)cfTJUgDJql_$Gk(t1i2pK)cvP(~ObRmO5rGvlj-xVuBH zz+|o7>$fg9DjwR}w?|P{4OOwKjTpZ71%(89E1z_pYUf@On<_ip$d0X#3At&l8#6WlT5*$c0IWACXL`PvVEY4og^gtS#^u)M^|L%tVHzRkhv zKH@QU4e<-tIm09iMrn)h<@#iWo>PE0`b6aRaHWX|ixE|#W(yNC6)Zj67Hb^ws1L>U zb+F0U&X?7w^;Uz#xKSEcE?S?A?;XgD1Ci7krJ9JFG_3P)_ob_mrd@EnU4A=nq1)cE zJ~P&T_8VbNgb2fyfp(v|CBb3*Yu21bRvKEK@^>6XKy|RIvK4TG&$XxEZtQ{&bHtc6#~hKFsFv*5KLOXBUJR~(M`y0>eOidLCH;V$lhxIanRmJf{AqGe ziF5N`XS)NeS$dgm@3Q+^gr{cRc{YC3#z5^#=(F*;GwxF5D|ODxnjF=i@9(nh=dmgL z{yWts=kOxV#~FO7GT!?P2|1|fa2)Tq;_!K72BD9rl5#M$QzvS-9Uu!!xbHt~J%E*e zDj!X#PSq=8v}KBT9AdbHZ!+QOQVkh)%sXS|=87IzjH8b`dMA@n#j0UG?3PVKAU-bx z7ixH~cQP&`cDtvg#mP*S-HEHCZ0A++xoBscTpdOOwmZn4@|4?d zP$V_xW3{zTTzsn`(P4nA5!Z3~u=9Yym{+Lqmw}Z~%NV$$lmkPf7ib17)md?!LEFds zO+~Fhbd@S2&p73f@1`zK<|=K)6-=H!h%GR1Ff|pp()KZv=(TJ@VT01#C--H4U(>;U zFWjbd)OZBjmpxN-l$em2yEmrh(TrU7+y@Dwc0xc1#dlC^bAYG?%65A?@^T&0Ei z?wo#W-I`=QZt^9Mh-Vog2XVxJmBuFA&_d$9jL@+s%1>fT-5MKe2F~h2;ApW?0@p~-!72Gd3S(6ks5{G`u zu};eCgnIe%_A;rD2WWDCw=mEq%r-e}88ic3J)~|50w!-C-lgdu<(-oM-Y0ppdYSC< zY%g&!+uAmPoq%-6 zbu6~)v#VmCRc3((0qtHjfVqUjQnJaL#9ZvSD?$CEuD2up%;h$J2UsK&eJE#t#uS&A zKT@ytA=m3-2OjZ~(_52$mQTJ+wxztKvJ|h%-9f%2%C9NqN(OSBAF6!Sic1od(S&WG zhG)=8mXKt1a+B@XTlpj$xcPdM`vD4e~h5bYq&^1WN$x}^LZvB8Xx1za!EB#nD86$@YCQiWTXfEw=KduF?{ouXCy*f$(!$Kh#$%<+erTCsG{=|k-zC=)!btGfE(G`>mB&yHO{!5WlfVfft7J!AwPW}rY$X5=RmRq7n|y2@D0MU)u#j%d{HYGV*Y9l+cI_A-Uz$ZY?)2 zPoF-VO%s)Wj>U=F&nkG2ukr*wTuvJ6 zq6Qr^u_~(7v1r7V>}P*=RKHBK!5zBGu+N~=Cpvv%(mOeAuf!qS7c0@+$Z~vBZr1@e zLj%o6L%tZz5v$y->`YjhduY1u<<;w~knNvR77<*`{73UP!DLWvz7{|~lbw87pk$_Y zNw&#~J-E1clCSa&bO*sT?6nUt@1^dSn_dDNf6g}*EHAh1ksc<_EEKRH0-PSJVU-j# z=cGza=H=DhOu!*~CIspad9+JUfD8$PesAZBW+#l8z~;+7?xwl&C~ zoG|h$F8f2Pn_X6Y=3xyjNbe7t8&{y7dcYm-D|wTl9cK;Tfe|)$#eQAh)$qT#E)KH# ze^WitK8LY%3i)l@R+Yu#7nRm@YLg${eYhN=L2a&6~Q5 zaXZt?QM-ObkJ8+*?7%p7DMi-GkM>KKmTi~cnDV-XrmgeRZOiR2PbAglhEe~kyZ zKWc;E)IM?)M|j&%9U?KUC`C)9NOg+Tx{hpO2|KprxiNNh38B#V4F*Yp3We+d)8e^oIeS|BNS zPPOMBR9&V-Lf}H8F1*;}d)YM0d)wd4S>4BlpE70B;5b&kb;f_A*A*LmCPje## z>i~dEF$w_IZ!SOnFb=RTkI)Ks>>lU?sg!B~>=+X^o;c2IuSg z6wXk=*7x$`wPO;NFe8LpS1z`)A0V=0bMu>@rA;pYOlusSh|TtKv7!!@Brp zya1|YwMs>0(1Gl8>LngyrWVw;K7YWq&+YLznMyut&dWd&e~yxXB>GkO0e-7D!GWrz z7sPOJY#&u6O!ZP#Cn`yj{6$GRkUrZuxRy_3;Ts7^KLz7avoMF_L?A-+RaFc+%L5i| z!chtkD_Z;7eEjexQNLZNzZIdIE>n!+fc0PGABX3EW%&jL&zs^FW!YDQVTnGJ(_{== zB~Df1)cF4!f1XX$Su^WD&Mt-)k57-tI0HwR_&%@-`#yrF?Ej8`Wz*oZvryrnPnE-n z6*R$}S`vL!x6K$0j1Xm0TLp<9VM5TJw(~<@|M>Cp8z)cLR_0W{ifk{ZalKt)*lFG0?FI!VRTsJOqY`kU@X7#HuqgQE|5q7I;f zF(LgNENXcj`bMzCI6DNI2`X_4i9zmrj7u+Be{zY(2^`Ct0;BA3R`r&C{!X@?zX=l) zvR*jAEZ|cxOu;$K7ABYSN3!@f`3a`$UQk@RHp zZSoH_-iIH&|NZa(se64UJ*(s34NR=&yYe<`FZtia*3;J6kkX5Y#ZubLzk)bCTh=YqQbW}*{S`8|Dk>2%AC~UvI@V%BPbv+OqvyYH3Ksse>$RA zG$7VW6^(IfJzcgx$2$rGy^B61h+2$bfo;3wWt-hY3)rm1`ufw4&b~Zo1M?dP>PCbt~!e+2S!NtR_QE)Q=AAo&{?>=sugLyR_RyX+Nc^bBUL=pI<yFE~7 z2aL|#Mh+@wiV6qwFeRRcAqVJ`sV^IxPT+Kpn(F*FYIPo$n%l@lO^pULYH43fO^pkv zr3|Cga8pZ0C1APyJeDq!e=0eVjU`Mk@mgZ9b?G)$=~{$>f7@jYjcp3O@CGO}Kf0?a z7zS#!#drEB0Mfu31$p~xK8!WdK=#-Vg>vupO$I|h;wo?M@_cVr3bDa`P86L>buqy} z13HDS6axeY5h2;E29}D-U$nT$Px|bO_<)qBQ?TJIEj$by#ZoX!f4UyP!?@^?(W}4S z!0N9WlmSe8YUC-=qaYi4q)VwhRe*LWEW9+AoJdysD1>4YwhOHM4rgqSv)1L=j7gU+ z(BEZy*>A9-vK&aaIaxc_UQgQV0ZPo*dtg9deXnk7*RC62!L+Lf6o_8l0ZmxgRcV(C z_SQD+(Yr z)O8CJLEi5*aePxYo%tUg9b#p6fHr34CWI*OR}2&+H}!qd6tdUzTQ*wb6T9|j1X%l1 zzb&nk|EGJCe}nMy9CnmdJEe*t(KFRl^jY2TsX_#_T3Q6I5pF}ckJ_A(Y1k-c_adKN zW&8DAv2JeEwj0jN6>^{${n?yP%y|wS*h~tFs<<(#xT*#cTc`TayMfzF@~fgRad>UE z<&|D0a_Wt8xzL?HyprHFWz5Yg4tg(XhI$u~e0tXUf2IAPN6^(t4vTgH&8c_B;X!}+ zaT+xGzH|>G$#f6(%ZwTl<{XED+JzQ>E3BRV44Xp_rV;qmU!9VN*tgQ(NBsTv&n!4t zwi^ouHVO(B`m$hl<$e>LDKyf<2k?`brQUHYYVVj99($6-qqAHtfC;MzS3bSA@tHfpFhhnam0-_d;xK?+JEYE9N)F)4Hmdlcl^e7}}e zru?3K3wz-rXtdMp(L~zC#A8%2MFbhWdj>bpoW_ZI@KU1*BN)OcE!x0-t`j+SeRx@R zlb*y=o_;7zd78;mri>uj>61O+h7SFB=?;^hf6V#hn>%n5!8r0J;}_sGl3?m51}P%Q zG!Tv*w@#l#$eMde*;IUD^+|kc4-G%|rYvDC&TVyI1Ggr+-Hyk}KaX2jW2i@bKIS+a z{GlpwX8-Gde#8lVkK2FUsju5U_OsYGRIL7X`SW|2XHPzrpYwh4_1}K^^bLL@bJ;~) ze=)w1umC-l3bE16fw2Yber)+ewqbOfXROb%&AxBmdISDfc* zcr!yzBQi78c&3GHD5Gp6J8Byk0nyvZLa5`GdfvjEHapS$_&!n7C__EY)>GCBfa@K4 z%(2%oI}NSxpKsi?_0SCh6P59SsTpFlbKd&ApTc@yqf*lL$eCjpG1H?r?=NNZf5V*f z!z!UyVT4}UxIH^PozPW{^&|I{-B5!yyk;q6uWy0zMwgKCo#kI-nR);jV{d9s?d>!& z!MQvCqQwK;NZIfT7wQAyppYr=1k*c>e+84MsDp zighkPi25P)tdo`-oI-)?XhxhRW+D5Q$LLJ(mxxAiuk$8fg+C0MGD!BW;41$e9$1?GoEF_Tr~UZce)zyZq5Z>{>rO(=L*uT>8#U~q zcXgBuiFJP1miMq2Ry6kqf6y4~ofyIgO-%dOoK5iTC}3YmE&xW7cb{fc8nv81CfBWQ z)*o@V@0w`JjWOwqc=?I+mAo80l7oEWl*<^hDW@n6Jo0)oj7E8o@o%Gu?dn;y*X$$I z$1jG8q(k$8UTIgN)VpHWzdCnYT%#CiX5?*$4hBigg3W7GAI(r^e=@i2DIHx;+!Qr# zYajUQB~ae}sZJ(;RVTHtyPM1%?80%mE75PH3S=_jeDQjlad@xZSC}|$ucgZ2;Ei{5Gl(Q&gJ68u*rY^2)alOqpW30?&9nJ@| znw0=(f~Q}&m+p2Oeq(6{3w)Cg_sxyGg?AOixvE|S7v1s}*!rLHNN5^&3f;SFPrUFi zj14S;rPlane>yjTjV9_ZfN$4kkL|GCD2re6x?2X!Q9QuL3C{Z9^2c? z+^Oq@^=$4P+qWCD0oRKi+PLf8jyv$yZtEVy!@qj62Yz;ZRN3`4jHk%=t~Zo6p33fr zfRl+nNxTYt=Y*{fN2}|3U04@Ah7b`g;BGPsxDWdC?AAK z+@G!NQy3`u=ecQ|-(P z+*#msI;3?I0K;*MVt-ve7!L<6(32l>I7Si=e}+CfFoSNp%ln|?PC+^zf*pFX>PND1 zbm=?pW%@u5Utpvr)oCYPeHTKfvrl0#ij#7YDi^+y05Kd1aG?#y0w|=snzlYzg9L*p zvo6$rXP#nGdv_n7LWnz1ds&y8hhnAH(J%T3MGRF>bXU#iIu4Jpj@)ij1)Hj)Uf1GtE*s%x z|79EwTi>lfXCAao?i5y8!d{CE+JBQox3Ea*rmo`~|If6Rv7 z)puRHs`7s#|9HYG7+ER60mAj-9uZ+#I4H|X|#a{F>FTXZzkQ~0ds~AGc!bt)@hARi= zAYkN366N(j;IvBO@U@4qXdnPvfBPM_>6F*^YTKbab~u$Euyimj?MY^5FGkD1`s%o9 z4p%C=f1>wNwF0MR*%H^!hOoAv>WTU11%)UQF1>PJBmS%toKLL2O4>@WvBje=&MVFhe;q z!!*W5U5uyJuk#qu7)OIF`p~iM1NF6hIbnOa3s8;CuX4+pcmS(6THwBn9%FY1TIt&3j3EwK6rc15{~$6LlSmF5AuQt`aB00_>M&r48$MvV-7==4Z@Kz zBl`t$MuGu`L;N`c>UqI*ct0A*j}?LK~Q`8-ULnQ_tsc7<_Le zxZYUMJfCx?E|7<{jbJz8&&ID~2dZ-hpYuZWq=iPQ(r|G;fBn#Nf8(*`22oBhc$ddF zCNQmTNDIAU6R$}&j|=&!#6jsmU>7fjBL)m79^H7z7B5;iu!}2pizC{G&tqfP$@n9* zc!RD40D93`0?}7S!i9t|Ohl1zd=*DX5TA1sF~lDmWF+8+&oPY&GKl&y8qCA%u@muw zM)eq^EcUahV74&_e~Gnwd1uSJI=RAX#@Fs75ky8HNc@m~m=IS85U(o}B1FXkA;W+U zt9IX*b8JwM5x3sJA{lf03eV{Hz>EPnVd7zod+!7&QSMD_3vHx_=QPL>K^0UFWS6bV z`b#8;HtK@Fl!E*K{e1bKJ%~G965=q7)c`~nEDeG~DdXX|f866s8j4&TjyO@zq8-Zg z_$PC#1OIA@)i=mF#PfvljDt0SI`JTY=2*t@07$i%1J^6I`{}<;2-GMY$4t~oM3kQa z1w8El=iT$}D4+vdyRpNA34YyGV8vb8Q~VvWAX3!OEEDa(j)xR?VB100J&YbX0Pf6z z`QNOTtDhkkf9C%dbHKvGxHnuCO|>^<<>5(?V#e~_tn0SWtm`LdUbjz1qO~_VZMSZ+ zxNe`7>$b<$uE64veP9_riSmhMxE6beYWeg$5eEKv*qlx!>b2)kQBzmXS{o_IltOkn zIOy74qYwE8p_6~Y4w|;PxpJ+L8sfvunP_L&JS=*1e{2pKWVWt90oTDa@*WS)w{>LI zLDY8_$J>LSUg@4}PkjiFc1E!`OW48d%o2LwI`f1MH0#GTh3G^rTz5Rw{~wo5=elr4 zMnz|@bLa|Z#2MKnd#^K6_7;7(s58nQmvxj`HX*WiGRnxxb~Ysm;dj6K{`r1C9?#e7 z`FXvb&)4Jg&+FYW77@&~Z;vS$%bC$AlSncj#;k^nb68I;!^+_1Wt^{{EQB1MT85JBXU*~OHHBB7d+}=)(o8sZDZTfZ+GpRSV>g`# z=I!3d2BkcfqKaIpGXoSc&lGFDDMFfEJTEyp5@Q9hGe#{+ zeedYb>qfrXnapOdc_?|_lpxD=)v_g<9uItP?WKN=o{uG_-csAH70w_vR+*N&YVQ2Qhdvk`g%aH%j0x^ z+r8X8R>L>)_;{-8ar04`ZAMo4q4}@?VFP`Yg-_0qC(ydeOD1naU~;f%yd$^ zJ;LJ`<*F3&P7Ht{;jRQ&)a(xLeinbR(>GasmCv zo5CSCZgIq1Biw#0?r|gV@Z#B!>D;igiw^7K=Q0cLKMy2zj;9{x^!gkBC`r>gOVhH} zPMNKP^Zn+hATpSiCVHx3O8E%$7n!Y_Q&lb%4Okr(Hr)f1sHIIc50VF`G~YF+puWxR z&>B`~GpqfsNiXkvxVycM6`7A5G*1J{wVff0mZwvmkFvG+F13kHB=LjnxORFyxf?$9 zV!O@}yfFD7?91Sjy<)+fa*N*z`Rne~iu@1UPew>0!5glMnrtzM<%#fCbnt5Q~b8Y&(-zQAl9|J{ki&ZUcWP5P)Fuz_CmhZ@biu1#~0{~8J!5Z#K z`5YI5x=oS0P(uV%v80HAblhtmC0z^mMz*0bQU%rtYY(muXr0jHtYWu>CHDfM&0Z2Y z4sz|2$f@h7!w*?ab{@3kD^3idT%)KW!;3Q?WY;H{*A*AC;RLen63XlH8i+_^#G?z6 zuC!CJ*kWMJ!+AIVG)(~EASbq1?}1JwdmmSN zQ;Nq1GC+}SIJk1-XjgD=rzt}Q?}00-i)4}Hjq`HrOMVQD-w|UbR}{nlV74Y=(EPV$ z6+~o_YR3}wU;N*?U(&YF)6T&NBa-!W@R;;73w^d zw!OgGcJGAdaJ+Z2^8L?1XsTiQ&=1ScIw`eAx85-#d%{D9Zl^R`-hW$QktP^M@Y0MI z&jLL496=6U!YW-p1D$D%F@eg07gP1qS| z{_ejpQOMwLZFxQ9Frt(6YG=^#qj9_b8mu-I?D+<>tq|oItppwQI z-b2acRxlY3!m3-}zc!MsGb2_uIt#<=jHP@TzwJKn<-%z8xoj$oMA&I!{Rm;PQ-t&N zr@KFNvWR0lN9g?a#{*LdfyB5@tx3y-R?OKUsGqW>1#e+mDq4o1B@7e50*c+A2_eHr z$cb!Uaq*!^S4i}h{A9fD-qWBnb}wXk?^nWB+46Ce1SZ6gP?PC_;R9iBq=`UV|NGt) zzXx)QZT-|Su1|DQuf?EQ>~+LeGCsRqVzCIP!iepB2~UpXWwBF-r|7{*7I<$;&r;#_ zwGObj9-u@aZ3-MWX_HH`lTNmwuJqQb;FCQ~i46!@$Fm@Nx#c!3GWQGGgJ@P1vMa`> zU&mT(A2kJnePi!B-6wgZrU z^r2OTchnda%QB8=F;vE|p%VBhB2e4LDqW)P^v83X{*hT2PG`*M2`anRiKdwRGbs`x&dt;R*t%ngC;b79tD%eQ0uJ{yi^3A1`=nI;8h0=z5JP2KZ*z% zMe&}t@?JElqC9#dhBySt5(GQ^bp*PWR=kVsU>p(Q%-)7G^|DO?Io{uZ8PUs->@pZ* zyp7UZO{_dPk<=X#u43iix`A!$nuXS)&JDZo~d-lohX?)?hs=Kmskw7BfJMpGt7H*3I42j+uXe zYsd8%6`M4E7sLkYS;=%V^i|l}$y198Q-?3gZL1+_tHG_RCb>2DB|*f*nwq)KBoPbf zWO&MMLX2V-iuKusLVBEyR*?Sdb1{M}_8MaSL+nu7E|oNT5XwuE@Sk?dx&1Y1J8;kn zI1IY&)W%w+xt2ggfPs)>ias>Uat3iIUS>3vaIA-{6JP-?=S~^2he?qeLiWh9M)q-I zg!TROA%@$v+%Bj0un2w&JS2?h8VUhB^r;XSk_2n>Y-5}(lGrc=&;O4#Z@m?e1BZT8 zTiH7@`^l#*RABqwP7{VpsFJNuoe@V@f0QNG?6gG;0_2HX4iTA+9`JZp8}A;&Z;?bc zpGS0@c$Sx5KFFa@NU_VSD)%2)>|x z=9Zv=)m(iztydoK68m&TOQgPG-1TkzWq&CX#`l9l@eb(g{!G~W32Z4+E7PXiXho)D z{hVic45qo!jgXk zjp&a6d$*2!rdI}(pHaR48Hx;Cm&Wz1mx~R`rIEygWN>PW0r3qBQphDj(EY%dXdHgI z+=HKE8ltbF3Ol^UygG7TdU}XYHwGx3W`e=Ka#~!@0G_5%JqD&vbf2m7! zIc+SQQWnu43HEL?SfN)=ct$0?AF2~*CWpiSEEjX)=x5XapbDGdU|yX*FHKrIq~=;` zZ?gyxV-de7k5fA?k4xoP;s<3%-faxbPBq2%&7gdL%Wx!n) z+2?S~QgCCMCFVNI%DrT4$^ex{^4jQn2F6&24D2V=?-TZ)T9ja71 z4qmB%7^w&Q4t#+!C=1{t>UV9BUoR@+@W4rcZG)FQ@~gLDT%eEwuIKD;oz=sUad4ah zVy*}r&}Vp&L7595A-zW&2$;m+@E0n?NF4oWb8R&;LD7fCN8MD9ax|D!>D6F@aOU-n zC(`pSAP5(2hhk>X-2#hIGGUxaA&PE2rhuCV9}0tdZ?|StB}ahbne2d8q+Iz&>Jm;2 zHmtG5(iYCL`@k@YEeM07rmFa-J&YP`4bHsTe_kefh>1a23r{6`KrvV^q=f5%R@`Ob z7@^YjSF2XU-0pc{5O@Brwvreq95j6Nozidgu|=GVMl?UU{6}<9zlE?&1#)71B20*ybgK!1Qat$2 zFzCGT2Q}54Yth1 zyxME zvkHR&P2{tUk6TNZ_4(EBhVgKeO13kW`Xmrvs>{w5(UFCt&!BuKGGgON=8yek5WUeq zIW{dM!0%}yrt87}1LJ@VgK}_WM8obJlFNVqinq5rCo0#XS*nu=r_@G_c+;(;_3PDP zy?o585#XP=H_JSDJR@|!2E2dEct#z@$tUV!9Mv09%)Bmk`6Vdc(eB?Ma!e%JY0KJJ zzu+&cyP<I49d5oB7){IALiCHAY1~SZNXcu!JiCX zGAJ|kK_`Ym{Lbh`#U#&X2d)g6Lw) zsadZ9>m|egTGSN6W!DM(^{A|`Fa}#WM5gqYpyH$>~;RxYM>ume5J7*=9Tn(cCK+fh+M$R_Dr%Iy< zV?!{ncAS@<->`>p#dNl126+?&cOo#aD^31i_?R*vhZi5f4E5h`%|sd2fa1UW%T_v& z0;e=bjO2lRd-cCDDD!-bo=EQXvMr`6*4js4uvPz6wSX5NM@Fx=X5P5`4irCT7d9YE zM3t_PxsuWUA9(un49a5vNB5J_$+=p*A{d_P|Ij78c%bk<_$V1Z@eiJVngVY#M-0`2 zeFhAd7?h);Bi`>4kpWjMb1#qn!1!Tnu7$c}Ng&?wn)xEGTc}?Dc&>nt(xlilH@W@? zr(byKHM~n)G(W&=_Ce@6R<)w1*^dN&+D8& z#MgnRr%pO&^bJ2%PX>q2rq|@{0NtI!GpZwNsT0w|1`*X0(c!btYiAkF2W}&0=p%Xm zqvpAt=qvj`*r{m7srBALPs|ap^`Z&sX8NFcN?;mE(sy!p8-Du9vE1j+i-pLlJ zx3-6RXHfr^Yr{`7Y@U;zA6dUw(!(2_W!+j5Uq72Ji54}}_B6U4Fme`Q{=lLd_vP(r zZ{gXv8>frs#|a{41ZO$mqyw=fAD&@&9$S`@bY`r{2Um=8pk*nGPM#I{ppP-|u`Es1 zxo>If;LfJ~TYQ|P1KhH(t--QA-xnWe(TOy*9kpf?SrQ*FOBUF)9DD0FwtNTsOwr)b z*J}CL@S87EG_SU*%CNo}DdE)Bc6t}KA`mfB^2++>8jJs-pYZLSj}0G8{=Qhm(9qaB zW}4_b%+z1eccu3?!hF{8^WagxeHo+&B*>@ciyNW*!&Sx$0M8!un)`|zs$0M6D2;Ya zG|qLa&quCZZEt)b)FL<+kAZ(j&F_Ri<8R z`H&+I32lP}K)#;9{D_bj@{X!eslIG4ph<~^V~o#pasgeR?h~m!{;{Rv!lvjrSg(&% z_rBM^J{HZ>SmLV)miI~RGq^99-RL#WX~f|rpqpS;uX~C*$$7h7>5^IYh0bsyDZzba zX+m1CEYZCt(3H2fQ%HB-cK~rb^P#ZMJ~29E0>AMCsIx}7{T?bzjF<_DO4ScU8{WQk zFcX3^W(eeM6$XO7J6L|4%;_7%io}y2wW$<(d7KLFzo6&vi$_1I|F*pKA~5SyZq)hy zM|-v4Q|gY%wZ>_;_oRRH_FP6$M-up-&9(%|tuyf{_ZJgx!o1-GON1-hzz1A+>uE`}8{HUWUu*f1r$c zzuF#zZ|=P$eCbMdUqp}j`B^h)!N zEztMw-j4uoBA0nc%IGM0Z}}PL_jQySu4vi+-YvByf6pyox>S;7{rc_g5} zd{_%nPyBjwyzNG!{75XpR$fNuOBHp*jEDWjp)YcqX5IQ z;sqgqLT6ICex!MBG^O+BE2OS3i*$2x#*0c6SGJYN@nEI-qq4_k-i8l#8k@U3HLd{# za^15VH_7CDzW7W`O0+)T+{sYu|E-oB+nnNGi6`~Cn5*4T59u-9WiLu>NPk=5Zc9^p zw+YqlBHuCb-JBGsN$Vr?GJfz>agCU7FX zS9g2R&|2(DUMBoHfP9GFQ< zC(u4-f05%$X}6}go}q4Lw56T4w;695ZoWWQ-tPnZ0o_}H!Y9Dj*|_+P;_2sxOx-$t UV9F=a$e(nUt7lMm(krU}0md14pP{r3Q6?nbmJOpUVO+_6U;__X16G(4B~LI~Y|4-ZTJ!@NBg`?eYZI+WCW;P! zQ-!j_9iw`toVf<4LYZWZOgZz*dViiemy_4XnK%Di)IA(ZD~`0P>kpNiKcutbPAXlH zG^DrcP1^4O>kXcZ5IPjy9SW`9*3Hhpylp@ZY{bxr&(r}pAoNyVe?PdMoCAm@+x#(k z$$RKXu3r9PQa&PoFL!0LDcfz)-g&lv#DAX5<^vw=oWJ}D%&wK$V{XG9&TXsTOGwq+ zt`@JaF8;{d;pD@=Ek7G`NcHafH(>6$Jz7~_MXx#DY=ipun0uW7ZovMyyNQJ5!#ryf z6_JL5j46DJSG}6`G#4`EFwP@TBZegfTj7jb(o# z!4XJ4?l&ofl|?e+A&`srHKt&*H}Tg~xEq@!_G6$i5fiBWROZ-2u4J)B+a%gPv@xr9kPh?k&3=7t)1_pGdFLOK2gkkd ziKpFsdAzCXjr?qt^UuIL@4-Sjc=;Dh_U^lRJTIT+n-u;YC4P&fv*W5uWGpRG850l( zV|n#7#>rox$FdK8eFiyyN&EI-9sRtO`T$~#|97~^eg@Rl%l&b?#owQSciw}Aa`5tJ znGApX@?shyq*3#sd127W~rrq-I>aOwJ13ksz?G|N zzy2&)0USSnc{P!=+57gn$eRst;Y^qvaH*NOtK06gG zEyx)D4PJR)Y!aB#;1HjMHH{A8&jF1Mk&+uVHu`xoqOtsWP-Cwt^tTXBvDkhd{szpS zfge?hlpL5>=<{TM!J_%|VBTX-ChYuQQ|vc^N(VY&kRndhPWCB}21QNry@PXpDz}YV zQeJEiCFoXlK|(qWi_+Ay!TjD{d8@AY>D?9W-J#xAtA`*jNG4P|7B}M7ApKRIdW)wx zZ0LmOGclucpbnOF{*%R6$&OWr=b~%)l#f z@|15$ut`(xyT$w|-%sMuVEh25B5$WOyI4Lk595&VxL8JbJayeLt`Va3fL-_udW>DU zd5GK(om!aWELKwx18)~rU+XIgOifFJQ~!?Q{vU!*oI=WHFpBE~kMf)^Ka}#FgQ3+R4NQAx)1KK&JUVEe!`PyN3e$om5m-l?-yRKM675YtbG25p zH&Z;#5tpmM&j~)0@wz;bGc`o{-|`W>2P}fj@e#^rQmsMII9wF#hCFwnTG~u%xDklJ z1UTutf^Bi`9o(1QecgOe4RaEdSAL~o$Ej_9vrjLv_-hz!$8)LMu(U~~00-sN)F=K* zre}laa(Kex7uakSdg>+(3e4!=0!mlChn+)71T7wen|`FRyQhSN1ZqFQUF`0f>K3;@ zs$WU`3?_r#aea6hiiPBOz|sW%f%xGkRQR?BNhGCJZjgCI#piy`K#}#VaT!V%CT6OC z9lMYMTNY?39Fyx_ZWb#*$i($lPX*UqwyAZ{zy}Q$&VSSK5?XqG4`%#CVrQvwaY1 zE=mNY_CJ4n5i^7Y^<-IY3C6NjH`S&BuFs6M z{UQ3{)F?+RmVojSM3D`XEYZ227Qj(ih#KVp|*W-bOLz~;AOwW7tWS_|x} zdZWuZ${P3(31i-no|d(0o%fhAt`eZW6@mDDPel-c^s0#{E9l$GlDldi4qYt*!<$dV zP66f$Fpm#x5|+^Xxh)%i*%}eWdJRM2pM$9d%?)UPEfG2-`f|_rwh)0viWg#dQ>7fi{$+`7y)qd;cb>hQQxb?d+q?_F}Zw`#zeH=K^K8scNHM2KHScVH30coESd zS2gSQxO+x@^fP&X2{%m{@b6oB)bcZj+%XNLf zPwOU}I`wR^bB&!7>?F>?bsLWxi=2|=tk%F~0tKWO*wJP8Sn1S%N{0rf?-gLK0YmI` z4pfLK)7KKMEIF%4@7fISYa1Rn4W}YZnO;_I;TTMxW0bk~9UyRDw7a0*HL1NJ?HMk@ zl{Op(kkBfBT~iXgHJbAKw~cWc`mmpEX^FS1^vuujR%_-!Aj(@NF4Jk2S_QC7Ac#m& zKKR{&Y5YV|V1h+|+a_?rv3Ep^5`)@X*y=lx-Ow1T<$BYGmo_*FgQL7>5oIlDmQ9n$ zR=s&z`vGbqK0l4e;Z=K-z@Cd2;6-_LtIt^aBp9=Q@IhUaNGV?5IXNNHlutkWW2Z%f zYv2g4ei&(D3j8J~Lu9S5exB61EN)Et<0@Jg%Oti5FMsRSi$I*)BK(r}v;ZhXg!3S| zPp(JxJT$Xr{bjM)H08#(r;$J#i!*Bx{caDJ$LBp%fY172U+k(CELylPnzibPvbi~} zQ+aBCC%JZ_y0#(C<+_c}{+U*b(LxvNs$Id=hX@_;ZUS~@*2pNHXJ3Bz?e$lFZt~@= zpsl#xKPN|x0&{2CkLq{!?a$YKT6XEp+9N2D*hC!hu?uIwec;K#@MOy_U6@))Wyv2R zg^&UYFo7YTYW0kdNCRBse<2=Cwd$U^xF`^REG^gg(_L7|R*xHbkGECA9u;DIri@2B zRkX*@!y}tKRBGQI6))eE9V!mS<9#Y8arW*{nX=yeUL8)n$6z|T!|09471HKuJWsv1 z$aH*Xles3`)!&Nl^!uuehj&)BM6_EbX_cCPi+`6KPoxZ{qoRP`m`ovUBF|Hc0x}(c z7lmZ5(V{Tg_w{mD?W^5!XFeb;e<)mk6?b_Mp3;84hhu{3cs-PZN0J^Qq7g!muX?*AZY`*me6@p449s{&Fs|kgm7B)zXh6 zkrUTUKBiZ(7Ni6|7efn13nc;jVZUlh4BNVe$D5ji&%}MqS_Mmj9rBxZZ{K9ptYu}P zzw=K@xxdFJ-B=h_Y#(GLwtG@m#O?P4A^E2DexB=*_XzZ9KZ^8|KVx+G=mOY(IE4ux zK?@ze`|kRE{%_a6{G1)+?L)nZAhZ5UP4*w*a~AqRtMx~mZAoN1lG}k9XP;gPeeZh& zvT+FuO0G+aXFpRYhiwHDmI*mh2BXY`X3aw6HTsmSnQqZrCG~2V$L-bRZ5khL(KTB& zHNKxH-qK=S$l~=8mJjV$#|9*SRUVS|#kd@(slVxY4t>1U#yF>}qB+@Bt(u&xVfbL1 zn2nLO%ERbnLI$*DZpg&gZ*ds8D{bx3iu4VIUN34+z;ltGE8`)z9=z}FFn9N@*cQ8+ zb&-}GYli&o5ufmVn`>MKL$d}H!q!jpgcwvu_J;H5bP{>93e-@FqMQ?kEGUEBvx_u&2zp1jMhtOrz3R%|G}U^OdM79LyvE^( zFM3fuUCkm&JW&(_3>3BS5Zj1Y(?*%J&21VJX`teHH@AEyfD&W!2M89ps0Q>39IQX1 zE|`OCWH@-p^px4|A!Xl;xAOu&6U;gAX>RIt`nBa-rxC1UjO0asY=HcpM9Ny7sYL34 zO-)|R6)1z9bh=6=2S!?}0aob14&vQ7d|Hel0!Ikm7RPN`lNBv+WCDJ$2`Qi$8#35& zLi2BGJ|00yY!=c%6J8T>vpQf|O4GTpRya{<4%jrMMXo?&>_D;U$z`Ejj1VZs2KOAL z(c$k}a|rOU$!O&o06pr4#D;n0U)vs2n;6YEoHqYq}g zjDR*Jh(ni;q(1cWxGZR2DOvePD=*^i1Q45R(sMC#fV;NrQCcRzRZh5Y_~YiZbk=Bg z>djV|9Cr8RO*%JeWHsgF?=)_J7-4pzauyOC*=pT1dWubdH>8i{XWi8z7;q&*oR|~t zyjBaMNqj0KbNc5!wrg?Y&>8aWYm&%N+2j!Fm$}jj=Op*V`to>rM@k zBRS^NG)vn7J;IcT-u!l~6YK1;u?bW|&#_q(IUL1*guy)hO&+|%C%nDv_ZAd&(wIdx zdt7t?CB`IaO+$k2y^9ZP{X$!IdZVlG2TSv6dX9`$G@;=+HeYF|PbhUy4)c?mH`LHD zPHxr!TW*u4{q0z^X>+s61li#)J}-q8WAw);xw`m&%&swGSx#Ahhg+9)L-wZ!yLv5|_t0EVX^}HY+?52C z<-%a!ZQDn@r~_-Phv|tV>Y(++c}uN|@3z7S5bZW;k(OD~Ix<(gYX_$|?5)NxnxZgq zWaXAaggB{^vHVB&dWdFlE-44Sa%`-Yu@c2f_dRuGgvr6kJZ+50Ditdgc*e2_%ldeK zb5p0SGO&z$Q=G2v50fZ^S#&jlpQtU!gY)+C{}g73#h`?7Gk5h(W!JAy)I|R% zy;4uclzuoe9xRRm6T_I1@zkXh_JN$dn!@IQv7o}x9IV#=9ATNyyDi^S zC%i4q*qpcF_y3EMetO~lP~3hf#7$Fwe@M?0wZ)-$a-6~*?Q+e5OL4jl~M6v(tXr=bV}a4EZuuCHDjqO)J--) z#_^jai+Gew_$^gZxQv$bk4M zOCb!?*ee22>-!LN&lkpS8V(*J*lE--Ju|ibsLwtyM%jG4qh<#G>G!2U&b0UBk3Y?< z79BaX$8GtzJW@~n5SBVl&5_>Otyl#!(yJj`MyF*%M=eNt0CvU-;}-btx(NWx5xTdD zncDU?y+_2p2WQvWmI1AQjagN&J; zG|RPFu4qh24+p++5k{+Kj~+Lst~{yPBRh>0qCFT$SF1rL%x ziinGKty*m=%+JeR@o87Z*S?O-&a|0|+>1ZM3~SYuRo1xybURE~?VhbXbA z6VaU6Re>mcuAQ1EsmUef>(lM+BXT3HArJDu6PW|Nhp%0BuAjbbHbWI%V&rMr#cO$D zmlGdn10QO=BRLVxj$PZXOI7bVH;!^-7t5XL46o|?L&eL?WRbqIRZG|47oWZ9^lEf2 z*@V|%Lbfk|`phyvA?q#YHcU}p722@JFN^KAzE@rLwZJi>XiH(@@66#u$wy93i})%+ z8y=~zc`e->U8-LGI@fd)b1Z*;tmeKI?OfEX-*+z?Z;w!EJ+uCDwZE>Lpn{oC9Dw87A;A4$w3Oo z!EcIfv0s6>5jG7fdFM@}YTa**67~2w8~Y*gh3vUdHgKff`pu1B8}{1r<7aJK7H!Af z3+~I;zzMcH>4{Tvgbo6m#C_S>6`O7O@zi`MYc-SszfNJU8Sp7L#eQAyPNl4@)x-z< zI)$=-W{^M`j_w#+0lln^10-$^PT~$nckZ2nUi$TM4_~gEc&K`B26`8I8pj--lnF4; z6(N2$Koq__`JGkPSpVa%Q+R6zQ_8Y+xVnSxypo4%^=Pz!2wda!#;)OkV*OEK$qLdq zvOk2(p_`;hk!wudO3gK9fTRlOE~@?E*tM{K)V^x!Jp_G_Uz?^XZsH?kmgd0Knah=m zFexvGZn({h@|P=rIq{cUYN3p&?iL>7h)7eF=IEIpg6^dm=p3NUQHV0;v}b(xaFGA4 ziaq-Y3>}K|iJ1eYI%beu%VvV?aO1s>s|h~mT_#L&r`%wOI!0vr zCn1kq81z*?rLB+6s{Etxz3Pf6F3h{Xm%W~Z>Sf5fZE+3D7s%|GNP_mB8|8t6F5&IG&sr)Y*-HYttpV^jTqQOy(y z9c!l8?i%zJK2QJB_Q;^C!01=@6y<%w)r1|6IWeS9eO=3`Cp{QFI?KA*eSWNcZuoJs zl~6g4vZ18Z!#e6yn&Eob(rS+TuG*F2mMK~8BRTeigKnd9X&KPg`K@2V+@Twy2Eodg z>upr;*`XSutmv7hPl!yc-hj4!x=M}v8?HIfy`NM)7<@btX3~q&BLLqg<3|Z`xSU49t{ps zoek{qZ5cQWWB|-_l{(jQdAdKL;XRCDAxr~d-rKHPQO%8NEpX*BCltNn?zTmx7Zl22 zK}GX8@Iq_gYMu9lJ-L;C&g)w}EM^99R(C~w$_aBwt+hT`?=8j~6Pt#-9s^o+Y!O}xyhlzQP$PS($a9~?ZR=E;Z*)uwSuqcyY+eu^VfflAd zXDahx4y4z+>8qy3+9+`2!YW?zftk!XMCixj8$Hy%H4|7qYeX{aG1Q!QA1|hqS z6}~I$`hIVPR894N*T%_>{G7S{;7%Opm2f_E&1Ssky^ox-CwrwTo`kE$(fn1c@TGXJ6P3LLnSNUi9%mv zlrHJ!43eHrY*B=Cd@`I9=`4q2bO9T<$d!v+aZ1V!p5$(S0~}{(U~KaEfroQr7f^!KY;{{f zGjMy9A4R5r0HNM5Kk<^TM5sL31(ViOuK4(s31b;ygjDh6qGiTciE(#_T!G12z1MGD zZ&WHBsNWPxRV{*5EF9Kddstsr3#%kIK`xD ztXO>S9$*34r)Rih#&Zx}`^Mf=<@wrAX^gCxgtS$EZm@#H>O;8~UAZm6>LKAV4h``O zH#oy23r2a5ALROMg`QJ@IQm57_Hd<%2n&xYQMZK&nFf{~ZmTtpc(jM=_BPsN?B~l` z)Ou?{V%#W=D;K>_#`g|n#(_v~jdD%IO&Zqu_xsY-NYgGl-Y&nLAEDbmus$=^_8VbN z#0bNGmVthsx+TG3{9DnTMphbHp7M7ZML>11Yl;q>W+tPle6HpAWI#}I@A;^dF9x$81%_G42bhEgK$;pBEhN=DUVQX=%mv3FV!Z+ zT0bvL%%~;6f|+PlY@f-FY^C)W$&VO6arcxiju^Wt+O`WXmd@uCde|2>hK!)napq zG9tD)8u6^KZBu6M*li9{s{=1eyMvm4F*PE$(N36y=IY=L*zO>E%2V#UL5b9ukJZ*X zY4NS5M5h6+Jg(F7Vdnv(F|SbJuK+8dmN9TgDJO>wjLvJWVxlT32 zqD1kJBV0`)MUS$WgP*M-M15dQ`i=-2u46ZeWNjUj8yZpe6aBF$S7~9sJEz}Twc5_}IAgVR+Pb{3|S)W!kfvQ&2j}=TD z!qw}HU+$0F=!cv;_6yhT0%mj^0snaQEYQ+z1o*z#HO~T#j-%l14t_in0qs0W;^Xd_ z2z=Z*!DYKckKUq84sYSTF*_TY>9vRCOqic?l53OPF0H>Uemj$(dU@u5#|6?^*DD}1 zzJ|WDzt;EJy3T&cRO{r}l-YNXAnWSv*p^v$S7y@BH*bIV*WSC$kJG=sN)gHyd}JTX z1}?#H4|aAX^9@st4;I0PskG9=<{F%#r4zNfl`cTe){){QEuYaUo+e4|> z#ZElpC8xKp_#mHroo#D*OJylu)w_dyNt9nx%9RY{x;!-Hsu!0eDBgs9p@wHLNS2Xg zZFX1e*IW4{9Ju9rQ-%Q{MdSr7+DsNObcOvy7tk$8$jMV(Q*Xn7G^H095@kA(3vOF} zEVrV&ufte286$^(879uar#F{&xF6R7w_)&6;!z!?fMKB!jPj#ya=)}#R!?`uvF&XA zH1yo=t1z@o;LY*%&p%|h62eIEw=M+uAl1iM4oxaAmPz1*kX5pyE^;RF1E}jR|G&%H zNfh_;-9fd#DAY8)7Zs9bm^9N#6@hhmbKGoTx7(d8<6SL(*2moemyinc&8=_MXMM~i z!L+9(>l{dy;bK#r48B2Fw#Jm8g0EWTLM&e`VOyqd+Z$oG*3XJ#-B*&8Dc^0mU&Ec3 z7A??y^UKdafA{8nW=w)4wXs;v7F=f3Bt%DEWzAAeGtFL4co0oY}-4lDh|`swwt$CvHIiElO)@G9a>hz}HJQW@=eO!sRja zr`v=0I8e`Y!VQu+@>(BY>ks5sep>%C|6WCdDcAX1Xd25~AZyBE4R6-h8cNomu%ar5 zW>BG6k+wH}xC*_Co-lGGQEYy|)|0GGip<)|+(X%a$aDmro%|eR#Trv}wSpWOtu3tOPumc-X$#|L=eQPnW%Wmw`=YZDcRwP!R5H+M)tEmII3T!+2ju07uaXe=`)=^GwGe2wm0HX?5maN?qoT>tGDY2o2h~3AzzH{ zh*j=?R(2+=%sn(+_wwp(QOWjCEsF>)X8srRHo;_2YrYmhzmT1LS)gR5c3H70sy(=P zaFTDz4Ri;=HSD#IFdwBJeZ+RqW>uV!Dmnx@@*RV%m&#gxR+O%vUO2ceHw`Q=x9yP; zCeAEWupk1Q9-C>E6g3#}RBhVKrprt^g>N)}2gWewyz=(_H~)dDw?yQ2BP!F7&{S zd^4Nl9$*>x$Gs?q3yXaW1}PPXHw9E*05*Br^1~}@hU+&!O)=Y2enhJ7A%teqI7Us3 zbyI>qFm^9xu~vBl%@FZ%p_ii$hr67ABTQe&v$+}w%e7JBn3KRBzvu@827u$?0K~(S zxw1$Y9uR}R2OVP{^t=Z6fxZ_~8C<8ko2|Yd zXYb94>>5uXtMQmk0C_xWgW%LYaTG@c+fV}{DXl0)OQpzlirl(RY+?yJwiLL3F?Mtb zq0u63n8^eMA2g{4fWAgGBAU(e#`?5|wwR1=1+qLD-%>;%9)j_}8SoPBhROgq-c&(s zc`1iWdDW_MeNO{M=q3oZaD^FU3S`v`Gg%Z6kN{(dtj!;L5;*cUkD(0*^AJ-TrjP51 zK?8CCI%q;7icPEAlh#;Z7c<3wMXoHqHCT7l?W!iLcEzS_t$+OB zRX1$+CjA@zKeoPmcat92$R{SZjUnT7VD)3QKYlvs}-WZ&JZ`)HiLj~JD$dA{KNm#;o2=}gBY-K+{6vyuF_b^MF%tuEi$d#Y( zKIy3sIINrMcg@4P_{v`Z)v{WpqB7|~jya7I`?hO6N6mQ|NJ1|O zNTOeb@8P$46C9{YdO=JV$Bt1|!dx#^b)u3a*&y(Kk&s=`2rJ^a;HbAXc>YwR!jUO{RXkP=BvtH(jRi;(+yE_N0<<_r|taEmp{I{{>sS{wiT?g z?WYD;L(Q_YX>vb3*j1oXsxjE?&)N$NB2C+zK5|v}AcjL|vAgIr3i6c6M-jK=eN^YO zU(0H92lE$rXa8&xxab}u#R}ABd^Du(kMf_lmXZm7#6-s?3OALm2@sejMoMqg+3eb! zb+Fq#w7+e`Y%qxrUD^NiaQz=Y`fovCPyZNfxbxZrI`sF8JbG`B{ME;yZo5rWMw(hC zl@Hr`d8>W|GY;aGIAX*o`F&y^bngwzc=pa5yO3{+RTvbdG<`^5{jQd6@jqku&u>;R zccl7%hO$}v<-0$B^X988$A*Fnyj)i;t6zmFy)59O=~umjgTS4}#V)(5Y3djx3UjHH zUSyI6M<2>e9Y6(RLi#yawDLOijbMp!afmb%RN@>GgWR`|%P3iPjmH@r%bNnD>~L1~ zmVWt0ww=F<6BDvtIKV96b1=-oIm{Mj*YZbycK&tt6HM2^5^9J(#JY2e;6~=@c;lR= zd`hYc#!PG?>Dlb->~Ctk4?lSS```c5j`~b`p7SqgoS35;`Z&*?A%&vy^XlvD8dujS zzN)00!80lF-A>GatN?-UvuDBJs7x%rmL*;_Nnm;z%DwcGnMA_@wn#?qcCiZbG4#=Y zJCdG%o&81WUzgB)`s+{Myn)I2rtG*(@$2w>RF!*mgTKz+G$k_4)SlT)g^8=>hZQE5 zw5v=-tkk{smYEI4gQh!<=67V zzRBQNEnfr5T%w1jULSSn43+g~Fk@K`WsPeCGAo&D`(#pU9oA3UG~-}lzjQOZ{NBe4D6_7b?R{I z?aJG^#LL>xuOP?y=KuU3f5+~=ym2j2n_!w`UR3Mn*YZX!MRVO#mulD;OUs3MfV7AI zQvdmDncpQw1V>)=qJZ^($v@U}SJzwqY_dv?llMxnGvRO?m6_vkeagm?&*%#>nDNm$iH<=KKsmoCuX6?@rlu%faYNVhpzJJw&%`s)!&%-4HjKwv|!Zg1DG8(_ios|OT_ zUfuysSlCr*mkWe^I5upv3<#_~;|Dq(U{71gyMZP#0WhbW2;}P&^FRKO#Xn=}oC;-E zZ|lti4}usdp!fZMi4e}6?2F(^*q!XOZ<$y_BaLTHW_a>N+KSg0aAd7)-=_?f`l*u> za7Tb_B-w0Qoqz^r9K`wt2Dplw3MQV<)QCC^9c~zJTF6#^1k~)CKF1(1I7-7G94Yu; z=qUy|2=-tY$TkhSx6@4`d&~fJNM5jw^ebe0Y7Vo$alUSUg=G&RxNKJ`%)Xs|@k z%sM5Yy6hDNkQy4gg^8fx_nJ7qtGmJc504J9GCM#!GjkI|l=mwI3X+@pzUnI3>-oL# z*7(G(|Cs>R|I}|w@8tg(-sC8}Jck`+)lO+rNQ_K16@6AWe3}>mt(KO6YlPbn?wvMg zWEwV#*}cerXE()ueP6A+JGJeG^K!);C`Nxa=QDGjLkBjKf}$#JjH+&$iNw~aKJ;$j z{*wHv=t~@4n{9ogmx-Kuqg*a@rw^}WI8B*yv#NvMOIo1bMIxV(wSH;e>JfBxlEb23 zKy&Jyad^-lew+qPzAwYWL^8ue{W7D5ggM8dpmw2urQZr`r$58yFo0EQ$T$;{H|I2CntObd@a$>K3sE|)-H zuz=32lH)<*g0ED(ehHacBj!XhQ*iwr1v--vK=T^v&0%IA!#8vvLzIHjh+301SWF57 z!x064dA8iIWtAzvCtt%}xC9#gG8Ad~+vm5*R1mWc&i0 zMiNZJ#2`finGV9SefH(Q z{rdhZ{6v|t#AL$+aboM)`hip{=;fhaZ=zl*-Cpsi@k%xSX|EsP%$HBB4c$bY#4d0?1mbw@ij{^dwmOxH@bwB@2vcPB+Jwz z$QXN5b82s=ks*hS`C6VAUA02u1=uu=inc0(R zyqc=!Gy!Awnu+m-(uZlaRSWW!taw8?Vc*T*W^OnlecCuHY*F9iCX4{+t%wH>bn++8HR zMlI)GvfExa>tArU@0w_T%K4a#MZCg9#!6le9@#-YamvMqY|1H01AAU?hLM*C8UHql z*sh*Mf6YEZef(moNIEs2=#>s7N`otQ!>e<*)h&vV7DnE7>R^(@EZDq7_0a-lCQI9% z($V$AUDe{Y_KCky0_E-R>SXp;byEAfzbo9qE*zJK68%Q1KqeD^&KGaA8He}keSwMd z{#vdaPTolOZ5?5{Z|oOv8iZd9qap0L)e}=4vT0qe>h)wtZyFpb)y;G3>ine%^aMMf zSDCrsAOZU}wag(@Oyu_`pR{cKHdky;}FZ$gFLhXi!hTK?8btp$|tNPt$ zXRH*V-dV0r=gJv>Su?vUy8`DF20nv${dO3)t;W_D^O579^!cGvNdz9pL>Kw+kedr; zjm-KeD{7^XJX(tB+9fwyc=^Sj=+aB5`HS_g+RK}v7a|fROUb;JM`1#UI7WigwAG$kv3-23m9dL;-bb`#9WO`N)GXy5sg( zB;zIwrd}1)(0?SJ+Dng~xRB!JLF|h5!_z_MH}VsI4<1t}fJ%t`kZi`yg~t*Bzq#_8 zLdV4Ycv>>n>nT)l2L9=vl#OsA-6uY_YL9bFqFlF6$Vq$6!E2BTZm)VIu<{XvQL~P! zE?1!_AB9NV<>-zLUVnH(==wtx`0x}}5{3no?=pacORcZhgOPDVU1|oyho(A?kFa`fx2b|n)zPkh zTk)Zkjqtj2L@gbKZX|0-8zV?39putMZXNu4`H-0dbgwJd@>8)pjDw>w0d%cuTzseh z@9yibv+HlKjS6d9!bIXoDh|(?c_h647GgVSRw}@*3L@L8xgAy7h|q)Y7KcP0zR;A3 z|J?Y`iGQsj`KC_>a@Lsa1sP3$A~q9$;1{6X1?!--rh*v)XAP#`m)ottwaOEFjf&1~ zuqNyL8qeF1O=f)kgCv$YE4SSH#Dl9}^LB5c<&-)-mYCgnb;ucMQdB40)Txc4P%_JRufi zkOVW7BQs26YShJaYU4VO5slFsWHE$3jKr7_Z7~u)F@))gwCIbY7zmDk7=#$FyrNMO z1ECK?KF1LfT`>^DQ4Ja1Y5YS^c*8)_BzS;B!HUk+hJPDh(GG@;44W=Z`HniE@I$ zy8^y3foXL^TIdypDf@s5kz%H)XEskg#A&;G1 zClij)5)8T$0T@MRi9}!VgbRsbn1~|r_$rQ&AR*@_Vn{ePh$rAj$T5uxGKu=(4dxN_ zIEeT`qk2qIR{OmKp4+7SQDrd4n4lq_Oo(--Hy206<9p7k1WF{Q9iK@w_*=bEuWt!!oZ&Zo71Tzz4ja`YU=8<)q3-nZZ1f9G?~^SSTq zx$f(_pFhsIPaZ?hfw!@sDy-a3@Ri=-%|4Rf?dtxbvvZaYp}{{dnh{5->}J13EEkRZ zdn+f()wu40CJoc?9G+2noOU>V-k{$rULgusD42Q;`D?r>RA=7--evb8(S9sz^ z$nqly%%#du>X=Ena`6!e=@hIv=HfRlOoXN3tuze{R`92X__5~go;6o+mEbrfurtxK zu_59(PgqhN=kel8Q$H&+UGGrYdx!o9aEQVcMM&@Tu~^>M!*hyA#iTD)7;K2iouJ#Y z(Ykp2QQRx#8uM-R^b!}bAvf<=6Yg@^y369bYTlmA{=h|lrlt#LCa83A$A#^(SsCH7 z`H6lujI19dD2n!A9BUmfNg!urn=SMIKS3Spc4 zepw!KRnRhMGvxCSAA{r@VFTG`I~LEUtUWp&qp(Rnrm=@)zdQ% z)5y`=&wdJkh53(Wl%H)+L|^^-X>UbW8~fiXd;hK0BR!d4nBxUH!Uy-)uR~OmH}g-2 zu?d}v;l?{-RdP3Ym!D?l36b^u1e1=El&7m% zEFd^#=0L*iF`krbteXS?2|~n}PPJ?$JH#_!AaK zXXT{zib(+8m#Jp*rQ%>@)Z^&v4hPPv>nlSYSLknhrP{a$ zt+T*eMj@C}Y;d9H1&q!8-`tx>pc;6Sr91d`;#29A>bkvmCb-NQ>BiuxWWLJ4*Rdx1 z<y zv<;afq))v9g&DYzs7DF;IAgh1PGrN8HJ9w0+Ca|y1B2su!9^F12`j+|(xWHA)WC>F z6@@`1HQkodi6<6ruLG>aR=~a5Kc=7D*EZx{-)iKqW*znDKpZ6-2VFzBZRPMOSBz)% zaPcLoz+FtV=10>SKh~Z<(`^C5yOf-CARbnn4E0|9FC+DBwAxus(vRKqs*DlT`&< zZzz=nS^!SrgbNI$n7o@KYY=r@rW&H!lC_X3 zZwI(@d|^~@OOTI5eB%Qp!Df=)7t-uBjxEdZMo-1s7$OWsySxFZ@G_@JuX zMRQa%9Q#iFsV}NU1+5oY6u-|1Um#54ynZS%)mVdK z8}BLQxrR_54aZHz#mWeS!EeRwOzvwFygZrt=ewbuZx47Q`nnGCgL!E2 z3h_#LZ$tFe#HM(ilURqep7;_Re!SZD=xN$Q42UqX)ji z(6!L2d{&&;%Fn1Enp^Idfu)xA-)KVJ`0#KKZG!9!2jX5QwIgoc*-bv5O#9ZkrMD9y z7CP+K5$>ottwJZ;dUQ8m-M^6wM`R4grdLlTGnCJ2BWdryMY- z#dYWZ0ZYy3{G4DJB6Rq4M5e<){S&w{tr4D#uUQnkrR50U6`oy(SIGoQ)WkM7IV&D7 z_g;K^N>`20!j#AD;yALJyp0`c_d}a50(-L}DrWPMU~;leH1~`zimZZGdG7~;%o8BK zXa|%Bf6FDOG+dk^!Mp3OJa@Mws*N+ers6Eq(BYq#v70X)>ID3vHMk9Tm7{1i8T=(j zlh(x8lhn=Rgdhr`Sld~BCZ7VrljG|AAY5nEo_pDU4_ymRLJ)hmkpdeTURKgSPY>Us)=+W8c9zfMuEh@kS$1fiH-%^|MYm zwz%#$1|oJmV^4rHG2awqqe@AjQgF*X%g?j8<9ND7mr&zCfi135FrFs%KOr*b{n{t_ zO2IUn%FQ$k45eph@bvVoa#kl)%pV=G$YCT~A^(rY98b9ng`nC9S|f@pQzz-Nwm!0{ zKzvn(Z+hg zxPZ-Z-+NN|h@x;OeZ*?#1S9ib1yM<_z|IuZTR3peslWPJF3!gc1wO-1G7rsg<(`X) z_0cgciL$j|ZKJgy23iZUI*^;iOPv^u(~R<#-wcu_^w;jd`wW!J;1hX1QFutb~N>aPF|mn~=leDiOG2OVUDMu5s4@+fj# zE~SQbgu9m{?n{t!*KlwqO==@SZG^or`=Ul!U^f0EzNwUU^0AIw{3p(y^#BWJp>8sJ zu(NUeW>D(*i-Ea2)9EkcNUUoo7Da*@(wDou5ae5lb480*%(+hma7d|N%n+~ z%SZ$HTSXZz^?Q%WJ}|Fm7oLu!;Y2F}rpW8!!EhtFR^JUqdnTfjF~^}0E$-sI#@FIz z&neJQjX*krs9`2G5L79UhhLuyw&3C$Q@R3-t}r(0y0+6V|F}IX=??`fCH88rbju2y ztdKvP2o67Y(}gcr9_V_v8zjWQ4&*=6T;pKjrg9)d&?5oS(M~9oX|{D#M=W9*h3vBi zJnXOxg<;kr8P!;usH3X1RiA*4(fLxS5mIftx)dCZ!C`AE0r<7J(lsICM+hW9peb8y zu^uF}(?V|t41*gna--kaV{^L!i^=c7!bENs$hLY|5cTF$7)Bt9F&GA@pIqty330c! z)Ui0&y%X;O2_3Lu1jr3FiSXsOb03;%I9pq?8NJ+%Lb=gT9I?6EfW@r$+``10ED&E< z6HZ3a1&D_|a|}l@BMZsA(Cm=-GD7o``Bwx z*mW`sSR8M{LO> zDLMe(=7F~`OkxycElU_G$Uu>xH&7(+wqi648JGTK_>N;PpLFst8wuHj! zCMCx~LU&qQY?-{$aK*f6j3>5Wfo{Mz=O7-Tt%#=N-HeB9Yrk+wCB!586_F!&Yk+@h z?j2!b`dLVz03c`E!V{QO?Kp*^uYr`T#UlvH$N^V?Uzd3c3_~9s86?o8?tX6uB;?Hg zg#XNFjf1oWdVYzP(&I1F;233s-b7j00^OvF6@xBxlb35*D6Dq!!!$@Jy0rzzB$Gy) zWkrkJCzM?^2ia8*3#OV6!0`1z6Oa#rFax;x4a#zd4}H^v5O$TH z7yjmD|8&`9ZXOri2vPK5CNEg7*j@0>Hq?pIon~Fv(b^YEV-y4ITga< z46UZLk{V6%p*ei9xp2BZ$h(vT#CcU@A5F-j<~Us_e->iV6#Rv;a6Te~9ijI85*;H?kNXLdbjb=Pj1suJUWDy}oU5EM= zOwVXrKvXVIUm+A6lxexs_3_xO>6&I=5bzYQuCy}3Ga}G$@qsnXEWI`PT7*Kn&9DVQ zns&tWpy3EEPmn*=>Kq$Jju`nFG>Q6iOPfFhm3z_5A!=`{2elUDS_=Z~siPn0LGuK; zHgo`g2t`^V#2qMPS1v5LYe0kz6GvnW<^XQzs?%8Oq$(trT64*m&?#STN^2}C*Q@oR z%@?`mg~CVD1_r=hdY%GnhKHrA1=Los$$nUX$HAeeWedv@J*P1I} zFS!&XO2l(;K=v+$J!&5yvSFwaAXScsKH-I~nS9w|XkZRDw-T@zS9&BuoMr#-fY`3e z1;WCe{%cPSsws#HQ{;D7KtR_(EE`5VhA~(Gps{dC4=nVX1H(_&Aejz2nY}xcg4m*E zw*~}s3`eqIpfQnZtdrCX12cjirX2s5Gc^Nf#(TuF>0(!8p{1$Z?kgbd@&DKtNnMvrI*V;-h-l7ql|`?AX9fPlxKZ4S|>>+!{7>SyY&e|luP+(wp#ae#m(1bQetxX} zj-x*I>KD7yg*7kHeE5-jyh_lqeGLO0(JnMgk$Z9%zho&2uXj@@(p{z@8A|4=cabfM zp58qjlX68~@lk?7b!^hMBhBY6LuOYw5pw^ah7I2<760iRg=sx;wk_$+zKgGz7IvS| zGz<14HF%L5&q(%=Z?_!c>{U5!A_vu66^24fO0(n&S}vu?-RitDT>8RmDfDac+R*Mb zj%Wjgx%;CjhZZUiQPOi4;O<4ic$@pwxX9x?1jX#!hex`N%ipBex_?=Qj12ERDt|lz z|9n@}-_`14YzcS1M7p#2!{klc`{xs^p5{trSp7u=`)oR580-VtD5^_6R>O5SSPWOZ z+EFxE{WRw9Dm+%QNWOVvz+YR>mh!?`d;XF7E8CfUd(RWIlG@+x#}g3+x6IZwhgkEa z_EzpT!UAjq(z}@lQf<}V_}d)$e?mP-$c#4%;8Gm=z9z6$*duFQ@_a5Gj~wQ1{Gs7u zkr~P?A(t(5xN^Anpm#pQV5`*g-JqS@9eWSDb<_26x$N};ov!wTrfH-{R%G zJm?O;{X!3Hsg$gdZF!u|fi*v==0#kn?RG7!OTBJK;)NEmil&I^7yocS&A{6$@99*c zhWE$ct=;Xa{zZ1m9;_Z6#*fu`Ry-Rlq)hptkcSfyx0ju*VnQdLjvv(6%X|JAxSbd| zaWx^$H%P_misIhH)hH{uAkkLI(}U@6wJ)l!n&))g%D?O%;Aif~D$SaHHlL4A@7>La ztB)JnP8-fU6c_rjM=hBf!H8Agg6b#8d$7BMK0g(cj%v)+v?lB(fG+r_#{*AkBe|cI zm^6)7=mP>DqPsD30u9`b17;NxlOxX{h7Ae3<<>783mgk2`a2CldLAnlf|?QPO^$ap zH#+Ynh8@w%RvIQVvtObp?|O) zzc`t@**EO3eJPc@KgNXrI=ov_@LQc;p?KQMR;gmB8}XhAMP&LZx6iM__c@hWMV5P( z|4PUxYoH>?hClH3#ccZhDJjY82V$&_iP+|&`}Px-bE=2>&Y|1CqY-w(uY9_$3hQ$Y zsaJF$dyXo+Q?}yH#x;s|Je&$!`#n!WNEA|_AyPXb9(%$=nY#*oKvl1%gg^GLtF{S40)%7er%Ad{`&uDX66~wgByEgiA z){n%kHu&#U9)8ZReBcPFHf@mq;WW!)J5?DL9qOYp=W zxk(7h8m+Dp*{fVF4h~_yDB$y@HMoE2`uy8$1ZRPcYp;1jc82ONy427H{tY}|#io;d z@pTroYcYz2pMR&t$5v~G&yiY)W9{l1whQbFPS=61pI618GBO7q6?;f3fQ35U9t l90SciS08kC-Zx(l;cWL#+)8SibOIe8Lk~{rn>pvw{ttPclc@jz diff --git a/docs/inventories/v1.30/objects.inv b/docs/inventories/v1.30/objects.inv index ddf589cf9db8cec55317f5fc8e8eb01ee2dcba19..b708ee4dfc5545eafc3c012f3733a6c61d1844c0 100644 GIT binary patch delta 21067 zcmV)AK*YbPz6ycA3XnqqGJ!;eL;gp|>uqtHhb0l8WGIi8{ zO{x#7cnw^x;PN=h<+g?XS-t*LHYw8=ASbSIic()CQRZT~1`m+M<0Afj#}Crv1BQ6? zQ6^1!*Hm5k$XpakfX$!#YGGHiYRPca?G2UZsB55uB~0K$e!H$!%iLhbxN?B{h6REV zJ`_O&l3PHctYDPRYwn`C-*q($%+)@B6gvT!3&1=+OKY)&?$1rx@ZJb3mP?q|e+s4= zG*_SjwwBjM%URyzy)BlvNgFT3@VZ!>P`=cV5qW_Ci3`t2`XuA&B>tNZI?BhIzCny! zVFbzGRtx6wZT<8^GD(OxE@9f@1!ufBRmJKU^bMfX4Aj(bA{T*@Ym^|Rj{k;#E$&Y; zPC+=_QBNG82!6Y!;^W<}dVFdYQiv4*$VnX7smcOdEFXp(2+2tq+!8MVV;mc`QqOdB z5w=_J`3n|%apZE|Nc?nF^S&DFc_`Mq+d^K_jcC*Yf38*Ll2O2zw%Q%HKA;t7;zlKf zdn=~iX$u1wcp^TVY(*6ip(;p!L@@Y8bn;jn#AHVv_w!Er<%A#4yb0cL5w5pX(2B^ z@s{lqbaZ+`r=m-@xz?|RN+46lQ+Jd_DRd?n{^n_ihWITO0yH90cN`vnT&Zola$UmP zXMK{z9BNJE*H@CjFz-p4Z?%C>HlZyxyLI`*#5HJ`ZvUk?S=M*kwBgF3>-icxm)N<5 zoy0k~8Rob-(J@KN78u-nQ9yo%18(xj+1^i)GO&U@5dtumfFX9$0ui@Mjrmy1YtAAv z%2&$_QijG&!ilh5YP6SsYd8Ya$C&Llx&s96igpupB5t+bkoOdKHgg#Y1IVb|hub7* z8(;bNZ=JfeMSixqBo80b~ zrbmmHnQ9w}0>4pWh^*FS){>q3Acv$sEu!BEo<$EY_;1;>Vu*87gkMra3xGmIC=ZhQ z!JN9cfd6`?iLLdwN`_VxF#ufFkf<1cRo z{n+L9F*!;USYOQEir?AmU#|SL^!B8s$Ad^>_jSNWZ!Q9V^?@e`Q@Aa;c42Pe8;$>9 zDTD-2fC)^CRf|V_L=xZ%{|oVGszvw6)kT0{ZMnjq?#g1F?YZVecrfW8&fWt_W7ey`s@;KQ45EV)Mr}l{kT*}`d1%=p z(_zUbb4@6J)jx~w@NyNx{gM?u5vi6)TI9}?NOPJ;u;$r)wA=6iambrzrxQPd^_cQ$ z3e^L69$JTy>97tbbB)&F!3o8aO|`8y`;GeYwePyv-&3MZp-ZR>#OX8no7;yN$nxZZQb2H3S&_G(1gJaFK!aS2*%6ySe~}N9Q5T> zWX4smeOuv=BavIGiG0*>XvwS^_*e`zm|ZC?us7R9Q(}POH9X!_3!aGkkhKWw1-=|s z@87+D%51TgWP&OA2c>Mv_@IJ?x!U%gH)5LyWo4=T!9qwbG^6LS9=SoVpY~CtAN(1k z!lNr-<8+~U1T9qf{`;#B`M+QN`b)Os#fP3zg3Qd9+BznO&zZ>wEjE`?PS07V=G+R@ z;>Gw*=zDJvtn+nPQF2*RJjd=r7Pb|v0w-91Bm@S=gq9*hpsN;?l z)7%^M50k4C#S-676pOT27QA|Wg1uqe#l8XOKle$8o?HsVdg?fxLLZA-3FkJes3n`K z6)T=K%zLa8%@{}v8wQC921vei!W>E~X(TEt?8=O*1Mr6JcEyzg#* z-@3b&t&7d|vPd&Wiy?n~MJIgM<`S2{P;KyStJQYo`HDbwAg?$Nl9R}*Exw9U#ER=; zyS%HG-7WLSKKEHLkj1I1eiE6vmJ=J!vl3VlMvrqQ-a&`-elM1PTehz2T^b=Ngq-AWI<4h4N5mL2a5B*D)4U2P zA+tYGCeZ0A*cd(7VAfV$ne49%rIfoayr7fQpagc0?oP}@&^yC5V2CFt#8CFSsg|qM zk{rp?I)?+ks6~u)C5tHWMBgnSAgG0>*apO!Hp*n|=HN-aC%9;9*<4eKSN{e*t_9AqPt+I^;nY&Pgq_H8(D-k=*n+=4K+rcS3{ zYrb_F!8*o>-^6azUrD62)TziMF38mQ&0K&I*m0$cWO5)R-HE^kDLkh#3|AOqh`Sd|vN@m=I~E_#10{BD?WrGHM|ZW@VO>gd#iFHfomgFv zX>y5NfX3K?d$b4l!RBIwKruFWEG~@>{a~Gol>m&|I)*O$pA}%Ok1xwH#-H)Rkh&KU;o(6A!L{XU8Vp zih%+=(5J7`i~v=+#f8Ej&rhbaMzKQ+TVPVy-Idqr+_;c-;K+ZcaRbB%vjg4KA;E#H zbW&(1k?Dq<(fL_-H46qfiV(NNE$&uoD*diDA&}x0Xm5_G+_fOhEu@>Tb`2S0TJ0br z)GBqR)`;PDb$vzVSnYCug&ScgTAc&y!cZ944%5h-MWaQp1^>M34(N4hsRP<9E=%ij zeXQ1N7K&r&V`mEjLHn9a6NGeB!w6I88|{n4kFFBZT{4wGrl?_K^zf{t4nUK}vM(!6 zK&hm})dF#d((s7U-lx4sJU`Og)87_=h_I-ku1=GP2oXFFowQ7UA)#V~wrmMec9vPv#47s7ac%}F-h8)kq~FX#fP*Dqb)mm2AcVU6MEgw z4unPYu6`PsFBg=53Bmy_h513<8(UB@PVUwKTWXi4{c~TmX)DY^1li#)Ixm40WAtU5 z4ZHY;d=0ype@^Rj^ZdpI(r0?75O5+?x*~1*rqn#o8ws5bHwrtagM^1qHSts-=$E*+RS&mtU zdzbck_|Nxx4_-3wzPMb{vQm<`3-4r_3xR#NZjX3Td(zkr+X+e3!3`MYE!B>u*A<3a zY1c_x&D6$!+<~|_+}ul|u-9&=Q51b=NA{mN{A4FJGMfKDUQW^U(vmDN&W&TOjP{}} zLKCF!j8GN~#M9=Ov@>lHsDQ97LbKjqU)O273^ZdSip%xpGKn&XMRyY@y5-`wPUQ(Q zxNgt?Pho~w1WFjEc2i$hdKdo4;q8}lsp~OJxVT4u#=R|GAfgx(3{Txkp&!V>yD78< zgbfvnX79|wQ^Y`XQ|^UAqtH%T?bdt`UGUa4V{6`u-~Sag{qV~DvAFqIub5P#=obQ3-3Pt*XCg~M*`mjRr-_+aHfuBxG z6c3qyo68i+C`N8jHuvqH>jUW;QKV}wljwpNneMJ`yA|)q9LBgWt!s5ZIfcBEB=bMG zULP@Kt;nX-;*^PNO)+uqTJs7|Uu#O}%jlrzZiyPNNXZWy#pq8OeXni)JdkluTFk-Q zDM`GgB$@i5GxOR7`~ywSD+3bd=d}sS=lr98yRY|9R$lBWT(^~SB$CY5h+-j|%OtYf zO8Fuo(miTFx=r4@E!}%F)v(kZ>dH*;qIr;6S`ept76()w@Tp0rX}Rcse0TpIzLwX2 z0_BuhcttgJB3@C8%*888f7QGrfF^#<<`{nM$L`i(SNQ-@T5NZH|EK1ZcJ+-uc z)Mp(D_y7T+Wopb&PH34nZgp+sTRre>4H_z5aWzm zGdj!+8PFo>0q7Mg3~AvjbrqwU5(7bJCWbxG`91Ls3^=*X)(q^{nA8MoE}>Y#`I0QK4X7MFoQvd?!k;2FH4%1k|-mAuzym$? zp@kgK4{3FaM=v0NAP~WRerf>;DKqPmQbZEbLCxWdLoKMu%TNH-kte}_foBe1*9TE{e z63R(Wzt;UV{-?Q}u~;eni$=&vjrf(~j$ZVCW~sdR^a%S&}8g zX}V}PXQc1cp+xu{9VN5)#!Vd%%zQR9-5gbliIrt8=_Xpte}1a!eS5IA0|oLev6GIq zQmH%8!nU~K?`u;-AXMI8iii6g+c;8z%aOyxW9Aa$-j~++FJ_OH zu||BfUBeJw8YQiNv7UnzEteeAe^S`5sd&k&`1Nwo9VP0~b2`*xMby%1Av17s zL+j0zUbVoTYkqXo)@9Ll)Z_O)yLw8n(`+XkVi76`oXhu_vnf{V^3$Pq2x&171HTSo zE*a1%H^p{YZw{p_q{Uhj{5piPWRO4^u5JmZFTJdl0wk_~E)L=jSGVrNnO^$!ehXi& z>uP(j0Y-&fIa8Mh83E$CAVfC7MAv|)mashxr z%(>wfZ`a&LvHZlbcmt^v=^u<+=q9Oh7NIW}`f^KOuBn9#6T>aM#_61;) z%*vyGGE?C`6cn9Y*A#rD`ZVk`7KNT%2YHghlM4+gr6=hZWk$ST{uv*I=yhG~-J>B7>oS0%2TfDDrZ`4V)d$IT5_g`i7QcO>#1N zkV-0SV~>{46+enu361l>4B@>#?7DtTGu#eqTFrjjRhyFC5WU3vNDlSjq}w1Z%>XH# z-}t@8J6yGhNwD%{xsJNnb~tL0(x+TnPe<_IA>t1AEK*Aj9Q>qVV8eST^_stKI2BQU zQVH9VrznZ5e(;0!v=V5;7)RyL1q&kOMbX4p@tl-VQUpeWgU?4%7A%OASFm|-*;Lz` z`XJ(#yx8}lFOM_A<~25DS9G`Hg~hK-S8jG|IK*mSxEuAcS)fb2!>6`5ZnGb$3)umR z#Jh^$A0^no@HhKtQ&ZkG)xeOxRWb*E%`BNYY?#7~nI`iEjMLo4*8RD!7AK45e%IAZ z>;aVH3cM2!BM7-pd-5W{m#r{h0Ep)zcP>2?%JGDR_b`TyFwLgg@~>O9u-%+gONQH& z3W5hR!QCy3&UMf?5eX`4V^53S0$0o25cXKhmDe|NT1-vgEN+YVk`r2JOG}r3a7J5< zH)d8Hc{yhzpgmmWsC3tqH)Ye5O9stF*}}xr=lE(j685POdz-ly=F36YZGcwEI;Q1w zDW40HW78~0&C626-G*3u<>y_rmbpbwK~#8f+68Nm<>}cH&k@7M&hnWh@^paooitLZCgPo!}0z&)C6%2oN9|6WzwvOQVV^=b{RVbr5~ zPY%C&S~SQH#BwQ?3z;0?V$8@3N_}DnE+h46h8B<1ykE6E+3%}moSn*k3fb{7^Z&zK z#B3jLyb!^~LC!(&p<{*b%Cf%OYTvc0`D^3k26|52dhqa*^G?_f*|I5r?YYsB+bk&t zil*tdj5{438)aloBL`P$UnEJ+U%)2z^Yk2NiqFY8^LZ&{WzF==x<$+H}Z=(!9nq5=Xya!#`0mGG+*WA&QO9ik2ACCdSc-Ji<@wr=X$+)@grroiu!6+uUAbjlxh_HK zKH)L;1@Q~lOCgef1)<#F2f034rKeONic2V~c&Of>EEXO|P-5ew9_ z0;+>uQ!IcJbRGETBJ@6DR_v#Uyg_)OHdkr`z|!wd-O;dhb`rew$5V$)hgNIB@9bL= z1CyP?z)~J&5N@_zBuK5Z%_Gz@JE@qlt~fAR%jboe39STZ>{C4*YgA7a^Rv*Nto0T4MNKvX{GBw>g zr7jn`a_1Ux#G)f>ux*j4x5ryVsiW=3`@3TO(YHZj*h-k}6ke8cKZDOrL2IEPBLx)| z`tkm%b{{?KGebm?m4YdiI#K&Q8(#h6Gb`i%5J-uC8sEjf5^~>>l*|#2LJT)ZP9`)* zYh#8z@!pDrx+AA;)9B-Q>B+?Ff^BSvJ`k%2q}wu5Atp};iD`!D!?cNkBzd%rG5B2`Qx0~|%08ts#;lRpiJ?q1&CQy|jB4QO2C&=Y4?vw3)9sQ6~XV&4q z-X4yMBjBGd9|fALjR4;jo90oVQE?Q!-N27WA|RE!Nbpa;oDxPqK(vzuO6;^RCO21F zDPfL6NiIcleRklc`16#5%KfLGW=Lm$SucRh@E+>UeyZ=XWu3jrLUH#@gs z@ImH58zq2YClo9lpjT_NcbYd(x5d8gbo}ZRenfw6azDxgKymey59e;PwonaJeoZpKISu@O<$*h9Fvb^4} zR&bo}me=vFX3PC%hkImynfm6|H{!ECRDSyK;oa#p6ZuG-EH{_JAww#&v4Q}9Kb1MtnLQ*o_rh%` z+twYs0S|9cT4RwOkGdIx8lz*_!k{m=Xd z5n-rY=I@|u%!`1xl*JO>tZyY0Z$DwnSPtEwLa?&>-uU4vq?-VXkt2&>^9FlQyg4Z{ zEz7Nkx{=AKP8}D(|NDQs?EU); zWHK$oM;W`aXy84*E;IOWIcY4b796#)Mb#|!RmZORQ1_#w`6Y^tO6ab@F@wyX$^4nh z@8q<;X1iirEm(KU>+x;9UPjoA1tbstVsv|Ka@Tw?p@R^AK-YE4@7@#@@Bh@iir{YT ze^qZ2tjDzKYXS5NKF;P9iW{}bid9i$?{IXE)W3H-BOzI^^KXCy-y8 zzACR4e~7$lP@Dbm=H1m84O(-Vqg=9^k`Ij?H6kv5p=_@fx2no2T~HSARnS5fyD=fF zm_sbAY@X*FM1ej{CC2_iW9@u{#|uN~o*MpUHv28W67ct1R`hpH`xx|c3J$LdXube! z^tAbhm$wX8uYMk5wk7|FRNZ|DP33XSnkea}1P91CyyVqd*al8f*r%D?K>r4Q;mqq8-S8NtfS3F zvs(d^NAp{P2n2jk8YmN9%-vDx0mr*4h|O>1a4WA`F|Y6Gz%bbb!5%JClS~0q)ijev z0S_4<4AI)@b59Is-lj3M{$w6vO2PDbJ=W_$_CR}GNJP3sK|xHYc(`~ zMRl%O#Xvc@<`oe{Z%>wb4x4Lat!x-t8Z7X(bDIXm~UE$%zQMJ zG^Q)F&-T0J$hED@Wt@Ib^e?8O0t-f#vy&V&=Th?zqAe3%P~b); zZaiC+Ti!J*2j}}#svP6OPoA^ua2~6FTsx!l+4lt-nUbT#kK!VOi)XayP|s7LO^ALa z+aJh=?gjLkWqo6{dXXtp+~6Qb>D|~9{^+(GC&1_9y&T(rfOJ&E3dvygqQglOj{MhZ2~wEaVl>K#DC!r953|%Qi-yE-r8MA zhrGzOz`LfzzHuk1y@q~OlU18yRkm6`e(z_~DeXzU zt+xBmr#}}9{u~=vgM4q@)j%!pt6kTh%a-@A>TX@Rz;vEdk*5!;@8(j&cle(JVNon@ zOJ_v+Ar9vB!Qf35BK{Rz$PSNx3$kw72vn__AO{jizxwt4RRExEZXy6YhZg{(UtPWb z!4I&k_s|P=Y~UXPiJW2&st?f4!)r2Jh`vPjHEt=`e0Y16m0LWh$lttgT!iG=I`P`G z;jXd~D|xtXG^T_8VZP(NeeZ+ri{>2ZwMZEZKE77HQPf7@eA6Dn85-Ds_MU&dHblZ4 z#zWYsaPPh1r$45g`1es0CLD_ zklDvf>}fSRf57d>=GsrD;g6c~GFTbCBp{1?6@Gx<;!SX%3f~fJygzk_Dl+DBx2O}< z%98ykd{$UvHb1!Z4@ts*S3WMo6qH8ELS6caK>Y6OrWy^FM=VT6F9nF?Ep@Hlzk8L5 z-!9a@s#u}(99|rd{*nK2X#QtWu6Q@pUcL8X^z-`AM6O(E{CB)mEBg)xB07B#ScJXz z;0gKvz`y+C`_cK>a9X7)VNM7-xlVT!LsYknj|K*r(z&%0inp+T z@t=WpL5mN2(QghVaqgvTyl6Z#%#{{EXPl=3%p!bEotO}IlV04rW(0_yoZZGN#k~sP1Q7Y3=(~N ziJV?!D-VwE%S>EA1EZn>I#{&)K8!wK6K}DLe27Kn95RE)a37aJw(JV8Gq~o30`u6A zD#jzfzT~~rZ{oxlZx?p3&h{7#b5IWJaoH9Bk)3^;{R|6Yut^yL7?Da&5L|Ja_SepZ zz=xzNVARroA(EaNmqHv-`HH;!2-5`M??0Noec7^*_ z1Yc28PT`pp_--c}z#Bl|`{*Vp9AU)lTVCTumjo7Yq26;Xnab3k!ipGjyNNZB$IwUZ zNP7Bh_M>p2T|)Qir=MTGg5~d~?5Irf>+pC~g?dqcz~5%Cni832;$UbdBF)wO!wSnc zQdK5`UdnNCOE+XyZ6DJRP>+VpzlCYFJ6Px!CnuKWCtY1IUjpZP-#sd6(2uMyzU3eG zRR-5$gBVbz66J&OD5U>lSm-8SCVZM7VvBEN^YEr15?XTktr+sy z78EUi?9-)M(3H4?^3ixp{mAy5SDWe+>>t9kMZGLx{b#$Pe`p`MF}K=q-h^M_6;u$I zh%BnDSmPKG^|<935i|aHe4LD?EA!JoP#8IN7(#;Q1`igmAU!&Ke+LGG#DF{YGECmFtOra;4D2X0!*u#Y67`a5DfMfPIDUJ|0Yi zl`9#UH~aePmp9+|;jv;)9)d(JG!4FRAItk)Q8i*KCGSSxsy267Z(!}T+%9)@~FkP&R!Q?v1*D<_D;Fs`1-fk?_3WCj;fLdb$Gyk zb>SUc;%%wt5Af*u>i_&7|G@69ymoB@8)2$!UR15;kMde<_Hvb}%Qe;zOZvp}07(u1 zCI0ibGJkQ52oAsNMFHvm;(xT}uCCYg^;(e|m&b*-DzULa>}uR!@3-B)uURby;p80d ze#i7apgd23-ef@N)Gg$oq85R0G7ocqVmeGYz@SWg*&sQCj448AVAyRzJWs=YX5 z@Q@Ng=StknFjj%gAsfXA!BJ32GOLlKpz@>ipz*UzeU={6@^A_|BBq9iiKA2sYSYva zJd}z78M(je73{BS;q1NYPkEk_eHCS6U&&g^lPYjI3R@f1Ehm!p?-N7O8O;`^u)-Nx zFIWnHq+8tDon`K4=6-|{@%0`FSg@g2*T|Wbc@m6SpBEr< z0|j(p;oPELkPqo_9Em0gSgEppu*j*4HVffA}Gbf5t?bh=y0M z>(xCC!V(ZbA520B=a!E}aIfix58Bs6tiF)MQ;8WSK1(pn z07ypS-KNC>Xdp&Gtgm5$tGKRU;rUd|sKeCZitwg|WaURdtiH)K0)gID8h-Ce!2fC| z5zxUU4n~q})1Zx<>=Nl~0;t2O1X)PFLVBd;5Hp?gWy8Px6r{0;JXk5OqCmY(cxu%t z0o5fhD1bAczFJra3VyGDEycHW*IWOg(Xph|8Ou(s+{6%hd8I%>cGccjUByQ|e-+*q z-;y&w6Tr+*`L-A(|4+Y=qwqWp2g;(K(xi|Wh)OClRd#%u7y;>%ECE-!NJF^yQk-Bk z9L3V7eNV58?eeZ#cDLeS3+3gCDG-dlQ|XyXr_h1pn&37Rj~-Qj*UiX@)uB4%d0TUf ze^uly3a{0=zLwiW4lO8`3OVS*I~guhrW97SlczNc^eB0R>xWNidswyGk|+h2-=B{3Mz*xK2eKT-%jLCz5#dw#y|D=xv}= zyX0t)xZx`m&uk$Rd&HbbY6;Gq05N2sn22M;J z$g!^@$g-dHAf7z=SeiVU$dczwAlVy}9Z{fTJwdwT><4o`xNt{m5*Q~IGJXLLBMG8m zVvr+&Oa~F&@o@4%gtT~&l+Hyb)*i&C4bTWuatA`~mE@V|~fS1+@iC z3d%izA;y6QtsOn-3u?Z+(_Pqjsg`FY9=Q4S+FWh4PP_LWx=kakViuN;(rczYGr zit9WbZDq(QWUUN!fn^~bucuh?f!YdYK=i?}Fsfftu3M<&?3G9kaaCJD-Uwt?@e z2)I0dp+}s1^~p(SS$?5;SEHdH=_JYUk*Jzt)N9@{JxpP_ZP6FedQVwn@QBITn_sSY z_d}(HVTCJIrd+9#aR+j8Iic^DHHabE5wzqOJ!Vm?r~L;E zs+Y{Lsg@-JSI@`L(@G3CNJ1CV$&4sl%wqC}hj%5|ry}5H`v^6178w#VotVa}scK4p z7ceF-nJ8`OTrjP*YQaGyY27@oK{le`JW^Xi3^bhNxxK;}bb9U+ftbh{!U6LNbc5q#cjXCqewXCncvyeQXI zLI+vqmko;)UY1?Ch<_L)WpIkSfV=d6cX%YJ#&goGy-E(_Q^(;W0fF|L5|@L7mWD>% zmFE>4p!Yo$DH6+ax32GDGc4=wci_cqkT8aiim3jtO3g6U3pjMu76G%e_n&50D{ndf zn%x-BRR4;neV3NkHXoCro2)RAp|30lk8H=ExXs0ftm-K`-}SuS1R*aEYyMk*`J8r* zEX+Oq2<7#&vCq%3_{bw@-w9}N$8LCc?z*}`cb|oFAUjqt%Az)G-k^Wa0v$P)x<4i3 z*Rk8G#bfOweS-`_+wUSV`$;59U3a&IJDh~$cHarUa#g^yG3SdnIE+Jk^}fQ)xw)4M zg`+ppU91C4cfo!I$>3VGP+od}9YsA7l|hk~<)U7WPV}b1p)a^;u2r3#s8C99@_CV| z8x9h1Zj;}Z-Wrc00S7lNRkIU3>Z<+AXY#C}$@0vcK49WByerCm-{G@v>Nl&6k`jcB zvRov~nDy44*-hCMxTet48N@TkVcfD3Ti(nEihq{reWjcTJdTm);oW_Iu5PG3GRr-0 zsD%&W!B$MyDY?PM%dh^$-g*fwf3e(DTV5D)BcdgWckK6dcu$>0vlWPdSU1I@v{mu` zZdcvFnpXdd>B%DJ=S)iWvpJMv%d9J(HtA3n_7QH-okhj_xiWgNt>UI-H|t{MW2J7( zfH~j|S-9BDF!`B#>u$e)RaU>knQp5QO;9CCDL z5}t)O#zr>5+S2%DIuC()GYvPuo4wh-6SgbiW2W{!A9rRwU2JUy*EgRLPxDd`T?c@D zXM2e!b%U^h!@a(9yD0~7gV?cyyTRkQBW>nbw+|2h%Ecb|+0)T~6gM|8pTgg}!Bm<* zmp%^xiJ45av@(3>m~IcfO|)rCzcQ&grshVP(98}I8MLYGak#nAkq2Sq!0lKhr6vp} zUKR9GeMb>y& zGFs~)RB#3U;h&U$jv>O;CqA}ni)&1*T(%F$NovhOYmf>auR0P~=m^4yRY%d5i(rpO zAzJEkR7VD{KRzII{V@u>dx$0p19QY1q+8wvZK}@<1J*EW_U#B{p)yC6xl|b>0q4ga z!e-Dzuvx=BP!p_SX$^4yH16TBMo$>dIu&~anA8%4vb*hnheIhX6e!quQ{CJ@3}P;o z1%Z1ToDPT7Y9e4bZ&7V;>Z9>+(gHPwA;)th>0qd%2QwPByR?r6?lwqILv%pT7QRK7$-@n$c2i~OaL3t1h~+~a{)FazndE!q)CEN z99I|Wa4>IwW7_iWF}@AK?!ep2vR)mFm1{+R$R9Rhta-BM6>I<{H=b&Uye*D723zlN z2GKR?z+3}Lrw=Q~kkST|_j9FZING$P2P5-_vexvb4^?vCY|w(ud?LlMUQiW4343Z2AMS838{7-(9c|TKZH_Q{be@^t*DsX1G^*Brj1>xekY< zm0w|hx({)x%#7Ap zq#rZOiil<6eT6Xz3LLY%-|AX*)6|=+7ZHqq%+iGAI)vgNK3I^S$r~ky@9rXnkdSba zz>nd=fjJ7v=*c3q{sWRl7Kg4a1Retiux(hO$tSe_B90yMYll+)0n10r(t&Jx^}<{K z#aGAGa=3EY=O+d)m2q&YW{%`L4NPPhVg4q590O4$1^EBFt+TG`*5&{A%pI691Ym`K zZd4H(olzG~dlJptA_o}5RYjwVMA8|MrK7lar%`tee=gSUnB4FkdXBfX7cj$kOQfNsp7`oNGE z#;7WK67Q^5{KJJ)#E~#7K@3Mk7D{wDe*l#rT0hK+E0T&M_C-izr+2M{cphN%$ zu~H(jP&~mqVi+pEM?BhzBS1%gNV$sN5suX13BM6iY)9xA#o+LU)(C3s#ow@FY>aYN z+u4*6dwiqJQavx(`mW8cv6%_=0$MpZjR@FY|YF}EC%#jE1e_# zV!<;sF*D5)AVr;KcF=WCY9~%dJ1tQEtG#RO z6Ih}CH<%+90mg&LqhzXs38Vl|3n^u0-Oai+m1fyKI`h^%8IEQOIyVO?NnF=Y%5^>9 zs`o?D$R1gT52Acv9d6kcVoW|hPlSO#0XBzINqX%mRK!BkleR{G0y4KD`y3o}^%>4% zzCozucR0^tx|<8vex)%!tYa~=-W6amSn+buu-49o%V9lCE-IJ!=lIVFzb7RHM<^OZ_O?M*ITu7plLm>1wAJs_AdAnX&(wY*9!H)c5%F0y8wjI zx5^Y^w(-PwJ%|;5D&xcRY_sq7g_r{P8rHqOfn(U^_NIn?{)=@5hi|PtgSZr9dBcqG zWm3aK($1w$FMH`;Omr{DCIk!F6F~MLvY_i7xa?U1(REkBrY4{&LbbM8(aHI>O$pHh ziscdlEC}9Lzr2a?eCV=;c&<{Vbi@XQS(p(5iI_YcHri`{b_6j?n8RCv&ZlJI*-HN)G>q<0 z$5IkGt|vm`b4brax>h*)T_G6^AayK`1IDf`9($QNH=OE86ydZh0hxR-=A+;I?JQKg>shNhRVx z4-hOpE3d|fBPlKNe~IOYQyw_dE>#O5alGA9)Avs@aDotxO5S&aU?UB-Z5h2jR|yZX z`TSpO?=A4^h`>wrmb$gw>n+AsW^ZnXIDq!O%-nm{?RmEkHbm#$Vu!E+{#9UK#bI;m;YPH4qVP--^lfU$g1+$@w=fhzb6Q0EkSQ>%^}p%2j1kTtVXNUw9QmsY=-b8q%BTZXViqTl>dFdzRxBT0^t)f z<%tC`;bpR4@nLf|nNdKdRFJ6!GR?mJbi(Y5cltD;S zo=pgxh778za{@3L~wkyvmFf%I5%z~L^U;m8960ssppNApl zREW6+F^3m|51KA12xnA;GYjENB8WyU260Y>IJY3qzn|bACA#L$jPG{7tXV}^m+kL^m*VE?&&hb}HXS7fzq*|&Zr&Nfk1u><6)KVo# zF8Y~;FeBAcB{`=;%q@sHrIsqeGb+NFg>aT!OXKgG&S>GBr{i2uaV{*J3#*wWmD(j0 z<A<*7xB}smNvU3Y!p7JEls0e2kLc&|E#LuZX=N8U6^=(ta7gUrB z3*|!9EBUT@FwVr^7=p}Z-nFxkw)nfIGit_Xw0BJjPN)bI3t>Wk!b*Hf#hF?-({!8} z6=!DQ%&0X<3D2o0a|>lYs7aGB2DvVsSr})~w^74IAHuo#8`CednYwy^9?}-Ct2(3R zdu|W1f~~5Oolrq07KlB@3im6O^ppxSwP2>yTBL+$RFs*8GNaWZB|4{K%q@(0LM@8d zQ=QR5IMb63(|HKwLi~+E$ZV#nUWBy8tE$eZDPP!wlW@T(;RzLGVxjQ5`a^l$h%wL^ z1!hWxnOZPYS}jtdGb+Y^%)*#aYLOD0QxWDC!kk!(F2&y%gv@4Y(PcjP)V`gEHDv%PKQxWDCLShAq_fvI73*_8>rEyY! zI(5Um5PxGJGMlNVFGAYl^;Bonq+ghc=UJ#+l;ng8F|i;f#KKp9vNI~k%mSIE7Jj@U z>5LYP?%T;WJA!dW#W=Gt)WqbA00Pbh#$ITVNmrk#Us1H~sBDTRPMeF_HS7XSF5EPq zi@!1LGMl0LJf!V}tk3X$Zg)XLO_`RRP(daZ$Yiuy7K52mVWt+$lv>-h^o$BKvtV=| zbb3y~m{T$47RH=^Qrnf_85QBoLQwD1gk;1y72@22I8TDOph8?&5Ertco=!p#6Y)0& zAhVhGZ4%NJf8TUQO?cAlM^6cyPspq%7RrSDCMfAC6=rI|=x)&Tf`T!lV$3Xz8Rbn- zf^#au+(JZFF#_@MSXS6UTlnSH;$!1J!J0{u?QHN#`6uKcjOU%$_LV+Pa;;99qruoG} zd91u zTv*?W3rf|0x}>69S}2$HSBdnZyr9CouwY)OKDhkyc`#<;Zwx_ZGtJ;Eq%GbI>WrH4 z%nUb!t)P;eP$4E3gf-lpkwD0XOl(Ug&t>yLDSARXXmZ*hlm75LhB76iOl_2D|Dzp2 zA%oZ$*2b{O1?91jtLw}{u@`uz6qGp?Wp1I&scndV5gaZGbqVRD#SHhc|26z8y}2nnv|?*k?b;-VM;=lEMsYmkfyg(mYG&G z$ljGk%vegcA?v*MN|>U_E<`hoEJ=tkLkL-4%kaA`zyH4He(rgm=RWuOe$V;bbI-{Y zyi=&RD`(}Wp!jHb_)izjgUeS~zXcSZ+8+Q`Dx*j0*eJtb90^<TL>RK>rzbc6bRZ&kKuB_)iyCc*F6S zo2Cg9QsSemh7(gizQ-D;*&k_Fi}b!AM>EwZ*e5B`Y<5;o#B&!|>Z4COkJ*K36ztGI zkR)^NwWj=cK;UtojkUi-r-UcW*Zj^u>JB6}RQ4_0fjhaIch3)24rn(?5p!8&5t;1IYnNX?qnm4yy4v zEI5p?U*KBc{G76N+ATPMX{=C}4|6K2{~gb5Cj z+VGL+wG?c_*o8ok?6gPC%X<^xM?ZvzyRZkyr7ZnuqNN$*6;^%ywBw$}@ZU*bsfO$Y z$&&jk2`21<@SAae%Vuyp6^qStNGJ@~%3Ve!0CUk4EjkweOXJ2J5*XoHwuNRto&F)b zlCiNEeiC*E({!Rjv%BnW2gUlQPTpm__G|;FOdFRvq>c?Szxr5|#J?mdIV6z7wX)A0 zQP9t&*{CI`6z)`Ml@fQs8?&xCqRH)r`@G?L@u2)?}7 zO#U@A9nAs9#<8P0r2OB({kZ*+V$3QK)`q*77v)+Jf0N4lf|?_Z;1I11AC1OZ-!VUN z4lNsBoUL6;ySC)eC)Sqondqmnem>)nAANkp(}uyqQ~!?cK581f>UAumFa`TW;jSh{ zhc5X1)=|LTfY1myK5pL>&9XK1x8H8Q>YxQ@{WE#m9emNifmrMewU=EBO){Mp?cZt z(k@9|AV!7iotZOlMCh2x591*I=8LwCi1()Q9RUy@%)*kPm_!@ zVhd!Aqh6Y`hy>GysvTFHbNkuuw48vZ68KnjF}4+pSZHc; zYL852SBM&t*(w5Ckqr$pUaDzCSXdf&*F2M#oJ{fmhmH1}lH%O{p=V2U#heX3ie#pU zzykIgW#st-IG3a#1(A6~sQ$m}mt2zGDTvJIhw9}Ay0|346-6d-kfmtuQ!>sYA&eSg zVWrOUwu`siOr&zolc9RBI*f?EI+EEV0`uPw$>2}pxNai6?n6I=^ zde;UD$ut&)Eg(Vz%$Elm5lYs^pKG2fNKS_r04=ClYZkH6v>_epr3BlgQaO=isJ;-y zwop2i%1I@YPQtU=ubp~I?jF8DVbt<|b9x_Rq z1VhE7pbNSnlL?bAz_3%%1$4+{G`AM~&G{*sx&Yy(O@9Q_oQ5tL=li?>1rFP?i0{Rq zGkR`BzFOhe)Zu(E!Kvu1c*q24ngLpViY}o;rV;hYqV%*AaiQaU!@50{qV$22aacBg z0RaWrvQAvvX}o+9r3-1&g4W*=r3WJf`PQA2Eey0g8=s*@MC}>*0ZWMAli?0BE+Y+H*9DnNm>ePF zYSRDaV3Cabn(-$G6Wb`b%}jI_0pRPGqxMCMK78mna}Kn-^}*53L-ta8N;< z9&8uqr6KSp8HOpMa?&Z#4Oz%W*km~cC!H5o5OUaF-Jvw@!AKwlcPjtyTZ*RiITu`e zbL6HHo4i>_{XwNXJb>~y@x?XWy6lJ=zP`iXYRRPuvEnE{AUvHg(@Vkaet~{Nhs;FJ zdw2O|0vwsA$7bSFiqo&KxF9KdU2X=9d<@dXZ6HG zXPsHZDYLMQZIAO!fy*r7q8Wd>pb^1g&lMJtXJ(SV9XoEC3F2+${08vQ47+L)l4-CH zwxk~#h*>(&jEJ}@-yH>+6l{y%N52>ur>P-8O*S2JVG)m-^M~w(gcvU#Z$`YoV%$-4 z6VQ|__SKlY*^ig$7wJUTH=UTtSbO zNYG)qx5I^&kOU;OU5C=f~FpKzR zgbx0X$DUd#`oX6l{{;%hQI7~LB1WQdx>CveDn+Qs^rbtC=w$v|&U_5%U2A7uW3Y7d8A!n&6L?%Di->E~8g{d)9PyBjJuvmssxNV?jb{w%_a-{kX3cUlhj+F<11M|x7#_P&}yr-G}6~?(IkQ$2VM#Uw$04WIJ#b{O+u5^F`=;kD>4O z{dmQ*O4orHr!Cf@`uLO%pPX=)mzxYdw?o=|nP{f8GmkK1gk_P=9%)MsT*|CG{ajPD z_dY6jtOxiOyHXeZGPv{L1E;Or{)kh6c5JuhsNu^^NssMQ)1n+mG)J&MH<;@a(-u8+ zqx1axXFNuK=CO5CT?<#nwYw^rdc5Ow+k`HyBzyPdN$vc6_hm9sv3Wjh`fSLIPYml- zze|H~eg0a*_crSF%XOssP}`;4E;G%puR$;augpowhQirPAJsgPmnhSx`z_*Tk#N1T`GRRqz>!cUj_O3GE= zY9%J9>B-7_a7{|hxZ7>m#egQScM3>~Dh|2p7sM_~O6<2?GPue7QTCU#WEWdJ|C)%+Jn=g4DUYyudT>Cztqk1kkoxwS?ohdEl>50Mw;`*a zZ(C0}MrlR#GCHPpR-PBGTxu=%eeW^JyKAXDSBhnm+CpM0yzl8ro>W+A3wh|Lbx-M? z-q!N;LK)}9X?q}J{v7SIUxlx|?~0Nh>fL-XM$}Ye0)-xKsFbMUJ;SH2t`Y87F_%_4#DSp3eKE?|+9EqBJSG$eN zk;B+W@Tl6@r5aqq)q;D)$W9~c9g#1l%<2svpz7{<7|t01Hy=Z``1iKGF58x+7INIf zuU6GoXO2((K=20VOUz`OvgYNUr$U~znysqQ$cg#Q+ohL&L^-B}@Fv$QhTP7+5~68L zow2)p9XeeV_Rg_y?wp2zwfIG0p&IuMy|R8kH6bD>nTJjV_Z_G@KzeZ(g-WyytqQOx zv)s%%vZw{80z>8C`8n$JOodP2E~hYmA>f08P}_&2C0|eTw-)7HOw1%JD?T&54;8WM zKejPt@t}0!!kg?$dZ%GIIyhr7ZXg zm-YO7@K{^V$f!T5{^u9_mN3Su-&r5Dzg7FfxbotB4*;ZXHf*^G&OR>SY6zgG)s6En z3WW>b2G5#%xlnagJ-{#j-PfQ3*P)~+gGJq83;Vkd@8>G_Qfjs?-nx4;7U)mlRHP&- zbp?3%XZ`@s%DT0*Ij9Qhr##xy?(fJc>oCb$;;lFxEbys=Kma1_FJ5I66F7B})I>;PZ!!x6T-|3kNe&Kesf8qI7UMrfK-;p+; z%y?2e+{5iztbT5L`|#V1M%^E)S4f7{NslsDOO~BCoRer@4xX>Ey5phJzMWE=9H6CY z`-$B}c(x9$2(I#$LFURyo&-0V`HO4No8FGNmECm$)Q9yt@qQI%`9lp7l*hH&>-edw zP`Q|S?R};CPb7C)B`EGgzZ2LPEg!%O0+GOsbyc=BVfih0?B2J36K9PZPkYq<6T6Mq zT9qS`!~a70aKLJZv~O_jFU};Q(r>xfeSfr27cEO*~}c N@LI=~p~4iQ<$p`j&sP8d delta 20957 zcmV)JK)b(zzY3|o3XnqqF@Z#dL;2v$MY{xtK)&5HYErp@xc_apfm2B}w z=WO`1XzJ6VS==%RTJ)(Z?@o*5CvJGTJXO;A2Q%}*QI4`X14IbEp5ft{?1%kU^mYiG zQJBdBdaR;EkX(aM%2KwNPg|T4RF>#3as-`TsBOD`TH+i~lD@inODC)f+4}5=7qU!$ z9bJ>^gDPGFmn*nDPI9?zp&wSSKb1|&GzG|sYn+PIS4ot)SgyeXWbs&te}D0VH0^*P z9(|NaQ{FXIS3WWqg%V)%=e}Cl)vQ`F99?@u;yLOX=wJyG^^o7LYt=G0m@%##puS;& zpo0%Z5P{@Yk0>h`jq{qjXzq7i%>r|OvJb^h0OkTPkI%+hETNloQ#QQU!HVS)=I@_^ zsRqpzXn?KdozZfN_jqrMIC9x;B!0T8d0!3oHWcgKZ6Pn|Ml@<6Ki4XA$tYk#Q{A{gW%I!P=JT(TpNb^1v_R`uNo@eoUv$2~h+ zhgk+qcz8m9{`|Uid$K5-j_Kw77-e*W1nllMnD->jx7xrbo6r`U-MV~Y;u0XKQ%Z10;$8Cb#I2LYH%zz{oWfr!tg#(b>h zHD?hS<*VfcDMRBX;Y4_UE;ZWAH5`HIW6WwB-2no3MY{<)3%6Qt$a{)An7Isv0c6x} z!fg_?4X*tAw@%%{B0t+)5{s+!tOwDyy5>M2%A02{(`lB{g<_dN5Ru}1#NM4p+Wg}t zCLyET@S6&oGZNW>2)>rkH#2qh7Ut;SDQJ!k{QOETXJA z&9X_|<|SGM+1rLzlFm=!QFzrJ7jKWn3-F@6y2TULJ`P5#e9%rNQi`|5PELq4=3}PF z^b&D!bECz}OtpmYRZ-Gu&M>EBu=$Nv`^7L_mCT?0H5Xkw%Ak)*uZgDG)pl^ zCUbL2r}C6aa;ZcNyF;AIWgA~1RXr-@eYvtS@G7#qaF(FIRqAdK=Qx<0&Mu`#Ru%qc;(O`oNQeDcqJ^yD+!# zfyRHZ6hZLHdA_;JX|Alxo)uMak>LNg}wp`&)cV)58@LcmUURMc+k1XL+ zAv`#OBsGQ}o^^Uh9Ayfles38>Yqh-c)1GUe#wfSh*V1?Epq2cq&dwaSo7>Y+HH7%IONT<(}^F! zdQAB=h3Wx353R$Zny=YHAFI%R!3x|ktRXJ)F=SkpF zY5u)vs$$!GPx~m+ z5B`i%;n5Yaak|Ppf)*-#|NYg6{NJyB{UzJ+;zJK8L1yMlZ5@Zh=gj1T7Msf`r{t{D zac%`_@nU=@^u0F-);T(?D7maDp5t{P3)>2RR)G^N5&{EbLQ9b$@)CVaR;^*lle=;& z)Nx0LX>JVqhsM>(VTtc2ibYy13tqiG!QQa#V&8z1pZlc4O)dpuJ$0N;p^rtagmar! z)RIlriWScq<~`PlW(=f-4THo41EgfG$i&#MasIn2b?ecx^z*G=En+dlbCYhB(vWL^ z4c>RRZ{1zX*2U&}S)`ex#gMzw@rr$Sn@CuR39jlYfhHHp5jVzg ze{)l&`5_V_$P9ncSqZENqsKY>?w~_|dcPOTEnC<1E{%{BLQe8Goz`-jBVvphI1_01 zY2JgBklCLo6X^65Y>XakFl(!>O!mu#QpznCUeL*DPy)M0cN^v*=$+vjFvOD+Vkmpv zRLfOrNsi=cox=fN)FMW@l0}qwqK_635Y)m`Yy)CV8)Y(fXYiz65?r*kY_2JP#jI$X zn>5;70D1EN>BImf#^eil0dY|cI0136enNkH4ziI+?LN~(HXHOQ`!*anZ_te(Zb2AZ zQ>W9fHQzdoU>#$`Z(=vQrPB7i4PuW-dSp?6}fJGC2^EZbM*$6rNKVh9!(K zMBoU)n_|CC>v*FC&ac1^G9dwf6k}se*&NV`9g7d=ff75n_S6rpqq|z{ur8&!T+vdv zPOL7-G`U1BKx6E{4cdeIT5~Z%pcorG7MDhcez4BPN&v=aLFekGD%VTK^T}`#X)IkD z4(O<12MzV)78446_%%7ChdGiyls4)h_AV{ZrUa+!<&o6;T8=Y8>Pj(x<)1CTi3eA} zgJY9!#Xtcb=+jqeMu4i^;zHq%=O@!yqu8N^EiftU?#k*@5op zkl;X8IwQ1`$aF(a=lra@ngs(HMTlGC7I!N(m3~*7P)BhKwAaQ|?pl!M{?W}>yMT-_ zt#%OkX_Y!tYs7H7y1pWRbF6l`!i_K#t2C`_L|D{NSEortgb1F0hfZ3ikWeu~TegH~ zI?FUpN(yPI3pzE}HG`lrcIxOr+oTQMZ9(K{UDIi*lLdetVQTqh`g30=w%J2r6#&Xk zBeS+VHVGyK=IL*8?;Sqi?Ipjr;PfetS=6nRiw>Z~m?Z7YNQful;zQbn(UzS&1I_$F z2)*uQ2f`vsS3iw^%$Ezw1mS3w!u+7_jV-7cCwFUrEwxM2{<$yOv=wF{g6!}YotMCh zG5Ru2gx$^in>tm+Fv3I< zp}r`(%;=?%X=<+&fYFQC-oR>kCW#l>o9PRa1&5}ZNWQN!uq2=cib1(zI=2F zg|AYk92+Yx1oj8^q&59uZ<_o$)&|f!)Y^jR(asQkTe#OpTefM$BPT%ptP^^bW7Y@u zEXSk7lFwCkjQt!8Rt?m%1|Ztf*f*lRb`D2l$bBm2)BezKDq8O?tnFQ;gFX-O6s=f<&C zMtji~AqY};Mkosg;%ReC+L^WpNI=*Yp;_;*uj{m32AZ)E#pQZ)nM4`HqPqzc-EwhT zr}6|DT({@{r!Ye-0ws(`yQ!}$y$gTj@b*i&)b$vDCS2Sj};S zE#~0ulqB9#l1%;3nR)F3{(&avl>rI!^V$S|<#Ybg-Pe04D=+pGuG>mE5=mxjM6r;~ zWfIwKrF@YP=^nKo-6rqdmhQcoYFO$Hb!8@a(LBg3Er?S+ivubT_|zoRv|RK*zPo=9 zU(4%1fpW?$yrP;q5wEC4=HeBlziQr_E_WD2>41V9h@+3Aeo_i&EgCIR-pU+!9I-{vSy%NM%>!;7l$AJX^PseDSKMP5&xbb8p% zO6fEUO^5SA3_GGTE3Gj5La!1wF`@#Ns9M2)_;YDsnbe+t{PDMqm9A5E?S5SzXQQpjOks)ZREy;KbipcU zh;c@(869SZ3}}(`0Q8C#hP3dNx{5_jiGiRq6T=?p{GRv*2AtexYX){}OlpEPmr#?d z2{q-P?5^r=dCS56+?R1r{f5~{RE&2Ai8QWUESF-rpfM)hpZLa27_C@6I&MyXU1$=k zM|v44M7uYUE_Q>oVik$X8V~9`2y4`&Ui3*aYe9s_8s}CYgf(i?6Vu781rZnPTJ$wV z+^UmJ@p)6lckB+tPNkR#iH$$Q3~Mm}7t*-`ba1f$_jPiL?g`+tL zmw*Ndece>WhC&KYpI#SjxeWJzF#QA_8j{H3Ay$+RwPC&UFXboPDF6ZVNa168hiNQf zav&M|AWe=6MEE{mNEsUty_k(W?**NS+u|Y|eS{NwDs#um7*c^Y;IXM7Mw|-C_546+ z;DMg{&_WLAhqSuIqZbfB5QtzuKed2_l$mu&DI$sJpyu$!p%&ESWhj7u>d2Gez%z$a zleNOuzY(;^;jKfhwW&)-;MIVYP;GFl%n|ktbg1fUB_-RWQnu*6jh1&<0 zrXvrB3SSTy5_fPklyFp399u+>HvHRTf7E0*f@GFUf$GLs8!)0)C_TMznqN+y_#o`=v1-^ufd3{srtk^KO*a`=Q=Qdmubg!F!YmRy{_-X zEXk7LG+nftGtzhJP$K+}j*?k?W;v37#O92Mr0=-5&SApNOW?@Md^ z7qdspSR+2#u3-p&FO8DcSkFNUmmaQ*b+KK5KPl|jRJ`O>{CYX)juQ3gIUVY;B5LWh zkQunRq4nlUuUg>FH9xv(>#}G&>hXJ@T|Fh(X|@v%u?Q6e&gJ{e*%YgF`RPzQgtVB4 zfnSF(mkj8Xn_|1HH-}Od(qgR%ejP$tGDx5dSGR=ImtNL?N&ymA7YA{Nt6TTsOfUU< zzlAT?b+tX%0HZ>#oTrSbpMIyn$4T^bf`@bdyv$a)~K>sky`qkX+Q84NtAMyM5Qf>4vJQ zw-7Cae{Gt7s<@6%?`fJnRi`dhBBm<89V&2bi_n(~eYvGC*VICWiQyJr<8)3_mFnt| zAA%gEDd-%a)m4ZxT51^I-tYK-i(*TD0(}LiT7ljLqEko@0M!{Y44;?{ybr$Ht)WK* zIjE-tJrzkUZt%?Vr>d=F8+a$%@Cls|9GbBx*9@kA#0BCbR~t!{W9EPESU%9YcBjO~@k`0(~(~Y3qHnD8F>si=l|%LM;6)AN3?OFMZZ+ ziwjV2V)OvETNo-)T9T9v%a2*Uk&Vhj+PJXJc12Tc+~e_%x2C7EZM6>hr_w^4T;jxO zsrVg#VrJ!0nW=Ce3W`pyYYIM6eHwNei$YJXgFH#$$%TfL(v$R_ruZbkg!03=(uQq| z%@=)<1&thO5J>+W+V+%2T@wD%UsFk$_#u^~p>qBWNFxmb>HRe=Hr-F`?>{OK0 zzeP7(vq|ZM@0;pNbyGNWtec{TYcN)Ln(?K79+AONfiSK#6nQz}2F?!WoCw}#eM8H! zCOH{BNF^1vu}912iXX+SgvNPbhVWh=c3nTF8E%I)t!BUNs!hpmh+g7-B!_x%(ru8I zW`LB=Z~R{49j;o$Bv^T}Tt{7OI~=u0=~FJPrz8095OIfl7O5o%4t~-wu;D$Fdd**d zH=K$nsf2CGQDmE7&}^ zY^v=|eGqX=UhMnOm&X}l^BSA7E4o|p!s1t^D>u6}9AdRE+>QF!EYKz1;Zs{2x7m-> zh3o)D;$6k>j}q)(_?vySsVVQ8YG6oz-zu4dW|qtxHcVm0Oq2Nn#%XS2>;BwVi<3oj zzw2ry_5jLp1>T8=5rkZ)J$Vt}%T^dL0K{{VJC_~`<#1;GQE;O>@1=Q`+{hy)e2v8TmufvaV12z#vM%Ih0BEv6=L7Pm!w$q6lgw56p> zIHN7b8#AkpyqvQU&>pUGRJv=*o3d%jC4=UoY+>T*b9}WM3Hwxtz0F(;^W`AyHbARn z9n4>EgIwpV!0H{g-i}`F=pfir9QC(my!B3LyJdh-mhAo?Dy3&&Q4`Nh3xp4 z`Tt=qVz!SrUWnl0Amm zZI%=RMbmU!#+?q2jWV*Pk%Ox=@+g-h0}5fiSyq;Iid=9?vIY~WCohNQ z^n|rNy;Q;%Ghi1c2F^Kuxee1>Ayc1GU$D2U^1!E~w^4>4&913x-h-}e@+OUDfy{q@ z>d??L#Uzs-ju9~yS^OMk!LCtxoC2$XJh!ndNiiyopa|=VLTs!qL>w5%(-OEmFGO<1 zz{9!G8z{lMZ*@~aH*me@A4Q_iN9m$qq$Sk?_mSk=SL_pr8x*Mb>rx%@_g;bGzL;cLQ*PMSV3a-uH3S& zT$dnqpYRy_g7}4h>!lFMf>3VogIpi2(o-rB#U+$gJXCK`77LFfD6#QTkM;;HR9}MF zED0esSoE|)-5M5$8rTN8u9i6a((bC8n`rmdtlhQf>aPV4^UBe@P+@cuzP;dO9*E>x zD3?Usq~XkgIp8m5y*AOgfBqf+1XBn-=~E@GzgH?DMi`ENFPH=WngoUMpG9{VSuUt~ z^51C`0oB2-DHgy9x(@tv5qcjnEA~@F-XOeCn=7>eVCi?K?r7LLI|*L;j+v6>w)Y10i{avyC=-VJMY$Z%~3NK5!pTXy*ptVqt zk%Ec}{dj*>yN{mrnIWRcO2L#$ov8hu4X^(3nU!&We+Z;RjqhS#3Ayh`O6G`1A%+_y zCli{ZwK2n35R<2a#56G znjLzTH5$YXT~-z##l$8N>KyP$P&vR<)4?K@-4xK93w~44`6a5#g-~D?c+7Xz)(3Ot zvg{gW8;)WN3>-{V11{V?sVxdE??Pc0S(TH2`*x`7puZPxQF~6_gB>zYRUP4HO6`nL zqUzCjZU@Ri1{Wo|GtyGR8_L*+k-ZgXSBGB^txSCA{f>+VT*vO^qD1kJBmPn%MUT@t z2S43HuzE*YQXddjT!(5B$?7&HS2UvRN3Q9jT)Bk$?i^mUZg+J9Zu})r6R64%5wVJi6XbFi_sMp@j(*6g zGwX0)Zx2Vs5%5o!j{?orMu6{%P4g(ws5lDVZs5lw5s=DVB>1OaP6?wQAlk_SC3e~u zlbb88lrTr3B$pz&K09zz{CUbj<^I!uPcx*mtQSCLcn@`FKh<~Hvd-RQqJyw+%Itf{ zkacyoZ_BK^Ei`z0&8#Ywg)i!H}Rv!!eXM;%)OdUwkogeegRXg`Mnz0Zn9 zJR0$=z#TUUJKPrlC9H7`4~ z$6pemSJiUiq;pyBnsQ-We_ANsjHXh^9FEC`vWF{Bfy8KkGS$9{5v2-?qg&Agl&OoQvO1*}F zo1NP*_#pG3jS|4H6AG3N(5toCJI$M?+hX5#x_#%+~%j9FilDcGU`DnEVr@a}Y)iF_nZmYYlAkRg?S*;qkeZR?KRfQPpzt+7at$mSkjFW#A{c?$`*htQvIcI;EnJ*X3|m(5{oy@Rt&U@iZ& z{%8J!h%i(x^LNlS=0(6;%3=v`*0&Ohx1X?OEQfATAy`>`Z~Sl-(oKNH$dN^`d4s(t z-kcPfmgUw%-Nx zWd2O$cXC=^vt6;R7OcDF_4u}4FC%Qm0+I)RF}giAxobXun9xB8pzFHjcW;V{_kU_$ zMR2$FzpA$h)?-@rwE+4BA7}Fl#f{oz#j2>bpyJ+1zA0BQ90b+iOd!I1kbCeE+d!XH za6+o65a{rC49=DcY5rLejtk|^$Yr@|VE?@Cj|?zza=(II7NB(BjGLsWz<{Ud(`Gtf zrt%4V&jv5h{P_|c#TUBM1E+`B5Drg~# z-I$P7%pn$5HqUboqClUf5@Y|Mv39<} zC~zYaH=eD^E$^C@gY$hVRgQ7tC(qe`bvTbzuAR~O?E8X^OvzE=M{$wC#WUJ;sOPEB zCPcrI?GI!__X2v&vc9oey~va)Zg7yJ^lt14e{@@p6X5glUXJZQKsqX7a!+)8hc*6{ zX5jV@DTs>Y#WVBVWJlY2WTM*f^gS*;!pr;&rmc$MHUXT7IF+{q;=k|*j+c6W0I5V- zZ|yFmLtf-s;9XN<-?)?1UPHgC$*N7UDqF1|KX}zOS-r~tO8$?%Z=-HB0vqVW?7B0g zl=h_FR@?pO)1Qk4e~t~TLB6-{YM_?))voK$Wy||lb+@iuU^>sK$kPYacXO%XJN(aq zuqYO{r8A=Z5C?PmVDP325&sH*E@X$t1z9(31gcg|kOK*%U;X<2Dge+nHxU4y!wUe? zudd$z;0IXNd*}r_Ht-LDL{6~>)dy(j;WZg9L|-EN8n+Z|KD@ok$}JvLg>MNq-k&-|6&Z86 zThxhaWyyXNJ}ay-n;%?%`iCUpD<79(3QD77p)UPIAb$6CQ;i18BNis3mjXocmbzB& z-@VGjZx`xcRjkl?4lfQ!|H%J1H2T5uH8= zEW+M<@Pzz-;9vgn{pfsbIIYr@Fee0^T&FvVA*x%(M+1XQ>D=0X3B_Aja?{Cr1?KCw z@2|db@`SzfOYB{V>B~^Z=X9LiPY<@vi=1MXGy6_zfoYm?ug*uV>h4*8k}6h;PN5)A zo_!Fsidv)zXzAhGFQ1r&>LjvjdHSg_y zhY>_yE?{*=jJ|ls@$2{By?pgemgA?`1zs+zmNas~oL&}xa8r$3-or(pq;WgRrfQlx z28q7CL{2ZVl?TW7WhO45fl*Nb9V}XYA4VUriMQBAKExt(4w=DYxR1*qTXu!l8C>&1 zfq86572}a#U-I7RH*sQ&w+lO1XL}5WIVgwqxa^Am$j-jaeuf1x*rW^rj7TLX2(GwI z`)lVy;6qY>6)|}8b%S1Zje9`_B{VK zyTbh|g0HA4r|?V)e76$~;0++~eRLBPjxb{OEwAySO9BhHQ17{xOl9g%VMPqN-NYKm zW9XxHBt88$`%$>iE}{GM)6Xwo!SZ)gc2uVLb$C2~szSXe;BT{6O^Hl1aWFI!k>+at zVTI)zsVWmeFXgznr5m!UwvTBDs7FKQ-@>%o9W3;VlM~DGlddk9FM)Ht?;e#j=ttHU z-|`RpDuZjWK@2EUiSohtaxbHA2-6>pvw_Mb6w-e&EOe7E6F$ukvBfvCd3aM02`#z& zRt$N6YzvAO_UTeBXiD5c`DnbQeq?*jt4;L@_77p&qF$D;{ARntZ|Hpdff7ih#7x8K2Ao{mHFu(D2$vs3?adDg9i&_&8CudaSuIUwcF|I z4{x1gd2sQ^Q+JND^$zG+^?&}4e_(f4Ub{AdjWAU&@jqH~SJ!L$dacNf%j3dZmDt!Ib~Wy=_uFpY*Q^$U zaB>cJzhinIP@bniZ!#cs>K1ZPQHwx-IGKkzF&(BHU{EH$Y>=Em@_?Ec{C8si81;l0nFEu!lJTRMyukwE%7422%!+uH&i249zd-CbF3 z)m|Jjcu0w$b0uzO7^^_$kd0!5;3%jhnbk;AQ29}M(D+%VK1+{jc{l|f5mUp%#8D~* zwQ1@I9!kZ4jND)K3iemEaQ0sHr#w%|zKXK3uVgLdNfo#pg{_V1mJ><)_lcqCjAjc{ zSmBJUab{GW&WLpFE%2LS%f}6W+7~PZ(k*W7&NBBib3a0f_+2NLKm3rzKVu?I zM8m7s_3EAmVF?JJ4<;dmbIZpfxYu;U2kmPjR$oZssl*HupQY_7jR8l0cB7gxWvJGV zl^lRO03;*vZqwobG!Ua8*4Hq>Ra{rF@O&y})M4syMR?OfvhpJ!R^MbAfk5vn4Zn9K z;D5D~2m>mg3vG>#hIL=vY$fjAf@*ZeobMyi%YbyK3*NuHvJf zzY1@QZ^@aT31H@@d|Ql?|EFKbQFxw)17*=qX;MfGL?soODmy++jDYk>mVm2Vq#@jU zDNZmNj$-N4zNgp4c6nDVyIXOvh4OO66bMG&sq{>xQ|Q2PO>i53ibs#C>t^J{>QEi> zysf##zbf(;h1Y6bU(0PGhZdAeg&g$ZoeY;LQwpov$*mx*Nhm-1ysFAXYFzc|HYykEa~Q$<`30W@k)b2(z>( z11F{qE9>(TBcppZFipKV=SZ!;Vh$2(*t1xv1U6<&+ zL8AE_WG3;HrNRhf9;UF|w&;s!y{D`(c*NxF z%`aEH`=Qdpu)>upQ?69WxC1%4oY42n8b%)aPD3xJ@jXj1d07NXo4tjUuB`mb>(mis zl)P#=rMAP!SeJzLTACKwv}y;Qmak#o`YJ(xaY_XWYEjd^gEPY^>f|}*J3N4VIIEr- zYc?9bXLsswbKg^Q64AP;Ruv9n+JjlzI^Ayy?I#LKRb;|odI$GW;AoHQJj*!PRx?Pcw3Aw$J2tMz%vyrQTvyp&S zUX<%9p@S^*%Z5b?FUzi6#6Jv@GC0M5UBF%XJ3Nw9<2mWpUL}X|spIgGfI$0AiOWGk zOGBgX%JT{i(EFZ>6p3ZITi5rn8J2bTJMiK)NEpLMMO6P+rDmAw1su9+i-1|#`%km0 zmA9OK&2Efms(;1PzDvt%n~%xRO;(u5&{vj&N4Dcn+~(p#R`nE}?|NQuf{>Sghc*AL zd``PY7UrIQg!1~?*yrb1eB=?d?*ufsV>i4zcU|3}yU#*7kR2-+Wl2>)378;<5ITzCi||?RSxw{Uj2luDjd99Zte=yYB>Fxhmk>Uk^YHFIS2xri zndP20)WQexU@NBUl-ywB++W@$=;wZSc+B-FLUV2=5w*B-K2NZn~vyunj+DPw*Fa z4mr9r3D3eCVzmJrr+F!e zt^>fnv%SQVxTDHOm<2>+0*hTVl@iGbc*=uM%c zB5OP>8LjmYD!2lF|L{*r#}MJ_6CYc(#Wf~YF53s>B(>(CHAn@IR~-o~bOd3#LfPH+!=aQG3KVR-sc!Bc z1~He)g225EPKQHkH4!kJx2U!^_0f1ZX@Q!;kmEU$bTHJ>gBcCmUD`(jcN?UqAv&OE zi(#x7$B@3Kp68E@@EOKJ5`%VYsP97P4fbsqjFTi(tsj;jlQbvT%}F>QJG7~h6qci`=1S+9=8%C(|D1dZN z`&dHM`pyv-bPycT+DqCfhjd&)&K2ZZ!GDwx;m<&aVZYfaRlQ=|DEU zdf~1A;;Z9oIb6By^Am%Y$~ZVxGe`2B1|~9$Fn<$2j)AC>0{s8o)>&6|>+=75<_^q% z7y__DH>wDY&Zvta9VRSeKklL{reaW!in-VeuILJ!7}TNyEA}ENhN2>dw2h)ClENb% z5bZEXf*3537zi;nu3v!1w%TAF$i5z2E%a!3EJ%lgPs6?f`O=jL4ZNG8Gz7!~h)L_I&gP z8Dz_^5}$=*1BTQ&q6iLy1UlpYg>;$#f&D;$w2*!w&C%_)zSw?|XkcN?ap15~a4=$= zy(o?+G+#X6o+G?n5KIQ!v%~6CykLa8kiv0)C_GQ7 zJ6{O9kaDM&o&akdPtK!HN0TFmjdO;A3qlMW{dlR`!P`RmhJoPxk={lzM=+FDKsRPk zePGB7V^kGAiFZ~j{^7zY;z*d4Aci9%3ne<7KY&UQtsiE^6-mVr`y!;V)4Nu}@lt{z zP$B?>SSgWMD4yUQF$@*oBOdL4#1Wt)q+G@C2uJGhgx?4$wj*?mVsLmvYXmj+;&0e7 zHbyzC?QF`3J-$(9sh*c?eb;8!*vy1_fmS>Z4}=xXV;Ho<6_6vSN(Jl?kvXhMIfs-v z#pRr-W7hoFV4BaG0s9J5WVBwwbQn|7Dg4@d;X;IZ6=^~W$;mPavPV#V0W|>G6pOn3 zoCqR?`XDgD&p$xFUj63);!f6tGz=v*0+AIn_3^Zt8*m-7yQxF*J0`n3-k?kfKgAJLtM6wG*eKoffG7 z)!w!C39L~68_W@l0OP^rQ8LxR1X6&fg_JV0?q=PZO0#Sqoq21X3`a8sotuM{B(Ccx z<+>hl)%&4nWRI-F2T?w-4!3LzF(x0MC&Iv=0Gq?9B)#?&Dq#f>3(6k=cf}Rr*dl&qPv=0THYlZq?yExvh zT>!%9TV)C{+j!!CyB@>}mGR+uw%K?4LQDaC4eMUtz%lG{dsD+c|HZn3!?)I+L0pQl zykSQ8GO6JqY3EX>m%Vf^Cc2kn6M}{82_SnAS1#-$R9duzlDpf&=}Z=;UM5SsPzy`%;Bv-=ToxqY^8q? z8b)`hV=0Lo*ApS}Ii%+yT`L^@@l$$U;kn6sjwgE{@`i-g6LaA(yQ8TaBR5#Zw4;;j z6##!=a=iwBFw`rys!&!2hKP)}Z52GEVtpKVZYg?ce)fDjM% zkileSZ-UYdh`U_twtNuaTq#ooBp;dVA>!>ECg2Mluh!uI%5=_Q^|`Ikd5ISqe0p`q zqYE|ahX-y|9g74SRN?+u`EdUE=aDd6!9V}>DBpa4+!bwmw>%O;t5H8ZaNDxuALb*$ zq!RI;2MCs)l~?1#k(3tszr=FHDGwZJm#T%3INoll>H8-cI6;U;CGWdIu#pDawv1k% ztAvNxeEu)C_ZE0{MBt@*OWoS;^%i3*vp2Ux96dGzqptNNlUcJMEC1MkRLD?@K66LqoHp-`&T7Z!MY1a#b0xfn`{%I!b$LT2Z zI7J|TnyDoT`JbgwULFEZ53%qYSsD=^K_eVy>xwy18d)oq>HN6LY0OIR}&1&qhinC(rSZy$bBjB-VuCmtPT8l~g7q zzP>7VUAejDZ^dN#ui5nZdeP<*KqNt|0OCFT9E&)!Fk9)#u`W zZwx_ZGjyMaw4IRk8O~V1FLzC~&d!kV3pZh5Y_xP`HbZz4(iSJIGit(E%KyG!-)EBv zf$#~L^2CCe@G{x2_^>&f%qSpJD#+9VnPy-Ay69@CESDjWsrVa%kl9SpPea<`MXxhz z${?gE&n5)U@zU2BEtHA%8c|B-gj_a%QwxO`|Lbxg+m+`Om>CsjX2Hy|uYbm4iCB@Q z&%+RND#YA^n8ORf2Thk0gfl9_nT2pB5k#XFgE*%`oLdm*@~yt0AY4!pE-Zu#`%5x? z5yF{?zcCb<&Af86khb_Mr!#8SGi>dO?Y`jO>*;d>=lCn9Gg>GUQY}@IQ!2!N)Pk5& zYN-+=7yZman2~C!lAKc^<`%@9QcIQK85QBoLO4sVrSbPoXS8t6({V1SI2RVqh1JZG zO6`)0a%rJlnzv+v{g~;*jWQR1V;C}pl7;^$PHa|`F3`nDs?lm3-Gc7-!;d3_)fy@7h^NTl`(q88zcG+PkI%Csc%qg)pH%VI@AL;!G`^ zX*$k~iZiotX4IObgy&S0xrH(x)TBumgIt%+EQ~Yg+o<8958+(=jp>(v*-TwM4{3|n zRh?1uJ+}v0!B$ntPN*Og3&b8{h5MCCdP;?vS};>;EmFcWD$2}4nbB&I5}i{q<`%|0 zp%%sKsm^F2oasr2={y8-A^yf7WHwV(FGAYlRaIxylrQYTNw{E?@Pvvou~2wj{h_>W z#2Dy|0yCw;Of8rxtrjVN(HRwEW?{@IwMYrhsR(lmVNR??m*Q^>LS{3y=rW`&UW;@_ zP5IKyK!yumiB6~(6ANRKhB2jLOf8IQ8pe!@F|#m86-WursR(lmA+Z9*`>8sk1#)h` z(m1IfmsqISejEZn(A*lChLNel<3UO{hoF_qCP$4cXhzr?JPbVRW ziTE1>klD=pHVJ8qzi&FDCOql&qo)MUCuG(W3uQun6O{Cn3Ny7}bT?>vLBW_&F=iIV zjPfQZ!8sLSZXu|5Z8E1ooKYdpEQm8zP0uJ0=TwMu3*y{=9_C+$V8mA|`oLv2^EOOF z+Tw46&ZtRGN8^Do|3f+*NC1z+dp}>$I@zerQ z)BIwgJXT&aD#Xl!n5nnnf&ww8Ld-3Qc@o4K72?c-Fq1gphFr;>Q$fxxkn?fRDFSmr zg}Ja`F0Ai=#Ra8mT~bjlEtE_9t3-NHUQl6PSTHYCA6$O)=R!~V!s1Oqi!WwSQNFZcGCblJ$=d$^r6g{CGG&yaMNq=}ALz$9MrZ&p7 z|Iv=1kU?w=Yh&2tg7R3%)pcf}*b6*U3d)>{GPh8F=F~Ps37=6>&McHOyA2`1oKs=W zEtqrNhLEb*1r_7M!nh#y8IlYte8I&zVa?xZR;i@ay#@NYr zDJCL=!A)d0q<4s{w@AidtXC*WHP*H@&k`W7?Rz3wW3btQZAW-f>Vl*MbmWeUid3H)OxN!<+YSc zl9Xhp;P36z7kB^UdKXlA+CVTcqcI8O$*oU?f4+jHTWB0e1cE&C4{yd8J5<_1bVNfolX2#&qC)SEs+E#QH*3_0lAQ(`m^^B>O?b+ zrGnJT=L@yqTvYK6(=cpw9I*6Z^S|elpm!o)_%plRFme{;igFL|#V4oi$>D#H|0ZE3 zw~ZeZ%ZGy`l=%;L)?58Y8z#?16%}W-_ru71;P*Hgn-~l8e3XjLL5~Ypdrw2=R}Gi% z)YR%+5ve?NQ@XS17~sSIYvJR^d0-DI0DH)ryw={-Yf>Iihia7r@x^!8#MqeUUr}-X zz;pS*PZrtpiPp; zR9vWlf!YNB2+AW0#D&a!`CsQ)I_v7`m*sNpM2ph;eYan3Pky%jE$+Ztcjv&w zioY~69bMBDdCIsFf$Sl;9T)#V>gvsR`zDefkw)R9m-m#tR4qUMmiBdg4twkT?ezFL zGbOtXN}Woejf$Edg}A zW485r%ZhsXn#<>74xhJ^Om<;5I;^ABK^D!ZcSU@|Q&l?P3{QT9ss~HL+}6a;*&vQn zfW@gaP+u(noyX4NmH`%*Hx0k9>`a;7++nBv*!F&-aDU?dV8?0dV2YzrZwZ4cF1x8U zg;M7IvR*p{DUw$inmS4%PVt(snV}xfJKfSpWifh9(8R5)v?j!(7razmN*LCZyNDN| z?CW)&5|(J1Saj%GmG3fW`-&$mHNOmfZEr66(SWAaSAPTwM2mG44aSda^?g19jh5*s zQi^%)jc1<_TY*CiPO*Bt@usq3vj&h0RoM65_zYRG85Cqab5=hEWgl);%HSFdFVH=S zArC2dd-7^&{=U1Q)hA&uLS7bnn`|JisnKtFaM-My%3L()(DkDv8a)e9#WkqffBz^+ zb~a1PAK;=$50Q?NJm#|ypf|n#Q4p}0rA6f?_^Xab6X;PXNw?$>Mh>fQyWo}vu;~oQ z8s9flVUovs)=f>Gro4y}LrPekc=wY2Sm^fw^XsD9&0X*meHg_W5{}&H?t)7hzy<>$ z5jN}nUGORc7zG6hx7hg91=lr%4Pqfcg#G$N7yPXuj6#CUpwcaW!I0M!6iN**q&U%z*3wvxH~vns{2vqM_XRgEw$%DpG;U_DHEb1aHEAoWw%sDoyHk=3j>tPW zx3IPPYK5U#Nks7whojXuBn*v~>nL(IglP9|3PU&HxLAvIXg7S@(1g;)1Nn^gK_^!;Wan{ahPiZ2=tAVBz?ijf$DJ_fqp18>4I?SK;T{ifqpzS z2`kxQa4Jp+jSnx851MGbFSvzt@hunyFTqy$ZqMD6!)uJ7gh9->ix1(dT&SQ8HA5Eh zIr^XVGNFWtnRNoh!&(gAWpJ@^Kn_;uYtM8U^HoSUeAE~=Yz>J-Zh~j^N5F;yA(1wl zV&J(!8wH89SU=GXPceZ}u#j;3Em${P3JDuqhb&?fLliJ%E&=ZClU!t+SL&6EE5hJw zW&~`|8WMrrI^GR;HGz$xAW`^9(^S*|aL1}+9kN24zm$q%CRtVca}Dbhlqg_id6MPh zo+ZfQ(9(&MTB)eE_(WX%ve6l<7Zv8=tS)$nF>K5l67^v6P%3H7Btp8yo9)Qp8rJK_ijZ0kW&g`5vuGq`?UWg}p)jsk{G^aJ&_@!1ZYV-x zO(OKV;mZhD3W5zFl673B&1I-vsX`jD$@FLSk$pHM-#?$e%d~|E(LY zY5Fq<5ysu{c2n3O2{J1^q##N%5Y0wVc}f}3GwB4noeDo~TX~urKjD^MsDSw)rC4Uq zT`gWP1t?(RWXluo1{E1pu@un_;)*gi2WimHhN7fhv20uw7fp4@8+0a-jqv9hlpabJ zCEb$B{#PnPk3~r>(qItoQh#4vOUhb^S)!qEN3rsGBQ*Z)DX|rE$W9_tAPr@dW!03& zMeE~bSW(u}&A4xiLT;Nx=5@oV7AC-OTgWM?Cu8qSLb2U&FUy~0ic0JbTDCk(%c{>| zn6+C`qEgMej>s64y|MwPNMJH6XL$`C#X6wzUtwaw{82XBRo!q)8yF1*iMH5&4pIlS zv5;u{Z8At5TNsT5VWJYHlrUs3MXr*=IO^lu`2>1aT2ibi0%(7}eA-(XG=+i4&2jav&-_ElD3bX+LXJKv*KC9g=++vqqe6 zNc+535YS;V%XO6LU->UdeB?24# zJ#be?*x)*3R&T`e7>Q__?FdGfa;bBogh`nrTZKwJKkQm*_az*IuA4*F`G!c~CIp`e zJGj)N6m2Rwq<@S=LuUWm;=XwuBb_uK!kJ1s6e~Anp&OP}WTFP=trJ~UG2|RY69=Am z&gCVl7(Iew!T@)v&JzXzejqBA4e(TlmusqF^vVhZ?}IzBxqRt{lU#i~K$ z|56j1H-~H{&b&@XJ*%-I=>FmXCp&?N{C!#)AC z3W8v`3uUZlX#7XeK8w5X2uniaPpFH{z;Sw~SdC}^@1y>2e^-ueLE~#c8wJ_OWb$O7 z2I{Pu7+kdQSFUgjxl8f8C+|&7a7u$fm(NIwlkRAR#SWnHIW=I!{)32LD`MI~LVyln zkPBt(E;RlWT#R`Zr~fGK9U6ZfwBb1Yzt|Kso&?$kIDO^VAvB&1z<)-Zcg;YZd7fAu zuk3clDN{R^w690)UssgZ#SereV85yE^pYn*|)N3tpfRhBvLbZ|-#v+W9E z?mU>4$Urfl|A$e|Kpp)*Mi2D;A7<7T^!v9}DOhge{M8I;Algk~u-FuSxWdggv$+?I zx2%!(oT$nbbXqKhkzb+l_tk&q@IpKKjr%43zZrG!xm1;AtUaF+LiHt(i+aUj^6A#f z1^Q~3T}!ZM$yLZ^pZ)q65eRf}AsN<8Fo+@SIwEhmH?G4CkcSyeL2bE53T2ykt+ z80bNOo|M9AcAB)mokDkUv(n-6onn!q^V(BwX*j)AANzpdqOmIIw}l7di{e3$}NTE~&T&tUPIQF{1{shoDcQ&$)Zhwc>D{p7EjhB}^I)_;Z6?b$Z!hGK}9b1Zy3IhwSAr=OMICqVnE zUfBMi%*_&{HM=W@?zytV8>7)%X<^LHu(^fbjiOmU{7O`+)Zxl4XGZ+(sV$4*bfIMW zxzR@~+MG>Zpl)N5LZZWF@<+DDN1)9IdWWlI-nPmXRoU7q7CUJ)qM0N<)?DsRJF_66 z9wl)w`KQ{L@v@FM@!q)KUTSlz@=B2!lw*phu8|EpVGg{T_K&8=_8SE+G;6qzhh*5* zUjwHf@#d}@L%ftW)e8eVf4kjQYbvG(B0R+=rNby$eM00zJ9j4f_IQ|pUtLe@qCb|Z z6y;po{L=e|YlNSgs(X}~6^)XN&Y%brim!UJ>PGkBQdilF%&p=}8tRQD^KqZ>U;XN@ zzWDm=tKAQtKl+|6IXA2B8C=DWvPaSx`hN=81~+bFM`j2bd7r~5c-u_l;APU2{0$3Uf*yM={7U_D*wM#j0xrTH7~mFp*K_@7_S55L2hezxX!VWI;UFE|E{ z89be#hK9NazZVIAxAQnx`sA+T)q!yOIbaQW^}buHdYMt^ADh3isk~jbc<-FYd;9#m@n7#A>aGdlhD~1-bWkVRd zQD-7-n1yViW#3)x!JfVu+dp9$aH;RcsM%sDBMSGb{uOjZhE4WqE?52ShR|)|C?9ih zHgP8Zy~Zu=&gwfsfKNx6oRlPR@?eU%%aP;ZQ;~od5pf^QNCXZA;yAS1lL?>Xx%T)J z3$FVeLAbMmXw2KL#le$29>4AE^wpmjDRjEl_C)u@xLN!@_O0lUIqNIB`HNJ|uWZ)H zc1U|~MZgI&uTO>LUq0}UVfj*OUVg9jJcL(kl9##Fw-fCPG%jC}nYJqR*T1#%Qa47I z-+d_{K&=FVUBBm>vukZS>nEwzexs)Fi3i5m_G#YoIPa>=-a}iNg4|2jT2pu*T(Tpp z8H+nzyqv}jg*|MzzB79NkPjobUf6jTF_gD}%X`l!7N_`3d9_bYc|-f#<70bXYxk~` zydu26O%W@d1b_e%0`+3&U13en+g1KMQ}tC@eBtZM6|-3@4~Yz_(=5Ga|2!vozgJ*f z!KvTtP}bPhM0>J~fWnvZpbMT1Y?sFx(>|qj+wpCihE&{ac*p(9MIjzrr)LuJ`{c33 zPqOJF9bv=yVTj8WIm^PFZ^NQ1;L`~D4O)_`AGGGc+fTxWC*RT>*vQ$>{M z+mgr0@_IeHy!&WMQiC*lsks{aecZ>L(p(l#@FUCJo32J6H~R5XKki1S3;f+*R#eYk zh#GPkrHh^?JB6JFUf1;3%*CNfz8FRPNDKe6{+b_%)%eAD-FvBZrxk8qulM{`?mx8O ziNePKO<>*ka`DrWD(dBJXz-611_Etkc_w1lWnxv$6O$3y)9btR%%dZ^qMsR@Vc(vt zaZCa5-|GL=#cv@)_t!#WVVcJus@Q2|-95bX_6E6MU{7S9mKS4cIItMm-^=N=RG+w$ znnJtXD0_sJgS9?w#dpQYXl**gU+IBi$1O@D7i#aVg1W|+fc(M3^x7~K;#c=`&B?Jv zQI^Brwc3aO#ClETzEUly8t*rH@l5}zD8*!@E2T01274*{NOJ9@F6V?0IdJ~Zy0WqD UFgd7@8f(yaqii1t+;ltsKM`-H7ytkO diff --git a/docs/versions.yaml b/docs/versions.yaml index 28287d3a0aa7..c8af4d36b78f 100644 --- a/docs/versions.yaml +++ b/docs/versions.yaml @@ -20,7 +20,7 @@ "1.24": 1.24.12 "1.25": 1.25.11 "1.26": 1.26.8 -"1.27": 1.27.5 -"1.28": 1.28.3 -"1.29": 1.29.4 -"1.30": 1.30.1 +"1.27": 1.27.6 +"1.28": 1.28.4 +"1.29": 1.29.5 +"1.30": 1.30.2 From db9d7eabd1ed1bbe9665fa80cfbcdeffcdfb6214 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 6 Jun 2024 08:40:40 -0400 Subject: [PATCH 35/61] subscription: making exception free (#34554) Signed-off-by: Alyssa Wilk --- envoy/config/subscription_factory.h | 10 +-- envoy/runtime/runtime.h | 3 +- .../config/subscription_factory_impl.cc | 69 +++++++++++-------- .../common/config/subscription_factory_impl.h | 11 ++- source/common/filter/config_discovery_impl.cc | 7 +- source/common/listener_manager/lds_api.cc | 12 ++-- .../rds/rds_route_config_subscription.cc | 5 +- source/common/router/scoped_rds.cc | 10 +-- source/common/router/vhds.cc | 5 +- source/common/runtime/runtime_impl.cc | 12 ++-- source/common/runtime/runtime_impl.h | 4 +- source/common/secret/sds_api.cc | 6 +- source/common/upstream/cds_api_impl.cc | 12 ++-- source/common/upstream/od_cds_api_impl.cc | 12 ++-- source/extensions/clusters/eds/eds.cc | 5 +- source/extensions/clusters/eds/leds.cc | 5 +- source/server/config_validation/server.cc | 2 +- source/server/server.cc | 2 +- test/common/runtime/runtime_impl_test.cc | 5 +- test/common/secret/sds_api_test.cc | 4 +- .../common/subscription_factory_impl_test.cc | 37 ++++++---- test/mocks/config/mocks.h | 4 +- test/mocks/runtime/mocks.h | 2 +- tools/code_format/config.yaml | 6 +- 24 files changed, 150 insertions(+), 100 deletions(-) diff --git a/envoy/config/subscription_factory.h b/envoy/config/subscription_factory.h index 1471e5f1752a..472fe9106295 100644 --- a/envoy/config/subscription_factory.h +++ b/envoy/config/subscription_factory.h @@ -54,9 +54,10 @@ class SubscriptionFactory { * @param resource_decoder how incoming opaque resource objects are to be decoded. * @param options subscription options. * - * @return SubscriptionPtr subscription object corresponding for config and type_url. + * @return SubscriptionPtr subscription object corresponding for config and type_url or error + * status. */ - virtual SubscriptionPtr subscriptionFromConfigSource( + virtual absl::StatusOr subscriptionFromConfigSource( const envoy::config::core::v3::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, SubscriptionCallbacks& callbacks, OpaqueResourceDecoderSharedPtr resource_decoder, const SubscriptionOptions& options) PURE; @@ -73,9 +74,10 @@ class SubscriptionFactory { * CollectionSubscription object. * @param resource_decoder how incoming opaque resource objects are to be decoded. * - * @return SubscriptionPtr subscription object corresponding for collection_locator. + * @return SubscriptionPtr subscription object corresponding for collection_locator or error + * status. */ - virtual SubscriptionPtr + virtual absl::StatusOr collectionSubscriptionFromUrl(const xds::core::v3::ResourceLocator& collection_locator, const envoy::config::core::v3::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, diff --git a/envoy/runtime/runtime.h b/envoy/runtime/runtime.h index a3c1f8bd1353..bd9d06a50a2b 100644 --- a/envoy/runtime/runtime.h +++ b/envoy/runtime/runtime.h @@ -227,8 +227,9 @@ class Loader { * the constructor is finished, with the exception of dynamic RTDS layers, * which require ClusterManager. * @param cm cluster manager reference. + * @return a status indicating if initialization was successful. */ - virtual void initialize(Upstream::ClusterManager& cm) PURE; + virtual absl::Status initialize(Upstream::ClusterManager& cm) PURE; /** * @return const Snapshot& the current snapshot. This reference is safe to use for the duration of diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 4e11b27da8b5..c0f9ce344767 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -23,11 +23,11 @@ SubscriptionFactoryImpl::SubscriptionFactoryImpl( validation_visitor_(validation_visitor), api_(api), server_(server), xds_resources_delegate_(xds_resources_delegate), xds_config_tracker_(xds_config_tracker) {} -SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( +absl::StatusOr SubscriptionFactoryImpl::subscriptionFromConfigSource( const envoy::config::core::v3::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, SubscriptionCallbacks& callbacks, OpaqueResourceDecoderSharedPtr resource_decoder, const SubscriptionOptions& options) { - THROW_IF_NOT_OK(Config::Utility::checkLocalInfo(type_url, local_info_)); + RETURN_IF_NOT_OK(Config::Utility::checkLocalInfo(type_url, local_info_)); SubscriptionStats stats = Utility::generateStats(scope); std::string subscription_type = ""; @@ -50,29 +50,29 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( switch (config.config_source_specifier_case()) { case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kPath: { - THROW_IF_NOT_OK(Utility::checkFilesystemSubscriptionBackingPath(config.path(), api_)); + RETURN_IF_NOT_OK(Utility::checkFilesystemSubscriptionBackingPath(config.path(), api_)); subscription_type = "envoy.config_subscription.filesystem"; break; } case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kPathConfigSource: { - THROW_IF_NOT_OK( + RETURN_IF_NOT_OK( Utility::checkFilesystemSubscriptionBackingPath(config.path_config_source().path(), api_)); subscription_type = "envoy.config_subscription.filesystem"; break; } case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kApiConfigSource: { const envoy::config::core::v3::ApiConfigSource& api_config_source = config.api_config_source(); - THROW_IF_NOT_OK(Utility::checkApiConfigSourceSubscriptionBackingCluster(cm_.primaryClusters(), - api_config_source)); - THROW_IF_NOT_OK(Utility::checkTransportVersion(api_config_source)); + RETURN_IF_NOT_OK(Utility::checkApiConfigSourceSubscriptionBackingCluster(cm_.primaryClusters(), + api_config_source)); + RETURN_IF_NOT_OK(Utility::checkTransportVersion(api_config_source)); switch (api_config_source.api_type()) { PANIC_ON_PROTO_ENUM_SENTINEL_VALUES; case envoy::config::core::v3::ApiConfigSource::AGGREGATED_GRPC: - throwEnvoyExceptionOrPanic("Unsupported config source AGGREGATED_GRPC"); + return absl::InvalidArgumentError("Unsupported config source AGGREGATED_GRPC"); case envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC: - throwEnvoyExceptionOrPanic("Unsupported config source AGGREGATED_DELTA_GRPC"); + return absl::InvalidArgumentError("Unsupported config source AGGREGATED_DELTA_GRPC"); case envoy::config::core::v3::ApiConfigSource::DEPRECATED_AND_UNAVAILABLE_DO_NOT_USE: - throwEnvoyExceptionOrPanic( + return absl::InvalidArgumentError( "REST_LEGACY no longer a supported ApiConfigSource. " "Please specify an explicit supported api_type in the following config:\n" + config.DebugString()); @@ -87,7 +87,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( break; } if (subscription_type.empty()) { - throwEnvoyExceptionOrPanic("Invalid API config source API type"); + return absl::InvalidArgumentError("Invalid API config source API type"); } break; } @@ -96,32 +96,32 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( break; } default: - throwEnvoyExceptionOrPanic( + return absl::InvalidArgumentError( "Missing config source specifier in envoy::config::core::v3::ConfigSource"); } ConfigSubscriptionFactory* factory = Registry::FactoryRegistry::getFactory(subscription_type); if (factory == nullptr) { - throwEnvoyExceptionOrPanic(fmt::format( + return absl::InvalidArgumentError(fmt::format( "Didn't find a registered config subscription factory implementation for name: '{}'", subscription_type)); } return factory->create(data); } -SubscriptionPtr createFromFactoryOrThrow(ConfigSubscriptionFactory::SubscriptionData& data, - absl::string_view subscription_type) { +absl::StatusOr createFromFactory(ConfigSubscriptionFactory::SubscriptionData& data, + absl::string_view subscription_type) { ConfigSubscriptionFactory* factory = Registry::FactoryRegistry::getFactory(subscription_type); if (factory == nullptr) { - throwEnvoyExceptionOrPanic(fmt::format( + return absl::InvalidArgumentError(fmt::format( "Didn't find a registered config subscription factory implementation for name: '{}'", subscription_type)); } return factory->create(data); } -SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( +absl::StatusOr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( const xds::core::v3::ResourceLocator& collection_locator, const envoy::config::core::v3::ConfigSource& config, absl::string_view resource_type, Stats::Scope& scope, SubscriptionCallbacks& callbacks, @@ -148,13 +148,15 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( switch (collection_locator.scheme()) { case xds::core::v3::ResourceLocator::FILE: { const std::string path = Http::Utility::localPathFromFilePath(collection_locator.id()); - THROW_IF_NOT_OK(Utility::checkFilesystemSubscriptionBackingPath(path, api_)); + RETURN_IF_NOT_OK(Utility::checkFilesystemSubscriptionBackingPath(path, api_)); factory_config.set_path(path); - return createFromFactoryOrThrow(data, "envoy.config_subscription.filesystem_collection"); + auto ptr_or_error = createFromFactory(data, "envoy.config_subscription.filesystem_collection"); + RETURN_IF_NOT_OK(ptr_or_error.status()); + return std::move(ptr_or_error.value()); } case xds::core::v3::ResourceLocator::XDSTP: { if (resource_type != collection_locator.resource_type()) { - throwEnvoyExceptionOrPanic( + return absl::InvalidArgumentError( fmt::format("xdstp:// type does not match {} in {}", resource_type, Config::XdsResourceIdentifier::encodeUrl(collection_locator))); } @@ -162,8 +164,8 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kApiConfigSource: { const envoy::config::core::v3::ApiConfigSource& api_config_source = config.api_config_source(); - THROW_IF_NOT_OK(Utility::checkApiConfigSourceSubscriptionBackingCluster(cm_.primaryClusters(), - api_config_source)); + RETURN_IF_NOT_OK(Utility::checkApiConfigSourceSubscriptionBackingCluster( + cm_.primaryClusters(), api_config_source)); // All Envoy collections currently are xDS resource graph roots and require node context // parameters. options.add_xdstp_node_context_params_ = true; @@ -171,17 +173,22 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( case envoy::config::core::v3::ApiConfigSource::DELTA_GRPC: { std::string type_url = TypeUtil::descriptorFullNameToTypeUrl(resource_type); data.type_url_ = type_url; - return createFromFactoryOrThrow(data, "envoy.config_subscription.delta_grpc_collection"); + auto ptr_or_error = + createFromFactory(data, "envoy.config_subscription.delta_grpc_collection"); + RETURN_IF_NOT_OK(ptr_or_error.status()); + return std::move(ptr_or_error.value()); } case envoy::config::core::v3::ApiConfigSource::AGGREGATED_GRPC: FALLTHRU; case envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC: { - return createFromFactoryOrThrow(data, - "envoy.config_subscription.aggregated_grpc_collection"); + auto ptr_or_error = + createFromFactory(data, "envoy.config_subscription.aggregated_grpc_collection"); + RETURN_IF_NOT_OK(ptr_or_error.status()); + return std::move(ptr_or_error.value()); } default: - throwEnvoyExceptionOrPanic(fmt::format("Unknown xdstp:// transport API type in {}", - api_config_source.DebugString())); + return absl::InvalidArgumentError(fmt::format("Unknown xdstp:// transport API type in {}", + api_config_source.DebugString())); } } case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kAds: { @@ -189,10 +196,12 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( // All Envoy collections currently are xDS resource graph roots and require node context // parameters. options.add_xdstp_node_context_params_ = true; - return createFromFactoryOrThrow(data, "envoy.config_subscription.ads_collection"); + auto ptr_or_error = createFromFactory(data, "envoy.config_subscription.ads_collection"); + RETURN_IF_NOT_OK(ptr_or_error.status()); + return std::move(ptr_or_error.value()); } default: - throwEnvoyExceptionOrPanic( + return absl::InvalidArgumentError( "Missing or not supported config source specifier in " "envoy::config::core::v3::ConfigSource for a collection. Only ADS and " "gRPC in delta-xDS mode are supported."); @@ -200,7 +209,7 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( } default: // TODO(htuch): Implement HTTP semantics for collection ResourceLocators. - throwEnvoyExceptionOrPanic("Unsupported code path"); + return absl::InvalidArgumentError("Unsupported code path"); } } diff --git a/source/common/config/subscription_factory_impl.h b/source/common/config/subscription_factory_impl.h index 2f37e08408e7..aff5a088340a 100644 --- a/source/common/config/subscription_factory_impl.h +++ b/source/common/config/subscription_factory_impl.h @@ -26,12 +26,11 @@ class SubscriptionFactoryImpl : public SubscriptionFactory, Logger::Loggable subscriptionFromConfigSource( + const envoy::config::core::v3::ConfigSource& config, absl::string_view type_url, + Stats::Scope& scope, SubscriptionCallbacks& callbacks, + OpaqueResourceDecoderSharedPtr resource_decoder, const SubscriptionOptions& options) override; + absl::StatusOr collectionSubscriptionFromUrl(const xds::core::v3::ResourceLocator& collection_locator, const envoy::config::core::v3::ConfigSource& config, absl::string_view resource_type, Stats::Scope& scope, diff --git a/source/common/filter/config_discovery_impl.cc b/source/common/filter/config_discovery_impl.cc index d3cb3e26b70d..5962df8e2345 100644 --- a/source/common/filter/config_discovery_impl.cc +++ b/source/common/filter/config_discovery_impl.cc @@ -78,8 +78,11 @@ FilterConfigSubscription::FilterConfigSubscription( filter_config_provider_manager_(filter_config_provider_manager), subscription_id_(subscription_id) { const auto resource_name = getResourceName(); - subscription_ = cluster_manager.subscriptionFactory().subscriptionFromConfigSource( - config_source, Grpc::Common::typeUrl(resource_name), *scope_, *this, resource_decoder_, {}); + subscription_ = + THROW_OR_RETURN_VALUE(cluster_manager.subscriptionFactory().subscriptionFromConfigSource( + config_source, Grpc::Common::typeUrl(resource_name), *scope_, *this, + resource_decoder_, {}), + Config::SubscriptionPtr); } void FilterConfigSubscription::start() { diff --git a/source/common/listener_manager/lds_api.cc b/source/common/listener_manager/lds_api.cc index b3b48a10c916..f77333ea0cce 100644 --- a/source/common/listener_manager/lds_api.cc +++ b/source/common/listener_manager/lds_api.cc @@ -32,11 +32,15 @@ LdsApiImpl::LdsApiImpl(const envoy::config::core::v3::ConfigSource& lds_config, init_target_("LDS", [this]() { subscription_->start({}); }) { const auto resource_name = getResourceName(); if (lds_resources_locator == nullptr) { - subscription_ = cm.subscriptionFactory().subscriptionFromConfigSource( - lds_config, Grpc::Common::typeUrl(resource_name), *scope_, *this, resource_decoder_, {}); + subscription_ = THROW_OR_RETURN_VALUE(cm.subscriptionFactory().subscriptionFromConfigSource( + lds_config, Grpc::Common::typeUrl(resource_name), + *scope_, *this, resource_decoder_, {}), + Config::SubscriptionPtr); } else { - subscription_ = cm.subscriptionFactory().collectionSubscriptionFromUrl( - *lds_resources_locator, lds_config, resource_name, *scope_, *this, resource_decoder_); + subscription_ = THROW_OR_RETURN_VALUE( + cm.subscriptionFactory().collectionSubscriptionFromUrl( + *lds_resources_locator, lds_config, resource_name, *scope_, *this, resource_decoder_), + Config::SubscriptionPtr); } init_manager.add(init_target_); } diff --git a/source/common/rds/rds_route_config_subscription.cc b/source/common/rds/rds_route_config_subscription.cc index 4bda9ae928c5..2f8eea06f982 100644 --- a/source/common/rds/rds_route_config_subscription.cc +++ b/source/common/rds/rds_route_config_subscription.cc @@ -31,10 +31,11 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( manager_identifier_(manager_identifier), config_update_info_(std::move(config_update)), resource_decoder_(std::move(resource_decoder)) { const auto resource_type = route_config_provider_manager_.protoTraits().resourceType(); - subscription_ = + subscription_ = THROW_OR_RETURN_VALUE( factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( config_source, Envoy::Grpc::Common::typeUrl(resource_type), *scope_, *this, - resource_decoder_, {}); + resource_decoder_, {}), + Envoy::Config::SubscriptionPtr); local_init_manager_.add(local_init_target_); } diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 1ad77bd8aecd..e459d0dc6be0 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -144,18 +144,20 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( route_config_provider_manager_(route_config_provider_manager) { const auto resource_name = getResourceName(); if (scoped_rds.srds_resources_locator().empty()) { - subscription_ = + subscription_ = THROW_OR_RETURN_VALUE( factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( scoped_rds.scoped_rds_config_source(), Grpc::Common::typeUrl(resource_name), *scope_, - *this, resource_decoder_, {}); + *this, resource_decoder_, {}), + Envoy::Config::SubscriptionPtr); } else { const auto srds_resources_locator = THROW_OR_RETURN_VALUE( Envoy::Config::XdsResourceIdentifier::decodeUrl(scoped_rds.srds_resources_locator()), xds::core::v3::ResourceLocator); - subscription_ = + subscription_ = THROW_OR_RETURN_VALUE( factory_context.clusterManager().subscriptionFactory().collectionSubscriptionFromUrl( srds_resources_locator, scoped_rds.scoped_rds_config_source(), resource_name, *scope_, - *this, resource_decoder_); + *this, resource_decoder_), + Envoy::Config::SubscriptionPtr); } // TODO(tony612): consider not using the callback here. diff --git a/source/common/router/vhds.cc b/source/common/router/vhds.cc index 823ee0b0f86d..b182c5b1f96c 100644 --- a/source/common/router/vhds.cc +++ b/source/common/router/vhds.cc @@ -58,10 +58,11 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, const auto resource_name = getResourceName(); Envoy::Config::SubscriptionOptions options; options.use_namespace_matching_ = true; - subscription_ = + subscription_ = THROW_OR_RETURN_VALUE( factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( config_update_info_->protobufConfigurationCast().vhds().config_source(), - Grpc::Common::typeUrl(resource_name), *scope_, *this, resource_decoder_, options); + Grpc::Common::typeUrl(resource_name), *scope_, *this, resource_decoder_, options), + Envoy::Config::SubscriptionPtr); } void VhdsSubscription::updateOnDemand(const std::string& with_route_config_name_prefix) { diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 253ea9710471..2d625947ccee 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -605,12 +605,13 @@ absl::Status LoaderImpl::initLayers(Event::Dispatcher& dispatcher, return loadNewSnapshot(); } -void LoaderImpl::initialize(Upstream::ClusterManager& cm) { +absl::Status LoaderImpl::initialize(Upstream::ClusterManager& cm) { cm_ = &cm; for (const auto& s : subscriptions_) { - s->createSubscription(); + RETURN_IF_NOT_OK(s->createSubscription()); } + return absl::OkStatus(); } void LoaderImpl::startRtdsSubscriptions(ReadyCallback on_done) { @@ -632,11 +633,14 @@ RtdsSubscription::RtdsSubscription( stats_scope_(store_.createScope("runtime")), resource_name_(rtds_layer.name()), init_target_("RTDS " + resource_name_, [this]() { start(); }) {} -void RtdsSubscription::createSubscription() { +absl::Status RtdsSubscription::createSubscription() { const auto resource_name = getResourceName(); - subscription_ = parent_.cm_->subscriptionFactory().subscriptionFromConfigSource( + auto subscription_or_error = parent_.cm_->subscriptionFactory().subscriptionFromConfigSource( config_source_, Grpc::Common::typeUrl(resource_name), *stats_scope_, *this, resource_decoder_, {}); + RETURN_IF_NOT_OK(subscription_or_error.status()); + subscription_ = std::move(*subscription_or_error); + return absl::OkStatus(); } absl::Status diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index 9c960a58cbd3..cf47d6f435d9 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -193,7 +193,7 @@ struct RtdsSubscription : Envoy::Config::SubscriptionBase& removed_resources); - void createSubscription(); + absl::Status createSubscription(); LoaderImpl& parent_; const envoy::config::core::v3::ConfigSource config_source_; @@ -223,7 +223,7 @@ class LoaderImpl : public Loader, Logger::Loggable { Api::Api& api); // Runtime::Loader - void initialize(Upstream::ClusterManager& cm) override; + absl::Status initialize(Upstream::ClusterManager& cm) override; const Snapshot& snapshot() override; SnapshotConstSharedPtr threadsafeSnapshot() override; absl::Status mergeValues(const absl::node_hash_map& values) override; diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index 1cce6941e2f9..a8ab37af3ee6 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -32,8 +32,10 @@ SdsApi::SdsApi(envoy::config::core::v3::ConfigSource sds_config, absl::string_vi time_source_.systemTime()} { const auto resource_name = getResourceName(); // This has to happen here (rather than in initialize()) as it can throw exceptions. - subscription_ = subscription_factory_.subscriptionFromConfigSource( - sds_config_, Grpc::Common::typeUrl(resource_name), *scope_, *this, resource_decoder_, {}); + subscription_ = THROW_OR_RETURN_VALUE( + subscription_factory_.subscriptionFromConfigSource( + sds_config_, Grpc::Common::typeUrl(resource_name), *scope_, *this, resource_decoder_, {}), + Config::SubscriptionPtr); } void SdsApi::resolveDataSource(const FileContentMap& files, diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index cfb0ee3e8b21..caa8c5de399a 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -25,11 +25,15 @@ CdsApiImpl::CdsApiImpl(const envoy::config::core::v3::ConfigSource& cds_config, helper_(cm, "cds"), cm_(cm), scope_(scope.createScope("cluster_manager.cds.")) { const auto resource_name = getResourceName(); if (cds_resources_locator == nullptr) { - subscription_ = cm_.subscriptionFactory().subscriptionFromConfigSource( - cds_config, Grpc::Common::typeUrl(resource_name), *scope_, *this, resource_decoder_, {}); + subscription_ = THROW_OR_RETURN_VALUE(cm_.subscriptionFactory().subscriptionFromConfigSource( + cds_config, Grpc::Common::typeUrl(resource_name), + *scope_, *this, resource_decoder_, {}), + Config::SubscriptionPtr); } else { - subscription_ = cm.subscriptionFactory().collectionSubscriptionFromUrl( - *cds_resources_locator, cds_config, resource_name, *scope_, *this, resource_decoder_); + subscription_ = THROW_OR_RETURN_VALUE( + cm.subscriptionFactory().collectionSubscriptionFromUrl( + *cds_resources_locator, cds_config, resource_name, *scope_, *this, resource_decoder_), + Config::SubscriptionPtr); } } diff --git a/source/common/upstream/od_cds_api_impl.cc b/source/common/upstream/od_cds_api_impl.cc index a9ec3c349693..f52fee9f193a 100644 --- a/source/common/upstream/od_cds_api_impl.cc +++ b/source/common/upstream/od_cds_api_impl.cc @@ -30,11 +30,15 @@ OdCdsApiImpl::OdCdsApiImpl(const envoy::config::core::v3::ConfigSource& odcds_co // class for CDS and ODCDS. const auto resource_name = getResourceName(); if (!odcds_resources_locator.has_value()) { - subscription_ = cm_.subscriptionFactory().subscriptionFromConfigSource( - odcds_config, Grpc::Common::typeUrl(resource_name), *scope_, *this, resource_decoder_, {}); + subscription_ = THROW_OR_RETURN_VALUE(cm_.subscriptionFactory().subscriptionFromConfigSource( + odcds_config, Grpc::Common::typeUrl(resource_name), + *scope_, *this, resource_decoder_, {}), + Config::SubscriptionPtr); } else { - subscription_ = cm.subscriptionFactory().collectionSubscriptionFromUrl( - *odcds_resources_locator, odcds_config, resource_name, *scope_, *this, resource_decoder_); + subscription_ = THROW_OR_RETURN_VALUE(cm.subscriptionFactory().collectionSubscriptionFromUrl( + *odcds_resources_locator, odcds_config, resource_name, + *scope_, *this, resource_decoder_), + Config::SubscriptionPtr); } } diff --git a/source/extensions/clusters/eds/eds.cc b/source/extensions/clusters/eds/eds.cc index 5ea525cd7b20..02a2b748298d 100644 --- a/source/extensions/clusters/eds/eds.cc +++ b/source/extensions/clusters/eds/eds.cc @@ -34,10 +34,11 @@ EdsClusterImpl::EdsClusterImpl(const envoy::config::cluster::v3::Cluster& cluste initialize_phase_ = InitializePhase::Secondary; } const auto resource_name = getResourceName(); - subscription_ = + subscription_ = THROW_OR_RETURN_VALUE( cluster_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( eds_config, Grpc::Common::typeUrl(resource_name), info_->statsScope(), *this, - resource_decoder_, {}); + resource_decoder_, {}), + Config::SubscriptionPtr); } EdsClusterImpl::~EdsClusterImpl() { diff --git a/source/extensions/clusters/eds/leds.cc b/source/extensions/clusters/eds/leds.cc index 5cad9d6c8b40..a3e5310b3e5b 100644 --- a/source/extensions/clusters/eds/leds.cc +++ b/source/extensions/clusters/eds/leds.cc @@ -24,10 +24,11 @@ LedsSubscription::LedsSubscription( Config::XdsResourceIdentifier::decodeUrl(leds_config.leds_collection_name()), xds::core::v3::ResourceLocator); const auto resource_name = getResourceName(); - subscription_ = + subscription_ = THROW_OR_RETURN_VALUE( factory_context.clusterManager().subscriptionFactory().collectionSubscriptionFromUrl( leds_resource_locator, leds_config.leds_config(), resource_name, *stats_scope_, *this, - resource_decoder_); + resource_decoder_), + Config::SubscriptionPtr); subscription_->start({}); } diff --git a/source/server/config_validation/server.cc b/source/server/config_validation/server.cc index 7aeb4e1d41a1..344bc8b5cd5a 100644 --- a/source/server/config_validation/server.cc +++ b/source/server/config_validation/server.cc @@ -144,7 +144,7 @@ void ValidationInstance::initialize(const Options& options, [this]() -> Network::DnsResolverSharedPtr { return this->dnsResolver(); }, sslContextManager(), *secret_manager_, quic_stat_names_, *this); THROW_IF_NOT_OK(config_.initialize(bootstrap_, *this, *cluster_manager_factory_)); - runtime().initialize(clusterManager()); + THROW_IF_NOT_OK(runtime().initialize(clusterManager())); clusterManager().setInitializedCb([this]() -> void { init_manager_.initialize(init_watcher_); }); } diff --git a/source/server/server.cc b/source/server/server.cc index ee3ab436b23e..0c02614b833e 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -768,7 +768,7 @@ absl::Status InstanceBase::initializeOrThrow(Network::Address::InstanceConstShar // We have to defer RTDS initialization until after the cluster manager is // instantiated (which in turn relies on runtime...). - runtime().initialize(clusterManager()); + RETURN_IF_NOT_OK(runtime().initialize(clusterManager())); clusterManager().setPrimaryClustersInitializedCb( [this]() { onClusterManagerPrimaryInitializationComplete(); }); diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index fa6415809cde..9d90992b54ab 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -962,7 +962,7 @@ class RtdsLoaderImplTest : public LoaderImplTest { dispatcher_, tls_, config, local_info_, store_, generator_, validation_visitor_, *api_); THROW_IF_NOT_OK(loader.status()); loader_ = std::move(loader.value()); - loader_->initialize(cm_); + THROW_IF_NOT_OK(loader_->initialize(cm_)); for (auto* sub : rtds_subscriptions_) { EXPECT_CALL(*sub, start(_)); } @@ -1298,7 +1298,8 @@ TEST_F(RtdsLoaderImplTest, BadConfigSource) { absl::StatusOr> loader = Runtime::LoaderImpl::create( dispatcher_, tls_, config, local_info_, store_, generator_, validation_visitor_, *api_); - EXPECT_THROW_WITH_MESSAGE(loader.value()->initialize(cm_), EnvoyException, "bad config"); + EXPECT_THROW_WITH_MESSAGE(loader.value()->initialize(cm_).IgnoreError(), EnvoyException, + "bad config"); } } // namespace diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index 4e1fab3c1042..1f73cacb7e39 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -112,7 +112,7 @@ TEST_F(SdsApiTest, InitManagerInitialised) { &stats](const envoy::config::core::v3::ConfigSource&, absl::string_view, Stats::Scope&, Config::SubscriptionCallbacks& cbs, Config::OpaqueResourceDecoderSharedPtr, - const Config::SubscriptionOptions&) -> Config::SubscriptionPtr { + const Config::SubscriptionOptions&) { return std::make_unique( *dispatcher_, Config::makePathConfigSource(sds_config_path), cbs, resource_decoder, stats, validation_visitor_, *api_); @@ -138,7 +138,7 @@ TEST_F(SdsApiTest, BadConfigSource) { ::testing::InSequence s; envoy::config::core::v3::ConfigSource config_source; EXPECT_CALL(subscription_factory_, subscriptionFromConfigSource(_, _, _, _, _, _)) - .WillOnce(InvokeWithoutArgs([]() -> Config::SubscriptionPtr { + .WillOnce(InvokeWithoutArgs([]() { throw EnvoyException("bad config"); return nullptr; })); diff --git a/test/extensions/config_subscription/common/subscription_factory_impl_test.cc b/test/extensions/config_subscription/common/subscription_factory_impl_test.cc index 04ff5951273b..ed4470656532 100644 --- a/test/extensions/config_subscription/common/subscription_factory_impl_test.cc +++ b/test/extensions/config_subscription/common/subscription_factory_impl_test.cc @@ -53,18 +53,21 @@ class SubscriptionFactoryTest : public testing::Test { SubscriptionPtr subscriptionFromConfigSource(const envoy::config::core::v3::ConfigSource& config) { - return subscription_factory_.subscriptionFromConfigSource( - config, Config::TypeUrl::get().ClusterLoadAssignment, *stats_store_.rootScope(), callbacks_, - resource_decoder_, {}); + return THROW_OR_RETURN_VALUE(subscription_factory_.subscriptionFromConfigSource( + config, Config::TypeUrl::get().ClusterLoadAssignment, + *stats_store_.rootScope(), callbacks_, resource_decoder_, {}), + SubscriptionPtr); } SubscriptionPtr collectionSubscriptionFromUrl(const std::string& xds_url, const envoy::config::core::v3::ConfigSource& config) { const auto resource_locator = XdsResourceIdentifier::decodeUrl(xds_url).value(); - return subscription_factory_.collectionSubscriptionFromUrl( - resource_locator, config, "envoy.config.endpoint.v3.ClusterLoadAssignment", - *stats_store_.rootScope(), callbacks_, resource_decoder_); + return THROW_OR_RETURN_VALUE(subscription_factory_.collectionSubscriptionFromUrl( + resource_locator, config, + "envoy.config.endpoint.v3.ClusterLoadAssignment", + *stats_store_.rootScope(), callbacks_, resource_decoder_), + SubscriptionPtr); } Upstream::MockClusterManager cm_; @@ -114,7 +117,7 @@ TEST_F(SubscriptionFactoryTest, RestClusterEmpty) { config.mutable_api_config_source()->set_api_type(envoy::config::core::v3::ApiConfigSource::REST); - EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); + EXPECT_CALL(cm_, primaryClusters()).WillRepeatedly(ReturnRef(primary_clusters)); EXPECT_THROW_WITH_REGEX(subscriptionFromConfigSource(config), EnvoyException, "API configs must have either a gRPC service or a cluster name defined:"); } @@ -125,7 +128,7 @@ TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, GrpcClusterEmpty) { config.mutable_api_config_source()->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC); - EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); + EXPECT_CALL(cm_, primaryClusters()).WillRepeatedly(ReturnRef(primary_clusters)); EXPECT_THROW_WITH_REGEX(subscriptionFromConfigSource(config), EnvoyException, "API configs must have either a gRPC service or a cluster name defined:"); } @@ -187,7 +190,7 @@ TEST_F(SubscriptionFactoryTest, RestClusterMultiton) { config.mutable_api_config_source()->add_cluster_names("static_cluster_bar"); primary_clusters.insert("static_cluster_bar"); - EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); + EXPECT_CALL(cm_, primaryClusters()).WillRepeatedly(ReturnRef(primary_clusters)); EXPECT_THROW_WITH_REGEX(subscriptionFromConfigSource(config), EnvoyException, fmt::format("{} must have a singleton cluster name specified:", config.mutable_api_config_source()->GetTypeName())); @@ -207,7 +210,7 @@ TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, GrpcClusterMultiton) { primary_clusters.insert("static_cluster_bar"); EXPECT_CALL(cm_, grpcAsyncClientManager()).WillRepeatedly(ReturnRef(cm_.async_client_manager_)); - EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); + EXPECT_CALL(cm_, primaryClusters()).WillRepeatedly(ReturnRef(primary_clusters)); EXPECT_THROW_WITH_REGEX(subscriptionFromConfigSource(config), EnvoyException, fmt::format("{}::.DELTA_.GRPC must have a " @@ -486,10 +489,13 @@ TEST_F(SubscriptionFactoryTest, LogWarningOnDeprecatedV2Transport) { primary_clusters.insert("static_cluster"); EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); - EXPECT_THROW_WITH_REGEX(subscription_factory_.subscriptionFromConfigSource( - config, Config::TypeUrl::get().ClusterLoadAssignment, - *stats_store_.rootScope(), callbacks_, resource_decoder_, {}), - EnvoyException, "V2 xDS transport protocol version is deprecated in"); + EXPECT_THAT(subscription_factory_ + .subscriptionFromConfigSource( + config, Config::TypeUrl::get().ClusterLoadAssignment, + *stats_store_.rootScope(), callbacks_, resource_decoder_, {}) + .status() + .message(), + testing::HasSubstr("V2 xDS transport protocol version is deprecated in")); } // Use of AUTO transport fails by default. This will encourage folks to upgrade to explicit V3. @@ -506,6 +512,7 @@ TEST_F(SubscriptionFactoryTest, AutoTransportIsAllowed) { Upstream::ClusterManager::ClusterSet primary_clusters; primary_clusters.insert("static_cluster"); EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); + EXPECT_CALL(cm_, grpcAsyncClientManager()).WillOnce(ReturnRef(cm_.async_client_manager_)); EXPECT_CALL(cm_.async_client_manager_, factoryForGrpcService(ProtoEq(expected_grpc_service), _, _)) @@ -539,7 +546,7 @@ TEST_P(SubscriptionFactoryTestApiConfigSource, NonExistentCluster) { api_config_source->add_cluster_names("static_cluster"); } Upstream::ClusterManager::ClusterSet primary_clusters; - EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); + EXPECT_CALL(cm_, primaryClusters()).WillRepeatedly(ReturnRef(primary_clusters)); EXPECT_THROW_WITH_MESSAGE(subscriptionFromConfigSource(config)->start({"static_cluster"}), EnvoyException, fmt::format("{} must have a statically defined " diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index ef9897f9cb2b..556ce0f7c0ff 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -79,12 +79,12 @@ class MockSubscriptionFactory : public SubscriptionFactory { MockSubscriptionFactory(); ~MockSubscriptionFactory() override; - MOCK_METHOD(SubscriptionPtr, subscriptionFromConfigSource, + MOCK_METHOD(absl::StatusOr, subscriptionFromConfigSource, (const envoy::config::core::v3::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, SubscriptionCallbacks& callbacks, OpaqueResourceDecoderSharedPtr resource_decoder, const SubscriptionOptions& options)); - MOCK_METHOD(SubscriptionPtr, collectionSubscriptionFromUrl, + MOCK_METHOD(absl::StatusOr, collectionSubscriptionFromUrl, (const xds::core::v3::ResourceLocator& collection_locator, const envoy::config::core::v3::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, SubscriptionCallbacks& callbacks, diff --git a/test/mocks/runtime/mocks.h b/test/mocks/runtime/mocks.h index 0329bf8154ca..9bb7e6c09c89 100644 --- a/test/mocks/runtime/mocks.h +++ b/test/mocks/runtime/mocks.h @@ -61,7 +61,7 @@ class MockLoader : public Loader { MockLoader(); ~MockLoader() override; - MOCK_METHOD(void, initialize, (Upstream::ClusterManager & cm)); + MOCK_METHOD(absl::Status, initialize, (Upstream::ClusterManager & cm)); MOCK_METHOD(const Snapshot&, snapshot, ()); MOCK_METHOD(SnapshotConstSharedPtr, threadsafeSnapshot, ()); MOCK_METHOD(absl::Status, mergeValues, ((const absl::node_hash_map&))); diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index ffa3c7582a2a..fa8766048cad 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -117,7 +117,11 @@ paths: - source/common/quic/quic_server_transport_socket_factory.cc - source/common/grpc/google_grpc_utils.cc - source/common/tcp_proxy/tcp_proxy.cc - - source/common/config/subscription_factory_impl.cc + - source/common/listener_manager/lds_api.cc + - source/common/upstream/od_cds_api_impl.cc + - source/common/upstream/cds_api_impl.cc + - source/common/router/vhds.cc + - source/common/rds/rds_route_config_subscription.cc - source/common/filter/config_discovery_impl.cc - source/common/json/json_internal.cc - source/common/router/scoped_rds.cc From e8448e89608e4823bd1bd531b8ec8743fc259e44 Mon Sep 17 00:00:00 2001 From: Nigel Brittain <108375408+nbaws@users.noreply.github.com> Date: Fri, 7 Jun 2024 00:28:29 +1000 Subject: [PATCH 36/61] aws: add nbaws as aws codeowner (#34376) Signed-off-by: Nigel Brittain --- CODEOWNERS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index e0d262de1d4d..030d1385a036 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -90,8 +90,8 @@ extensions/filters/common/original_src @klarose @mattklein123 /*/extensions/filters/http/cache @toddmgreer @jmarantz @penguingao @mpwarres @capoferro /*/extensions/http/cache/simple_http_cache @toddmgreer @jmarantz @penguingao @mpwarres @capoferro # aws_iam grpc credentials -/*/extensions/grpc_credentials/aws_iam @suniltheta @lavignes @mattklein123 -/*/extensions/common/aws @suniltheta @lavignes @mattklein123 +/*/extensions/grpc_credentials/aws_iam @suniltheta @mattklein123 @nbaws +/*/extensions/common/aws @suniltheta @mattklein123 @nbaws # adaptive concurrency limit extension. /*/extensions/filters/http/adaptive_concurrency @tonya11en @mattklein123 # admission control extension. @@ -153,8 +153,8 @@ extensions/filters/common/original_src @klarose @mattklein123 # support for on-demand VHDS requests /*/extensions/filters/http/on_demand @dmitri-d @htuch @kyessenov /*/extensions/filters/network/connection_limit @mattklein123 @alyssawilk @delong-coder -/*/extensions/filters/http/aws_request_signing @derekargueta @suniltheta @mattklein123 @marcomagdy -/*/extensions/filters/http/aws_lambda @suniltheta @mattklein123 @marcomagdy @lavignes +/*/extensions/filters/http/aws_request_signing @derekargueta @suniltheta @mattklein123 @marcomagdy @nbaws +/*/extensions/filters/http/aws_lambda @suniltheta @mattklein123 @marcomagdy @nbaws /*/extensions/filters/http/buffer @alyssawilk @mattklein123 /*/extensions/transport_sockets/raw_buffer @alyssawilk @mattklein123 # Watchdog Extensions From 8a6570ed0706b410fb7032c00f1d0d01795824e2 Mon Sep 17 00:00:00 2001 From: Sankalp <125449768+sankalpjha555@users.noreply.github.com> Date: Thu, 6 Jun 2024 20:53:37 +0530 Subject: [PATCH 37/61] Fix mongo_proxy:codec_impl_test and mongo_proxy:bson_impl_test on big endian (#34453) Signed-off-by: sankalpjha555 --- .../extensions/filters/network/mongo_proxy/bson_impl.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.cc b/source/extensions/filters/network/mongo_proxy/bson_impl.cc index 8ea6cf028376..669f26e08917 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.cc +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.cc @@ -23,7 +23,11 @@ int32_t BufferHelper::peekInt32(Buffer::Instance& data) { int32_t val; val = data.peekLEInt(); +#ifdef ABSL_IS_BIG_ENDIAN + return val; +#else return le32toh(val); +#endif } uint8_t BufferHelper::removeByte(Buffer::Instance& data) { @@ -88,7 +92,11 @@ int64_t BufferHelper::removeInt64(Buffer::Instance& data) { int64_t val; val = data.drainLEInt(); +#ifdef ABSL_IS_BIG_ENDIAN + return val; +#else return le64toh(val); +#endif } std::string BufferHelper::removeString(Buffer::Instance& data) { From bb3b8834be96aa07e0780ba5ff4534393ce3e445 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Thu, 6 Jun 2024 21:37:58 +0530 Subject: [PATCH 38/61] add support for dynamic direct response for files (#32789) * add support for dynamic direct response for files Signed-off-by: Rama Chavali * add locking Signed-off-by: Rama Chavali * fix test and add release notes Signed-off-by: Rama Chavali * improve lamda Signed-off-by: Rama Chavali * move to tls storage Signed-off-by: Rama Chavali * compile errors Signed-off-by: Rama Chavali * move to shared pointer with mutex Signed-off-by: Rama Chavali * fix it Signed-off-by: Rama Chavali * use new datasource provider Signed-off-by: Rama Chavali * fix format Signed-off-by: Rama Chavali * fix test Signed-off-by: Rama Chavali * fix condition Signed-off-by: Rama Chavali * change false Signed-off-by: Rama Chavali * move to create Signed-off-by: Rama Chavali * fix test Signed-off-by: Rama Chavali * fix test Signed-off-by: Rama Chavali --------- Signed-off-by: Rama Chavali --- changelogs/current.yaml | 6 ++ source/common/config/datasource.cc | 26 ++++++--- source/common/config/datasource.h | 14 +++-- source/common/router/config_impl.cc | 14 +++-- source/common/router/config_impl.h | 8 ++- test/common/config/datasource_test.cc | 16 ++--- .../direct_response_integration_test.cc | 58 +++++++++++++++++++ 7 files changed, 113 insertions(+), 29 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 19bbdf4287c9..7c20910cd10c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -248,6 +248,12 @@ new_features: added :ref:`stat_prefix ` configuration to support additional stat prefix for the OpenTelemetry logger. +- area: routing + change: | + added support in :ref:`file datasource ` implementation + to listen to file changes and dynamically update the response when :ref:`watched_directory + ` + is configured in :ref:`DataSource `. - area: listener change: | Added :ref:`bypass_overload_manager ` diff --git a/source/common/config/datasource.cc b/source/common/config/datasource.cc index 95816710b1b2..8cb923f74aeb 100644 --- a/source/common/config/datasource.cc +++ b/source/common/config/datasource.cc @@ -100,29 +100,36 @@ DynamicData::~DynamicData() { } } -absl::string_view DynamicData::data() const { +const std::string& DynamicData::data() const { const auto thread_local_data = slot_->get(); return thread_local_data.has_value() ? *thread_local_data->data_ : EMPTY_STRING; } -absl::string_view DataSourceProvider::data() const { +const std::string& DataSourceProvider::data() const { if (absl::holds_alternative(data_)) { return absl::get(data_); } return absl::get(data_).data(); } -absl::StatusOr DataSourceProvider::create(const ProtoDataSource& source, - Event::Dispatcher& main_dispatcher, - ThreadLocal::SlotAllocator& tls, - Api::Api& api, bool allow_empty, - uint64_t max_size) { +absl::StatusOr DataSourceProvider::create(const ProtoDataSource& source, + Event::Dispatcher& main_dispatcher, + ThreadLocal::SlotAllocator& tls, + Api::Api& api, bool allow_empty, + uint64_t max_size) { auto initial_data_or_error = read(source, allow_empty, api, max_size); RETURN_IF_STATUS_NOT_OK(initial_data_or_error); if (!source.has_watched_directory() || source.specifier_case() != envoy::config::core::v3::DataSource::kFilename) { - return DataSourceProvider(std::move(initial_data_or_error).value()); + if (source.specifier_case() != envoy::config::core::v3::DataSource::kFilename && + initial_data_or_error.value().length() > max_size) { + return absl::InvalidArgumentError(fmt::format("response body size is {} bytes; maximum is {}", + initial_data_or_error.value().length(), + max_size)); + } + return std::unique_ptr( + new DataSourceProvider(std::move(initial_data_or_error).value())); } auto slot = ThreadLocal::TypedSlot::makeUnique(tls); @@ -157,7 +164,8 @@ absl::StatusOr DataSourceProvider::create(const ProtoDataSou }); RETURN_IF_NOT_OK(watcher_status); - return DataSourceProvider(DynamicData(main_dispatcher, std::move(slot), std::move(watcher))); + return std::unique_ptr( + new DataSourceProvider(DynamicData(main_dispatcher, std::move(slot), std::move(watcher)))); } } // namespace DataSource diff --git a/source/common/config/datasource.h b/source/common/config/datasource.h index 1c0f47879392..6dc6131338f9 100644 --- a/source/common/config/datasource.h +++ b/source/common/config/datasource.h @@ -20,8 +20,11 @@ namespace Envoy { namespace Config { namespace DataSource { +class DataSourceProvider; + using ProtoDataSource = envoy::config::core::v3::DataSource; using ProtoWatchedDirectory = envoy::config::core::v3::WatchedDirectory; +using DataSourceProviderPtr = std::unique_ptr; /** * Read contents of the DataSource. @@ -54,7 +57,7 @@ class DynamicData { Filesystem::WatcherPtr watcher); ~DynamicData(); - absl::string_view data() const; + const std::string& data() const; private: Event::Dispatcher& dispatcher_; @@ -85,12 +88,11 @@ class DataSourceProvider { * NOTE: If file watch is enabled and the new file content does not meet the * requirements (allow_empty, max_size), the provider will keep the old content. */ - static absl::StatusOr create(const ProtoDataSource& source, - Event::Dispatcher& main_dispatcher, - ThreadLocal::SlotAllocator& tls, Api::Api& api, - bool allow_empty, uint64_t max_size = 0); + static absl::StatusOr + create(const ProtoDataSource& source, Event::Dispatcher& main_dispatcher, + ThreadLocal::SlotAllocator& tls, Api::Api& api, bool allow_empty, uint64_t max_size = 0); - absl::string_view data() const; + const std::string& data() const; private: DataSourceProvider(std::string&& data) : data_(std::move(data)) {} diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 612efe3489f4..6384850d3b4b 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -572,10 +572,16 @@ RouteEntryImplBase::RouteEntryImplBase(const CommonVirtualHostSharedPtr& vhost, using_new_timeouts_(route.route().has_max_stream_duration()), match_grpc_(route.match().has_grpc()), case_sensitive_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(route.match(), case_sensitive, true)) { - auto body_or_error = ConfigUtility::parseDirectResponseBody( - route, factory_context.api(), vhost_->globalRouteConfig().maxDirectResponseBodySizeBytes()); - SET_AND_RETURN_IF_NOT_OK(body_or_error.status(), creation_status); - direct_response_body_ = body_or_error.value(); + + if (route.has_direct_response() && route.direct_response().has_body()) { + + auto provider_or_error = Envoy::Config::DataSource::DataSourceProvider::create( + route.direct_response().body(), factory_context.mainThreadDispatcher(), + factory_context.threadLocal(), factory_context.api(), true, + vhost_->globalRouteConfig().maxDirectResponseBodySizeBytes()); + SET_AND_RETURN_IF_NOT_OK(provider_or_error.status(), creation_status); + direct_response_body_provider_ = std::move(provider_or_error.value()); + } if (!route.request_headers_to_add().empty() || !route.request_headers_to_remove().empty()) { request_headers_parser_ = THROW_OR_RETURN_VALUE( HeaderParser::configure(route.request_headers_to_add(), route.request_headers_to_remove()), diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 38ea31d46b41..a70e5c8669b0 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -24,6 +24,7 @@ #include "source/common/common/matchers.h" #include "source/common/common/packed_struct.h" +#include "source/common/config/datasource.h" #include "source/common/config/metadata.h" #include "source/common/http/hash_policy.h" #include "source/common/http/header_utility.h" @@ -798,7 +799,10 @@ class RouteEntryImplBase : public RouteEntryAndRoute, std::string newUri(const Http::RequestHeaderMap& headers) const override; void rewritePathHeader(Http::RequestHeaderMap&, bool) const override {} Http::Code responseCode() const override { return direct_response_code_.value(); } - const std::string& responseBody() const override { return direct_response_body_; } + const std::string& responseBody() const override { + return direct_response_body_provider_ != nullptr ? direct_response_body_provider_->data() + : EMPTY_STRING; + } // Router::Route const DirectResponseEntry* directResponseEntry() const override; @@ -1227,7 +1231,7 @@ class RouteEntryImplBase : public RouteEntryAndRoute, const DecoratorConstPtr decorator_; const RouteTracingConstPtr route_tracing_; - std::string direct_response_body_; + Envoy::Config::DataSource::DataSourceProviderPtr direct_response_body_provider_; PerFilterConfigs per_filter_configs_; const std::string route_name_; TimeSource& time_source_; diff --git a/test/common/config/datasource_test.cc b/test/common/config/datasource_test.cc index 015613eaa695..9c4ee31aa56c 100644 --- a/test/common/config/datasource_test.cc +++ b/test/common/config/datasource_test.cc @@ -176,8 +176,8 @@ TEST(DataSourceProviderTest, NonFileDataSourceTest) { NiceMock tls; auto provider_or_error = - DataSource::DataSourceProvider::create(config, *dispatcher, tls, *api, false, 0); - EXPECT_EQ(provider_or_error.value().data(), "Hello, world!"); + DataSource::DataSourceProvider::create(config, *dispatcher, tls, *api, false, 15); + EXPECT_EQ(provider_or_error.value()->data(), "Hello, world!"); } TEST(DataSourceProviderTest, FileDataSourceButNoWatch) { @@ -218,7 +218,7 @@ TEST(DataSourceProviderTest, FileDataSourceButNoWatch) { auto provider_or_error = DataSource::DataSourceProvider::create(config, *dispatcher, tls, *api, false, 0); - EXPECT_EQ(provider_or_error.value().data(), "Hello, world!"); + EXPECT_EQ(provider_or_error.value()->data(), "Hello, world!"); // Update the symlink to point to the new file. TestEnvironment::renameFile(TestEnvironment::temporaryPath("envoy_test/watcher_new_link"), @@ -227,7 +227,7 @@ TEST(DataSourceProviderTest, FileDataSourceButNoWatch) { dispatcher->run(Event::Dispatcher::RunType::NonBlock); // The provider should still return the old content. - EXPECT_EQ(provider_or_error.value().data(), "Hello, world!"); + EXPECT_EQ(provider_or_error.value()->data(), "Hello, world!"); // Remove the file. unlink(TestEnvironment::temporaryPath("envoy_test/watcher_target").c_str()); @@ -278,7 +278,7 @@ TEST(DataSourceProviderTest, FileDataSourceAndWithWatch) { // Create a provider with watch. auto provider_or_error = DataSource::DataSourceProvider::create(config, *dispatcher, tls, *api, false, 0); - EXPECT_EQ(provider_or_error.value().data(), "Hello, world!"); + EXPECT_EQ(provider_or_error.value()->data(), "Hello, world!"); // Update the symlink to point to the new file. TestEnvironment::renameFile(TestEnvironment::temporaryPath("envoy_test/watcher_new_link"), @@ -287,7 +287,7 @@ TEST(DataSourceProviderTest, FileDataSourceAndWithWatch) { dispatcher->run(Event::Dispatcher::RunType::NonBlock); // The provider should return the updated content. - EXPECT_EQ(provider_or_error.value().data(), "Hello, world! Updated!"); + EXPECT_EQ(provider_or_error.value()->data(), "Hello, world! Updated!"); // Remove the file. unlink(TestEnvironment::temporaryPath("envoy_test/watcher_target").c_str()); @@ -339,7 +339,7 @@ TEST(DataSourceProviderTest, FileDataSourceAndWithWatchButUpdateError) { // ignored. auto provider_or_error = DataSource::DataSourceProvider::create(config, *dispatcher, tls, *api, false, 15); - EXPECT_EQ(provider_or_error.value().data(), "Hello, world!"); + EXPECT_EQ(provider_or_error.value()->data(), "Hello, world!"); // Update the symlink to point to the new file. TestEnvironment::renameFile(TestEnvironment::temporaryPath("envoy_test/watcher_new_link"), @@ -348,7 +348,7 @@ TEST(DataSourceProviderTest, FileDataSourceAndWithWatchButUpdateError) { dispatcher->run(Event::Dispatcher::RunType::NonBlock); // The provider should return the old content because the updated content is ignored. - EXPECT_EQ(provider_or_error.value().data(), "Hello, world!"); + EXPECT_EQ(provider_or_error.value()->data(), "Hello, world!"); // Remove the file. unlink(TestEnvironment::temporaryPath("envoy_test/watcher_target").c_str()); diff --git a/test/integration/direct_response_integration_test.cc b/test/integration/direct_response_integration_test.cc index bc79926acaff..7000502d0b74 100644 --- a/test/integration/direct_response_integration_test.cc +++ b/test/integration/direct_response_integration_test.cc @@ -44,6 +44,62 @@ class DirectResponseIntegrationTest : public testing::TestWithParambody().size()); EXPECT_EQ(body_content, response->body()); } + + // Test direct response with a file as the body. + void testDirectResponseFile() { + TestEnvironment::writeStringToFileForTest("file_direct.txt", "dummy"); + const std::string filename = TestEnvironment::temporaryPath("file_direct.txt"); + config_helper_.addConfigModifier( + [&](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) -> void { + auto* route_config = hcm.mutable_route_config(); + auto* route = route_config->mutable_virtual_hosts(0)->mutable_routes(0); + + route->mutable_match()->set_prefix("/direct"); + + auto* direct_response = route->mutable_direct_response(); + direct_response->set_status(200); + direct_response->mutable_body()->set_filename(filename); + direct_response->mutable_body()->mutable_watched_directory()->set_path( + TestEnvironment::temporaryDirectory()); + }); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto encoder_decoder = codec_client_->startRequest(Http::TestRequestHeaderMapImpl{ + {":method", "POST"}, + {":path", "/direct"}, + {":scheme", "http"}, + {":authority", "host"}, + }); + auto response = std::move(encoder_decoder.second); + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + EXPECT_EQ("dummy", response->body()); + + codec_client_->close(); + + // Update the file and validate that the response is updated. + TestEnvironment::writeStringToFileForTest("file_direct_updated.txt", "dummy-updated"); + TestEnvironment::renameFile(TestEnvironment::temporaryPath("file_direct_updated.txt"), + TestEnvironment::temporaryPath("file_direct.txt")); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto encoder_decoder_updated = codec_client_->startRequest(Http::TestRequestHeaderMapImpl{ + {":method", "POST"}, + {":path", "/direct"}, + {":scheme", "http"}, + {":authority", "host"}, + }); + auto updated_response = std::move(encoder_decoder_updated.second); + ASSERT_TRUE(updated_response->waitForEndStream()); + ASSERT_TRUE(updated_response->complete()); + EXPECT_EQ("200", updated_response->headers().getStatusValue()); + EXPECT_EQ("dummy-updated", updated_response->body()); + codec_client_->close(); + } }; INSTANTIATE_TEST_SUITE_P(IpVersions, DirectResponseIntegrationTest, @@ -65,4 +121,6 @@ TEST_P(DirectResponseIntegrationTest, DirectResponseBodySizeSmall) { testDirectResponseBodySize(1); } +TEST_P(DirectResponseIntegrationTest, DefaultDirectResponseFile) { testDirectResponseFile(); } + } // namespace Envoy From fbce85914421145b5ae3210c9313eced63e535b0 Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Thu, 6 Jun 2024 11:58:20 -0500 Subject: [PATCH 39/61] mobile: Remove unnecessary copies in the JvmFilterContext (#34590) The filter implementations don't typically run in a separate thread, so creating a copy is unnecessary. Risk Level: low Testing: unit tests Docs Changes: n/a Release Notes: n/a Platform Specific Features: mobile Signed-off-by: Fredy Wijaya --- .../envoymobile/engine/JvmFilterContext.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/mobile/library/java/io/envoyproxy/envoymobile/engine/JvmFilterContext.java b/mobile/library/java/io/envoyproxy/envoymobile/engine/JvmFilterContext.java index 59822278408f..9c665a699689 100644 --- a/mobile/library/java/io/envoyproxy/envoymobile/engine/JvmFilterContext.java +++ b/mobile/library/java/io/envoyproxy/envoymobile/engine/JvmFilterContext.java @@ -68,11 +68,8 @@ public Object onRequestHeaders(long headerCount, boolean endStream, long[] strea * @return Object[], pair of HTTP filter status and optional modified data. */ public Object onRequestData(ByteBuffer data, boolean endStream, long[] streamIntel) { - // Create a copy of the `data` because the `data` uses direct `ByteBuffer` and the `data` will - // be destroyed after calling this callback. - ByteBuffer copiedData = ByteBuffers.copy(data); return toJniFilterDataStatus( - filter.onRequestData(copiedData, endStream, new EnvoyStreamIntel(streamIntel))); + filter.onRequestData(data, endStream, new EnvoyStreamIntel(streamIntel))); } /** @@ -113,11 +110,8 @@ public Object onResponseHeaders(long headerCount, boolean endStream, long[] stre * @return Object[], pair of HTTP filter status and optional modified data. */ public Object onResponseData(ByteBuffer data, boolean endStream, long[] streamIntel) { - // Create a copy of the `data` because the `data` uses direct `ByteBuffer` and the `data` will - // be destroyed after calling this callback. - ByteBuffer copiedData = ByteBuffers.copy(data); return toJniFilterDataStatus( - filter.onResponseData(copiedData, endStream, new EnvoyStreamIntel(streamIntel))); + filter.onResponseData(data, endStream, new EnvoyStreamIntel(streamIntel))); } /** From 84d8fdd11e78013cd50596fa3b704e152512455e Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Thu, 6 Jun 2024 13:57:57 -0500 Subject: [PATCH 40/61] mobile: Fix RtdsIntegrationTest (#34592) Fixes #34537 Risk Level: low Testing: unit test Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a Signed-off-by: Fredy Wijaya --- mobile/test/common/integration/rtds_integration_test.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mobile/test/common/integration/rtds_integration_test.cc b/mobile/test/common/integration/rtds_integration_test.cc index 60c72c66cc93..8845abd36188 100644 --- a/mobile/test/common/integration/rtds_integration_test.cc +++ b/mobile/test/common/integration/rtds_integration_test.cc @@ -29,7 +29,7 @@ class RtdsIntegrationTest : public XdsIntegrationTest { void createEnvoy() override { Platform::XdsBuilder xds_builder( /*xds_server_address=*/Network::Test::getLoopbackAddressUrlString(ipVersion()), - /*xds_server_port=*/fake_upstreams_[1]->localAddress()->ip()->port()); + /*xds_server_port=*/fake_upstreams_.back()->localAddress()->ip()->port()); // Add the layered runtime config, which includes the RTDS layer. xds_builder.addRuntimeDiscoveryService("some_rtds_resource", /*timeout_in_seconds=*/1) .setSslRootCerts(getUpstreamCert()); @@ -40,6 +40,7 @@ class RtdsIntegrationTest : public XdsIntegrationTest { void SetUp() override { initialize(); } void runReloadTest() { + stream_ = createNewStream(createDefaultStreamCallbacks()); // Send a request on the data plane. stream_->sendHeaders(std::make_unique(default_request_headers_), true); @@ -100,14 +101,12 @@ INSTANTIATE_TEST_SUITE_P( // Envoy Mobile's xDS APIs only support state-of-the-world, not delta. testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::UnifiedSotw))); -// https://github.com/envoyproxy/envoy/issues/34537 -TEST_P(RtdsIntegrationTest, DISABLED_RtdsReloadWithDfpMixedScheme) { +TEST_P(RtdsIntegrationTest, RtdsReloadWithDfpMixedScheme) { TestScopedStaticReloadableFeaturesRuntime scoped_runtime({{"dfp_mixed_scheme", true}}); runReloadTest(); } -// https://github.com/envoyproxy/envoy/issues/34537 -TEST_P(RtdsIntegrationTest, DISABLED_RtdsReloadWithoutDfpMixedScheme) { +TEST_P(RtdsIntegrationTest, RtdsReloadWithoutDfpMixedScheme) { TestScopedStaticReloadableFeaturesRuntime scoped_runtime({{"dfp_mixed_scheme", false}}); runReloadTest(); } From 139fc0fb504850e6d16d7d5e5838956f76a1b27b Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Thu, 6 Jun 2024 14:30:21 -0500 Subject: [PATCH 41/61] mobile: Add protocol info in the error details (#34571) Risk Level: low Testing: unit tests Docs Changes: n/a Release Notes: n/a Platform Specific Features: mobile Signed-off-by: Fredy Wijaya --- mobile/library/common/http/client.cc | 7 ++- .../java/org/chromium/net/impl/Errors.java | 2 +- mobile/test/common/http/client_test.cc | 6 +-- .../chromium/net/CronetUrlRequestTest.java | 52 ++++++++++--------- 4 files changed, 38 insertions(+), 29 deletions(-) diff --git a/mobile/library/common/http/client.cc b/mobile/library/common/http/client.cc index 9db7e5db94f5..463bebdd97d0 100644 --- a/mobile/library/common/http/client.cc +++ b/mobile/library/common/http/client.cc @@ -454,12 +454,17 @@ void Client::DirectStreamCallbacks::latchError() { std::transform(info.responseFlags().begin(), info.responseFlags().end(), response_flags.begin(), [](StreamInfo::ResponseFlag flag) { return std::to_string(flag.value()); }); error_msg_details.push_back(absl::StrCat("RESPONSE_FLAGS: ", absl::StrJoin(response_flags, ","))); + if (info.protocol().has_value()) { + // https://github.com/envoyproxy/envoy/blob/fbce85914421145b5ae3210c9313eced63e535b0/envoy/http/protocol.h#L13 + error_msg_details.push_back(absl::StrCat("PROTOCOL: ", *info.protocol())); + } if (std::string resp_code_details = info.responseCodeDetails().value_or(""); !resp_code_details.empty()) { error_msg_details.push_back(absl::StrCat("DETAILS: ", std::move(resp_code_details))); } // The format of the error message propogated to callbacks is: - // RESPONSE_CODE: {value}|ERROR_CODE: {value}|RESPONSE_FLAGS: {value}|DETAILS: {value} + // RESPONSE_CODE: {value}|ERROR_CODE: {value}|RESPONSE_FLAGS: {value}|PROTOCOL: {value}|DETAILS: + // {value} // // Where RESPONSE_CODE is the HTTP response code from StreamInfo::responseCode(). // ERROR_CODE is of the envoy_error_code_t enum type, and gets mapped from RESPONSE_CODE. diff --git a/mobile/library/java/org/chromium/net/impl/Errors.java b/mobile/library/java/org/chromium/net/impl/Errors.java index 7c6aa9132b7e..c5709aba3b78 100644 --- a/mobile/library/java/org/chromium/net/impl/Errors.java +++ b/mobile/library/java/org/chromium/net/impl/Errors.java @@ -71,7 +71,7 @@ public String toString() { /** * Maps Envoymobile's errorcode to chromium's net errorcode - * @param responseFlag envoymobile's finalStreamIntel responseFlag + * @param finalStreamIntel envoymobile's finalStreamIntel * @return the NetError that the EnvoyMobileError maps to */ public static NetError mapEnvoyMobileErrorToNetError(EnvoyFinalStreamIntel finalStreamIntel) { diff --git a/mobile/test/common/http/client_test.cc b/mobile/test/common/http/client_test.cc index a65e38333642..8d3f8d286314 100644 --- a/mobile/test/common/http/client_test.cc +++ b/mobile/test/common/http/client_test.cc @@ -477,9 +477,8 @@ TEST_P(ClientTest, EnvoyLocalError) { stream_callbacks.on_error_ = [&](const EnvoyError& error, envoy_stream_intel, envoy_final_stream_intel) -> void { EXPECT_EQ(error.error_code_, ENVOY_CONNECTION_FAILURE); - EXPECT_THAT( - error.message_, - Eq("RESPONSE_CODE: 503|ERROR_CODE: 2|RESPONSE_FLAGS: 4,26|DETAILS: failed miserably")); + EXPECT_THAT(error.message_, Eq("RESPONSE_CODE: 503|ERROR_CODE: 2|RESPONSE_FLAGS: " + "4,26|PROTOCOL: 3|DETAILS: failed miserably")); EXPECT_EQ(error.attempt_count_, 123); callbacks_called.on_error_calls_++; }; @@ -503,6 +502,7 @@ TEST_P(ClientTest, EnvoyLocalError) { stream_info_.setResponseFlag(StreamInfo::ResponseFlag(StreamInfo::UpstreamRemoteReset)); stream_info_.setResponseFlag(StreamInfo::ResponseFlag(StreamInfo::DnsResolutionFailed)); stream_info_.setAttemptCount(123); + EXPECT_CALL(stream_info_, protocol()).WillRepeatedly(Return(Http::Protocol::Http3)); response_encoder_->getStream().resetStream(Http::StreamResetReason::LocalConnectionFailure); ASSERT_EQ(callbacks_called.on_headers_calls_, 0); // Ensure that the callbacks on the EnvoyStreamCallbacks were called. diff --git a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java index a442014b3315..7e8c6d8f39ec 100644 --- a/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java +++ b/mobile/test/java/org/chromium/net/CronetUrlRequestTest.java @@ -2055,25 +2055,29 @@ public void testDestroyUploadDataStreamAdapterOnSucceededCallback() throws Excep @SmallTest @Feature({"Cronet"}) public void testErrorCodes() throws Exception { - checkSpecificErrorCode(EnvoyMobileError.DNS_RESOLUTION_FAILED, NetError.ERR_NAME_NOT_RESOLVED, - NetworkException.ERROR_HOSTNAME_NOT_RESOLVED, "NAME_NOT_RESOLVED", false, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 26"); - checkSpecificErrorCode(EnvoyMobileError.UPSTREAM_CONNECTION_TERMINATION, - NetError.ERR_CONNECTION_CLOSED, NetworkException.ERROR_CONNECTION_CLOSED, - "CONNECTION_CLOSED", true, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 6"); - checkSpecificErrorCode(EnvoyMobileError.UPSTREAM_CONNECTION_FAILURE, - NetError.ERR_CONNECTION_REFUSED, - NetworkException.ERROR_CONNECTION_REFUSED, "CONNECTION_REFUSED", false, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 5"); - checkSpecificErrorCode(EnvoyMobileError.UPSTREAM_REMOTE_RESET, NetError.ERR_CONNECTION_RESET, - NetworkException.ERROR_CONNECTION_RESET, "CONNECTION_RESET", true, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 4"); - checkSpecificErrorCode(EnvoyMobileError.STREAM_IDLE_TIMEOUT, NetError.ERR_TIMED_OUT, - NetworkException.ERROR_TIMED_OUT, "TIMED_OUT", true, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 16"); - checkSpecificErrorCode(0x2000, NetError.ERR_OTHER, NetworkException.ERROR_OTHER, "OTHER", false, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 13"); + checkSpecificErrorCode( + EnvoyMobileError.DNS_RESOLUTION_FAILED, NetError.ERR_NAME_NOT_RESOLVED, + NetworkException.ERROR_HOSTNAME_NOT_RESOLVED, "NAME_NOT_RESOLVED", false, + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 26|PROTOCOL: 1"); + checkSpecificErrorCode( + EnvoyMobileError.UPSTREAM_CONNECTION_TERMINATION, NetError.ERR_CONNECTION_CLOSED, + NetworkException.ERROR_CONNECTION_CLOSED, "CONNECTION_CLOSED", true, + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 6|PROTOCOL: 1"); + checkSpecificErrorCode( + EnvoyMobileError.UPSTREAM_CONNECTION_FAILURE, NetError.ERR_CONNECTION_REFUSED, + NetworkException.ERROR_CONNECTION_REFUSED, "CONNECTION_REFUSED", false, + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 5|PROTOCOL: 1"); + checkSpecificErrorCode( + EnvoyMobileError.UPSTREAM_REMOTE_RESET, NetError.ERR_CONNECTION_RESET, + NetworkException.ERROR_CONNECTION_RESET, "CONNECTION_RESET", true, + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 4|PROTOCOL: 1"); + checkSpecificErrorCode( + EnvoyMobileError.STREAM_IDLE_TIMEOUT, NetError.ERR_TIMED_OUT, + NetworkException.ERROR_TIMED_OUT, "TIMED_OUT", true, + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 16|PROTOCOL: 1"); + checkSpecificErrorCode( + 0x2000, NetError.ERR_OTHER, NetworkException.ERROR_OTHER, "OTHER", false, + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 13|PROTOCOL: 1"); } /* @@ -2098,7 +2102,7 @@ public void testInternetDisconnectedError() throws Exception { checkSpecificErrorCode( EnvoyMobileError.DNS_RESOLUTION_FAILED, NetError.ERR_INTERNET_DISCONNECTED, NetworkException.ERROR_INTERNET_DISCONNECTED, "INTERNET_DISCONNECTED", false, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 26"); + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 26|PROTOCOL: 1"); // bring back online since the AndroidNetworkMonitor class is a singleton connectivityManager.setActiveNetworkInfo(networkInfo); @@ -2316,10 +2320,10 @@ public void test404Head() throws Exception { NativeTestServer.getFileURL("/notfound.html"), callback, callback.getExecutor()); builder.setHttpMethod("HEAD").build().start(); callback.blockForDone(); - checkSpecificErrorCode(EnvoyMobileError.UPSTREAM_CONNECTION_FAILURE, - NetError.ERR_CONNECTION_REFUSED, - NetworkException.ERROR_CONNECTION_REFUSED, "CONNECTION_REFUSED", false, - /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 5"); + checkSpecificErrorCode( + EnvoyMobileError.UPSTREAM_CONNECTION_FAILURE, NetError.ERR_CONNECTION_REFUSED, + NetworkException.ERROR_CONNECTION_REFUSED, "CONNECTION_REFUSED", false, + /*error_details=*/"RESPONSE_CODE: 400|ERROR_CODE: 0|RESPONSE_FLAGS: 5|PROTOCOL: 1"); } @Test From 7fcc47414c9ebc3915616730612b0608031ea8e9 Mon Sep 17 00:00:00 2001 From: Dennis Kniep Date: Fri, 7 Jun 2024 03:45:45 +0200 Subject: [PATCH 42/61] oauth filter: preserve existing auth header (#34470) Allows to preserve the exsting authorization header in oauth2 filter Signed-off-by: Dennis Kniep --- .../filters/http/oauth2/v3/oauth.proto | 8 ++- .../extensions/filters/http/oauth2/config.cc | 7 +++ .../extensions/filters/http/oauth2/filter.cc | 12 ++-- .../extensions/filters/http/oauth2/filter.h | 2 + .../filters/http/oauth2/config_test.cc | 58 +++++++++++++++++++ .../filters/http/oauth2/filter_test.cc | 51 +++++++++++++++- 6 files changed, 132 insertions(+), 6 deletions(-) diff --git a/api/envoy/extensions/filters/http/oauth2/v3/oauth.proto b/api/envoy/extensions/filters/http/oauth2/v3/oauth.proto index aa5f70b2897a..22696745f630 100644 --- a/api/envoy/extensions/filters/http/oauth2/v3/oauth.proto +++ b/api/envoy/extensions/filters/http/oauth2/v3/oauth.proto @@ -74,7 +74,7 @@ message OAuth2Credentials { // OAuth config // -// [#next-free-field: 16] +// [#next-free-field: 17] message OAuth2Config { enum AuthType { // The ``client_id`` and ``client_secret`` will be sent in the URL encoded request body. @@ -111,6 +111,12 @@ message OAuth2Config { // Forward the OAuth token as a Bearer to upstream web service. bool forward_bearer_token = 7; + // If set to true, preserve the existing authorization header. + // By default Envoy strips the existing authorization header before forwarding upstream. + // Can not be set to true if forward_bearer_token is already set to true. + // Default value is false. + bool preserve_authorization_header = 16; + // Any request that matches any of the provided matchers will be passed through without OAuth validation. repeated config.route.v3.HeaderMatcher pass_through_matcher = 8; diff --git a/source/extensions/filters/http/oauth2/config.cc b/source/extensions/filters/http/oauth2/config.cc index 771b4d307457..322c6e2907db 100644 --- a/source/extensions/filters/http/oauth2/config.cc +++ b/source/extensions/filters/http/oauth2/config.cc @@ -64,6 +64,13 @@ Http::FilterFactoryCb OAuth2Config::createFilterFactoryFromProtoTyped( throw EnvoyException("invalid HMAC secret configuration"); } + if (proto_config.preserve_authorization_header() && proto_config.forward_bearer_token()) { + throw EnvoyException( + "invalid combination of forward_bearer_token and preserve_authorization_header " + "configuration. If forward_bearer_token is set to true, then " + "preserve_authorization_header must be false"); + } + auto secret_reader = std::make_shared( std::move(secret_provider_token_secret), std::move(secret_provider_hmac_secret), context.serverFactoryContext().threadLocal(), context.serverFactoryContext().api()); diff --git a/source/extensions/filters/http/oauth2/filter.cc b/source/extensions/filters/http/oauth2/filter.cc index 8ccf894bfa9e..d5d97279cf7c 100644 --- a/source/extensions/filters/http/oauth2/filter.cc +++ b/source/extensions/filters/http/oauth2/filter.cc @@ -198,6 +198,7 @@ FilterConfig::FilterConfig( stats_(FilterConfig::generateStats(stats_prefix, scope)), encoded_resource_query_params_(encodeResourceList(proto_config.resources())), forward_bearer_token_(proto_config.forward_bearer_token()), + preserve_authorization_header_(proto_config.preserve_authorization_header()), pass_through_header_matchers_(headerMatchers(proto_config.pass_through_matcher(), context)), deny_redirect_header_matchers_(headerMatchers(proto_config.deny_redirect_matcher(), context)), cookie_names_(proto_config.credentials().cookie_names()), @@ -289,10 +290,13 @@ Http::FilterHeadersStatus OAuth2Filter::decodeHeaders(Http::RequestHeaderMap& he } } - // Sanitize the Authorization header, since we have no way to validate its content. Also, - // if token forwarding is enabled, this header will be set based on what is on the HMAC cookie - // before forwarding the request upstream. - headers.removeInline(authorization_handle.handle()); + // Only sanitize the Authorization header if preserveAuthorizationHeader is false + if (!config_->preserveAuthorizationHeader()) { + // Sanitize the Authorization header, since we have no way to validate its content. Also, + // if token forwarding is enabled, this header will be set based on what is on the HMAC cookie + // before forwarding the request upstream. + headers.removeInline(authorization_handle.handle()); + } // The following 2 headers are guaranteed for regular requests. The asserts are helpful when // writing test code to not forget these important variables in mock requests diff --git a/source/extensions/filters/http/oauth2/filter.h b/source/extensions/filters/http/oauth2/filter.h index 69b1acfad215..f1ed914a931b 100644 --- a/source/extensions/filters/http/oauth2/filter.h +++ b/source/extensions/filters/http/oauth2/filter.h @@ -119,6 +119,7 @@ class FilterConfig { const std::string& clusterName() const { return oauth_token_endpoint_.cluster(); } const std::string& clientId() const { return client_id_; } bool forwardBearerToken() const { return forward_bearer_token_; } + bool preserveAuthorizationHeader() const { return preserve_authorization_header_; } const std::vector& passThroughMatchers() const { return pass_through_header_matchers_; } @@ -164,6 +165,7 @@ class FilterConfig { const std::string encoded_auth_scopes_; const std::string encoded_resource_query_params_; const bool forward_bearer_token_ : 1; + const bool preserve_authorization_header_ : 1; const std::vector pass_through_header_matchers_; const std::vector deny_redirect_header_matchers_; const CookieNames cookie_names_; diff --git a/test/extensions/filters/http/oauth2/config_test.cc b/test/extensions/filters/http/oauth2/config_test.cc index 8abe960cd904..e16fb0ca9b9d 100644 --- a/test/extensions/filters/http/oauth2/config_test.cc +++ b/test/extensions/filters/http/oauth2/config_test.cc @@ -201,6 +201,64 @@ TEST(ConfigTest, WrongCookieName) { EnvoyException, "value does not match regex pattern"); } +TEST(ConfigTest, WrongCombinationOfPreserveAuthorizationAndForwardBearer) { + const std::string yaml = R"EOF( +config: + forward_bearer_token: true + preserve_authorization_header: true + token_endpoint: + cluster: foo + uri: oauth.com/token + timeout: 3s + credentials: + client_id: "secret" + token_secret: + name: token + hmac_secret: + name: hmac + cookie_names: + bearer_token: BearerToken + oauth_hmac: OauthHMAC + oauth_expires: OauthExpires + id_token: IdToken + refresh_token: RefreshToken + authorization_endpoint: https://oauth.com/oauth/authorize/ + redirect_uri: "%REQ(x-forwarded-proto)%://%REQ(:authority)%/callback" + redirect_path_matcher: + path: + exact: /callback + signout_path: + path: + exact: /signout + auth_scopes: + - user + - openid + - email + resources: + - oauth2-resource + - http://example.com + - https://example.com + )EOF"; + + OAuth2Config factory; + ProtobufTypes::MessagePtr proto_config = factory.createEmptyConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + NiceMock context; + context.server_factory_context_.cluster_manager_.initializeClusters({"foo"}, {}); + + // This returns non-nullptr for token_secret and hmac_secret. + auto& secret_manager = + context.server_factory_context_.cluster_manager_.cluster_manager_factory_.secretManager(); + ON_CALL(secret_manager, findStaticGenericSecretProvider(_)) + .WillByDefault(Return(std::make_shared( + envoy::extensions::transport_sockets::tls::v3::GenericSecret()))); + + EXPECT_THROW_WITH_REGEX( + factory.createFilterFactoryFromProto(*proto_config, "stats", context).status().IgnoreError(), + EnvoyException, + "invalid combination of forward_bearer_token and preserve_authorization_header"); +} + } // namespace Oauth2 } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/http/oauth2/filter_test.cc b/test/extensions/filters/http/oauth2/filter_test.cc index 99ee26e79585..78a49df1e710 100644 --- a/test/extensions/filters/http/oauth2/filter_test.cc +++ b/test/extensions/filters/http/oauth2/filter_test.cc @@ -112,7 +112,7 @@ class OAuth2Test : public testing::TestWithParam { ::envoy::extensions::filters::http::oauth2::v3::OAuth2Config_AuthType auth_type = ::envoy::extensions::filters::http::oauth2::v3::OAuth2Config_AuthType:: OAuth2Config_AuthType_URL_ENCODED_BODY, - int default_refresh_token_expires_in = 0) { + int default_refresh_token_expires_in = 0, bool preserve_authorization_header = false) { envoy::extensions::filters::http::oauth2::v3::OAuth2Config p; auto* endpoint = p.mutable_token_endpoint(); endpoint->set_cluster("auth.example.com"); @@ -123,6 +123,7 @@ class OAuth2Test : public testing::TestWithParam { p.set_authorization_endpoint("https://auth.example.com/oauth/authorize/"); p.mutable_signout_path()->mutable_path()->set_exact("/_signout"); p.set_forward_bearer_token(forward_bearer_token); + p.set_preserve_authorization_header(preserve_authorization_header); auto* useRefreshToken = p.mutable_use_refresh_token(); useRefreshToken->set_value(use_refresh_token); @@ -618,6 +619,54 @@ TEST_F(OAuth2Test, OAuthOkPassButInvalidToken) { EXPECT_EQ(scope_.counterFromString("test.oauth_success").value(), 1); } +/** + * Scenario: The OAuth filter receives a request with a foreign token in the Authorization + * header. This header should be forwarded when preserve authorization header is enabled + * and forwarding bearer token is disabled. + * + * Expected behavior: the filter should forward the foreign token and let the request proceed. + */ +TEST_F(OAuth2Test, OAuthOkPreserveForeignAuthHeader) { + init(getConfig(false /* forward_bearer_token */, true /* use_refresh_token */, + ::envoy::extensions::filters::http::oauth2::v3::OAuth2Config_AuthType:: + OAuth2Config_AuthType_URL_ENCODED_BODY /* encoded_body_type */, + 1200 /* default_refresh_token_expires_in */, + true /* preserve_authorization_header */)); + + Http::TestRequestHeaderMapImpl mock_request_headers{ + {Http::Headers::get().Path.get(), "/anypath"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Get}, + {Http::Headers::get().Scheme.get(), "https"}, + {Http::CustomHeaders::get().Authorization.get(), "Bearer ValidAuthorizationHeader"}, + }; + + Http::TestRequestHeaderMapImpl expected_headers{ + {Http::Headers::get().Path.get(), "/anypath"}, + {Http::Headers::get().Host.get(), "traffic.example.com"}, + {Http::Headers::get().Method.get(), Http::Headers::get().MethodValues.Get}, + {Http::Headers::get().Scheme.get(), "https"}, + {Http::CustomHeaders::get().Authorization.get(), "Bearer ValidAuthorizationHeader"}, + }; + + // cookie-validation mocking + EXPECT_CALL(*validator_, setParams(_, _)); + EXPECT_CALL(*validator_, isValid()).WillOnce(Return(true)); + + // Sanitized return reference mocking + std::string legit_token{"legit_token"}; + EXPECT_CALL(*validator_, token()).WillRepeatedly(ReturnRef(legit_token)); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->decodeHeaders(mock_request_headers, false)); + + // Ensure that existing OAuth forwarded headers got sanitized. + EXPECT_EQ(mock_request_headers, expected_headers); + + EXPECT_EQ(scope_.counterFromString("test.oauth_failure").value(), 0); + EXPECT_EQ(scope_.counterFromString("test.oauth_success").value(), 1); +} + /** * Scenario: The OAuth filter receives a request without valid OAuth cookies to a non-callback URL * (indicating that the user needs to re-validate cookies or get 401'd). From 74f327dc8b5a0f8ab67d7a47535c88cc89fed681 Mon Sep 17 00:00:00 2001 From: "Antonio V. Leonti" <53806445+antoniovleonti@users.noreply.github.com> Date: Thu, 6 Jun 2024 21:46:20 -0400 Subject: [PATCH 43/61] Add decoder header mutation rules (#34182) * add header mutation rules Signed-off-by: antoniovleonti * fix spelling errors Signed-off-by: antoniovleonti * use proto field link in proto doc string Signed-off-by: antoniovleonti * Fix proto link Signed-off-by: antoniovleonti * fix changelog proto link too Signed-off-by: antoniovleonti * use correct link in changelog Signed-off-by: antoniovleonti * fix integration test Signed-off-by: antoniovleonti * remove redundant LowerCaseString conversion Signed-off-by: antoniovleonti * use decoder_header_mutation_rules as optional field Signed-off-by: antoniovleonti * remove lambda from check header func Signed-off-by: antoniovleonti * fix doc reference/link Signed-off-by: antoniovleonti * formatting fixes Signed-off-by: antoniovleonti --------- Signed-off-by: antoniovleonti Signed-off-by: Antonio V. Leonti <53806445+antoniovleonti@users.noreply.github.com> --- .../filters/http/ext_authz/v3/BUILD | 1 + .../filters/http/ext_authz/v3/ext_authz.proto | 22 +- changelogs/current.yaml | 6 + .../extensions/filters/http/ext_authz/BUILD | 1 + .../filters/http/ext_authz/ext_authz.cc | 142 +++++--- .../filters/http/ext_authz/ext_authz.h | 29 ++ .../ext_authz/ext_authz_integration_test.cc | 139 +++++--- .../filters/http/ext_authz/ext_authz_test.cc | 323 ++++++++++++++++++ 8 files changed, 566 insertions(+), 97 deletions(-) diff --git a/api/envoy/extensions/filters/http/ext_authz/v3/BUILD b/api/envoy/extensions/filters/http/ext_authz/v3/BUILD index cabe849e71d1..7bbb39173ac6 100644 --- a/api/envoy/extensions/filters/http/ext_authz/v3/BUILD +++ b/api/envoy/extensions/filters/http/ext_authz/v3/BUILD @@ -7,6 +7,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/annotations:pkg", + "//envoy/config/common/mutation_rules/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/type/matcher/v3:pkg", "//envoy/type/v3:pkg", diff --git a/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto b/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto index f4750864c91b..986fa2c9166c 100644 --- a/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto +++ b/api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.ext_authz.v3; +import "envoy/config/common/mutation_rules/v3/mutation_rules.proto"; import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/config_source.proto"; import "envoy/config/core/v3/grpc_service.proto"; @@ -28,10 +29,10 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // External Authorization :ref:`configuration overview `. // [#extension: envoy.filters.http.ext_authz] -// [#next-free-field: 26] +// [#next-free-field: 27] message ExtAuthz { option (udpa.annotations.versioning).previous_message_type = - "envoy.config.filter.http.ext_authz.v2.ExtAuthz"; + "envoy.config.filter.http.ext_authz.v3.ExtAuthz"; reserved 4; @@ -261,6 +262,23 @@ message ExtAuthz { // It's recommended you set this to true unless you already rely on the old behavior. False is the // default only for backwards compatibility. bool encode_raw_headers = 23; + + // Rules for what modifications an ext_authz server may make to the request headers before + // continuing decoding / forwarding upstream. + // + // If set to anything, enables header mutation checking against configured rules. Note that + // :ref:`HeaderMutationRules ` + // has defaults that change ext_authz behavior. Also note that if this field is set to anything, + // ext_authz can no longer append to :-prefixed headers. + // + // If empty, header mutation rule checking is completely disabled. + // + // Regardless of what is configured here, ext_authz cannot remove :-prefixed headers. + // + // This field and ``validate_mutations`` have different use cases. ``validate_mutations`` enables + // correctness checks for all header / query parameter mutations (e.g. for invalid characters). + // This field allows the filter to reject mutations to specific headers. + config.common.mutation_rules.v3.HeaderMutationRules decoder_header_mutation_rules = 26; } // Configuration for buffering the request data. diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 7c20910cd10c..0f340c48d22c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -238,6 +238,12 @@ new_features: change: | Added support to healthcheck with ProxyProtocol in TCP Healthcheck by setting :ref:`health_check_config `. +- area: ext_authz + change: | + added + :ref:`decoder_header_mutation_rules ` + which allows you to configure what decoder header mutations are allowed from the ext_authz + service as well as whether to fail the downstream request if disallowed mutations are requested. - area: access_log change: | added new ``access_log`` command operators to retrieve upstream connection information change: ``%UPSTREAM_PEER_URI_SAN%``, diff --git a/source/extensions/filters/http/ext_authz/BUILD b/source/extensions/filters/http/ext_authz/BUILD index 6fbb0b871ed1..02970d791e31 100644 --- a/source/extensions/filters/http/ext_authz/BUILD +++ b/source/extensions/filters/http/ext_authz/BUILD @@ -31,6 +31,7 @@ envoy_cc_library( "//source/common/runtime:runtime_protos_lib", "//source/extensions/filters/common/ext_authz:ext_authz_grpc_lib", "//source/extensions/filters/common/ext_authz:ext_authz_http_lib", + "//source/extensions/filters/common/mutation_rules:mutation_rules_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/ext_authz/v3:pkg_cc_proto", "@envoy_api//envoy/service/auth/v3:pkg_cc_proto", diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index 824682477bc3..88b27ec663a1 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -21,6 +21,8 @@ namespace ExtAuthz { namespace { using MetadataProto = ::envoy::config::core::v3::Metadata; +using Filters::Common::MutationRules::CheckOperation; +using Filters::Common::MutationRules::CheckResult; void fillMetadataContext(const std::vector& source_metadata, const std::vector& metadata_context_namespaces, @@ -289,6 +291,17 @@ void Filter::onDestroy() { } } +CheckResult Filter::validateAndCheckDecoderHeaderMutation( + Filters::Common::MutationRules::CheckOperation operation, absl::string_view key, + absl::string_view value) const { + if (config_->validateMutations() && (!Http::HeaderUtility::headerNameIsValid(key) || + !Http::HeaderUtility::headerValueIsValid(value))) { + return CheckResult::FAIL; + } + // Check header mutation is valid according to configured decoder mutation rules. + return config_->checkDecoderHeaderMutation(operation, Http::LowerCaseString(key), value); +} + void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { state_ = State::Complete; using Filters::Common::ExtAuthz::CheckStatus; @@ -325,69 +338,116 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { ENVOY_STREAM_LOG(trace, "ext_authz filter added header(s) to the request:", *decoder_callbacks_); for (const auto& [key, value] : response->headers_to_set) { - if (config_->validateMutations() && (!Http::HeaderUtility::headerNameIsValid(key) || - !Http::HeaderUtility::headerValueIsValid(value))) { - ENVOY_STREAM_LOG(trace, "Rejected invalid header '{}':'{}'.", *decoder_callbacks_, key, - value); + CheckResult check_result = validateAndCheckDecoderHeaderMutation( + Filters::Common::MutationRules::CheckOperation::SET, key, value); + switch (check_result) { + case CheckResult::OK: + ENVOY_STREAM_LOG(trace, "'{}':'{}'", *decoder_callbacks_, key, value); + request_headers_->setCopy(Http::LowerCaseString(key), value); + break; + case CheckResult::IGNORE: + ENVOY_STREAM_LOG(trace, "Ignoring invalid header to set '{}':'{}'.", *decoder_callbacks_, + key, value); + break; + case CheckResult::FAIL: + ENVOY_STREAM_LOG(trace, "Rejecting invalid header to set '{}':'{}'.", *decoder_callbacks_, + key, value); rejectResponse(); return; } - ENVOY_STREAM_LOG(trace, "'{}':'{}'", *decoder_callbacks_, key, value); - request_headers_->setCopy(Http::LowerCaseString(key), value); } for (const auto& [key, value] : response->headers_to_add) { - if (config_->validateMutations() && (!Http::HeaderUtility::headerNameIsValid(key) || - !Http::HeaderUtility::headerValueIsValid(value))) { - ENVOY_STREAM_LOG(trace, "Rejected invalid header '{}':'{}'.", *decoder_callbacks_, key, - value); + CheckResult check_result = validateAndCheckDecoderHeaderMutation( + Filters::Common::MutationRules::CheckOperation::SET, key, value); + switch (check_result) { + case CheckResult::OK: + ENVOY_STREAM_LOG(trace, "'{}':'{}'", *decoder_callbacks_, key, value); + request_headers_->addCopy(Http::LowerCaseString(key), value); + break; + case CheckResult::IGNORE: + ENVOY_STREAM_LOG(trace, "Ignoring invalid header to add '{}':'{}'.", *decoder_callbacks_, + key, value); + break; + case CheckResult::FAIL: + ENVOY_STREAM_LOG(trace, "Rejecting invalid header to add '{}':'{}'.", *decoder_callbacks_, + key, value); rejectResponse(); return; } - ENVOY_STREAM_LOG(trace, "'{}':'{}'", *decoder_callbacks_, key, value); - request_headers_->addCopy(Http::LowerCaseString(key), value); } for (const auto& [key, value] : response->headers_to_append) { - if (config_->validateMutations() && (!Http::HeaderUtility::headerNameIsValid(key) || - !Http::HeaderUtility::headerValueIsValid(value))) { - ENVOY_STREAM_LOG(trace, "Rejected invalid header '{}':'{}'.", *decoder_callbacks_, key, - value); + CheckResult check_result = validateAndCheckDecoderHeaderMutation( + Filters::Common::MutationRules::CheckOperation::APPEND, key, value); + switch (check_result) { + case CheckResult::OK: { + ENVOY_STREAM_LOG(trace, "'{}':'{}'", *decoder_callbacks_, key, value); + Http::LowerCaseString lowercase_key(key); + const auto header_to_modify = request_headers_->get(lowercase_key); + // TODO(dio): Add a flag to allow appending non-existent headers, without setting it + // first (via `headers_to_add`). For example, given: + // 1. Original headers {"original": "true"} + // 2. Response headers from the authorization servers {{"append": "1"}, {"append": + // "2"}} + // + // Currently it is not possible to add {{"append": "1"}, {"append": "2"}} (the + // intended combined headers: {{"original": "true"}, {"append": "1"}, {"append": + // "2"}}) to the request to upstream server by only sets `headers_to_append`. + if (!header_to_modify.empty()) { + ENVOY_STREAM_LOG(trace, "'{}':'{}'", *decoder_callbacks_, key, value); + // The current behavior of appending is by combining entries with the same key, + // into one entry. The value of that combined entry is separated by ",". + // TODO(dio): Consider to use addCopy instead. + request_headers_->appendCopy(lowercase_key, value); + } + break; + } + case CheckResult::IGNORE: + ENVOY_STREAM_LOG(trace, "Ignoring invalid header to append '{}':'{}'.", *decoder_callbacks_, + key, value); + break; + case CheckResult::FAIL: + ENVOY_STREAM_LOG(trace, "Rejecting invalid header to append '{}':'{}'.", + *decoder_callbacks_, key, value); rejectResponse(); return; } - Http::LowerCaseString lowercase_key(key); - const auto header_to_modify = request_headers_->get(lowercase_key); - // TODO(dio): Add a flag to allow appending non-existent headers, without setting it first - // (via `headers_to_add`). For example, given: - // 1. Original headers {"original": "true"} - // 2. Response headers from the authorization servers {{"append": "1"}, {"append": "2"}} - // - // Currently it is not possible to add {{"append": "1"}, {"append": "2"}} (the intended - // combined headers: {{"original": "true"}, {"append": "1"}, {"append": "2"}}) to the request - // to upstream server by only sets `headers_to_append`. - if (!header_to_modify.empty()) { - ENVOY_STREAM_LOG(trace, "'{}':'{}'", *decoder_callbacks_, key, value); - // The current behavior of appending is by combining entries with the same key, into one - // entry. The value of that combined entry is separated by ",". - // TODO(dio): Consider to use addCopy instead. - request_headers_->appendCopy(lowercase_key, value); - } } ENVOY_STREAM_LOG(trace, "ext_authz filter removed header(s) from the request:", *decoder_callbacks_); for (const auto& key : response->headers_to_remove) { - // We don't allow removing any :-prefixed headers, nor Host, as removing them would make the - // request malformed. - // // If the response contains an invalid header to remove, it's the same as trying to remove a // header that doesn't exist, so just ignore it. - if (!Http::HeaderUtility::isRemovableHeader(key) || - (config_->validateMutations() && !Http::HeaderUtility::headerNameIsValid(key))) { - ENVOY_STREAM_LOG(trace, "Not removing header '{}'.", *decoder_callbacks_, key); + if (config_->validateMutations() && !Http::HeaderUtility::headerNameIsValid(key)) { + ENVOY_STREAM_LOG(trace, "Ignoring invalid header removal '{}'.", *decoder_callbacks_, key); + continue; + } + // We don't allow removing any :-prefixed headers, nor Host, as removing them would make the + // request malformed. checkDecoderHeaderMutation also performs this check, however, so only + // perform this check explicitly if decoder header mutation rules is empty. + if (!config_->hasDecoderHeaderMutationRules() && + !Http::HeaderUtility::isRemovableHeader(key)) { + ENVOY_STREAM_LOG(trace, "Ignoring invalid header removal '{}'.", *decoder_callbacks_, key); continue; } - ENVOY_STREAM_LOG(trace, "'{}'", *decoder_callbacks_, key); - request_headers_->remove(Http::LowerCaseString(key)); + // Check header mutation is valid according to configured decoder header mutation rules. + Http::LowerCaseString lowercase_key(key); + switch (config_->checkDecoderHeaderMutation(CheckOperation::REMOVE, lowercase_key, + EMPTY_STRING)) { + case CheckResult::OK: + ENVOY_STREAM_LOG(trace, "'{}'", *decoder_callbacks_, key); + request_headers_->remove(lowercase_key); + break; + case CheckResult::IGNORE: + ENVOY_STREAM_LOG(trace, "Ignoring disallowed header removal '{}'.", *decoder_callbacks_, + key); + break; + case CheckResult::FAIL: + ENVOY_STREAM_LOG(trace, "Rejecting disallowed header removal '{}'.", *decoder_callbacks_, + key); + rejectResponse(); + return; + } } if (!response->response_headers_to_add.empty()) { diff --git a/source/extensions/filters/http/ext_authz/ext_authz.h b/source/extensions/filters/http/ext_authz/ext_authz.h index ad2b14f92408..81e6cee23d81 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.h +++ b/source/extensions/filters/http/ext_authz/ext_authz.h @@ -24,6 +24,7 @@ #include "source/extensions/filters/common/ext_authz/ext_authz.h" #include "source/extensions/filters/common/ext_authz/ext_authz_grpc_impl.h" #include "source/extensions/filters/common/ext_authz/ext_authz_http_impl.h" +#include "source/extensions/filters/common/mutation_rules/mutation_rules.h" namespace Envoy { namespace Extensions { @@ -74,6 +75,12 @@ class FilterConfig { status_on_error_(toErrorCode(config.status_on_error().code())), validate_mutations_(config.validate_mutations()), scope_(scope), + decoder_header_mutation_checker_( + config.has_decoder_header_mutation_rules() + ? absl::optional( + Filters::Common::MutationRules::Checker( + config.decoder_header_mutation_rules(), factory_context.regexEngine())) + : absl::nullopt), runtime_(factory_context.runtime()), http_context_(factory_context.httpContext()), filter_enabled_(config.has_filter_enabled() ? absl::optional( @@ -162,6 +169,20 @@ class FilterConfig { bool headersAsBytes() const { return encode_raw_headers_; } + Filters::Common::MutationRules::CheckResult + checkDecoderHeaderMutation(const Filters::Common::MutationRules::CheckOperation& operation, + const Http::LowerCaseString& key, absl::string_view value) const { + if (!decoder_header_mutation_checker_.has_value()) { + return Filters::Common::MutationRules::CheckResult::OK; + } + return decoder_header_mutation_checker_->check(operation, key, value); + } + + // Used for headers_to_remove to avoid a redundant pseudo header check. + bool hasDecoderHeaderMutationRules() const { + return decoder_header_mutation_checker_.has_value(); + } + Http::Code statusOnError() const { return status_on_error_; } bool validateMutations() const { return validate_mutations_; } @@ -252,6 +273,7 @@ class FilterConfig { const Http::Code status_on_error_; const bool validate_mutations_; Stats::Scope& scope_; + const absl::optional decoder_header_mutation_checker_; Runtime::Loader& runtime_; Http::Context& http_context_; LabelsMap destination_labels_; @@ -372,6 +394,13 @@ class Filter : public Logger::Loggable, void onComplete(Filters::Common::ExtAuthz::ResponsePtr&&) override; private: + // Convenience function for the following: + // 1. If `validate_mutations` is set to true, validate header key and value. + // 2. If `decoder_header_mutation_rules` is set, check that mutation is allowed. + Filters::Common::MutationRules::CheckResult + validateAndCheckDecoderHeaderMutation(Filters::Common::MutationRules::CheckOperation operation, + absl::string_view key, absl::string_view value) const; + // Called when the filter is configured to reject invalid responses & the authz response contains // invalid header or query parameters. Sends a local response with the configured rejection status // code. diff --git a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc index 0495698266bf..a31789636957 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc @@ -176,6 +176,7 @@ class ExtAuthzGrpcIntegrationTest {"not-allowed", "nope"}, {"regex-food", "food"}, {"regex-fool", "fool"}, + {"disallow-mutation-downstream-req", "authz resp cannot set or append to this header"}, // If the below header exists in the downstream request, it is NOT copied in authz request. {Envoy::Extensions::Filters::Common::ExtAuthz::Headers::get().EnvoyAuthPartialBody.get(), "shouldn't be visible in authz request"}, @@ -330,6 +331,11 @@ class ExtAuthzGrpcIntegrationTest EXPECT_THAT(upstream_request_->headers(), Http::HeaderValueOf("x-envoy-auth-failure-mode-allowed", "true")); } + // Check that ext_authz didn't remove this downstream header which should be immune to + // mutations. + EXPECT_THAT(upstream_request_->headers(), + Http::HeaderValueOf("disallow-mutation-downstream-req", + "authz resp cannot set or append to this header")); for (const auto& header_to_add : opts.headers_to_add) { EXPECT_THAT(upstream_request_->headers(), @@ -616,6 +622,10 @@ class ExtAuthzGrpcIntegrationTest patterns: - prefix: allowed-prefix-denied + decoder_header_mutation_rules: + disallow_expression: + regex: ^disallow-mutation.* + with_request_body: max_request_bytes: 1024 allow_partial_message: true @@ -645,26 +655,27 @@ class ExtAuthzHttpIntegrationTest void initiateClientConnection() { auto conn = makeClientConnection(lookupPort("http")); codec_client_ = makeHttpConnection(std::move(conn)); - const auto headers = - Http::TestRequestHeaderMapImpl{{":method", "GET"}, - {":path", "/"}, - {":scheme", "http"}, - {":authority", "host"}, - {"x-case-sensitive-header", case_sensitive_header_value_}, - {"baz", "foo"}, - {"bat", "foo"}, - {"remove-me", "upstream-should-not-see-me"}, - {"x-duplicate", "one"}, - {"x-duplicate", "two"}, - {"x-duplicate", "three"}, - {"allowed-prefix-one", "one"}, - {"allowed-prefix-two", "two"}, - {"allowed-prefix-denied", "blah"}, - {"not-allowed", "nope"}, - {"authorization", "legit"}, - {"regex-food", "food"}, - {"regex-fool", "fool"}, - {"x-forwarded-for", "1.2.3.4"}}; + const auto headers = Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-case-sensitive-header", case_sensitive_header_value_}, + {"baz", "foo"}, + {"bat", "foo"}, + {"remove-me", "upstream-should-not-see-me"}, + {"x-duplicate", "one"}, + {"x-duplicate", "two"}, + {"x-duplicate", "three"}, + {"allowed-prefix-one", "one"}, + {"allowed-prefix-two", "two"}, + {"allowed-prefix-denied", "blah"}, + {"not-allowed", "nope"}, + {"authorization", "legit"}, + {"regex-food", "food"}, + {"regex-fool", "fool"}, + {"disallow-mutation-downstream-req", "authz resp cannot set or append to this header"}, + {"x-forwarded-for", "1.2.3.4"}}; if (client_request_body_.empty()) { response_ = codec_client_->makeHeaderOnlyRequest(headers); } else { @@ -681,32 +692,19 @@ class ExtAuthzHttpIntegrationTest result = ext_authz_request_->waitForEndStream(*dispatcher_); RELEASE_ASSERT(result, result.message()); - EXPECT_EQ("one", ext_authz_request_->headers() - .get(Http::LowerCaseString(std::string("allowed-prefix-one")))[0] - ->value() - .getStringView()); - EXPECT_EQ("two", ext_authz_request_->headers() - .get(Http::LowerCaseString(std::string("allowed-prefix-two")))[0] - ->value() - .getStringView()); - EXPECT_EQ("legit", ext_authz_request_->headers() - .get(Http::LowerCaseString(std::string("authorization")))[0] - ->value() - .getStringView()); + EXPECT_THAT(ext_authz_request_->headers(), Http::HeaderValueOf("allowed-prefix-one", "one")); + EXPECT_THAT(ext_authz_request_->headers(), Http::HeaderValueOf("allowed-prefix-two", "two")); + EXPECT_THAT(ext_authz_request_->headers(), Http::HeaderValueOf("authorization", "legit")); + EXPECT_THAT(ext_authz_request_->headers(), Http::HeaderValueOf("regex-food", "food")); + EXPECT_THAT(ext_authz_request_->headers(), Http::HeaderValueOf("regex-fool", "fool")); + EXPECT_TRUE(ext_authz_request_->headers() .get(Http::LowerCaseString(std::string("not-allowed"))) .empty()); EXPECT_TRUE(ext_authz_request_->headers() .get(Http::LowerCaseString(std::string("allowed-prefix-denied"))) .empty()); - EXPECT_EQ("food", ext_authz_request_->headers() - .get(Http::LowerCaseString(std::string("regex-food")))[0] - ->value() - .getStringView()); - EXPECT_EQ("fool", ext_authz_request_->headers() - .get(Http::LowerCaseString(std::string("regex-fool")))[0] - ->value() - .getStringView()); + if (encodeRawHeaders()) { // Duplicate headers should NOT be merged. const auto duplicate = @@ -740,9 +738,12 @@ class ExtAuthzHttpIntegrationTest {":status", "200"}, {"baz", "baz"}, {"bat", "bar"}, + {"authz-add-disallow-mutation", "this should not be allowed due to disallow_expression"}, {"x-append-bat", "append-foo"}, {"x-append-bat", "append-bar"}, {"x-envoy-auth-headers-to-remove", "remove-me"}, + // Try to remove this header that should not be able to be removed. + {"x-envoy-auth-headers-to-remove", "disallow-mutation-downstream-req"}, }; ext_authz_request_->encodeHeaders(response_headers, true); } @@ -831,6 +832,11 @@ class ExtAuthzHttpIntegrationTest EXPECT_TRUE(upstream_request_->headers() .get(Http::LowerCaseString{"x-envoy-auth-headers-to-remove"}) .empty()); + // The side stream tried to add this header that violates the disallow_expression header + // mutation rule. Make sure it did not get added. + EXPECT_TRUE(upstream_request_->headers() + .get(Http::LowerCaseString{"authz-add-disallow-mutation"}) + .empty()); upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); ASSERT_TRUE(response_->waitForEndStream()); @@ -847,10 +853,16 @@ class ExtAuthzHttpIntegrationTest std::string client_request_body_; const Http::LowerCaseString case_sensitive_header_name_{"x-case-sensitive-header"}; const std::string case_sensitive_header_value_{"Case-Sensitive"}; + // TODO: mutation rule const std::string legacy_default_config_ = R"EOF( disallowed_headers: patterns: - prefix: allowed-prefix-denied + + decoder_header_mutation_rules: + disallow_expression: + regex: disallow-mutation.* + http_service: server_uri: uri: "ext_authz:9000" @@ -889,6 +901,10 @@ class ExtAuthzHttpIntegrationTest patterns: - prefix: allowed-prefix-denied + decoder_header_mutation_rules: + disallow_expression: + regex: disallow-mutation.* + http_service: server_uri: uri: "ext_authz:9000" @@ -977,13 +993,20 @@ TEST_P(ExtAuthzGrpcIntegrationTest, CheckAfterBufferingComplete) { // Start a client connection and start request. Http::TestRequestHeaderMapImpl headers{ - {":method", "POST"}, {":path", "/test"}, - {":scheme", "http"}, {":authority", "host"}, - {"x-duplicate", "one"}, {"x-duplicate", "two"}, - {"x-duplicate", "three"}, {"allowed-prefix-one", "one"}, - {"allowed-prefix-two", "two"}, {"allowed-prefix-denied", "will not be sent"}, - {"not-allowed", "nope"}, {"regex-food", "food"}, - {"regex-fool", "fool"}}; + {":method", "POST"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-duplicate", "one"}, + {"x-duplicate", "two"}, + {"x-duplicate", "three"}, + {"allowed-prefix-one", "one"}, + {"allowed-prefix-two", "two"}, + {"allowed-prefix-denied", "will not be sent"}, + {"not-allowed", "nope"}, + {"regex-food", "food"}, + {"regex-fool", "fool"}, + {"disallow-mutation-downstream-req", "authz resp cannot set or append to this header"}}; auto conn = makeClientConnection(lookupPort("http")); codec_client_ = makeHttpConnection(std::move(conn)); @@ -1156,13 +1179,21 @@ TEST_P(ExtAuthzGrpcIntegrationTest, FailureModeAllowNonUtf8) { invalid_unicode.append(1, char(0x28)); invalid_unicode.append("valid_suffix"); Http::TestRequestHeaderMapImpl headers{ - {":method", "POST"}, {":path", "/test"}, - {":scheme", "http"}, {":authority", "host"}, - {"x-bypass", invalid_unicode}, {"allowed-prefix-one", "one"}, - {"allowed-prefix-two", "two"}, {"allowed-prefix-denied", "denied"}, - {"not-allowed", "nope"}, {"x-duplicate", "one"}, - {"x-duplicate", "two"}, {"x-duplicate", "three"}, - {"regex-food", "food"}, {"regex-fool", "fool"}}; + {":method", "POST"}, + {":path", "/test"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-bypass", invalid_unicode}, + {"allowed-prefix-one", "one"}, + {"allowed-prefix-two", "two"}, + {"allowed-prefix-denied", "denied"}, + {"not-allowed", "nope"}, + {"x-duplicate", "one"}, + {"x-duplicate", "two"}, + {"x-duplicate", "three"}, + {"regex-food", "food"}, + {"regex-fool", "fool"}, + {"disallow-mutation-downstream-req", "authz resp cannot set or append to this header"}}; response_ = codec_client_->makeRequestWithBody(headers, {}); diff --git a/test/extensions/filters/http/ext_authz/ext_authz_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_test.cc index 042d640bcd8c..53f2fdf33583 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_test.cc @@ -388,6 +388,329 @@ TEST_F(InvalidMutationTest, QueryParametersToSetValue) { testResponse(response); } +struct DecoderHeaderMutationRulesTestOpts { + absl::optional rules; + bool expect_reject_response = false; + Filters::Common::ExtAuthz::UnsafeHeaderVector allowed_headers_to_add; + Filters::Common::ExtAuthz::UnsafeHeaderVector disallowed_headers_to_add; + Filters::Common::ExtAuthz::UnsafeHeaderVector allowed_headers_to_append; + Filters::Common::ExtAuthz::UnsafeHeaderVector disallowed_headers_to_append; + Filters::Common::ExtAuthz::UnsafeHeaderVector allowed_headers_to_set; + Filters::Common::ExtAuthz::UnsafeHeaderVector disallowed_headers_to_set; + std::vector allowed_headers_to_remove; + std::vector disallowed_headers_to_remove; +}; +class DecoderHeaderMutationRulesTest + : public HttpFilterTestBase> { +public: + void runTest(DecoderHeaderMutationRulesTestOpts opts) { + InSequence s; + + envoy::extensions::filters::http::ext_authz::v3::ExtAuthz proto_config{}; + TestUtility::loadFromYaml(R"( + transport_api_version: V3 + grpc_service: + envoy_grpc: + cluster_name: "ext_authz_server" + failure_mode_allow: false + )", + proto_config); + if (opts.rules != std::nullopt) { + *(proto_config.mutable_decoder_header_mutation_rules()) = *opts.rules; + } + + initialize(proto_config); + + // Simulate a downstream request. + populateRequestHeadersFromOpts(opts); + + ON_CALL(decoder_filter_callbacks_, connection()) + .WillByDefault(Return(OptRef{connection_})); + connection_.stream_info_.downstream_connection_info_provider_->setRemoteAddress(addr_); + connection_.stream_info_.downstream_connection_info_provider_->setLocalAddress(addr_); + EXPECT_CALL(*client_, check(_, _, _, _)) + .WillOnce(Invoke( + [&](Filters::Common::ExtAuthz::RequestCallbacks& callbacks, + const envoy::service::auth::v3::CheckRequest&, Tracing::Span&, + const StreamInfo::StreamInfo&) -> void { request_callbacks_ = &callbacks; })); + + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + if (opts.expect_reject_response) { + EXPECT_CALL(decoder_filter_callbacks_, encodeHeaders_(_, true)) + .WillOnce(Invoke([&](const Http::ResponseHeaderMap& headers, bool) -> void { + EXPECT_EQ(headers.getStatusValue(), + std::to_string(enumToInt(Http::Code::InternalServerError))); + })); + } else { + EXPECT_CALL(decoder_filter_callbacks_, continueDecoding()); + } + + // Construct authz response from opts. + Filters::Common::ExtAuthz::Response response = getResponseFromOpts(opts); + + request_callbacks_->onComplete(std::make_unique(response)); + + if (!opts.expect_reject_response) { + // Now make sure the downstream header is / is not there depending on the test. + checkRequestHeadersFromOpts(opts); + } + } + + void populateRequestHeadersFromOpts(const DecoderHeaderMutationRulesTestOpts& opts) { + for (const auto& [key, _] : opts.allowed_headers_to_set) { + request_headers_.addCopy(Http::LowerCaseString(key), "will be overridden"); + } + for (const auto& [key, _] : opts.disallowed_headers_to_set) { + request_headers_.addCopy(Http::LowerCaseString(key), "will not be overridden"); + } + + for (const auto& [key, _] : opts.allowed_headers_to_append) { + request_headers_.addCopy(Http::LowerCaseString(key), "will be appended to"); + } + for (const auto& [key, _] : opts.disallowed_headers_to_append) { + request_headers_.addCopy(Http::LowerCaseString(key), "will not be appended to"); + } + + for (const auto& key : opts.allowed_headers_to_remove) { + request_headers_.addCopy(Http::LowerCaseString(key), "will be removed"); + } + for (const auto& key : opts.disallowed_headers_to_remove) { + request_headers_.addCopy(Http::LowerCaseString(key), "will not be removed"); + } + } + + void checkRequestHeadersFromOpts(const DecoderHeaderMutationRulesTestOpts& opts) { + for (const auto& [key, value] : opts.allowed_headers_to_add) { + EXPECT_EQ(request_headers_.get_(Http::LowerCaseString(key)), value) + << "(key: '" << key << "')"; + } + for (const auto& [key, _] : opts.disallowed_headers_to_add) { + EXPECT_FALSE(request_headers_.has(Http::LowerCaseString(key))) << "(key: '" << key << "')"; + } + + for (const auto& [key, value] : opts.allowed_headers_to_set) { + EXPECT_EQ(request_headers_.get_(Http::LowerCaseString(key)), value) + << "(key: '" << key << "')"; + } + for (const auto& [key, _] : opts.disallowed_headers_to_set) { + EXPECT_EQ(request_headers_.get_(Http::LowerCaseString(key)), "will not be overridden") + << "(key: '" << key << "')"; + } + + for (const auto& [key, value] : opts.allowed_headers_to_append) { + EXPECT_EQ(request_headers_.get_(Http::LowerCaseString(key)), + absl::StrCat("will be appended to,", value)) + << "(key: '" << key << "')"; + } + for (const auto& [key, value] : opts.disallowed_headers_to_append) { + EXPECT_EQ(request_headers_.get_(Http::LowerCaseString(key)), "will not be appended to") + << "(key: '" << key << "')"; + } + + for (const auto& key : opts.allowed_headers_to_remove) { + EXPECT_FALSE(request_headers_.has(Http::LowerCaseString(key))) << "(key: '" << key << "')"; + } + for (const auto& key : opts.disallowed_headers_to_remove) { + EXPECT_EQ(request_headers_.get_(Http::LowerCaseString(key)), "will not be removed") + << "(key: '" << key << "')"; + } + } + + static Filters::Common::ExtAuthz::Response + getResponseFromOpts(const DecoderHeaderMutationRulesTestOpts& opts) { + Filters::Common::ExtAuthz::Response response; + response.status = Filters::Common::ExtAuthz::CheckStatus::OK; + + for (const auto& vec : {opts.allowed_headers_to_add, opts.disallowed_headers_to_add}) { + for (const auto& [key, value] : vec) { + response.headers_to_add.emplace_back(key, value); + } + } + + for (const auto& vec : {opts.allowed_headers_to_set, opts.disallowed_headers_to_set}) { + for (const auto& [key, value] : vec) { + response.headers_to_set.emplace_back(key, value); + } + } + + for (const auto& vec : {opts.allowed_headers_to_append, opts.disallowed_headers_to_append}) { + for (const auto& [key, value] : vec) { + response.headers_to_append.emplace_back(key, value); + } + } + + for (const auto& vec : {opts.allowed_headers_to_remove, opts.disallowed_headers_to_remove}) { + for (const auto& key : vec) { + response.headers_to_remove.emplace_back(key); + } + } + + return response; + } +}; + +// If decoder_header_mutation_rules is empty, there should be no additional restrictions to +// ext_authz header mutations. +TEST_F(DecoderHeaderMutationRulesTest, EmptyConfig) { + DecoderHeaderMutationRulesTestOpts opts; + opts.allowed_headers_to_add = {{":authority", "google"}}; + opts.allowed_headers_to_append = {{"normal", "one"}, {":fake-pseudo-header", "append me"}}; + opts.allowed_headers_to_set = {{"x-envoy-whatever", "override"}}; + opts.allowed_headers_to_remove = {"delete-me"}; + // These are not allowed not because of the header mutation rules config, but because ext_authz + // doesn't allow the removable of headers starting with `:` anyway. + opts.disallowed_headers_to_remove = {":method", ":fake-pseudoheader"}; + runTest(opts); +} + +// Test behavior when the rules field exists but all sub-fields are their default values (should be +// exactly the same as above) +TEST_F(DecoderHeaderMutationRulesTest, ExplicitDefaultConfig) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.disallowed_headers_to_add = {{":authority", "google"}}; + opts.allowed_headers_to_append = {{"normal", "one"}}; + opts.disallowed_headers_to_append = {{":fake-pseudo-header", "append me"}}; + opts.disallowed_headers_to_set = {{"x-envoy-whatever", "override"}}; + opts.allowed_headers_to_remove = {"delete-me"}; + // These are not allowed not because of the header mutation rules config, but because ext_authz + // doesn't allow the removable of headers starting with `:` anyway. + opts.disallowed_headers_to_remove = {":method", ":fake-pseudoheader"}; + runTest(opts); +} + +TEST_F(DecoderHeaderMutationRulesTest, DisallowAll) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_all()->set_value(true); + + opts.disallowed_headers_to_add = {{"cant-add-me", "sad"}}; + opts.disallowed_headers_to_append = {{"cant-append-to-me", "fail"}}; + opts.disallowed_headers_to_set = {{"cant-override-me", "nope"}}; + opts.disallowed_headers_to_remove = {"cant-delete-me"}; + runTest(opts); +} + +TEST_F(DecoderHeaderMutationRulesTest, RejectResponseAdd) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_all()->set_value(true); + opts.rules->mutable_disallow_is_error()->set_value(true); + opts.expect_reject_response = true; + + opts.disallowed_headers_to_add = {{"cant-add-me", "sad"}}; + runTest(opts); +} + +TEST_F(DecoderHeaderMutationRulesTest, RejectResponseAppend) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_all()->set_value(true); + opts.rules->mutable_disallow_is_error()->set_value(true); + opts.expect_reject_response = true; + + opts.disallowed_headers_to_append = {{"cant-append-to-me", "fail"}}; + runTest(opts); +} + +TEST_F(DecoderHeaderMutationRulesTest, RejectResponseAppendPseudoheader) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_is_error()->set_value(true); + opts.expect_reject_response = true; + + opts.disallowed_headers_to_append = {{":fake-pseudo-header", "fail"}}; + runTest(opts); +} + +TEST_F(DecoderHeaderMutationRulesTest, RejectResponseSet) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_all()->set_value(true); + opts.rules->mutable_disallow_is_error()->set_value(true); + opts.expect_reject_response = true; + + opts.disallowed_headers_to_set = {{"cant-override-me", "nope"}}; + runTest(opts); +} + +TEST_F(DecoderHeaderMutationRulesTest, RejectResponseRemove) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_all()->set_value(true); + opts.rules->mutable_disallow_is_error()->set_value(true); + opts.expect_reject_response = true; + + opts.disallowed_headers_to_remove = {"cant-delete-me"}; + runTest(opts); +} + +TEST_F(DecoderHeaderMutationRulesTest, RejectResponseRemovePseudoHeader) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_is_error()->set_value(true); + opts.expect_reject_response = true; + + opts.disallowed_headers_to_remove = {":fake-pseudo-header"}; + runTest(opts); +} + +TEST_F(DecoderHeaderMutationRulesTest, DisallowExpression) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_expression()->set_regex("^x-example-.*"); + + opts.allowed_headers_to_add = {{"add-me-one", "one"}, {"add-me-two", "two"}}; + opts.disallowed_headers_to_add = {{"x-example-add", "nope"}}; + opts.allowed_headers_to_append = {{"append-to-me", "appended value"}}; + opts.disallowed_headers_to_append = {{"x-example-append", "no sir"}}; + opts.allowed_headers_to_set = {{"override-me", "new value"}}; + opts.disallowed_headers_to_set = {{"x-example-set", "no can do"}}; + opts.allowed_headers_to_remove = {"delete-me"}; + opts.disallowed_headers_to_remove = {"x-example-remove"}; + runTest(opts); +} + +// Tests that allow_expression overrides other settings (except disallow, which is tested +// elsewhere). +TEST_F(DecoderHeaderMutationRulesTest, AllowExpression) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_all()->set_value(true); + opts.rules->mutable_allow_expression()->set_regex("^x-allow-.*"); + + opts.allowed_headers_to_add = {{"x-allow-add-me-one", "one"}, {"x-allow-add-me-two", "two"}}; + opts.disallowed_headers_to_add = {{"not-allowed", "nope"}}; + opts.allowed_headers_to_append = {{"x-allow-append-to-me", "appended value"}}; + opts.disallowed_headers_to_append = {{"xx-allow-wrong-prefix", "no sir"}}; + opts.allowed_headers_to_set = {{"x-allow-override-me", "new value"}}; + opts.disallowed_headers_to_set = {{"cant-set-me", "no can do"}}; + opts.allowed_headers_to_remove = {"x-allow-delete-me"}; + opts.disallowed_headers_to_remove = {"cannot-remove"}; + runTest(opts); +} + +// Tests that disallow_expression overrides allow_expression. +TEST_F(DecoderHeaderMutationRulesTest, OverlappingAllowAndDisallowExpressions) { + DecoderHeaderMutationRulesTestOpts opts; + opts.rules = envoy::config::common::mutation_rules::v3::HeaderMutationRules(); + opts.rules->mutable_disallow_all()->set_value(true); + // Note the disallow expression's matches are a subset of the allow expression's matches. + opts.rules->mutable_allow_expression()->set_regex(".*allowed.*"); + opts.rules->mutable_disallow_expression()->set_regex(".*disallowed.*"); + + opts.allowed_headers_to_add = {{"allowed-add", "yes"}}; + opts.disallowed_headers_to_add = {{"disallowed-add", "nope"}}; + opts.allowed_headers_to_append = {{"allowed-append", "appended value"}}; + opts.disallowed_headers_to_append = {{"disallowed-append", "no sir"}}; + opts.allowed_headers_to_set = {{"allowed-set", "new value"}}; + opts.disallowed_headers_to_set = {{"disallowed-set", "no can do"}}; + opts.allowed_headers_to_remove = {"allowed-remove"}; + opts.disallowed_headers_to_remove = {"disallowed-remove"}; + runTest(opts); +} + // Test that the per route config is properly merged: more specific keys override previous keys. TEST_F(HttpFilterTest, MergeConfig) { envoy::extensions::filters::http::ext_authz::v3::ExtAuthzPerRoute settings; From 244feec4af4cceb5e331301f89dbbdeca6b84291 Mon Sep 17 00:00:00 2001 From: zirain Date: Fri, 7 Jun 2024 10:06:45 +0800 Subject: [PATCH 44/61] OpenTelemetry access logger support extension formatters (#34469) * api change Signed-off-by: zirain * implement Signed-off-by: zirain * format Signed-off-by: zirain * fix test Signed-off-by: zirain * use Formatter::CommandParserPtr Signed-off-by: zirain * changelogs Signed-off-by: zirain * format Signed-off-by: zirain --------- Signed-off-by: zirain --- .../access_loggers/open_telemetry/v3/BUILD | 1 + .../open_telemetry/v3/logs_service.proto | 8 +- changelogs/current.yaml | 5 ++ .../open_telemetry/access_log_impl.cc | 8 +- .../open_telemetry/access_log_impl.h | 3 +- .../access_loggers/open_telemetry/config.cc | 10 ++- .../open_telemetry/substitution_formatter.cc | 7 +- .../open_telemetry/substitution_formatter.h | 8 +- .../access_loggers/open_telemetry/BUILD | 22 +++++- .../open_telemetry/access_log_impl_test.cc | 9 ++- .../substitution_formatter_speed_test.cc | 3 +- .../substitution_formatter_test.cc | 74 ++++++++++++++----- 12 files changed, 125 insertions(+), 33 deletions(-) diff --git a/api/envoy/extensions/access_loggers/open_telemetry/v3/BUILD b/api/envoy/extensions/access_loggers/open_telemetry/v3/BUILD index 95eff6986b7f..dad7cd1aef35 100644 --- a/api/envoy/extensions/access_loggers/open_telemetry/v3/BUILD +++ b/api/envoy/extensions/access_loggers/open_telemetry/v3/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/core/v3:pkg", "//envoy/extensions/access_loggers/grpc/v3:pkg", "@com_github_cncf_xds//udpa/annotations:pkg", "@opentelemetry_proto//:common", diff --git a/api/envoy/extensions/access_loggers/open_telemetry/v3/logs_service.proto b/api/envoy/extensions/access_loggers/open_telemetry/v3/logs_service.proto index 0e4e89cdb6ac..641276a64bd6 100644 --- a/api/envoy/extensions/access_loggers/open_telemetry/v3/logs_service.proto +++ b/api/envoy/extensions/access_loggers/open_telemetry/v3/logs_service.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.access_loggers.open_telemetry.v3; +import "envoy/config/core/v3/extension.proto"; import "envoy/extensions/access_loggers/grpc/v3/als.proto"; import "opentelemetry/proto/common/v1/common.proto"; @@ -22,7 +23,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // populate `opentelemetry.proto.collector.v1.logs.ExportLogsServiceRequest.resource_logs `_. // In addition, the request start time is set in the dedicated field. // [#extension: envoy.access_loggers.open_telemetry] -// [#next-free-field: 7] +// [#next-free-field: 8] message OpenTelemetryAccessLogConfig { // [#comment:TODO(itamarkam): add 'filter_state_objects_to_log' to logs.] grpc.v3.CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message = {required: true}]; @@ -51,4 +52,9 @@ message OpenTelemetryAccessLogConfig { // ``access_logs.open_telemetry_access_log.``. If non-empty, stats will be rooted at // ``access_logs.open_telemetry_access_log..``. string stat_prefix = 6; + + // Specifies a collection of Formatter plugins that can be called from the access log configuration. + // See the formatters extensions documentation for details. + // [#extension-category: envoy.formatter] + repeated config.core.v3.TypedExtensionConfig formatters = 7; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 0f340c48d22c..6da420210a7e 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -254,6 +254,11 @@ new_features: added :ref:`stat_prefix ` configuration to support additional stat prefix for the OpenTelemetry logger. +- area: open_telemetry + change: | + added :ref:`formatters + ` + configuration to support extension formatter for the OpenTelemetry logger. - area: routing change: | added support in :ref:`file datasource ` implementation diff --git a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc index b630c9c0cf53..f86fda60b50b 100644 --- a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc @@ -9,6 +9,7 @@ #include "source/common/common/assert.h" #include "source/common/config/utility.h" +#include "source/common/formatter/substitution_formatter.h" #include "source/common/http/headers.h" #include "source/common/network/utility.h" #include "source/common/protobuf/message_validator_impl.h" @@ -58,7 +59,8 @@ AccessLog::ThreadLocalLogger::ThreadLocalLogger(GrpcAccessLoggerSharedPtr logger AccessLog::AccessLog( ::Envoy::AccessLog::FilterPtr&& filter, envoy::extensions::access_loggers::open_telemetry::v3::OpenTelemetryAccessLogConfig config, - ThreadLocal::SlotAllocator& tls, GrpcAccessLoggerCacheSharedPtr access_logger_cache) + ThreadLocal::SlotAllocator& tls, GrpcAccessLoggerCacheSharedPtr access_logger_cache, + const std::vector& commands) : Common::ImplBase(std::move(filter)), tls_slot_(tls.allocateSlot()), access_logger_cache_(std::move(access_logger_cache)) { @@ -71,9 +73,9 @@ AccessLog::AccessLog( // Packing the body "AnyValue" to a "KeyValueList" only if it's not empty, otherwise the // formatter would fail to parse it. if (config.body().value_case() != ::opentelemetry::proto::common::v1::AnyValue::VALUE_NOT_SET) { - body_formatter_ = std::make_unique(packBody(config.body())); + body_formatter_ = std::make_unique(packBody(config.body()), commands); } - attributes_formatter_ = std::make_unique(config.attributes()); + attributes_formatter_ = std::make_unique(config.attributes(), commands); } void AccessLog::emitLog(const Formatter::HttpFormatterContext& log_context, diff --git a/source/extensions/access_loggers/open_telemetry/access_log_impl.h b/source/extensions/access_loggers/open_telemetry/access_log_impl.h index 7d1013488eb4..308ec8ac1bdb 100644 --- a/source/extensions/access_loggers/open_telemetry/access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/access_log_impl.h @@ -36,7 +36,8 @@ class AccessLog : public Common::ImplBase { AccessLog( ::Envoy::AccessLog::FilterPtr&& filter, envoy::extensions::access_loggers::open_telemetry::v3::OpenTelemetryAccessLogConfig config, - ThreadLocal::SlotAllocator& tls, GrpcAccessLoggerCacheSharedPtr access_logger_cache); + ThreadLocal::SlotAllocator& tls, GrpcAccessLoggerCacheSharedPtr access_logger_cache, + const std::vector& commands); private: /** diff --git a/source/extensions/access_loggers/open_telemetry/config.cc b/source/extensions/access_loggers/open_telemetry/config.cc index c371b5b724a4..237c5c9fa839 100644 --- a/source/extensions/access_loggers/open_telemetry/config.cc +++ b/source/extensions/access_loggers/open_telemetry/config.cc @@ -7,6 +7,7 @@ #include "source/common/common/assert.h" #include "source/common/common/macros.h" +#include "source/common/formatter/substitution_format_string.h" #include "source/common/grpc/async_client_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/extensions/access_loggers/open_telemetry/access_log_impl.h" @@ -40,9 +41,12 @@ AccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, const envoy::extensions::access_loggers::open_telemetry::v3::OpenTelemetryAccessLogConfig&>( config, context.messageValidationVisitor()); - return std::make_shared(std::move(filter), proto_config, - context.serverFactoryContext().threadLocal(), - getAccessLoggerCacheSingleton(context.serverFactoryContext())); + auto commands = + Formatter::SubstitutionFormatStringUtils::parseFormatters(proto_config.formatters(), context); + + return std::make_shared( + std::move(filter), proto_config, context.serverFactoryContext().threadLocal(), + getAccessLoggerCacheSingleton(context.serverFactoryContext()), commands); } ProtobufTypes::MessagePtr AccessLogFactory::createEmptyConfigProto() { diff --git a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc index e48522bc54ef..a99b80878dd2 100644 --- a/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc +++ b/source/extensions/access_loggers/open_telemetry/substitution_formatter.cc @@ -21,8 +21,9 @@ namespace AccessLoggers { namespace OpenTelemetry { OpenTelemetryFormatter::OpenTelemetryFormatter( - const ::opentelemetry::proto::common::v1::KeyValueList& format_mapping) - : kv_list_output_format_(FormatBuilder().toFormatMapValue(format_mapping)) {} + const ::opentelemetry::proto::common::v1::KeyValueList& format_mapping, + const std::vector& commands) + : kv_list_output_format_(FormatBuilder(commands).toFormatMapValue(format_mapping)) {} OpenTelemetryFormatter::OpenTelemetryFormatMapWrapper OpenTelemetryFormatter::FormatBuilder::toFormatMapValue( @@ -77,7 +78,7 @@ OpenTelemetryFormatter::FormatBuilder::toFormatListValue( std::vector OpenTelemetryFormatter::FormatBuilder::toFormatStringValue(const std::string& string_format) const { - return Formatter::SubstitutionFormatParser::parse(string_format, {}); + return Formatter::SubstitutionFormatParser::parse(string_format, commands_); } ::opentelemetry::proto::common::v1::AnyValue OpenTelemetryFormatter::providersCallback( diff --git a/source/extensions/access_loggers/open_telemetry/substitution_formatter.h b/source/extensions/access_loggers/open_telemetry/substitution_formatter.h index ea797fff13e2..bc97fe91d5b9 100644 --- a/source/extensions/access_loggers/open_telemetry/substitution_formatter.h +++ b/source/extensions/access_loggers/open_telemetry/substitution_formatter.h @@ -26,7 +26,8 @@ OpenTelemetryFormatMapVisitorHelper(Ts...) -> OpenTelemetryFormatMapVisitorHelpe */ class OpenTelemetryFormatter { public: - OpenTelemetryFormatter(const ::opentelemetry::proto::common::v1::KeyValueList& format_mapping); + OpenTelemetryFormatter(const ::opentelemetry::proto::common::v1::KeyValueList& format_mapping, + const std::vector& commands); ::opentelemetry::proto::common::v1::KeyValueList format(const Formatter::HttpFormatterContext& context, const StreamInfo::StreamInfo& info) const; @@ -62,12 +63,17 @@ class OpenTelemetryFormatter { // Methods for building the format map. class FormatBuilder { public: + explicit FormatBuilder(const std::vector& commands) + : commands_(commands) {} std::vector toFormatStringValue(const std::string& string_format) const; OpenTelemetryFormatMapWrapper toFormatMapValue(const ::opentelemetry::proto::common::v1::KeyValueList& struct_format) const; OpenTelemetryFormatListWrapper toFormatListValue( const ::opentelemetry::proto::common::v1::ArrayValue& list_value_format) const; + + private: + const std::vector& commands_; }; // Methods for doing the actual formatting. diff --git a/test/extensions/access_loggers/open_telemetry/BUILD b/test/extensions/access_loggers/open_telemetry/BUILD index 659be55579cf..9764ee5ecd18 100644 --- a/test/extensions/access_loggers/open_telemetry/BUILD +++ b/test/extensions/access_loggers/open_telemetry/BUILD @@ -43,9 +43,11 @@ envoy_extension_cc_test( "//test/mocks/access_log:access_log_mocks", "//test/mocks/grpc:grpc_mocks", "//test/mocks/local_info:local_info_mocks", + "//test/mocks/server:server_mocks", "//test/mocks/ssl:ssl_mocks", "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/thread_local:thread_local_mocks", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/data/accesslog/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/access_loggers/grpc/v3:pkg_cc_proto", "@opentelemetry_proto//:logs_cc_proto", @@ -89,18 +91,36 @@ envoy_extension_cc_test( envoy_extension_cc_test( name = "substitution_formatter_test", srcs = ["substitution_formatter_test.cc"], + copts = select({ + "//bazel:windows_x86_64": [], # TODO: fix the windows ANTLR build + "//conditions:default": [ + "-DUSE_CEL_PARSER", + ], + }), extension_names = ["envoy.access_loggers.open_telemetry"], + tags = ["skip_on_windows"], deps = [ "//source/common/formatter:substitution_formatter_lib", "//source/common/http:exception_lib", "//source/common/http:header_map_lib", "//source/common/router:string_accessor_lib", + "//source/extensions/access_loggers/open_telemetry:config", "//source/extensions/access_loggers/open_telemetry:substitution_formatter_lib", + "//source/extensions/formatter/cel:cel_lib", + "//source/extensions/formatter/cel:config", + "//test/mocks/server:factory_context_mocks", "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/upstream:cluster_info_mocks", "//test/test_common:utility_lib", "@opentelemetry_proto//:logs_cc_proto", - ], + ] + select( + { + "//bazel:windows_x86_64": [], + "//conditions:default": [ + "@com_google_cel_cpp//parser", + ], + }, + ), ) envoy_cc_benchmark_binary( diff --git a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc index 14c19f5bd185..532d2b288eef 100644 --- a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc @@ -2,10 +2,12 @@ #include #include "envoy/common/time.h" +#include "envoy/config/core/v3/extension.pb.h" #include "envoy/data/accesslog/v3/accesslog.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" #include "source/common/buffer/zero_copy_input_stream_impl.h" +#include "source/common/formatter/substitution_format_string.h" #include "source/common/network/address_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/common/router/string_accessor_impl.h" @@ -15,6 +17,7 @@ #include "test/mocks/common.h" #include "test/mocks/grpc/mocks.h" #include "test/mocks/local_info/mocks.h" +#include "test/mocks/server/mocks.h" #include "test/mocks/ssl/mocks.h" #include "test/mocks/stream_info/mocks.h" #include "test/mocks/thread_local/mocks.h" @@ -80,7 +83,10 @@ class AccessLogTest : public testing::Test { EXPECT_EQ(Common::GrpcAccessLoggerType::HTTP, logger_type); return logger_; }); - return std::make_unique(FilterPtr{filter_}, config_, tls_, logger_cache_); + auto commands = + Formatter::SubstitutionFormatStringUtils::parseFormatters(config_.formatters(), context_); + + return std::make_unique(FilterPtr{filter_}, config_, tls_, logger_cache_, commands); } void expectLog(const std::string& expected_log_entry_yaml) { @@ -94,6 +100,7 @@ class AccessLogTest : public testing::Test { MockFilter* filter_{new NiceMock()}; NiceMock tls_; + NiceMock context_; envoy::extensions::access_loggers::open_telemetry::v3::OpenTelemetryAccessLogConfig config_; std::shared_ptr logger_{new MockGrpcAccessLogger()}; std::shared_ptr logger_cache_{new MockGrpcAccessLoggerCache()}; diff --git a/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc b/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc index 11557702df2b..31975e697e2b 100644 --- a/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc +++ b/test/extensions/access_loggers/open_telemetry/substitution_formatter_speed_test.cc @@ -53,7 +53,8 @@ std::unique_ptr makeOpenTelemetryFormatter() { string_value: '%REQ(USER-AGENT)%' )EOF"; TestUtility::loadFromYaml(format_yaml, otel_log_format); - return std::make_unique(otel_log_format); + std::vector commands = {}; + return std::make_unique(otel_log_format, commands); } std::unique_ptr makeStreamInfo() { diff --git a/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc b/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc index 2adedb958afc..54fe710a47f3 100644 --- a/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc +++ b/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc @@ -5,11 +5,14 @@ #include "envoy/common/exception.h" #include "envoy/stream_info/stream_info.h" +#include "source/common/formatter/substitution_format_string.h" #include "source/common/formatter/substitution_formatter.h" #include "source/common/http/header_map_impl.h" #include "source/common/router/string_accessor_impl.h" +#include "source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h" #include "source/extensions/access_loggers/open_telemetry/substitution_formatter.h" +#include "test/mocks/server/factory_context.h" #include "test/mocks/stream_info/mocks.h" #include "test/mocks/upstream/cluster_info.h" #include "test/test_common/utility.h" @@ -127,7 +130,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterPlainStringTest) { string_value: "plain_string_value" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } @@ -162,7 +165,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterTypesTest) { - string_value: "%PROTOCOL%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); KeyValueList expected; TestUtility::loadFromYaml(R"EOF( @@ -298,7 +301,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterNestedObjectsTest) { - string_value: "%PROTOCOL%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); KeyValueList expected; TestUtility::loadFromYaml(R"EOF( values: @@ -416,7 +419,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterSingleOperatorTest) { string_value: "%PROTOCOL%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } @@ -437,7 +440,7 @@ TEST(SubstitutionFormatterTest, EmptyOpenTelemetryFormatterTest) { string_value: "" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } @@ -469,7 +472,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterNonExistentHeaderTest) { string_value: "%RESP(some_response_header)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); @@ -508,7 +511,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterAlternateHeaderTest) { string_value: "%RESP(response_present_header?response_absent_header)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); @@ -546,7 +549,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterDynamicMetadataTest) { string_value: "%DYNAMIC_METADATA(com.test:test_obj:inner_key)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput( formatter.format({&request_header, &response_header, &response_trailer}, stream_info), @@ -591,7 +594,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataTest) { string_value: "%CLUSTER_METADATA(com.test:test_obj:non_existing_key)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput( formatter.format({&request_header, &response_header, &response_trailer}, stream_info), @@ -614,7 +617,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterClusterMetadataNoClusterIn string_value: "%CLUSTER_METADATA(com.test:test_key)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); // Empty optional (absl::nullopt) { @@ -656,7 +659,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateTest) { string_value: "%FILTER_STATE(test_obj)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } @@ -696,7 +699,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateTest) { string_value: "%UPSTREAM_FILTER_STATE(test_obj)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } @@ -726,7 +729,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateSpeciferTest) { string_value: "%FILTER_STATE(test_key:TYPED)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } @@ -762,7 +765,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateSpecife string_value: "%UPSTREAM_FILTER_STATE(test_key:TYPED)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } @@ -787,7 +790,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterFilterStateErrorSpeciferTe string_value: "%FILTER_STATE(test_key:TYPED)%" )EOF", key_mapping); - EXPECT_THROW_WITH_MESSAGE(OpenTelemetryFormatter formatter(key_mapping), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(OpenTelemetryFormatter formatter(key_mapping, {}), EnvoyException, "Invalid filter state serialize type, only support PLAIN/TYPED/FIELD."); } @@ -818,7 +821,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterUpstreamFilterStateErrorSp string_value: "%UPSTREAM_FILTER_STATE(test_key:TYPED)%" )EOF", key_mapping); - EXPECT_THROW_WITH_MESSAGE(OpenTelemetryFormatter formatter(key_mapping), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(OpenTelemetryFormatter formatter(key_mapping, {}), EnvoyException, "Invalid filter state serialize type, only support PLAIN/TYPED/FIELD."); } @@ -855,7 +858,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterStartTimeTest) { string_value: "%START_TIME(%f.%1f.%2f.%3f)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); verifyOpenTelemetryOutput(formatter.format({}, stream_info), expected); } @@ -878,7 +881,7 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterMultiTokenTest) { string_value: "%PROTOCOL% plainstring %REQ(some_request_header)% %RESP(some_response_header)%" )EOF", key_mapping); - OpenTelemetryFormatter formatter(key_mapping); + OpenTelemetryFormatter formatter(key_mapping, {}); absl::optional protocol = Http::Protocol::Http11; EXPECT_CALL(stream_info, protocol()).WillRepeatedly(Return(protocol)); @@ -888,6 +891,41 @@ TEST(SubstitutionFormatterTest, OpenTelemetryFormatterMultiTokenTest) { } } +#ifdef USE_CEL_PARSER +TEST(SubstitutionFormatterTest, CELFormatterTest) { + { + NiceMock context; + StreamInfo::MockStreamInfo stream_info; + Http::TestRequestHeaderMapImpl request_header{{"some_request_header", "SOME_REQUEST_HEADER"}}; + Http::TestResponseHeaderMapImpl response_header{ + {"some_response_header", "SOME_RESPONSE_HEADER"}}; + + OpenTelemetryFormatMap expected = {{"cel_field", "SOME_REQUEST_HEADER SOME_RESPONSE_HEADER"}}; + + envoy::extensions::access_loggers::open_telemetry::v3::OpenTelemetryAccessLogConfig otel_config; + TestUtility::loadFromYaml(R"EOF( + resource_attributes: + values: + - key: "cel_field" + value: + string_value: "%CEL(request.headers['some_request_header'])% %CEL(response.headers['some_response_header'])%" + formatters: + - name: envoy.formatter.cel + typed_config: + "@type": type.googleapis.com/envoy.extensions.formatter.cel.v3.Cel + )EOF", + otel_config); + auto commands = Formatter::SubstitutionFormatStringUtils::parseFormatters( + otel_config.formatters(), context); + + OpenTelemetryFormatter formatter(otel_config.resource_attributes(), commands); + + verifyOpenTelemetryOutput(formatter.format({&request_header, &response_header}, stream_info), + expected); + } +} +#endif + } // namespace } // namespace OpenTelemetry } // namespace AccessLoggers From f039f4d305916a7ac10a9ab83503ae51554b2ca9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:20:00 +0100 Subject: [PATCH 45/61] build(deps): bump elasticsearch from 8.13.4 to 8.14.0 in /examples/skywalking (#34576) build(deps): bump elasticsearch in /examples/skywalking Bumps elasticsearch from 8.13.4 to 8.14.0. --- updated-dependencies: - dependency-name: elasticsearch dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/skywalking/Dockerfile-elasticsearch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/skywalking/Dockerfile-elasticsearch b/examples/skywalking/Dockerfile-elasticsearch index 3942fa1cf21f..4c2b5e562218 100644 --- a/examples/skywalking/Dockerfile-elasticsearch +++ b/examples/skywalking/Dockerfile-elasticsearch @@ -1 +1 @@ -FROM elasticsearch:8.13.4@sha256:11e1717dd78f46fbecf64f7f27a358d331abb5e95b5451c00730243ee44fb665 +FROM elasticsearch:8.14.0@sha256:dfd92a2938094bdd0fc4203796d7ad3426b72b19b4a29d8b26bb032510c5bed6 From 62645ec64cba62aabd2206f1e581a535cb051f7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:20:10 +0100 Subject: [PATCH 46/61] build(deps): bump otel/opentelemetry-collector from `1cffbc3` to `8473e4b` in /examples/opentelemetry (#34575) build(deps): bump otel/opentelemetry-collector Bumps otel/opentelemetry-collector from `1cffbc3` to `8473e4b`. --- updated-dependencies: - dependency-name: otel/opentelemetry-collector dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/opentelemetry/Dockerfile-opentelemetry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/opentelemetry/Dockerfile-opentelemetry b/examples/opentelemetry/Dockerfile-opentelemetry index e27962f834f3..9e7f6e5ecced 100644 --- a/examples/opentelemetry/Dockerfile-opentelemetry +++ b/examples/opentelemetry/Dockerfile-opentelemetry @@ -1,7 +1,7 @@ FROM alpine:3.20@sha256:77726ef6b57ddf65bb551896826ec38bc3e53f75cdde31354fbffb4f25238ebd as otelc_curl RUN apk --update add curl -FROM otel/opentelemetry-collector:latest@sha256:1cffbc33556826c5185cebc1b1dbe1e056b3417bd422cdd4a03747dc6d19388f +FROM otel/opentelemetry-collector:latest@sha256:8473e4b0e81ecc8aa238b5e10a53659ce0e6559d5159d77f8f01f3ecbb1f3391 COPY --from=otelc_curl / / From 54266a082654f21b409f9d0af53a009bdd31b9de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:20:19 +0100 Subject: [PATCH 47/61] build(deps): bump actions/dependency-review-action from 4.3.2 to 4.3.3 (#34574) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 4.3.2 to 4.3.3. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/0c155c5e8556a497adf53f2c18edabf945ed8e70...72eb03d02c7872a771aacd928f3123ac62ad6d3a) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/_precheck_deps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_precheck_deps.yml b/.github/workflows/_precheck_deps.yml index 17facaf412d0..14ccb10ff127 100644 --- a/.github/workflows/_precheck_deps.yml +++ b/.github/workflows/_precheck_deps.yml @@ -55,4 +55,4 @@ jobs: ref: ${{ fromJSON(inputs.request).request.sha }} persist-credentials: false - name: Dependency Review - uses: actions/dependency-review-action@0c155c5e8556a497adf53f2c18edabf945ed8e70 # v4.3.2 + uses: actions/dependency-review-action@72eb03d02c7872a771aacd928f3123ac62ad6d3a # v4.3.3 From 328ea27c978f902502b1cf4e7ca75e4a81e113c5 Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Fri, 7 Jun 2024 13:29:14 -0500 Subject: [PATCH 48/61] mobile: Remove sandboxNetwork=standard (#34606) sandboxNetwork=standard is no longer needed. Risk Level: low (tests only) Testing: CI Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a Signed-off-by: Fredy Wijaya --- mobile/test/common/integration/BUILD | 16 ---------------- mobile/test/java/integration/BUILD | 16 ---------------- .../envoyproxy/envoymobile/engine/testing/BUILD | 4 ---- mobile/test/java/org/chromium/net/BUILD | 16 ---------------- mobile/test/java/org/chromium/net/impl/BUILD | 4 ---- mobile/test/java/org/chromium/net/testing/BUILD | 8 -------- .../java/org/chromium/net/urlconnection/BUILD | 4 ---- mobile/test/kotlin/integration/BUILD | 2 -- mobile/test/kotlin/integration/proxying/BUILD | 12 ------------ 9 files changed, 82 deletions(-) diff --git a/mobile/test/common/integration/BUILD b/mobile/test/common/integration/BUILD index 9ee2a0c7cbea..e7d938ce874e 100644 --- a/mobile/test/common/integration/BUILD +++ b/mobile/test/common/integration/BUILD @@ -14,10 +14,6 @@ envoy_mobile_package() envoy_cc_test( name = "client_integration_test", srcs = ["client_integration_test.cc"], - exec_properties = { - # TODO(willengflow): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, linkopts = select({ "@envoy//bazel:apple": [ # For the TestProxyResolutionApi test. @@ -52,10 +48,6 @@ envoy_cc_test( data = [ "@envoy//test/config/integration/certs", ], - exec_properties = { - # TODO(willengflow): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, external_deps = [ "abseil_strings", ], @@ -79,10 +71,6 @@ envoy_cc_test( data = [ "@envoy//test/config/integration/certs", ], - exec_properties = { - # TODO(willengflow): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, external_deps = [ "abseil_strings", ], @@ -105,10 +93,6 @@ envoy_cc_test( data = [ "@envoy//test/config/integration/certs", ], - exec_properties = { - # TODO(willengflow): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, repository = "@envoy", deps = [ ":xds_integration_test_lib", diff --git a/mobile/test/java/integration/BUILD b/mobile/test/java/integration/BUILD index a45c135d28a9..a08f3616130a 100644 --- a/mobile/test/java/integration/BUILD +++ b/mobile/test/java/integration/BUILD @@ -10,10 +10,6 @@ envoy_mobile_android_test( srcs = [ "AndroidEngineStartUpTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ @@ -33,10 +29,6 @@ envoy_mobile_android_test( srcs = [ "AndroidEngineFlowTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ @@ -57,10 +49,6 @@ envoy_mobile_android_test( srcs = [ "AndroidEngineExplicitFlowTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ @@ -81,10 +69,6 @@ envoy_mobile_android_test( srcs = [ "AndroidEngineSocketTagTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ diff --git a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD index 367f4261dc73..fb431ad3148f 100644 --- a/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD +++ b/mobile/test/java/io/envoyproxy/envoymobile/engine/testing/BUILD @@ -74,10 +74,6 @@ envoy_mobile_android_test( srcs = [ "QuicTestServerTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ diff --git a/mobile/test/java/org/chromium/net/BUILD b/mobile/test/java/org/chromium/net/BUILD index f22d692ee86c..8e615ab387b2 100644 --- a/mobile/test/java/org/chromium/net/BUILD +++ b/mobile/test/java/org/chromium/net/BUILD @@ -17,10 +17,6 @@ envoy_mobile_android_test( "UploadDataProvidersTest.java", "UrlResponseInfoTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ @@ -47,10 +43,6 @@ envoy_mobile_android_test( srcs = [ "CronetUrlRequestContextTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ @@ -76,10 +68,6 @@ envoy_mobile_android_test( srcs = [ "CronetUrlRequestTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ @@ -105,10 +93,6 @@ envoy_mobile_android_test( srcs = [ "BidirectionalStreamTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ diff --git a/mobile/test/java/org/chromium/net/impl/BUILD b/mobile/test/java/org/chromium/net/impl/BUILD index 9605525ebffd..3c34f8e7c9f3 100644 --- a/mobile/test/java/org/chromium/net/impl/BUILD +++ b/mobile/test/java/org/chromium/net/impl/BUILD @@ -16,10 +16,6 @@ envoy_mobile_android_test( "ErrorsTest.java", "UrlRequestCallbackTester.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ diff --git a/mobile/test/java/org/chromium/net/testing/BUILD b/mobile/test/java/org/chromium/net/testing/BUILD index ff20d29a8295..96b8b41c1724 100644 --- a/mobile/test/java/org/chromium/net/testing/BUILD +++ b/mobile/test/java/org/chromium/net/testing/BUILD @@ -84,10 +84,6 @@ envoy_mobile_android_test( srcs = [ "Http2TestServerTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ @@ -108,10 +104,6 @@ envoy_mobile_android_test( srcs = [ "AndroidEnvoyExplicitH2FlowTest.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ diff --git a/mobile/test/java/org/chromium/net/urlconnection/BUILD b/mobile/test/java/org/chromium/net/urlconnection/BUILD index 2872d85568cd..8505f860f917 100644 --- a/mobile/test/java/org/chromium/net/urlconnection/BUILD +++ b/mobile/test/java/org/chromium/net/urlconnection/BUILD @@ -17,10 +17,6 @@ envoy_mobile_android_test( "MessageLoopTest.java", "TestUtil.java", ], - exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", - }, native_deps = [ "//test/jni:libenvoy_jni_with_test_extensions.so", ] + select({ diff --git a/mobile/test/kotlin/integration/BUILD b/mobile/test/kotlin/integration/BUILD index a648eb74c43e..2c376e6322aa 100644 --- a/mobile/test/kotlin/integration/BUILD +++ b/mobile/test/kotlin/integration/BUILD @@ -332,8 +332,6 @@ envoy_mobile_android_test( "FilterThrowingExceptionTest.kt", ], exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", "dockerNetwork": "standard", }, native_deps = [ diff --git a/mobile/test/kotlin/integration/proxying/BUILD b/mobile/test/kotlin/integration/proxying/BUILD index ef4c24c53327..0f8f5864cfa1 100644 --- a/mobile/test/kotlin/integration/proxying/BUILD +++ b/mobile/test/kotlin/integration/proxying/BUILD @@ -11,8 +11,6 @@ envoy_mobile_android_test( "ProxyInfoIntentPerformHTTPRequestUsingProxyTest.kt", ], exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", "dockerNetwork": "standard", }, native_deps = [ @@ -38,8 +36,6 @@ envoy_mobile_android_test( "ProxyInfoIntentPerformHTTPSRequestUsingProxyTest.kt", ], exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", "dockerNetwork": "standard", }, native_deps = [ @@ -65,8 +61,6 @@ envoy_mobile_android_test( "ProxyInfoIntentPerformHTTPSRequestUsingAsyncProxyTest.kt", ], exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", "dockerNetwork": "standard", }, native_deps = [ @@ -92,8 +86,6 @@ envoy_mobile_android_test( "ProxyInfoIntentPerformHTTPSRequestBadHostnameTest.kt", ], exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", "dockerNetwork": "standard", }, native_deps = [ @@ -119,8 +111,6 @@ envoy_mobile_android_test( "ProxyPollPerformHTTPRequestUsingProxyTest.kt", ], exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", "dockerNetwork": "standard", }, native_deps = [ @@ -146,8 +136,6 @@ envoy_mobile_android_test( "ProxyPollPerformHTTPRequestWithoutUsingPACProxyTest.kt", ], exec_properties = { - # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. - "sandboxNetwork": "standard", "dockerNetwork": "standard", }, native_deps = [ From fd19754daaf158830541729d1b10676a113fdc60 Mon Sep 17 00:00:00 2001 From: RenjieTang Date: Fri, 7 Jun 2024 11:50:51 -0700 Subject: [PATCH 49/61] Update QUICHE from cea6f57f9 to c5d1ed730 (#34526) Signed-off-by: Renjie Tang --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index ddf8a30a8a96..a675c338594c 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1192,12 +1192,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "cea6f57f9ce03a5aa2bb0e0d8adcdf3ab452c0c3", - sha256 = "d0187c4c3c74a709727549b020ef90471113d70047dff7d8fd9f2bfd37a6da5b", + version = "c5d1ed730f6062c6647aebe130d78f088028e52f", + sha256 = "d4d61c3189d989d7e51323823ce6f9de6970a09803624e80283e9f8135c9fe4b", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["controlplane", "dataplane_core"], - release_date = "2024-05-29", + release_date = "2024-06-04", cpe = "N/A", license = "BSD-3-Clause", license_url = "https://github.com/google/quiche/blob/{version}/LICENSE", From ff862a246420cf5cb6628346e8ac384a02d7ae36 Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Fri, 7 Jun 2024 14:34:25 -0500 Subject: [PATCH 50/61] mobile: Remove no-remote-exec (#34610) This PR removes the -no-remote-exec tag and replaces it with sandboxNetwork=standard exec_properties to allow binding a socket to localhost so that the tests are executed with EngFlow instead of the GitHub runner. This should speed up the iOS tests significantly. Risk Level: low Testing: CI Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a Signed-off-by: Fredy Wijaya --- mobile/bazel/apple.bzl | 3 +- mobile/test/objective-c/EnvoyTestServer.h | 7 +- mobile/test/swift/BUILD | 4 +- mobile/test/swift/integration/BUILD | 76 +++++++++++++------- mobile/test/swift/integration/proxying/BUILD | 7 +- mobile/test/swift/stats/BUILD | 4 +- 6 files changed, 66 insertions(+), 35 deletions(-) diff --git a/mobile/bazel/apple.bzl b/mobile/bazel/apple.bzl index 309f9aa10670..e3600e1eee32 100644 --- a/mobile/bazel/apple.bzl +++ b/mobile/bazel/apple.bzl @@ -34,7 +34,7 @@ def envoy_objc_library(name, hdrs = [], visibility = [], data = [], deps = [], m # ], # ) # -def envoy_mobile_swift_test(name, srcs, size = None, data = [], deps = [], tags = [], repository = "", visibility = [], flaky = False): +def envoy_mobile_swift_test(name, srcs, size = None, data = [], deps = [], tags = [], repository = "", visibility = [], flaky = False, exec_properties = {}): test_lib_name = name + "_lib" swift_library( name = test_lib_name, @@ -57,6 +57,7 @@ def envoy_mobile_swift_test(name, srcs, size = None, data = [], deps = [], tags tags = tags, visibility = visibility, flaky = flaky, + exec_properties = exec_properties, ) def envoy_mobile_objc_test(name, srcs, data = [], deps = [], tags = [], visibility = [], flaky = False): diff --git a/mobile/test/objective-c/EnvoyTestServer.h b/mobile/test/objective-c/EnvoyTestServer.h index 13b0ea1f48bf..dbc9758d9e64 100644 --- a/mobile/test/objective-c/EnvoyTestServer.h +++ b/mobile/test/objective-c/EnvoyTestServer.h @@ -4,9 +4,10 @@ // Interface for starting and managing a test server. Calls into to test_server.cc // -// NB: Any test that utilizes this class must have a `no-remote-exec` tag in its BUILD target. -// EnvoyTestServer binds to a listening socket on the machine it runs on, and on CI, this -// operation is not permitted in remote execution environments. +// NB: Any test that utilizes this class must have a `"sandboxNetwork": "standard"` +// `exec_properties` in its BUILD target to allow binding a listening socket on +// the EngFlow machines +// (https://docs.engflow.com/re/client/platform-options-reference.html#sandboxallowed). @interface EnvoyTestServer : NSObject // Get the port of the upstream server. diff --git a/mobile/test/swift/BUILD b/mobile/test/swift/BUILD index 80b407888809..7077c25b6368 100644 --- a/mobile/test/swift/BUILD +++ b/mobile/test/swift/BUILD @@ -20,8 +20,10 @@ envoy_mobile_swift_test( "RetryPolicyMapperTests.swift", "RetryPolicyTests.swift", ], + exec_properties = { + "sandboxNetwork": "standard", + }, flaky = True, # TODO(jpsim): Fix timeouts when running these tests on CI - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec visibility = ["//visibility:public"], deps = [ "//library/objective-c:envoy_engine_objc_lib", diff --git a/mobile/test/swift/integration/BUILD b/mobile/test/swift/integration/BUILD index 5eeac3b43fdf..fb0f71f20fd0 100644 --- a/mobile/test/swift/integration/BUILD +++ b/mobile/test/swift/integration/BUILD @@ -10,11 +10,9 @@ envoy_mobile_swift_test( srcs = [ "EndToEndNetworkingTest.swift", ], - # The test starts an Envoy server, which binds to a local socket. Binding to a local socket is - # not permitted on remote execution hosts, so we have to add the `no-remote-exec` tag. - tags = [ - "no-remote-exec", - ], + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -28,7 +26,9 @@ envoy_mobile_swift_test( srcs = [ "CancelStreamTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -42,7 +42,9 @@ envoy_mobile_swift_test( srcs = [ "EngineApiTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -55,7 +57,9 @@ envoy_mobile_swift_test( srcs = [ "FilterResetIdleTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -69,7 +73,9 @@ envoy_mobile_swift_test( srcs = [ "GRPCReceiveErrorTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -83,11 +89,9 @@ envoy_mobile_swift_test( srcs = [ "IdleTimeoutTest.swift", ], - # The test starts an Envoy server, which binds to a local socket. Binding to a local socket is - # not permitted on remote execution hosts, so we have to add the `no-remote-exec` tag. - tags = [ - "no-remote-exec", - ], + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -101,7 +105,9 @@ envoy_mobile_swift_test( srcs = [ "KeyValueStoreTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -115,7 +121,9 @@ envoy_mobile_swift_test( srcs = [ "ReceiveDataTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -129,7 +137,9 @@ envoy_mobile_swift_test( srcs = [ "ReceiveErrorTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -142,7 +152,9 @@ envoy_mobile_swift_test( srcs = [ "ResetConnectivityStateTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -156,7 +168,9 @@ envoy_mobile_swift_test( srcs = [ "SendDataTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -170,7 +184,9 @@ envoy_mobile_swift_test( srcs = [ "SendHeadersTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -184,7 +200,9 @@ envoy_mobile_swift_test( srcs = [ "SendTrailersTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -198,7 +216,9 @@ envoy_mobile_swift_test( srcs = [ "SetEventTrackerTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -212,7 +232,9 @@ envoy_mobile_swift_test( srcs = [ "SetEventTrackerTestNoTracker.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -226,7 +248,9 @@ envoy_mobile_swift_test( srcs = [ "SetLoggerTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", @@ -240,7 +264,9 @@ envoy_mobile_swift_test( srcs = [ "CancelGRPCStreamTest.swift", ], - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ ":test_extensions", diff --git a/mobile/test/swift/integration/proxying/BUILD b/mobile/test/swift/integration/proxying/BUILD index d15f5a6f6526..b891bf3b39c7 100644 --- a/mobile/test/swift/integration/proxying/BUILD +++ b/mobile/test/swift/integration/proxying/BUILD @@ -10,10 +10,9 @@ envoy_mobile_swift_test( srcs = [ "HTTPRequestUsingProxyTest.swift", ], - # The test starts an Envoy test server, which requires the `no-remote-exec` tag. - tags = [ - "no-remote-exec", - ], + exec_properties = { + "sandboxNetwork": "standard", + }, visibility = ["//visibility:public"], deps = [ "//library/objective-c:envoy_engine_objc_lib", diff --git a/mobile/test/swift/stats/BUILD b/mobile/test/swift/stats/BUILD index 14e1d31bd8d0..bbecf8d2e710 100644 --- a/mobile/test/swift/stats/BUILD +++ b/mobile/test/swift/stats/BUILD @@ -12,8 +12,10 @@ envoy_mobile_swift_test( "ElementTests.swift", "TagsBuilderTests.swift", ], + exec_properties = { + "sandboxNetwork": "standard", + }, flaky = True, # TODO(jpsim): Fix timeouts when running these tests on CI - tags = ["no-remote-exec"], # TODO(32551): Re-enable remote exec visibility = ["//visibility:public"], deps = [ "//library/objective-c:envoy_engine_objc_lib", From 208dac8607f6223224ed105a995933bf4961726d Mon Sep 17 00:00:00 2001 From: botengyao Date: Fri, 7 Jun 2024 15:50:56 -0400 Subject: [PATCH 51/61] coverage: add datadog test and bring back `extensions/tracers` threshhold (#34587) Signed-off-by: Boteng Yao --- .../tracers/datadog/agent_http_client_test.cc | 37 +++++++++++++++++++ test/per_file_coverage.sh | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/test/extensions/tracers/datadog/agent_http_client_test.cc b/test/extensions/tracers/datadog/agent_http_client_test.cc index 57e4bcceb4d3..9b367cca6067 100644 --- a/test/extensions/tracers/datadog/agent_http_client_test.cc +++ b/test/extensions/tracers/datadog/agent_http_client_test.cc @@ -352,6 +352,43 @@ TEST_F(DatadogAgentHttpClientTest, OnErrorStreamReset) { callbacks_->onFailure(request_, Http::AsyncClient::FailureReason::Reset); } +TEST_F(DatadogAgentHttpClientTest, OnErrorExceedResponseBufferLimit) { + // When `onFailure` is invoked on the `Http::AsyncClient::Callbacks` with + // `FailureReason::ExceedResponseBufferLimit`, the associated `on_error` callback is invoked + // with a corresponding `datadog::tracing::Error`. + + EXPECT_CALL(cluster_manager_.instance_.thread_local_cluster_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([this](Http::RequestMessagePtr&, Http::AsyncClient::Callbacks& callbacks_arg, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callbacks_ = &callbacks_arg; + return &request_; + })); + + // `callbacks_->onFailure(...)` will cause `on_error_` to be called. + // `on_response_` will not be called. + EXPECT_CALL(on_error_, Call(_)).WillOnce(Invoke([](datadog::tracing::Error error) { + EXPECT_EQ(error.code, datadog::tracing::Error::ENVOY_HTTP_CLIENT_FAILURE); + })); + EXPECT_CALL(on_response_, Call(_, _, _)).Times(0); + + // The request will not be canceled; neither explicitly nor in + // `~AgentHTTPClient`, because it will have been fulfilled. + EXPECT_CALL(request_, cancel()).Times(0); + + const auto ignore = [](auto&&...) {}; + datadog::tracing::Expected result = + client_.post(url_, ignore, "{}", on_response_.AsStdFunction(), on_error_.AsStdFunction(), + time_.monotonicTime() + std::chrono::seconds(1)); + EXPECT_TRUE(result) << result.error(); + + Http::ResponseMessagePtr msg(new Http::ResponseMessageImpl( + Http::ResponseHeaderMapPtr{new Http::TestResponseHeaderMapImpl{{":status", "200"}}})); + msg->body().add("{}"); + + callbacks_->onFailure(request_, Http::AsyncClient::FailureReason::ExceedResponseBufferLimit); +} + TEST_F(DatadogAgentHttpClientTest, OnErrorOther) { // When `onFailure` is invoked on the `Http::AsyncClient::Callbacks` with any // value other than `FailureReason::Reset`, the associated `on_error` callback diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index f48c76c288a0..2d2b536f3949 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -44,7 +44,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/rate_limit_descriptors/expr:95.0" "source/extensions/stat_sinks/graphite_statsd:82.8" # Death tests don't report LCOV "source/extensions/stat_sinks/statsd:85.2" # Death tests don't report LCOV -"source/extensions/tracers:96.4" +"source/extensions/tracers:96.5" "source/extensions/tracers/common:74.8" "source/extensions/tracers/common/ot:72.9" "source/extensions/tracers/opencensus:93.9" From 89d7b57c5d8563c5aec98345a9ea10fd6ee3ed15 Mon Sep 17 00:00:00 2001 From: code Date: Sat, 8 Jun 2024 08:52:44 +0800 Subject: [PATCH 52/61] runtime: remove refresh rtt runtime flag (#34599) Signed-off-by: wbpcode Co-authored-by: wbpcode --- changelogs/current.yaml | 3 ++ source/common/http/conn_manager_impl.cc | 17 +------- source/common/http/conn_manager_impl.h | 2 - source/common/runtime/runtime_features.cc | 2 - test/common/http/conn_manager_impl_test.cc | 38 ----------------- test/integration/filter_integration_test.cc | 45 --------------------- 6 files changed, 5 insertions(+), 102 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 6da420210a7e..8b2cfad31f1a 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -82,6 +82,9 @@ minor_behavior_changes: - area: filters change: | Set ``WWW-Authenticate`` header for 401 responses from the Basic Auth filter. +- area: http + change: | + Removed runtime guard ``envoy.reloadable_features.refresh_rtt_after_request`` and legacy code path. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index d8ddacb89e9b..b217c1e2f7b2 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -126,10 +126,8 @@ ConnectionManagerImpl::ConnectionManagerImpl(ConnectionManagerConfigSharedPtr co /*node_id=*/local_info_.node().id(), /*server_name=*/config_->serverName(), /*proxy_status_config=*/config_->proxyStatusConfig())), - max_requests_during_dispatch_( - runtime_.snapshot().getInteger(ConnectionManagerImpl::MaxRequestsPerIoCycle, UINT32_MAX)), - refresh_rtt_after_request_( - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.refresh_rtt_after_request")) { + max_requests_during_dispatch_(runtime_.snapshot().getInteger( + ConnectionManagerImpl::MaxRequestsPerIoCycle, UINT32_MAX)) { ENVOY_LOG_ONCE_IF( trace, accept_new_http_stream_ == nullptr, "LoadShedPoint envoy.load_shed_points.http_connection_manager_decode_headers is not " @@ -337,17 +335,6 @@ void ConnectionManagerImpl::doDeferredStreamDestroy(ActiveStream& stream) { } stream.completeRequest(); - - // If refresh rtt after request is required explicitly, then try to get rtt again set it into - // connection info. - if (refresh_rtt_after_request_) { - // Set roundtrip time in connectionInfoSetter before OnStreamComplete - absl::optional t = read_callbacks_->connection().lastRoundTripTime(); - if (t.has_value()) { - read_callbacks_->connection().connectionInfoSetter().setRoundTripTime(t.value()); - } - } - stream.filter_manager_.onStreamComplete(); // For HTTP/3, skip access logging here and add deferred logging info diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 5f546ed9be3d..a325dff0dfb2 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -638,8 +638,6 @@ class ConnectionManagerImpl : Logger::Loggable, uint32_t requests_during_dispatch_count_{0}; const uint32_t max_requests_during_dispatch_{UINT32_MAX}; Event::SchedulableCallbackPtr deferred_request_processing_callback_; - - const bool refresh_rtt_after_request_{}; }; } // namespace Http diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 708f029cdac2..f1a9815ec6eb 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -132,8 +132,6 @@ FALSE_RUNTIME_GUARD(envoy_reloadable_features_runtime_initialized); // TODO(mattklein123): Also unit test this if this sticks and this becomes the default for Apple & // Android. FALSE_RUNTIME_GUARD(envoy_reloadable_features_always_use_v6); -// TODO(wbpcode) complete remove this feature is no one use it. -FALSE_RUNTIME_GUARD(envoy_reloadable_features_refresh_rtt_after_request); // TODO(vikaschoudhary16) flip this to true only after all the // TcpProxy::Filter::HttpStreamDecoderFilterCallbacks are implemented or commented as unnecessary FALSE_RUNTIME_GUARD(envoy_restart_features_upstream_http_filters_with_tcp_proxy); diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 27e876a43669..51cc25624285 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -3570,44 +3570,6 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterBidiData) { EXPECT_EQ("world", response_body); } -TEST_F(HttpConnectionManagerImplTest, RoundTripTimeHasValue) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.refresh_rtt_after_request", "true"}}); - - setup(false, ""); - - // Set up the codec. - Buffer::OwnedImpl fake_input("input"); - conn_manager_->createCodec(fake_input); - - startRequest(true); - - EXPECT_CALL(filter_callbacks_.connection_, lastRoundTripTime()) - .WillOnce(Return(absl::optional(300))); - EXPECT_CALL(filter_callbacks_.connection_, connectionInfoSetter()); - - filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); -} - -TEST_F(HttpConnectionManagerImplTest, RoundTripTimeHasNoValue) { - TestScopedRuntime scoped_runtime; - scoped_runtime.mergeValues({{"envoy.reloadable_features.refresh_rtt_after_request", "true"}}); - - setup(false, ""); - - // Set up the codec. - Buffer::OwnedImpl fake_input("input"); - conn_manager_->createCodec(fake_input); - - startRequest(true); - - EXPECT_CALL(filter_callbacks_.connection_, lastRoundTripTime()) - .WillOnce(Return(absl::optional())); - EXPECT_CALL(filter_callbacks_.connection_, connectionInfoSetter()).Times(0); - - filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); -} - TEST_F(HttpConnectionManagerImplTest, RequestTimeoutDisabledByDefault) { setup(false, ""); diff --git a/test/integration/filter_integration_test.cc b/test/integration/filter_integration_test.cc index dad67eb89edd..3d1af43a8c7e 100644 --- a/test/integration/filter_integration_test.cc +++ b/test/integration/filter_integration_test.cc @@ -224,51 +224,6 @@ TEST_P(FilterIntegrationTest, MissingHeadersLocalReplyDownstreamBytesCount) { } } -TEST_P(FilterIntegrationTest, RoundTripTimeForDownstreamConnection) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.refresh_rtt_after_request", "true"); - - config_helper_.prependFilter(R"EOF( - name: stream-info-to-headers-filter - )EOF"); - initialize(); - - codec_client_ = makeHttpConnection(lookupPort("http")); - - // Send first request. - { - // Send a headers only request. - auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); - waitForNextUpstreamRequest(); - - // Make sure that the body was injected to the request. - EXPECT_TRUE(upstream_request_->complete()); - - // Send a headers only response. - upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); - - // Make sure that round trip time was populated - EXPECT_FALSE(response->headers().get(Http::LowerCaseString("round_trip_time")).empty()); - } - - // Send second request. - { - // Send a headers only request. - auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); - waitForNextUpstreamRequest(); - - // Make sure that the body was injected to the request. - EXPECT_TRUE(upstream_request_->complete()); - - // Send a headers only response. - upstream_request_->encodeHeaders(default_response_headers_, true); - ASSERT_TRUE(response->waitForEndStream()); - - // Make sure that round trip time was populated - EXPECT_FALSE(response->headers().get(Http::LowerCaseString("round_trip_time")).empty()); - } -} - TEST_P(FilterIntegrationTest, MissingHeadersLocalReplyUpstreamBytesCount) { useAccessLog("%UPSTREAM_WIRE_BYTES_SENT% %UPSTREAM_WIRE_BYTES_RECEIVED% " "%UPSTREAM_HEADER_BYTES_SENT% %UPSTREAM_HEADER_BYTES_RECEIVED%"); From a6792927ff71015069326490af5306ad56d769c9 Mon Sep 17 00:00:00 2001 From: Jonathan Albrecht Date: Sat, 8 Jun 2024 13:17:35 -0400 Subject: [PATCH 53/61] Add big endian support to the dns parser (#34456) Fixes: //test/extensions/filters/udp/dns_filter:dns_filter_integration_test //test/extensions/filters/udp/dns_filter:dns_filter_test on big endian platforms. Signed-off-by: Jonathan Albrecht --- .../filters/udp/dns_filter/dns_parser.cc | 5 +++++ .../filters/udp/dns_filter/dns_parser.h | 21 ++++++++++++++++--- .../filters/udp/dns_filter/dns_filter_test.cc | 6 ++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.cc b/source/extensions/filters/udp/dns_filter/dns_parser.cc index b63f69278d89..ed4b850a5c8b 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.cc +++ b/source/extensions/filters/udp/dns_filter/dns_parser.cc @@ -91,8 +91,13 @@ bool DnsAnswerRecord::serialize(Buffer::OwnedImpl& output) { // Store the 128bit address with 2 64 bit writes const absl::uint128 addr6 = ip_address->ipv6()->address(); output.writeBEInt(sizeof(addr6)); +#ifdef ABSL_IS_BIG_ENDIAN + output.writeBEInt(absl::Uint128High64(addr6)); + output.writeBEInt(absl::Uint128Low64(addr6)); +#else output.writeLEInt(absl::Uint128Low64(addr6)); output.writeLEInt(absl::Uint128High64(addr6)); +#endif } else if (ip_address->ipv4() != nullptr) { output.writeBEInt(4); output.writeLEInt(ip_address->ipv4()->address()); diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.h b/source/extensions/filters/udp/dns_filter/dns_parser.h index 80ebabd398ae..3449c19d6478 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.h +++ b/source/extensions/filters/udp/dns_filter/dns_parser.h @@ -137,9 +137,23 @@ struct DnsParserCounters { queries_with_ans_or_authority_rrs(queries_with_ans_or_authority_rrs) {} }; -// The flags have been verified with dig and this structure should not be modified. The flag -// order here does not match the RFC, but takes byte ordering into account so that serialization -// does not bitwise operations. +// The flags have been verified with dig and this structure should not be modified. On little +// endian platforms, the flag order here does not match the RFC, but takes byte ordering into +// account so that serialization does not bitwise operations. +#if ABSL_IS_BIG_ENDIAN +PACKED_STRUCT(struct DnsHeaderFlags { + unsigned qr : 1; // query or response + unsigned opcode : 4; // operation code + unsigned aa : 1; // authoritative answer + unsigned tc : 1; // truncated response + unsigned rd : 1; // recursion desired + unsigned ra : 1; // recursion available + unsigned z : 1; // z - bit (must be zero in queries per RFC1035) + unsigned ad : 1; // authenticated data + unsigned cd : 1; // checking disabled + unsigned rcode : 4; // return code +}); +#else PACKED_STRUCT(struct DnsHeaderFlags { unsigned rcode : 4; // return code unsigned cd : 1; // checking disabled @@ -152,6 +166,7 @@ PACKED_STRUCT(struct DnsHeaderFlags { unsigned opcode : 4; // operation code unsigned qr : 1; // query or response }); +#endif /** * Structure representing the DNS header as it appears in a packet diff --git a/test/extensions/filters/udp/dns_filter/dns_filter_test.cc b/test/extensions/filters/udp/dns_filter/dns_filter_test.cc index 58620fc5191a..ba7a0e61aa86 100644 --- a/test/extensions/filters/udp/dns_filter/dns_filter_test.cc +++ b/test/extensions/filters/udp/dns_filter/dns_filter_test.cc @@ -1910,6 +1910,12 @@ TEST_F(DnsFilterTest, InvalidShortBufferTest) { } TEST_F(DnsFilterTest, RandomizeFirstAnswerTest) { +#if defined(__linux__) && defined(__s390x__) + // Skip on s390x because this test incorrectly depends on the ordering of + // addresses that happens to work on other platforms. + // See https://github.com/envoyproxy/envoy/pull/24330 + GTEST_SKIP() << "Skipping RandomizeFirstAnswerTest on s390x"; +#endif InSequence s; setup(forward_query_off_config); From c2a2cfe43e5c842ba0f62ac55ae3e56ad136fb74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 08:38:22 +0100 Subject: [PATCH 54/61] build(deps-dev): bump @vitejs/plugin-react from 4.3.0 to 4.3.1 in /examples/single-page-app/ui (#34623) build(deps-dev): bump @vitejs/plugin-react Bumps [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/tree/HEAD/packages/plugin-react) from 4.3.0 to 4.3.1. - [Release notes](https://github.com/vitejs/vite-plugin-react/releases) - [Changelog](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite-plugin-react/commits/v4.3.1/packages/plugin-react) --- updated-dependencies: - dependency-name: "@vitejs/plugin-react" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/single-page-app/ui/package.json | 2 +- examples/single-page-app/ui/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/single-page-app/ui/package.json b/examples/single-page-app/ui/package.json index e57c089c2230..a8989ee685f3 100644 --- a/examples/single-page-app/ui/package.json +++ b/examples/single-page-app/ui/package.json @@ -25,7 +25,7 @@ "@types/react-dom": "^18.3.0", "@typescript-eslint/eslint-plugin": "^6.21.0", "@typescript-eslint/parser": "^6.21.0", - "@vitejs/plugin-react": "^4.3.0", + "@vitejs/plugin-react": "^4.3.1", "eslint": "^8.57.0", "eslint-config-standard-with-typescript": "^43.0.0", "eslint-plugin-import": "^2.25.2", diff --git a/examples/single-page-app/ui/yarn.lock b/examples/single-page-app/ui/yarn.lock index 9684f150be14..3fcbe07b6d4e 100644 --- a/examples/single-page-app/ui/yarn.lock +++ b/examples/single-page-app/ui/yarn.lock @@ -1700,10 +1700,10 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== -"@vitejs/plugin-react@^4.3.0": - version "4.3.0" - resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.3.0.tgz#f20ec2369a92d8abaaefa60da8b7157819d20481" - integrity sha512-KcEbMsn4Dpk+LIbHMj7gDPRKaTMStxxWRkRmxsg/jVdFdJCZWt1SchZcf0M4t8lIKdwwMsEyzhrcOXRrDPtOBw== +"@vitejs/plugin-react@^4.3.1": + version "4.3.1" + resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.3.1.tgz#d0be6594051ded8957df555ff07a991fb618b48e" + integrity sha512-m/V2syj5CuVnaxcUJOQRel/Wr31FFXRFlnOoq1TVtkCxsY5veGMTEmpWHndrhB2U8ScHtCQB1e+4hWYExQc6Lg== dependencies: "@babel/core" "^7.24.5" "@babel/plugin-transform-react-jsx-self" "^7.24.5" From 783d7510e15041b95109fa5e3a953afd136d2388 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 08:38:36 +0100 Subject: [PATCH 55/61] build(deps-dev): bump vite from 5.2.12 to 5.2.13 in /examples/single-page-app/ui (#34622) build(deps-dev): bump vite in /examples/single-page-app/ui Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 5.2.12 to 5.2.13. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v5.2.13/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v5.2.13/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/single-page-app/ui/package.json | 2 +- examples/single-page-app/ui/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/single-page-app/ui/package.json b/examples/single-page-app/ui/package.json index a8989ee685f3..3df85522663a 100644 --- a/examples/single-page-app/ui/package.json +++ b/examples/single-page-app/ui/package.json @@ -35,6 +35,6 @@ "eslint-plugin-react-hooks": "^4.6.2", "eslint-plugin-react-refresh": "^0.4.7", "typescript": "*", - "vite": "^5.2.12" + "vite": "^5.2.13" } } diff --git a/examples/single-page-app/ui/yarn.lock b/examples/single-page-app/ui/yarn.lock index 3fcbe07b6d4e..b2125546276a 100644 --- a/examples/single-page-app/ui/yarn.lock +++ b/examples/single-page-app/ui/yarn.lock @@ -4290,10 +4290,10 @@ use-sidecar@^1.1.2: detect-node-es "^1.1.0" tslib "^2.0.0" -vite@^5.2.12: - version "5.2.12" - resolved "https://registry.yarnpkg.com/vite/-/vite-5.2.12.tgz#3536c93c58ba18edea4915a2ac573e6537409d97" - integrity sha512-/gC8GxzxMK5ntBwb48pR32GGhENnjtY30G4A0jemunsBkiEZFw60s8InGpN8gkhHEkjnRK1aSAxeQgwvFhUHAA== +vite@^5.2.13: + version "5.2.13" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.2.13.tgz#945ababcbe3d837ae2479c29f661cd20bc5e1a80" + integrity sha512-SSq1noJfY9pR3I1TUENL3rQYDQCFqgD+lM6fTRAM8Nv6Lsg5hDLaXkjETVeBt+7vZBCMoibD+6IWnT2mJ+Zb/A== dependencies: esbuild "^0.20.1" postcss "^8.4.38" From a17fd267503b8b1cc865f433d0c7c000aa358a21 Mon Sep 17 00:00:00 2001 From: botengyao Date: Mon, 10 Jun 2024 03:42:05 -0400 Subject: [PATCH 56/61] sec-release: updated 2024 q2 release (#34551) Signed-off-by: Boteng Yao --- RELEASES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index 3d645c62ce9f..bab49dae35e8 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -65,6 +65,7 @@ actual mechanics of the release itself. | 2022 Q4 | Can Cecen ([cancecen](https://github.com/cancecen)) | Tony Allen ([tonya11en](https://github.com/tonya11en)) | | 2023 Q3 | Boteng Yao ([botengyao](https://github.com/botengyao)) | Kateryna Nezdolii ([nezdolik](https://github.com/nezdolik)) | | 2023 Q4 | Paul Merrison ([pmerrison](https://github.com/pmerrison)) | Brian Sonnenberg ([briansonnenberg](https://github.com/briansonnenberg)) | +| 2024 Q2 | Ryan Northey ([phlax](https://github.com/phlax)) | Boteng Yao ([botengyao](https://github.com/botengyao)) | ## Major release schedule @@ -135,6 +136,7 @@ Security releases are published on a 3-monthly cycle, around the mid point betwe | Quarter | Expected | Actual | Difference | |:-------:|:----------:|:----------:|:----------:| -| 2024 Q2 | 2024/06/04 | | | +| 2024 Q2 | 2024/06/04 | 2024/06/04 | 0 days | +| 2024 Q3 | 2024/09/03 | NOTE: Zero-day vulnerabilities, and upstream vulnerabilities disclosed to us under embargo, may necessitate an emergency release with little or no warning. From 79881c21e152bb033ebe048ba05a0709a3b0f2da Mon Sep 17 00:00:00 2001 From: Shivam7-1 <55046031+Shivam7-1@users.noreply.github.com> Date: Mon, 10 Jun 2024 18:30:16 +0530 Subject: [PATCH 57/61] Update searchtools.js DOM text reinterpreted as HTML (#34607) Signed-off-by: Shivam7-1 <55046031+Shivam7-1@users.noreply.github.com> --- docs/root/_static/searchtools.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/_static/searchtools.js b/docs/root/_static/searchtools.js index d460d8033155..02d0473059b3 100644 --- a/docs/root/_static/searchtools.js +++ b/docs/root/_static/searchtools.js @@ -129,7 +129,7 @@ const _displayItem = (item, searchTerms) => { }; const _finishSearch = (resultCount) => { Search.stopPulse(); - Search.title.innerText = _("Search Results"); + Search.title.textContent = _("Search Results"); if (!resultCount) Search.status.innerText = Documentation.gettext( "Your search did not match any documents. Please make sure that all words are spelled correctly and that you've selected enough categories." From cd23d24304511de10ba3113e70b8338762557399 Mon Sep 17 00:00:00 2001 From: Namrata Bhave Date: Mon, 10 Jun 2024 18:34:43 +0530 Subject: [PATCH 58/61] Fix //test/common/websocket:codec_test on big endian (#34443) Fix //test/common/websocket:codec_test on big endian(s390x) As length is set using memcpy and further processed as per LE format, the test fails on s390x. Signed-off-by: namrata-ibm --- source/common/websocket/codec.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/common/websocket/codec.cc b/source/common/websocket/codec.cc index 1c1fc4e40b1a..e95d8ab3d256 100644 --- a/source/common/websocket/codec.cc +++ b/source/common/websocket/codec.cc @@ -128,7 +128,11 @@ uint8_t Decoder::doDecodeExtendedLength(absl::Span& data) { num_remaining_extended_length_bytes_ -= bytes_to_decode; if (num_remaining_extended_length_bytes_ == 0) { +#if ABSL_IS_BIG_ENDIAN + length_ = state_ == State::FrameHeaderExtendedLength16Bit ? htole16(le64toh(length_)) : length_; +#else length_ = state_ == State::FrameHeaderExtendedLength16Bit ? htobe16(length_) : htobe64(length_); +#endif if (num_remaining_masking_key_bytes_ > 0) { state_ = State::FrameHeaderMaskingKey; } else { From 69d4ef8d04678710ec1633e1e7effbda6623cc8d Mon Sep 17 00:00:00 2001 From: Teju Nareddy Date: Mon, 10 Jun 2024 08:07:40 -0500 Subject: [PATCH 59/61] proxy_protocol_filter: Add field `stat_prefix` to the filter configuration (#34414) Commit Message: proxy_protocol_filter: Add field stat_prefix to the filter configuration Additional Description: This field allows for differentiating statistics when multiple proxy protocol listener filters are configured. This PR is a follow-up from previous conversation: #32861 (comment) Risk Level: Low All client-facing behavior changes are guarded by new filter config field. Testing: Stats unit tests Proxy protocol listener filter integration tests Docs Changes: Done Release Notes: Done Platform Specific Features: None Signed-off-by: Teju Nareddy --- .../proxy_protocol/v3/proxy_protocol.proto | 7 ++++++ changelogs/current.yaml | 5 ++++ .../listener_filters/proxy_protocol.rst | 6 ++--- source/common/config/well_known_names.cc | 14 ++++++++--- source/common/config/well_known_names.h | 2 ++ .../listener/proxy_protocol/proxy_protocol.cc | 15 ++++++++---- .../listener/proxy_protocol/proxy_protocol.h | 2 +- test/common/stats/tag_extractor_impl_test.cc | 11 +++++++++ .../proxy_proto_integration_test.cc | 23 ++++++++++++++----- 9 files changed, 67 insertions(+), 18 deletions(-) diff --git a/api/envoy/extensions/filters/listener/proxy_protocol/v3/proxy_protocol.proto b/api/envoy/extensions/filters/listener/proxy_protocol/v3/proxy_protocol.proto index 31ca8a6950e4..cc96c4822e23 100644 --- a/api/envoy/extensions/filters/listener/proxy_protocol/v3/proxy_protocol.proto +++ b/api/envoy/extensions/filters/listener/proxy_protocol/v3/proxy_protocol.proto @@ -18,6 +18,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // PROXY protocol listener filter. // [#extension: envoy.filters.listener.proxy_protocol] +// [#next-free-field: 6] message ProxyProtocol { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.listener.proxy_protocol.v2.ProxyProtocol"; @@ -85,4 +86,10 @@ message ProxyProtocol { // and an incoming request matches the V2 signature, the filter will allow the request through without any modification. // The filter treats this request as if it did not have any PROXY protocol information. repeated config.core.v3.ProxyProtocolConfig.Version disallowed_versions = 4; + + // The human readable prefix to use when emitting statistics for the filter. + // If not configured, statistics will be emitted without the prefix segment. + // See the :ref:`filter's statistics documentation ` for + // more information. + string stat_prefix = 5; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 8b2cfad31f1a..3c3e6ba20947 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -225,6 +225,11 @@ new_features: - area: redis change: | Added support for `inline commands `_. +- area: proxy_protocol + change: | + Added field :ref:`stat_prefix ` + to the proxy protocol listener filter configuration, allowing for differentiating statistics when multiple proxy + protocol listener filters are configured. - area: aws_lambda change: | The ``aws_lambda`` filter now supports the diff --git a/docs/root/configuration/listeners/listener_filters/proxy_protocol.rst b/docs/root/configuration/listeners/listener_filters/proxy_protocol.rst index e96d457e7725..c545f117c570 100644 --- a/docs/root/configuration/listeners/listener_filters/proxy_protocol.rst +++ b/docs/root/configuration/listeners/listener_filters/proxy_protocol.rst @@ -33,7 +33,7 @@ If there is a protocol error or an unsupported address family Statistics ---------- -This filter emits the following general statistics, rooted at *downstream_proxy_proto* +This filter emits the following general statistics, rooted at *proxy_proto.[.]* .. csv-table:: :header: Name, Type, Description @@ -42,7 +42,7 @@ This filter emits the following general statistics, rooted at *downstream_proxy_ not_found_disallowed, Counter, "Total number of connections that don't contain the PROXY protocol header and are rejected." not_found_allowed, Counter, "Total number of connections that don't contain the PROXY protocol header, but are allowed due to :ref:`allow_requests_without_proxy_protocol `." -The filter also emits the statistics rooted at *downstream_proxy_proto.versions.* +The filter also emits the statistics rooted at *proxy_proto.[.]versions.* for each matched PROXY protocol version. Proxy protocol versions include ``v1`` and ``v2``. .. csv-table:: @@ -53,7 +53,7 @@ for each matched PROXY protocol version. Proxy protocol versions include ``v1`` disallowed, Counter, "Total number of ``found`` connections that are rejected due to :ref:`disallowed_versions `." error, Counter, "Total number of connections where the PROXY protocol header was malformed (and the connection was rejected)." -The filter also emits the following legacy statistics, rooted at its own scope: +The filter also emits the following legacy statistics, rooted at its own scope and **not** including the *stat_prefix*: .. csv-table:: :header: Name, Type, Description diff --git a/source/common/config/well_known_names.cc b/source/common/config/well_known_names.cc index 1cc7913d77f4..c9b73b1ab089 100644 --- a/source/common/config/well_known_names.cc +++ b/source/common/config/well_known_names.cc @@ -27,7 +27,8 @@ std::string expandRegex(const std::string& regex) { // alphanumerics, underscores, and dashes. {"", R"([\w-\./]+)"}, // Match a prefix that is either a listener plus name or cluster plus name - {"", R"((?:listener|cluster)\..*?)"}}); + {"", R"((?:listener|cluster)\..*?)"}, + {"", R"(\d)"}}); } const Regex::CompiledGoogleReMatcher& validTagValueRegex() { @@ -218,8 +219,15 @@ TagNameValues::TagNameValues() { // http..rbac.(.)* but excluding policy addRe2(RBAC_HTTP_PREFIX, R"(^http\.\.rbac\.(()\.).*?)", "", "policy"); - // proxy_proto.(versions.v.)** - addRe2(PROXY_PROTOCOL_VERSION, R"(^proxy_proto\.(versions\.v(\d)\.)\w+)", "proxy_proto.versions"); + // proxy_proto.(.)** + addRe2(PROXY_PROTOCOL_PREFIX, R"(^proxy_proto\.(()\.).+$)", "", "versions"); + + // proxy_proto.([.]versions.v().)(found|disallowed|error) + // + // Strips out: [.]versions.v(). + // Leaving: proxy_proto.(found|disallowed|error) + addRe2(PROXY_PROTOCOL_VERSION, + R"(^proxy_proto\.((?:\.)?versions\.v()\.)\w+$)"); } void TagNameValues::addRe2(const std::string& name, const std::string& regex, diff --git a/source/common/config/well_known_names.h b/source/common/config/well_known_names.h index 86b522b9678f..334e50cc6aa2 100644 --- a/source/common/config/well_known_names.h +++ b/source/common/config/well_known_names.h @@ -165,6 +165,8 @@ class TagNameValues { const std::string REDIS_PREFIX = "envoy.redis_prefix"; // Proxy Protocol version for a connection (Proxy Protocol listener filter). const std::string PROXY_PROTOCOL_VERSION = "envoy.proxy_protocol_version"; + // Stats prefix for the proxy protocol listener filter. + const std::string PROXY_PROTOCOL_PREFIX = "envoy.proxy_protocol_prefix"; // Mapping from the names above to their respective regex strings. const std::vector> name_regex_pairs_; diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index 847ce6d45e7b..5af1c2ca4094 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -55,17 +55,22 @@ namespace ProxyProtocol { constexpr absl::string_view kProxyProtoStatsPrefix = "proxy_proto."; constexpr absl::string_view kVersionStatsPrefix = "versions."; -ProxyProtocolStats ProxyProtocolStats::create(Stats::Scope& scope) { +ProxyProtocolStats ProxyProtocolStats::create(Stats::Scope& scope, absl::string_view stat_prefix) { + std::string filter_stat_prefix = std::string(kProxyProtoStatsPrefix); + if (!stat_prefix.empty()) { + filter_stat_prefix = absl::StrCat(kProxyProtoStatsPrefix, stat_prefix, "."); + } + return { /*legacy_=*/{LEGACY_PROXY_PROTOCOL_STATS(POOL_COUNTER(scope))}, /*general_=*/ - {GENERAL_PROXY_PROTOCOL_STATS(POOL_COUNTER_PREFIX(scope, kProxyProtoStatsPrefix))}, + {GENERAL_PROXY_PROTOCOL_STATS(POOL_COUNTER_PREFIX(scope, filter_stat_prefix))}, /*v1_=*/ {VERSIONED_PROXY_PROTOCOL_STATS(POOL_COUNTER_PREFIX( - scope, absl::StrCat(kProxyProtoStatsPrefix, kVersionStatsPrefix, "v1.")))}, + scope, absl::StrCat(filter_stat_prefix, kVersionStatsPrefix, "v1.")))}, /*v2_=*/ {VERSIONED_PROXY_PROTOCOL_STATS(POOL_COUNTER_PREFIX( - scope, absl::StrCat(kProxyProtoStatsPrefix, kVersionStatsPrefix, "v2.")))}, + scope, absl::StrCat(filter_stat_prefix, kVersionStatsPrefix, "v2.")))}, }; } @@ -105,7 +110,7 @@ void VersionedProxyProtocolStats::increment(ReadOrParseState decision) { Config::Config( Stats::Scope& scope, const envoy::extensions::filters::listener::proxy_protocol::v3::ProxyProtocol& proto_config) - : stats_(ProxyProtocolStats::create(scope)), + : stats_(ProxyProtocolStats::create(scope, proto_config.stat_prefix())), allow_requests_without_proxy_protocol_(proto_config.allow_requests_without_proxy_protocol()), pass_all_tlvs_(proto_config.has_pass_through_tlvs() ? proto_config.pass_through_tlvs().match_type() == diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.h b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.h index b3507e1a6071..3a508813891a 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.h +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.h @@ -96,7 +96,7 @@ struct ProxyProtocolStats { * For backwards compatibility, the legacy stats are rooted under their own scope. * The general and versioned stats are correctly rooted at this filter's own scope. */ - static ProxyProtocolStats create(Stats::Scope& scope); + static ProxyProtocolStats create(Stats::Scope& scope, absl::string_view stat_prefix); }; /** diff --git a/test/common/stats/tag_extractor_impl_test.cc b/test/common/stats/tag_extractor_impl_test.cc index e7ed185c9b1b..6bb2e8443b1e 100644 --- a/test/common/stats/tag_extractor_impl_test.cc +++ b/test/common/stats/tag_extractor_impl_test.cc @@ -485,12 +485,23 @@ TEST(TagExtractorTest, DefaultTagExtractors) { "http.rbac.policy.shadow_denied", {rbac_http_hcm_prefix, rbac_http_prefix, rbac_policy_name}); + // Proxy Protocol stat prefix + Tag proxy_protocol_prefix; + proxy_protocol_prefix.name_ = tag_names.PROXY_PROTOCOL_PREFIX; + proxy_protocol_prefix.value_ = "test_stat_prefix"; + regex_tester.testRegex("proxy_proto.not_found_disallowed", "proxy_proto.not_found_disallowed", + {}); + regex_tester.testRegex("proxy_proto.test_stat_prefix.not_found_disallowed", + "proxy_proto.not_found_disallowed", {proxy_protocol_prefix}); + // Proxy Protocol version prefix Tag proxy_protocol_version; proxy_protocol_version.name_ = tag_names.PROXY_PROTOCOL_VERSION; proxy_protocol_version.value_ = "2"; regex_tester.testRegex("proxy_proto.versions.v2.error", "proxy_proto.error", {proxy_protocol_version}); + regex_tester.testRegex("proxy_proto.test_stat_prefix.versions.v2.error", "proxy_proto.error", + {proxy_protocol_prefix, proxy_protocol_version}); } TEST(TagExtractorTest, ExtAuthzTagExtractors) { diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_proto_integration_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_proto_integration_test.cc index 8ad20b803384..4ca41df0b729 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_proto_integration_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_proto_integration_test.cc @@ -116,8 +116,9 @@ TEST_P(ProxyProtoIntegrationTest, V2RouterRequestAndResponseWithBodyNoBufferV6) testRouterRequestAndResponseWithBody(1024, 512, false, false, &creator); - // Verify stats (with tags for proxy protocol version). + // Verify stats (with tags for proxy protocol version, but no stat prefix). const auto found_counter = test_server_->counter("proxy_proto.versions.v2.found"); + ASSERT_NE(found_counter.get(), nullptr); EXPECT_EQ(found_counter->value(), 1UL); EXPECT_EQ(found_counter->tagExtractedName(), "proxy_proto.found"); EXPECT_THAT(found_counter->tags(), IsSupersetOf(Stats::TagVector{ @@ -378,6 +379,7 @@ ProxyProtoDisallowedVersionsIntegrationTest::ProxyProtoDisallowedVersionsIntegra // V1 is disallowed. ::envoy::extensions::filters::listener::proxy_protocol::v3::ProxyProtocol proxy_protocol; proxy_protocol.add_disallowed_versions(::envoy::config::core::v3::ProxyProtocolConfig::V1); + proxy_protocol.set_stat_prefix("test_stat_prefix"); auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); auto* ppv_filter = listener->mutable_listener_filters(0); @@ -400,19 +402,25 @@ TEST_P(ProxyProtoDisallowedVersionsIntegrationTest, V1Disallowed) { /*end_stream=*/false, /*verify=*/false)); tcp_client->waitForDisconnect(); - // Verify stats (with tags for proxy protocol version). - const auto found_counter = test_server_->counter("proxy_proto.versions.v1.found"); + // Verify stats (with tags for proxy protocol version and stat prefix). + const auto found_counter = + test_server_->counter("proxy_proto.test_stat_prefix.versions.v1.found"); + ASSERT_NE(found_counter.get(), nullptr); EXPECT_EQ(found_counter->value(), 1UL); EXPECT_EQ(found_counter->tagExtractedName(), "proxy_proto.found"); EXPECT_THAT(found_counter->tags(), IsSupersetOf(Stats::TagVector{ {"envoy.proxy_protocol_version", "1"}, + {"envoy.proxy_protocol_prefix", "test_stat_prefix"}, })); - const auto disallowed_counter = test_server_->counter("proxy_proto.versions.v1.disallowed"); + const auto disallowed_counter = + test_server_->counter("proxy_proto.test_stat_prefix.versions.v1.disallowed"); + ASSERT_NE(disallowed_counter.get(), nullptr); EXPECT_EQ(disallowed_counter->value(), 1UL); EXPECT_EQ(disallowed_counter->tagExtractedName(), "proxy_proto.disallowed"); EXPECT_THAT(disallowed_counter->tags(), IsSupersetOf(Stats::TagVector{ {"envoy.proxy_protocol_version", "1"}, + {"envoy.proxy_protocol_prefix", "test_stat_prefix"}, })); } @@ -430,12 +438,15 @@ TEST_P(ProxyProtoDisallowedVersionsIntegrationTest, V2Error) { ASSERT_TRUE(tcp_client->write(buf.toString(), /*end_stream=*/false, /*verify=*/false)); tcp_client->waitForDisconnect(); - // Verify stats (with tags for proxy protocol version). - const auto found_counter = test_server_->counter("proxy_proto.versions.v2.error"); + // Verify stats (with tags for proxy protocol version and stat prefix). + const auto found_counter = + test_server_->counter("proxy_proto.test_stat_prefix.versions.v2.error"); + ASSERT_NE(found_counter.get(), nullptr); EXPECT_EQ(found_counter->value(), 1UL); EXPECT_EQ(found_counter->tagExtractedName(), "proxy_proto.error"); EXPECT_THAT(found_counter->tags(), IsSupersetOf(Stats::TagVector{ {"envoy.proxy_protocol_version", "2"}, + {"envoy.proxy_protocol_prefix", "test_stat_prefix"}, })); } From 0650db1380f2ebf654c5b629764b2a6e8e0818e9 Mon Sep 17 00:00:00 2001 From: Paul Sohn Date: Mon, 10 Jun 2024 09:13:54 -0400 Subject: [PATCH 60/61] Fix QUIC deferred access logging for black-holed client (#34433) Commit Message: Fix QUIC deferred access logging when client terminates connection before final ack Additional Description: Addresses bug reported in #29930. Risk Level: Low Testing: Integration test Docs Changes: N/A Signed-off-by: Paul Sohn --- source/common/quic/quic_stats_gatherer.h | 5 ++ source/common/runtime/runtime_features.cc | 2 +- .../integration/quic_http_integration_test.cc | 55 +++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/source/common/quic/quic_stats_gatherer.h b/source/common/quic/quic_stats_gatherer.h index f704eeebc238..8a7b55fdc448 100644 --- a/source/common/quic/quic_stats_gatherer.h +++ b/source/common/quic/quic_stats_gatherer.h @@ -17,6 +17,11 @@ namespace Quic { class QuicStatsGatherer : public quic::QuicAckListenerInterface { public: explicit QuicStatsGatherer(Envoy::TimeSource* time_source) : time_source_(time_source) {} + ~QuicStatsGatherer() override { + if (!logging_done_) { + maybeDoDeferredLog(false); + } + } // QuicAckListenerInterface void OnPacketAcked(int acked_bytes, quic::QuicTime::Delta delta_largest_observed) override; diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index f1a9815ec6eb..50bc1655a634 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -148,7 +148,7 @@ FALSE_RUNTIME_GUARD(envoy_restart_features_use_eds_cache_for_ads); // For more information about Universal Header Validation, please see // https://github.com/envoyproxy/envoy/issues/10646 FALSE_RUNTIME_GUARD(envoy_reloadable_features_enable_universal_header_validator); -// TODO(pksohn): enable after fixing https://github.com/envoyproxy/envoy/issues/29930 +// TODO(pksohn): enable after canarying fix for https://github.com/envoyproxy/envoy/issues/29930 FALSE_RUNTIME_GUARD(envoy_reloadable_features_quic_defer_logging_to_ack_listener); // TODO(panting): flip this to true after some test time. FALSE_RUNTIME_GUARD(envoy_reloadable_features_use_config_in_happy_eyeballs); diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 33a28f2dda61..ce687bb7a122 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -1359,6 +1359,61 @@ TEST_P(QuicHttpIntegrationTest, DeferredLogging) { EXPECT_EQ(/* request headers */ metrics.at(19), metrics.at(20)); } +TEST_P(QuicHttpIntegrationTest, DeferredLoggingWithBlackholedClient) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.quic_defer_logging_to_ack_listener", + "true"); + useAccessLog( + "%PROTOCOL%,%ROUNDTRIP_DURATION%,%REQUEST_DURATION%,%RESPONSE_DURATION%,%RESPONSE_" + "CODE%,%BYTES_RECEIVED%,%ROUTE_NAME%,%VIRTUAL_CLUSTER_NAME%,%RESPONSE_CODE_DETAILS%,%" + "CONNECTION_TERMINATION_DETAILS%,%START_TIME%,%UPSTREAM_HOST%,%DURATION%,%BYTES_SENT%,%" + "RESPONSE_FLAGS%,%DOWNSTREAM_LOCAL_ADDRESS%,%UPSTREAM_CLUSTER%,%STREAM_ID%,%DYNAMIC_" + "METADATA(" + "udp.proxy.session:bytes_sent)%,%REQ(:path)%,%STREAM_INFO_REQ(:path)%"); + initialize(); + + // Make a header-only request and delay the response by 1ms to ensure that the ROUNDTRIP_DURATION + // metric is > 0 if exists. + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + IntegrationStreamDecoderPtr response = + codec_client_->makeHeaderOnlyRequest(default_request_headers_); + absl::SleepFor(absl::Milliseconds(1)); + waitForNextUpstreamRequest(0, TestUtility::DefaultTimeout); + upstream_request_->encodeHeaders(default_response_headers_, true); + + // Prevent the client's dispatcher from running by not calling waitForEndStream or closing the + // client's connection, then wait for server to tear down connection due to too many + // retransmissions. + int iterations = 0; + std::string contents = TestEnvironment::readFileToStringForTest(access_log_name_); + while (iterations < 20) { + timeSystem().advanceTimeWait(std::chrono::seconds(1)); + // check for deferred logs from connection teardown. + contents = TestEnvironment::readFileToStringForTest(access_log_name_); + if (!contents.empty()) { + break; + } + iterations++; + } + + std::vector entries = absl::StrSplit(contents, '\n', absl::SkipEmpty()); + EXPECT_EQ(entries.size(), 1); + std::string log = entries[0]; + + std::vector metrics = absl::StrSplit(log, ','); + ASSERT_EQ(metrics.size(), 21); + EXPECT_EQ(/* PROTOCOL */ metrics.at(0), "HTTP/3"); + EXPECT_EQ(/* ROUNDTRIP_DURATION */ metrics.at(1), "-"); + EXPECT_GE(/* REQUEST_DURATION */ std::stoi(metrics.at(2)), 0); + EXPECT_GE(/* RESPONSE_DURATION */ std::stoi(metrics.at(3)), 0); + EXPECT_EQ(/* RESPONSE_CODE */ metrics.at(4), "200"); + EXPECT_EQ(/* BYTES_RECEIVED */ metrics.at(5), "0"); + // Ensure that request headers from top-level access logger parameter and stream info are + // consistent. + EXPECT_EQ(/* request headers */ metrics.at(19), metrics.at(20)); + + codec_client_->close(); +} + TEST_P(QuicHttpIntegrationTest, DeferredLoggingDisabled) { config_helper_.addRuntimeOverride("envoy.reloadable_features.quic_defer_logging_to_ack_listener", "false"); From b3b2c1afacbd2513a65989cc3cc75a625562ee1b Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 10 Jun 2024 09:48:32 -0400 Subject: [PATCH 61/61] tooling: fixing a string (#34580) Signed-off-by: Alyssa Wilk --- tools/code_format/check_format.py | 4 +++- tools/code_format/config.yaml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index e0ae1fdf1a35..99c15b031ddf 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -806,7 +806,9 @@ def has_non_comment_throw(line): return False if self.deny_listed_for_exceptions(file_path): - if has_non_comment_throw(line) or "THROW" in line or "throwExceptionOrPanic" in line: + if has_non_comment_throw( + line) or "THROW" in line or "throwEnvoyExceptionOrPanic" in line: + report_error( "Don't introduce throws into exception-free files, use error " "statuses instead.") diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index fa8766048cad..fccba5e6cdd3 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -161,6 +161,7 @@ paths: - source/common/event/file_event_impl.cc - source/common/http/async_client_impl.cc - source/common/grpc/google_async_client_impl.cc + - source/common/formatter/substitution_format_utility.cc # Extensions can exempt entire directories but new extensions # points should ideally use StatusOr - source/extensions/access_loggers