From a4ad9c26197ee93b20a5df1aba91aa522f6820c8 Mon Sep 17 00:00:00 2001 From: code Date: Tue, 17 Sep 2024 20:51:19 +0800 Subject: [PATCH 01/28] route: use reference wrapper for get all filter config (#36079) Commit Message: route: use reference wrapper for get all filter config Additional Description: @alyssawilk I think we have a information gap at #36028. The pointer is better for the `perFilterConfigs()` because we always need to do a dynamic_cast. pointer is simpler and won't result in potential exception throwing. But the `Utility::getAllPerFilterConfigs()` could use the reference wrapper because it take a template parameter and will return the typed configs. Risk Level: low. Testing: n/a. Docs Changes: n/a. Release Notes: n/a. Platform Specific Features: n/a. --------- Signed-off-by: wangbaiping --- .../filters/http/source/golang_filter.cc | 11 +-- source/common/http/utility.h | 7 +- .../filters/http/cors/cors_filter.cc | 93 +++++++++---------- .../filters/http/cors/cors_filter.h | 12 +-- .../custom_response/custom_response_filter.cc | 5 +- .../filters/http/ext_authz/ext_authz.cc | 7 +- .../filters/http/ext_proc/ext_proc.cc | 7 +- .../http/header_mutation/header_mutation.cc | 15 ++- .../http/header_mutation/header_mutation.h | 2 +- .../filters/http/cors/cors_filter_test.cc | 20 ++-- 10 files changed, 84 insertions(+), 95 deletions(-) diff --git a/contrib/golang/filters/http/source/golang_filter.cc b/contrib/golang/filters/http/source/golang_filter.cc index 226a6656ec..cc4bd5c294 100644 --- a/contrib/golang/filters/http/source/golang_filter.cc +++ b/contrib/golang/filters/http/source/golang_filter.cc @@ -1377,16 +1377,13 @@ void Filter::deferredDeleteRequest(HttpRequestInternal* req) { uint64_t Filter::getMergedConfigId() { Http::StreamFilterCallbacks* callbacks = decoding_state_.getFilterCallbacks(); + auto id = config_->getConfigId(); + // get all of the per route config auto route_config_list = Http::Utility::getAllPerFilterConfig(callbacks); - ENVOY_LOG(debug, "golang filter route config list length: {}.", route_config_list.size()); - - auto id = config_->getConfigId(); - for (auto it : route_config_list) { - ASSERT(it != nullptr, "route config should not be null"); - auto route_config = *it; - id = route_config.getPluginConfigId(id, config_->pluginName()); + for (const FilterConfigPerRoute& typed_config : route_config_list) { + id = typed_config.getPluginConfigId(id, config_->pluginName()); } return id; diff --git a/source/common/http/utility.h b/source/common/http/utility.h index d1b2d0d093..eaee0c34fa 100644 --- a/source/common/http/utility.h +++ b/source/common/http/utility.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -572,11 +573,11 @@ const ConfigType* resolveMostSpecificPerFilterConfig(const Http::StreamFilterCal * and their lifetime is the same as the matched route. */ template -absl::InlinedVector +absl::InlinedVector, 4> getAllPerFilterConfig(const Http::StreamFilterCallbacks* callbacks) { ASSERT(callbacks != nullptr); - absl::InlinedVector all_configs; + absl::InlinedVector, 4> all_configs; for (const auto* config : callbacks->perFilterConfigs()) { const ConfigType* typed_config = dynamic_cast(config); @@ -584,7 +585,7 @@ getAllPerFilterConfig(const Http::StreamFilterCallbacks* callbacks) { ENVOY_LOG_MISC(debug, "Failed to retrieve the correct type of route specific filter config"); continue; } - all_configs.push_back(typed_config); + all_configs.push_back(*typed_config); } return all_configs; diff --git a/source/extensions/filters/http/cors/cors_filter.cc b/source/extensions/filters/http/cors/cors_filter.cc index 98ecb2a17c..49fba6d4a5 100644 --- a/source/extensions/filters/http/cors/cors_filter.cc +++ b/source/extensions/filters/http/cors/cors_filter.cc @@ -66,10 +66,17 @@ void CorsFilter::initializeCorsPolicies() { // If no cors policy is configured in the per filter config, then the cors policy fields in the // route configuration will be ignored. if (policies_.empty()) { - policies_ = { - decoder_callbacks_->route()->routeEntry()->corsPolicy(), - decoder_callbacks_->route()->virtualHost().corsPolicy(), - }; + const auto route = decoder_callbacks_->route(); + ASSERT(route != nullptr); + ASSERT(route->routeEntry() != nullptr); + + if (auto* typed_cfg = route->routeEntry()->corsPolicy(); typed_cfg != nullptr) { + policies_.push_back(*typed_cfg); + } + + if (auto* typed_cfg = route->virtualHost().corsPolicy(); typed_cfg != nullptr) { + policies_.push_back(*typed_cfg); + } } } @@ -204,11 +211,7 @@ void CorsFilter::setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& c } bool CorsFilter::isOriginAllowed(const Http::HeaderString& origin) { - const auto allow_origins = allowOrigins(); - if (allow_origins == nullptr) { - return false; - } - for (const auto& allow_origin : *allow_origins) { + for (const auto& allow_origin : allowOrigins()) { if (allow_origin->match("*") || allow_origin->match(origin.getStringView())) { return true; } @@ -216,92 +219,88 @@ bool CorsFilter::isOriginAllowed(const Http::HeaderString& origin) { return false; } -const std::vector* CorsFilter::allowOrigins() { - for (const auto policy : policies_) { - if (policy && !policy->allowOrigins().empty()) { - return &policy->allowOrigins(); +absl::Span CorsFilter::allowOrigins() { + for (const Router::CorsPolicy& policy : policies_) { + if (!policy.allowOrigins().empty()) { + return policy.allowOrigins(); } } - return nullptr; + return {}; } bool CorsFilter::forwardNotMatchingPreflights() { - for (const auto policy : policies_) { - if (policy && policy->forwardNotMatchingPreflights()) { - return policy->forwardNotMatchingPreflights().value(); + for (const Router::CorsPolicy& policy : policies_) { + if (policy.forwardNotMatchingPreflights()) { + return policy.forwardNotMatchingPreflights().value(); } } return true; } -const std::string& CorsFilter::allowMethods() { - for (const auto policy : policies_) { - if (policy && !policy->allowMethods().empty()) { - return policy->allowMethods(); +absl::string_view CorsFilter::allowMethods() { + for (const Router::CorsPolicy& policy : policies_) { + if (!policy.allowMethods().empty()) { + return policy.allowMethods(); } } return EMPTY_STRING; } -const std::string& CorsFilter::allowHeaders() { - for (const auto policy : policies_) { - if (policy && !policy->allowHeaders().empty()) { - return policy->allowHeaders(); +absl::string_view CorsFilter::allowHeaders() { + for (const Router::CorsPolicy& policy : policies_) { + if (!policy.allowHeaders().empty()) { + return policy.allowHeaders(); } } return EMPTY_STRING; } -const std::string& CorsFilter::exposeHeaders() { - for (const auto policy : policies_) { - if (policy && !policy->exposeHeaders().empty()) { - return policy->exposeHeaders(); +absl::string_view CorsFilter::exposeHeaders() { + for (const Router::CorsPolicy& policy : policies_) { + if (!policy.exposeHeaders().empty()) { + return policy.exposeHeaders(); } } return EMPTY_STRING; } -const std::string& CorsFilter::maxAge() { - for (const auto policy : policies_) { - if (policy && !policy->maxAge().empty()) { - return policy->maxAge(); +absl::string_view CorsFilter::maxAge() { + for (const Router::CorsPolicy& policy : policies_) { + if (!policy.maxAge().empty()) { + return policy.maxAge(); } } return EMPTY_STRING; } bool CorsFilter::allowCredentials() { - for (const auto policy : policies_) { - if (policy && policy->allowCredentials()) { - return policy->allowCredentials().value(); + for (const Router::CorsPolicy& policy : policies_) { + if (policy.allowCredentials()) { + return policy.allowCredentials().value(); } } return false; } bool CorsFilter::allowPrivateNetworkAccess() { - for (const auto policy : policies_) { - if (policy && policy->allowPrivateNetworkAccess()) { - return policy->allowPrivateNetworkAccess().value(); + for (const Router::CorsPolicy& policy : policies_) { + if (policy.allowPrivateNetworkAccess()) { + return policy.allowPrivateNetworkAccess().value(); } } return false; } bool CorsFilter::shadowEnabled() { - for (const auto policy : policies_) { - if (policy) { - return policy->shadowEnabled(); - } + for (const Router::CorsPolicy& policy : policies_) { + return policy.shadowEnabled(); } return false; } bool CorsFilter::enabled() { - for (const auto policy : policies_) { - if (policy) { - return policy->enabled(); - } + for (const Router::CorsPolicy& policy : policies_) { + return policy.enabled(); } return false; } diff --git a/source/extensions/filters/http/cors/cors_filter.h b/source/extensions/filters/http/cors/cors_filter.h index 0c8cf535a9..a89a488b3f 100644 --- a/source/extensions/filters/http/cors/cors_filter.h +++ b/source/extensions/filters/http/cors/cors_filter.h @@ -88,11 +88,11 @@ class CorsFilter : public Http::StreamFilter { private: friend class CorsFilterTest; - const std::vector* allowOrigins(); - const std::string& allowMethods(); - const std::string& allowHeaders(); - const std::string& exposeHeaders(); - const std::string& maxAge(); + absl::Span allowOrigins(); + absl::string_view allowMethods(); + absl::string_view allowHeaders(); + absl::string_view exposeHeaders(); + absl::string_view maxAge(); bool allowCredentials(); bool allowPrivateNetworkAccess(); bool shadowEnabled(); @@ -102,7 +102,7 @@ class CorsFilter : public Http::StreamFilter { Http::StreamDecoderFilterCallbacks* decoder_callbacks_{}; Http::StreamEncoderFilterCallbacks* encoder_callbacks_{}; - absl::InlinedVector policies_; + absl::InlinedVector, 4> policies_; bool is_cors_request_{}; std::string latched_origin_; diff --git a/source/extensions/filters/http/custom_response/custom_response_filter.cc b/source/extensions/filters/http/custom_response/custom_response_filter.cc index d133c8b753..52e5aefe3e 100644 --- a/source/extensions/filters/http/custom_response/custom_response_filter.cc +++ b/source/extensions/filters/http/custom_response/custom_response_filter.cc @@ -48,13 +48,12 @@ Http::FilterHeadersStatus CustomResponseFilter::encodeHeaders(Http::ResponseHead // policy. Note that since the traversal is least to most specific, we can't // return early when a match is found. PolicySharedPtr policy; - for (const auto* typed_config : + for (const FilterConfig& typed_config : Http::Utility::getAllPerFilterConfig(encoder_callbacks_)) { - ASSERT(typed_config != nullptr); // Check if a match is found first to avoid overwriting policy with an // empty shared_ptr. - auto maybe_policy = typed_config->getPolicy(headers, encoder_callbacks_->streamInfo()); + auto maybe_policy = typed_config.getPolicy(headers, encoder_callbacks_->streamInfo()); if (maybe_policy) { policy = maybe_policy; } diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index b1713614ca..962aede2c2 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -192,13 +192,12 @@ void Filter::initiateCall(const Http::RequestHeaderMap& headers) { } absl::optional maybe_merged_per_route_config; - for (const auto* cfg : + for (const FilterConfigPerRoute& cfg : Http::Utility::getAllPerFilterConfig(decoder_callbacks_)) { - ASSERT(cfg != nullptr); if (maybe_merged_per_route_config.has_value()) { - maybe_merged_per_route_config.value().merge(*cfg); + maybe_merged_per_route_config.value().merge(cfg); } else { - maybe_merged_per_route_config = *cfg; + maybe_merged_per_route_config = cfg; } } diff --git a/source/extensions/filters/http/ext_proc/ext_proc.cc b/source/extensions/filters/http/ext_proc/ext_proc.cc index 8af46dc4da..3013ea1d82 100644 --- a/source/extensions/filters/http/ext_proc/ext_proc.cc +++ b/source/extensions/filters/http/ext_proc/ext_proc.cc @@ -1279,13 +1279,12 @@ void Filter::mergePerRouteConfig() { route_config_merged_ = true; absl::optional merged_config; - for (const auto* typed_cfg : + for (const FilterConfigPerRoute& typed_cfg : Http::Utility::getAllPerFilterConfig(decoder_callbacks_)) { - ASSERT(typed_cfg != nullptr); if (!merged_config.has_value()) { - merged_config.emplace(*typed_cfg); + merged_config.emplace(typed_cfg); } else { - merged_config.emplace(FilterConfigPerRoute(merged_config.value(), *typed_cfg)); + merged_config.emplace(FilterConfigPerRoute(merged_config.value(), typed_cfg)); } } diff --git a/source/extensions/filters/http/header_mutation/header_mutation.cc b/source/extensions/filters/http/header_mutation/header_mutation.cc index e61635a580..259f180d17 100644 --- a/source/extensions/filters/http/header_mutation/header_mutation.cc +++ b/source/extensions/filters/http/header_mutation/header_mutation.cc @@ -46,12 +46,11 @@ Http::FilterHeadersStatus HeaderMutation::decodeHeaders(Http::RequestHeaderMap& // `getAllPerFilterConfig` above returns). // Thus, here we reverse iterate the vector when `most_specific_wins` is false. for (auto it = route_configs_.rbegin(); it != route_configs_.rend(); ++it) { - (*it)->mutations().mutateRequestHeaders(headers, ctx, decoder_callbacks_->streamInfo()); + (*it).get().mutations().mutateRequestHeaders(headers, ctx, decoder_callbacks_->streamInfo()); } } else { - for (const auto* route_config : route_configs_) { - route_config->mutations().mutateRequestHeaders(headers, ctx, - decoder_callbacks_->streamInfo()); + for (const PerRouteHeaderMutation& route_config : route_configs_) { + route_config.mutations().mutateRequestHeaders(headers, ctx, decoder_callbacks_->streamInfo()); } } @@ -70,12 +69,12 @@ Http::FilterHeadersStatus HeaderMutation::encodeHeaders(Http::ResponseHeaderMap& if (!config_->mostSpecificHeaderMutationsWins()) { for (auto it = route_configs_.rbegin(); it != route_configs_.rend(); ++it) { - (*it)->mutations().mutateResponseHeaders(headers, ctx, encoder_callbacks_->streamInfo()); + (*it).get().mutations().mutateResponseHeaders(headers, ctx, encoder_callbacks_->streamInfo()); } } else { - for (const auto* route_config : route_configs_) { - route_config->mutations().mutateResponseHeaders(headers, ctx, - encoder_callbacks_->streamInfo()); + for (const PerRouteHeaderMutation& route_config : route_configs_) { + route_config.mutations().mutateResponseHeaders(headers, ctx, + encoder_callbacks_->streamInfo()); } } diff --git a/source/extensions/filters/http/header_mutation/header_mutation.h b/source/extensions/filters/http/header_mutation/header_mutation.h index 5286d83404..254c9ca076 100644 --- a/source/extensions/filters/http/header_mutation/header_mutation.h +++ b/source/extensions/filters/http/header_mutation/header_mutation.h @@ -82,7 +82,7 @@ class HeaderMutation : public Http::PassThroughFilter, public Logger::Loggable route_configs_{}; + absl::InlinedVector, 4> route_configs_{}; }; } // namespace HeaderMutation diff --git a/test/extensions/filters/http/cors/cors_filter_test.cc b/test/extensions/filters/http/cors/cors_filter_test.cc index c699fe543e..17c1c5818d 100644 --- a/test/extensions/filters/http/cors/cors_filter_test.cc +++ b/test/extensions/filters/http/cors/cors_filter_test.cc @@ -87,8 +87,8 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, true)); EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(2, filter_.policiesForTest().size()); - EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(0)); - EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(1)); + EXPECT_EQ(cors_policy_.get(), &filter_.policiesForTest().at(0).get()); + EXPECT_EQ(cors_policy_.get(), &filter_.policiesForTest().at(1).get()); } // Only 'typed_per_filter_config' of virtual host has cors policy. @@ -104,7 +104,7 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, true)); EXPECT_EQ(false, isCorsRequest()); EXPECT_EQ(1, filter_.policiesForTest().size()); - EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(0)); + EXPECT_EQ(cors_policy_.get(), &filter_.policiesForTest().at(0).get()); } // No cors policy in the 'typed_per_filter_config'. @@ -121,9 +121,7 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, true)); EXPECT_EQ(false, isCorsRequest()); - EXPECT_EQ(2, filter_.policiesForTest().size()); - EXPECT_EQ(nullptr, filter_.policiesForTest().at(0)); - EXPECT_EQ(nullptr, filter_.policiesForTest().at(1)); + EXPECT_EQ(0, filter_.policiesForTest().size()); } { filter_ = CorsFilter(config_); @@ -139,9 +137,8 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, true)); EXPECT_EQ(false, isCorsRequest()); - EXPECT_EQ(2, filter_.policiesForTest().size()); - EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(0)); - EXPECT_EQ(nullptr, filter_.policiesForTest().at(1)); + EXPECT_EQ(1, filter_.policiesForTest().size()); + EXPECT_EQ(cors_policy_.get(), &filter_.policiesForTest().at(0).get()); } { filter_ = CorsFilter(config_); @@ -157,9 +154,8 @@ TEST_F(CorsFilterTest, InitializeCorsPoliciesTest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); EXPECT_EQ(false, isCorsRequest()); - EXPECT_EQ(2, filter_.policiesForTest().size()); - EXPECT_EQ(nullptr, filter_.policiesForTest().at(0)); - EXPECT_EQ(cors_policy_.get(), filter_.policiesForTest().at(1)); + EXPECT_EQ(1, filter_.policiesForTest().size()); + EXPECT_EQ(cors_policy_.get(), &filter_.policiesForTest().at(0).get()); } } From 634f71e38b706b10d3591098dbda5eb65fc34097 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 17 Sep 2024 10:02:41 -0400 Subject: [PATCH 02/28] formatter: removing exceptions from substitution format string (#36168) Risk Level: low Testing: updated tests Docs Changes: n/a Release Notes: n/a https://github.com/envoyproxy/envoy-mobile/issues/176 Signed-off-by: Alyssa Wilk --- .../formatter/substitution_format_string.h | 24 ++++++----- source/common/local_reply/local_reply.cc | 4 +- source/common/tcp_proxy/tcp_proxy.cc | 6 ++- .../common/stream_access_log_common_impl.h | 5 ++- .../extensions/access_loggers/file/config.cc | 13 ++++-- .../access_loggers/fluentd/config.cc | 6 ++- .../access_loggers/open_telemetry/config.cc | 5 ++- .../common/set_filter_state/filter_config.cc | 6 ++- .../network/generic_proxy/file_access_log.h | 16 ++++--- .../filters/udp/udp_proxy/config.cc | 12 ++++-- .../local_response_policy.cc | 6 ++- .../matching/actions/format_string/config.cc | 5 ++- .../substitution_format_string_test.cc | 43 +++++++++---------- .../open_telemetry/access_log_impl_test.cc | 2 +- .../substitution_formatter_test.cc | 2 +- .../network/generic_proxy/proxy_test.cc | 2 +- test/extensions/formatter/cel/cel_test.cc | 14 +++--- .../formatter/metadata/metadata_test.cc | 4 +- .../req_without_query_test.cc | 15 ++++--- tools/code_format/config.yaml | 1 - 20 files changed, 109 insertions(+), 82 deletions(-) diff --git a/source/common/formatter/substitution_format_string.h b/source/common/formatter/substitution_format_string.h index fc1d6f5e6e..c66c77eaff 100644 --- a/source/common/formatter/substitution_format_string.h +++ b/source/common/formatter/substitution_format_string.h @@ -29,7 +29,7 @@ class SubstitutionFormatStringUtils { * Parse list of formatter configurations to commands. */ template - static std::vector> + static absl::StatusOr>> parseFormatters(const FormattersConfig& formatters, Server::Configuration::GenericFactoryContext& context) { std::vector> commands; @@ -37,13 +37,13 @@ class SubstitutionFormatStringUtils { auto* factory = Envoy::Config::Utility::getFactory>(formatter); if (!factory) { - throwEnvoyExceptionOrPanic(absl::StrCat("Formatter not found: ", formatter.name())); + return absl::InvalidArgumentError(absl::StrCat("Formatter not found: ", formatter.name())); } auto typed_config = Envoy::Config::Utility::translateAnyToFactoryConfig( formatter.typed_config(), context.messageValidationVisitor(), *factory); auto parser = factory->createCommandParserFromProto(*typed_config, context); if (!parser) { - throwEnvoyExceptionOrPanic( + return absl::InvalidArgumentError( absl::StrCat("Failed to create command parser: ", formatter.name())); } commands.push_back(std::move(parser)); @@ -56,26 +56,28 @@ class SubstitutionFormatStringUtils { * Generate a formatter object from config SubstitutionFormatString. */ template - static FormatterBasePtr + static absl::StatusOr> fromProtoConfig(const envoy::config::core::v3::SubstitutionFormatString& config, Server::Configuration::GenericFactoryContext& context) { // Instantiate formatter extensions. auto commands = parseFormatters(config.formatters(), context); + RETURN_IF_NOT_OK_REF(commands.status()); switch (config.format_case()) { case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormat: return std::make_unique>( - config.text_format(), config.omit_empty_values(), commands); + config.text_format(), config.omit_empty_values(), *commands); case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kJsonFormat: return createJsonFormatter( config.json_format(), true, config.omit_empty_values(), config.has_json_format_options() ? config.json_format_options().sort_properties() : false, - commands); - case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormatSource: + *commands); + case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormatSource: { + auto data_source_or_error = Config::DataSource::read(config.text_format_source(), true, + context.serverFactoryContext().api()); + RETURN_IF_NOT_OK(data_source_or_error.status()); return std::make_unique>( - THROW_OR_RETURN_VALUE(Config::DataSource::read(config.text_format_source(), true, - context.serverFactoryContext().api()), - std::string), - config.omit_empty_values(), commands); + *data_source_or_error, config.omit_empty_values(), *commands); + } case envoy::config::core::v3::SubstitutionFormatString::FormatCase::FORMAT_NOT_SET: PANIC_DUE_TO_PROTO_UNSET; } diff --git a/source/common/local_reply/local_reply.cc b/source/common/local_reply/local_reply.cc index 8de608a225..10bd821388 100644 --- a/source/common/local_reply/local_reply.cc +++ b/source/common/local_reply/local_reply.cc @@ -22,7 +22,9 @@ class BodyFormatter { BodyFormatter(const envoy::config::core::v3::SubstitutionFormatString& config, Server::Configuration::GenericFactoryContext& context) - : formatter_(Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config, context)), + : formatter_(THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config, context), + Formatter::FormatterBasePtr)), content_type_( !config.content_type().empty() ? config.content_type() : config.format_case() == diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index 6de501b25d..c4e2e58722 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -707,8 +707,10 @@ TunnelingConfigHelperImpl::TunnelingConfigHelperImpl( envoy::config::core::v3::SubstitutionFormatString substitution_format_config; substitution_format_config.mutable_text_format_source()->set_inline_string( config_message.tunneling_config().hostname()); - hostname_fmt_ = Formatter::SubstitutionFormatStringUtils::fromProtoConfig( - substitution_format_config, context); + hostname_fmt_ = + THROW_OR_RETURN_VALUE(Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + substitution_format_config, context), + Formatter::FormatterBasePtr); } std::string TunnelingConfigHelperImpl::host(const StreamInfo::StreamInfo& stream_info) const { diff --git a/source/extensions/access_loggers/common/stream_access_log_common_impl.h b/source/extensions/access_loggers/common/stream_access_log_common_impl.h index 1713b91825..5b07233124 100644 --- a/source/extensions/access_loggers/common/stream_access_log_common_impl.h +++ b/source/extensions/access_loggers/common/stream_access_log_common_impl.h @@ -18,8 +18,9 @@ createStreamAccessLogInstance(const Protobuf::Message& config, AccessLog::Filter MessageUtil::downcastAndValidate(config, context.messageValidationVisitor()); Formatter::FormatterPtr formatter; if (fal_config.access_log_format_case() == T::AccessLogFormatCase::kLogFormat) { - formatter = - Formatter::SubstitutionFormatStringUtils::fromProtoConfig(fal_config.log_format(), context); + formatter = THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::fromProtoConfig(fal_config.log_format(), context), + Formatter::FormatterBasePtr); } else if (fal_config.access_log_format_case() == T::AccessLogFormatCase::ACCESS_LOG_FORMAT_NOT_SET) { formatter = Formatter::HttpSubstitutionFormatUtils::defaultSubstitutionFormatter(); diff --git a/source/extensions/access_loggers/file/config.cc b/source/extensions/access_loggers/file/config.cc index a21c693436..2761aa5064 100644 --- a/source/extensions/access_loggers/file/config.cc +++ b/source/extensions/access_loggers/file/config.cc @@ -35,7 +35,9 @@ FileAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, } else { envoy::config::core::v3::SubstitutionFormatString sff_config; sff_config.mutable_text_format_source()->set_inline_string(fal_config.format()); - formatter = Formatter::SubstitutionFormatStringUtils::fromProtoConfig(sff_config, context); + formatter = THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::fromProtoConfig(sff_config, context), + Formatter::FormatterBasePtr); } break; case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase::kJsonFormat: @@ -46,12 +48,15 @@ FileAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, kTypedJsonFormat: { envoy::config::core::v3::SubstitutionFormatString sff_config; *sff_config.mutable_json_format() = fal_config.typed_json_format(); - formatter = Formatter::SubstitutionFormatStringUtils::fromProtoConfig(sff_config, context); + formatter = THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::fromProtoConfig(sff_config, context), + Formatter::FormatterBasePtr); break; } case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase::kLogFormat: - formatter = - Formatter::SubstitutionFormatStringUtils::fromProtoConfig(fal_config.log_format(), context); + formatter = THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::fromProtoConfig(fal_config.log_format(), context), + Formatter::FormatterBasePtr); break; case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase:: ACCESS_LOG_FORMAT_NOT_SET: diff --git a/source/extensions/access_loggers/fluentd/config.cc b/source/extensions/access_loggers/fluentd/config.cc index 877bd69de2..a6614137e6 100644 --- a/source/extensions/access_loggers/fluentd/config.cc +++ b/source/extensions/access_loggers/fluentd/config.cc @@ -60,8 +60,10 @@ FluentdAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config // payload. // TODO(ohadvano): Improve the formatting operation by creating a dedicated formatter that // will directly serialize the record to msgpack payload. - auto commands = - Formatter::SubstitutionFormatStringUtils::parseFormatters(proto_config.formatters(), context); + auto commands = THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::parseFormatters(proto_config.formatters(), context), + std::vector>); + Formatter::FormatterPtr json_formatter = Formatter::SubstitutionFormatStringUtils::createJsonFormatter(proto_config.record(), true, false, false, commands); diff --git a/source/extensions/access_loggers/open_telemetry/config.cc b/source/extensions/access_loggers/open_telemetry/config.cc index 237c5c9fa8..5b9e817f52 100644 --- a/source/extensions/access_loggers/open_telemetry/config.cc +++ b/source/extensions/access_loggers/open_telemetry/config.cc @@ -41,8 +41,9 @@ AccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, const envoy::extensions::access_loggers::open_telemetry::v3::OpenTelemetryAccessLogConfig&>( config, context.messageValidationVisitor()); - auto commands = - Formatter::SubstitutionFormatStringUtils::parseFormatters(proto_config.formatters(), context); + auto commands = THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::parseFormatters(proto_config.formatters(), context), + std::vector>); return std::make_shared( std::move(filter), proto_config, context.serverFactoryContext().threadLocal(), diff --git a/source/extensions/filters/common/set_filter_state/filter_config.cc b/source/extensions/filters/common/set_filter_state/filter_config.cc index 5188b097af..a14e672e89 100644 --- a/source/extensions/filters/common/set_filter_state/filter_config.cc +++ b/source/extensions/filters/common/set_filter_state/filter_config.cc @@ -50,8 +50,10 @@ Config::parse(const Protobuf::RepeatedPtrField& proto_val break; } value.skip_if_empty_ = proto_value.skip_if_empty(); - value.value_ = Formatter::SubstitutionFormatStringUtils::fromProtoConfig( - proto_value.format_string(), context); + value.value_ = + THROW_OR_RETURN_VALUE(Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + proto_value.format_string(), context), + Formatter::FormatterBasePtr); values.push_back(std::move(value)); } return values; diff --git a/source/extensions/filters/network/generic_proxy/file_access_log.h b/source/extensions/filters/network/generic_proxy/file_access_log.h index 779b4c08fc..30eabb239e 100644 --- a/source/extensions/filters/network/generic_proxy/file_access_log.h +++ b/source/extensions/filters/network/generic_proxy/file_access_log.h @@ -59,8 +59,9 @@ class FileAccessLogFactoryBase : public AccessLog::AccessLogInstanceFactoryBase< } else { envoy::config::core::v3::SubstitutionFormatString sff_config; sff_config.mutable_text_format_source()->set_inline_string(typed_config.format()); - formatter = - Formatter::SubstitutionFormatStringUtils::fromProtoConfig(sff_config, context); + formatter = THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::fromProtoConfig(sff_config, context), + Formatter::FormatterBasePtr); } break; case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase:: @@ -72,14 +73,17 @@ class FileAccessLogFactoryBase : public AccessLog::AccessLogInstanceFactoryBase< kTypedJsonFormat: { envoy::config::core::v3::SubstitutionFormatString sff_config; *sff_config.mutable_json_format() = typed_config.typed_json_format(); - formatter = - Formatter::SubstitutionFormatStringUtils::fromProtoConfig(sff_config, context); + formatter = THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::fromProtoConfig(sff_config, context), + Formatter::FormatterBasePtr); break; } case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase:: kLogFormat: - formatter = Formatter::SubstitutionFormatStringUtils::fromProtoConfig( - typed_config.log_format(), context); + formatter = + THROW_OR_RETURN_VALUE(Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + typed_config.log_format(), context), + Formatter::FormatterBasePtr); break; case envoy::extensions::access_loggers::file::v3::FileAccessLog::AccessLogFormatCase:: ACCESS_LOG_FORMAT_NOT_SET: diff --git a/source/extensions/filters/udp/udp_proxy/config.cc b/source/extensions/filters/udp/udp_proxy/config.cc index 436368faf0..200e7099c7 100644 --- a/source/extensions/filters/udp/udp_proxy/config.cc +++ b/source/extensions/filters/udp/udp_proxy/config.cc @@ -61,8 +61,10 @@ TunnelingConfigImpl::TunnelingConfigImpl(const TunnelingConfig& config, envoy::config::core::v3::SubstitutionFormatString proxy_substitution_format_config; proxy_substitution_format_config.mutable_text_format_source()->set_inline_string( config.proxy_host()); - proxy_host_formatter_ = Formatter::SubstitutionFormatStringUtils::fromProtoConfig( - proxy_substitution_format_config, context); + proxy_host_formatter_ = + THROW_OR_RETURN_VALUE(Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + proxy_substitution_format_config, context), + Formatter::FormatterBasePtr); if (config.has_proxy_port()) { uint32_t port = config.proxy_port().value(); @@ -76,8 +78,10 @@ TunnelingConfigImpl::TunnelingConfigImpl(const TunnelingConfig& config, envoy::config::core::v3::SubstitutionFormatString target_substitution_format_config; target_substitution_format_config.mutable_text_format_source()->set_inline_string( config.target_host()); - target_host_formatter_ = Formatter::SubstitutionFormatStringUtils::fromProtoConfig( - target_substitution_format_config, context); + target_host_formatter_ = + THROW_OR_RETURN_VALUE(Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + target_substitution_format_config, context), + Formatter::FormatterBasePtr); } UdpProxyFilterConfigImpl::UdpProxyFilterConfigImpl( diff --git a/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc b/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc index 4880fb2098..4777165781 100644 --- a/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc +++ b/source/extensions/http/custom_response/local_response_policy/local_response_policy.cc @@ -39,8 +39,10 @@ LocalResponsePolicy::LocalResponsePolicy( // by this PR and will be fixed in the future. Server::GenericFactoryContextImpl generic_context(context, context.messageValidationVisitor()); if (config.has_body_format()) { - formatter_ = Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config.body_format(), - generic_context); + formatter_ = + THROW_OR_RETURN_VALUE(Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + config.body_format(), generic_context), + Formatter::FormatterBasePtr); } } diff --git a/source/extensions/matching/actions/format_string/config.cc b/source/extensions/matching/actions/format_string/config.cc index ff8cb31418..3de72f9ec1 100644 --- a/source/extensions/matching/actions/format_string/config.cc +++ b/source/extensions/matching/actions/format_string/config.cc @@ -32,8 +32,9 @@ ActionFactory::createActionFactoryCb(const Protobuf::Message& proto_config, proto_config, validator); Server::GenericFactoryContextImpl generic_context(context, validator); - Formatter::FormatterConstSharedPtr formatter = - Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config, generic_context); + Formatter::FormatterConstSharedPtr formatter = THROW_OR_RETURN_VALUE( + Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config, generic_context), + Formatter::FormatterBasePtr); return [formatter]() { return std::make_unique(formatter); }; } diff --git a/test/common/formatter/substitution_format_string_test.cc b/test/common/formatter/substitution_format_string_test.cc index 6d74620d2a..e2d01dd133 100644 --- a/test/common/formatter/substitution_format_string_test.cc +++ b/test/common/formatter/substitution_format_string_test.cc @@ -47,7 +47,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigText) { )EOF"; TestUtility::loadFromYaml(yaml, config_); - auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + auto formatter = *SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("plain text, path=/bar/foo, code=200", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -63,7 +63,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigJson) { )EOF"; TestUtility::loadFromYaml(yaml, config_); - auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + auto formatter = *SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); const auto out_json = formatter->formatWithContext(formatter_context_, stream_info_); const std::string expected = R"EOF({ @@ -86,10 +86,11 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestInvalidConfigs) { }; for (const auto& yaml : invalid_configs) { TestUtility::loadFromYaml(yaml, config_); - EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatStringUtils::fromProtoConfig(config_, context_), - EnvoyException, - "Only string values, nested structs, list values and number values " - "are supported in structured access log format."); + EXPECT_THROW_WITH_MESSAGE( + SubstitutionFormatStringUtils::fromProtoConfig(config_, context_).IgnoreError(), + EnvoyException, + "Only string values, nested structs, list values and number values " + "are supported in structured access log format."); } } @@ -107,7 +108,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigFormatterExtension) )EOF"; TestUtility::loadFromYaml(yaml, config_); - auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + auto formatter = *SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("plain text TestFormatter", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -127,9 +128,8 @@ TEST_F(SubstitutionFormatStringUtilsTest, )EOF"; TestUtility::loadFromYaml(yaml, config_); - EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatStringUtils::fromProtoConfig(config_, context_), - EnvoyException, - "Failed to create command parser: envoy.formatter.FailFormatter"); + EXPECT_EQ(SubstitutionFormatStringUtils::fromProtoConfig(config_, context_).status().message(), + "Failed to create command parser: envoy.formatter.FailFormatter"); } TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigFormatterExtensionUnknown) { @@ -143,9 +143,8 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigFormatterExtensionU )EOF"; TestUtility::loadFromYaml(yaml, config_); - EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatStringUtils::fromProtoConfig(config_, context_), - EnvoyException, - "Formatter not found: envoy.formatter.TestFormatterUnknown"); + EXPECT_EQ(SubstitutionFormatStringUtils::fromProtoConfig(config_, context_).status().message(), + "Formatter not found: envoy.formatter.TestFormatterUnknown"); } TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigJsonWithExtension) { @@ -166,7 +165,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigJsonWithExtension) )EOF"; TestUtility::loadFromYaml(yaml, config_); - auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + auto formatter = *SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); const auto out_json = formatter->formatWithContext(formatter_context_, stream_info_); const std::string expected = R"EOF({ @@ -201,7 +200,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigJsonWithMultipleExt )EOF"; TestUtility::loadFromYaml(yaml, config_); - auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + auto formatter = *SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); const auto out_json = formatter->formatWithContext(formatter_context_, stream_info_); const std::string expected = R"EOF({ @@ -225,9 +224,8 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestParseFormattersWithUnknownExtensio TestUtility::loadFromYaml(yaml, proto); *entry1 = proto; - EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatStringUtils::parseFormatters(config, context_), - EnvoyException, - "Formatter not found: envoy.formatter.TestFormatterUnknown"); + EXPECT_EQ(SubstitutionFormatStringUtils::parseFormatters(config, context_).status().message(), + "Formatter not found: envoy.formatter.TestFormatterUnknown"); } TEST_F(SubstitutionFormatStringUtilsTest, TestParseFormattersWithInvalidFormatter) { @@ -246,9 +244,8 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestParseFormattersWithInvalidFormatte TestUtility::loadFromYaml(yaml, proto); *entry1 = proto; - EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatStringUtils::parseFormatters(config, context_), - EnvoyException, - "Failed to create command parser: envoy.formatter.FailFormatter"); + EXPECT_EQ(SubstitutionFormatStringUtils::parseFormatters(config, context_).status().message(), + "Failed to create command parser: envoy.formatter.FailFormatter"); } TEST_F(SubstitutionFormatStringUtilsTest, TestParseFormattersWithSingleExtension) { @@ -267,7 +264,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestParseFormattersWithSingleExtension TestUtility::loadFromYaml(yaml, proto); *entry1 = proto; - auto commands = SubstitutionFormatStringUtils::parseFormatters(config, context_); + auto commands = *SubstitutionFormatStringUtils::parseFormatters(config, context_); ASSERT_EQ(1, commands.size()); absl::optional max_length = {}; @@ -306,7 +303,7 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestParseFormattersWithMultipleExtensi TestUtility::loadFromYaml(additional_command_yaml, additional_command_proto); *entry2 = additional_command_proto; - auto commands = SubstitutionFormatStringUtils::parseFormatters(config, context_); + auto commands = *SubstitutionFormatStringUtils::parseFormatters(config, context_); ASSERT_EQ(2, commands.size()); absl::optional max_length = {}; diff --git a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc index 532d2b288e..db253e6680 100644 --- a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc @@ -84,7 +84,7 @@ class AccessLogTest : public testing::Test { return logger_; }); auto commands = - Formatter::SubstitutionFormatStringUtils::parseFormatters(config_.formatters(), context_); + *Formatter::SubstitutionFormatStringUtils::parseFormatters(config_.formatters(), context_); return std::make_unique(FilterPtr{filter_}, config_, tls_, logger_cache_, commands); } diff --git a/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc b/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc index 54fe710a47..ab51c3eb2e 100644 --- a/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc +++ b/test/extensions/access_loggers/open_telemetry/substitution_formatter_test.cc @@ -915,7 +915,7 @@ TEST(SubstitutionFormatterTest, CELFormatterTest) { "@type": type.googleapis.com/envoy.extensions.formatter.cel.v3.Cel )EOF", otel_config); - auto commands = Formatter::SubstitutionFormatStringUtils::parseFormatters( + auto commands = *Formatter::SubstitutionFormatStringUtils::parseFormatters( otel_config.formatters(), context); OpenTelemetryFormatter formatter(otel_config.resource_attributes(), commands); diff --git a/test/extensions/filters/network/generic_proxy/proxy_test.cc b/test/extensions/filters/network/generic_proxy/proxy_test.cc index 6f6121a64d..31900ba301 100644 --- a/test/extensions/filters/network/generic_proxy/proxy_test.cc +++ b/test/extensions/filters/network/generic_proxy/proxy_test.cc @@ -108,7 +108,7 @@ class FilterConfigTest : public testing::Test { envoy::config::core::v3::SubstitutionFormatString sff_config; sff_config.mutable_text_format_source()->set_inline_string(format); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig( + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig( sff_config, factory_context_); return std::make_shared( diff --git a/test/extensions/formatter/cel/cel_test.cc b/test/extensions/formatter/cel/cel_test.cc index 77f9d1b69d..ff4e4425cd 100644 --- a/test/extensions/formatter/cel/cel_test.cc +++ b/test/extensions/formatter/cel/cel_test.cc @@ -77,7 +77,7 @@ TEST_F(CELFormatterTest, TestRequestHeader) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("GET", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -93,7 +93,7 @@ TEST_F(CELFormatterTest, TestMissingRequestHeader) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("-", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -109,7 +109,7 @@ TEST_F(CELFormatterTest, TestWithoutMaxLength) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("/original/path?secret=parameter", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -126,7 +126,7 @@ TEST_F(CELFormatterTest, TestMaxLength) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("/original", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -142,7 +142,7 @@ TEST_F(CELFormatterTest, TestContains) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("true", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -158,7 +158,7 @@ TEST_F(CELFormatterTest, TestComplexCelExpression) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("true /original false", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -174,7 +174,7 @@ TEST_F(CELFormatterTest, TestInvalidExpression) { TestUtility::loadFromYaml(yaml, config_); EXPECT_THROW_WITH_REGEX( - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_), + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_), EnvoyException, "Not able to parse filter expression: .*"); } #endif diff --git a/test/extensions/formatter/metadata/metadata_test.cc b/test/extensions/formatter/metadata/metadata_test.cc index acb03df661..f6d434036f 100644 --- a/test/extensions/formatter/metadata/metadata_test.cc +++ b/test/extensions/formatter/metadata/metadata_test.cc @@ -37,7 +37,9 @@ class MetadataFormatterTest : public ::testing::Test { )EOF", tag, type); TestUtility::loadFromYaml(yaml, config_); - return Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + return THROW_OR_RETURN_VALUE( + Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_), + Envoy::Formatter::FormatterBasePtr); } Http::TestRequestHeaderMapImpl request_headers_; diff --git a/test/extensions/formatter/req_without_query/req_without_query_test.cc b/test/extensions/formatter/req_without_query/req_without_query_test.cc index bda43e2626..e019ef4e91 100644 --- a/test/extensions/formatter/req_without_query/req_without_query_test.cc +++ b/test/extensions/formatter/req_without_query/req_without_query_test.cc @@ -44,7 +44,7 @@ TEST_F(ReqWithoutQueryTest, TestStripQueryString) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("/request/path", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -61,7 +61,7 @@ TEST_F(ReqWithoutQueryTest, TestSelectMainHeader) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("/original/path", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -78,7 +78,7 @@ TEST_F(ReqWithoutQueryTest, TestSelectAlternativeHeader) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("/request/path", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -95,7 +95,7 @@ TEST_F(ReqWithoutQueryTest, TestTruncateHeader) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("/requ", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -112,7 +112,7 @@ TEST_F(ReqWithoutQueryTest, TestNonExistingHeader) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); EXPECT_EQ("-", formatter->formatWithContext(formatter_context_, stream_info_)); } @@ -139,7 +139,7 @@ TEST_F(ReqWithoutQueryTest, TestFormatJson) { TestUtility::loadFromYaml(yaml, config_); auto formatter = - Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); + *Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_); const std::string actual = formatter->formatWithContext(formatter_context_, stream_info_); EXPECT_TRUE(TestUtility::jsonStringEqual(actual, expected)); } @@ -156,7 +156,8 @@ TEST_F(ReqWithoutQueryTest, TestParserNotRecognizingCommand) { )EOF"; TestUtility::loadFromYaml(yaml, config_); - EXPECT_THROW(Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_), + EXPECT_THROW(Envoy::Formatter::SubstitutionFormatStringUtils::fromProtoConfig(config_, context_) + .IgnoreError(), EnvoyException); } diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index 286208a3ba..bbc5d3f81d 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -110,7 +110,6 @@ paths: - source/common/formatter/stream_info_formatter.h - source/common/formatter/stream_info_formatter.cc - source/common/formatter/substitution_formatter.h - - source/common/formatter/substitution_format_string.h - source/common/stats/tag_extractor_impl.cc - source/common/protobuf/yaml_utility.cc - source/common/protobuf/utility.cc From 6c2e7ebef4aa41d9700d59e2316b118dcbf912b0 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 17 Sep 2024 11:59:00 -0400 Subject: [PATCH 03/28] docs: update owners to reflect RedHat access (#36183) Removing an employee who has moved on. Signed-off-by: Alyssa Wilk --- OWNERS.md | 1 - 1 file changed, 1 deletion(-) diff --git a/OWNERS.md b/OWNERS.md index c001eb66a1..3ba38ffed2 100644 --- a/OWNERS.md +++ b/OWNERS.md @@ -76,7 +76,6 @@ without further review. * All senior maintainers * Tony Allen ([tonya11en](https://github.com/tonya11en)) (tony@allen.gg) -* Otto van der Schaaf ([oschaaf](https://github.com/oschaaf)) (oschaaf@redhat.com) * Tim Walsh ([twghu](https://github.com/twghu)) (twalsh@redhat.com) * Pradeep Rao ([pradeepcrao](https://github.com/pradeepcrao)) (pcrao@google.com) * Kateryna Nezdolii ([nezdolik](https://github.com/nezdolik)) (kateryna.nezdolii@gmail.com) From a1e12cd10f26eaa741e31bbf0770e8baee75dfe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20R=2E=20Sede=C3=B1o?= Date: Tue, 17 Sep 2024 12:38:37 -0400 Subject: [PATCH 04/28] bump googleapis (#36182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Risk Level: low Testing: Signed-off-by: Alejandro R SedeƱo --- api/bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 6c353c8591..dc3c4389c3 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -79,9 +79,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Google APIs", project_desc = "Public interface definitions of Google APIs", project_url = "https://github.com/googleapis/googleapis", - version = "114a745b2841a044e98cdbb19358ed29fcf4a5f1", - sha256 = "9b4e0d0a04a217c06b426aefd03b82581a9510ca766d2d1c70e52bb2ad4a0703", - release_date = "2023-01-10", + version = "fd52b5754b2b268bc3a22a10f29844f206abb327", + sha256 = "97fc354dddfd3ea03e7bf2ad74129291ed6fad7ff39d3bd8daec738a3672eb8a", + release_date = "2024-09-16", strip_prefix = "googleapis-{version}", urls = ["https://github.com/googleapis/googleapis/archive/{version}.tar.gz"], use_category = ["api"], From cea046f058c29d727ada8aea92316089e6451694 Mon Sep 17 00:00:00 2001 From: bsurber <73970703+bsurber@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:54:19 -0700 Subject: [PATCH 05/28] rlqs: Implement RLQS stream restarts if the stream goes down mid-use. (#36170) Commit Message: Implement RLQS stream restarts if the stream goes down mid-use. Additional Description: Stream restarts are done during periodic usage reporting, which limits retry spam while backends are offline. Risk Level: Testing: Integration testing updated to exercise the filter before and after stream closure. Docs Changes: Release Notes: Platform Specific Features: --------- Signed-off-by: Brian Surber --- .../filters/http/rate_limit_quota/client.h | 2 +- .../http/rate_limit_quota/client_impl.cc | 36 +++++++----- .../http/rate_limit_quota/client_impl.h | 2 +- .../filters/http/rate_limit_quota/filter.cc | 2 +- .../http/rate_limit_quota/client_test.cc | 33 ++++++++++- .../http/rate_limit_quota/client_test_utils.h | 12 ++-- .../http/rate_limit_quota/integration_test.cc | 55 +++++++++++-------- .../filters/http/rate_limit_quota/mocks.h | 2 +- 8 files changed, 95 insertions(+), 49 deletions(-) diff --git a/source/extensions/filters/http/rate_limit_quota/client.h b/source/extensions/filters/http/rate_limit_quota/client.h index a8d31c39b0..198b919d5b 100644 --- a/source/extensions/filters/http/rate_limit_quota/client.h +++ b/source/extensions/filters/http/rate_limit_quota/client.h @@ -37,7 +37,7 @@ class RateLimitClient { public: virtual ~RateLimitClient() = default; - virtual absl::Status startStream(const StreamInfo::StreamInfo& stream_info) PURE; + virtual absl::Status startStream(const StreamInfo::StreamInfo* stream_info) PURE; virtual void closeStream() PURE; virtual void sendUsageReport(absl::optional bucket_id) PURE; diff --git a/source/extensions/filters/http/rate_limit_quota/client_impl.cc b/source/extensions/filters/http/rate_limit_quota/client_impl.cc index 7d886a49e7..876060592e 100644 --- a/source/extensions/filters/http/rate_limit_quota/client_impl.cc +++ b/source/extensions/filters/http/rate_limit_quota/client_impl.cc @@ -64,14 +64,17 @@ RateLimitQuotaUsageReports RateLimitClientImpl::buildReport(absl::optional bucket_id) { - if (stream_ != nullptr) { - // Build the report and then send the report to RLQS server. - // `end_stream` should always be set to false as we don't want to close the stream locally. - stream_->sendMessage(buildReport(bucket_id), /*end_stream=*/false); - } else { - // Don't send any reports if stream has already been closed. - ENVOY_LOG(debug, "The stream has already been closed; no reports will be sent."); + if (stream_ == nullptr) { + ENVOY_LOG(debug, "The RLQS stream has been closed and must be restarted to send reports."); + if (absl::Status err = startStream(nullptr); !err.ok()) { + ENVOY_LOG(error, "Failed to start the stream to send reports."); + return; + } } + + // Build the report and then send the report to RLQS server. + // `end_stream` should always be set to false as we don't want to close the stream locally. + stream_->sendMessage(buildReport(bucket_id), /*end_stream=*/false); } void RateLimitClientImpl::onReceiveMessage(RateLimitQuotaResponsePtr&& response) { @@ -165,20 +168,27 @@ void RateLimitClientImpl::onRemoteClose(Grpc::Status::GrpcStatus status, stream_ = nullptr; } -absl::Status RateLimitClientImpl::startStream(const StreamInfo::StreamInfo& stream_info) { +absl::Status RateLimitClientImpl::startStream(const StreamInfo::StreamInfo* stream_info) { // Starts stream if it has not been opened yet. if (stream_ == nullptr) { ENVOY_LOG(debug, "Trying to start the new gRPC stream"); + auto stream_options = Http::AsyncClient::RequestOptions(); + if (stream_info) { + stream_options.setParentContext(Http::AsyncClient::ParentContext{stream_info}); + } stream_ = aync_client_.start( *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.rate_limit_quota.v3.RateLimitQuotaService.StreamRateLimitQuotas"), - *this, - Http::AsyncClient::RequestOptions().setParentContext( - Http::AsyncClient::ParentContext{&stream_info})); + *this, stream_options); + } + + // If still null after attempting a start. + if (stream_ == nullptr) { + return absl::InternalError("Failed to start the stream"); } - // Returns error status if start failed (i.e., stream_ is nullptr). - return stream_ == nullptr ? absl::InternalError("Failed to start the stream") : absl::OkStatus(); + ENVOY_LOG(debug, "gRPC stream has been started"); + return absl::OkStatus(); } } // namespace RateLimitQuota diff --git a/source/extensions/filters/http/rate_limit_quota/client_impl.h b/source/extensions/filters/http/rate_limit_quota/client_impl.h index b471755d24..c4584c26b8 100644 --- a/source/extensions/filters/http/rate_limit_quota/client_impl.h +++ b/source/extensions/filters/http/rate_limit_quota/client_impl.h @@ -45,7 +45,7 @@ class RateLimitClientImpl : public RateLimitClient, void onRemoteClose(Grpc::Status::GrpcStatus status, const std::string& message) override; // RateLimitClient methods. - absl::Status startStream(const StreamInfo::StreamInfo& stream_info) override; + absl::Status startStream(const StreamInfo::StreamInfo* stream_info) override; void closeStream() override; // Send the usage report to RLQS server void sendUsageReport(absl::optional bucket_id) override; diff --git a/source/extensions/filters/http/rate_limit_quota/filter.cc b/source/extensions/filters/http/rate_limit_quota/filter.cc index 65535753c3..e5671033cc 100644 --- a/source/extensions/filters/http/rate_limit_quota/filter.cc +++ b/source/extensions/filters/http/rate_limit_quota/filter.cc @@ -185,7 +185,7 @@ RateLimitQuotaFilter::sendImmediateReport(const size_t bucket_id, // Start the streaming on the first request. // It will be a no-op if the stream is already active. - auto status = client_.rate_limit_client->startStream(callbacks_->streamInfo()); + auto status = client_.rate_limit_client->startStream(&callbacks_->streamInfo()); if (!status.ok()) { ENVOY_LOG(error, "Failed to start the gRPC stream: ", status.message()); // TODO(tyxia) Check `NoAssignmentBehavior` behavior instead of fail-open here. diff --git a/test/extensions/filters/http/rate_limit_quota/client_test.cc b/test/extensions/filters/http/rate_limit_quota/client_test.cc index 1fee6b6f46..336b47b7ff 100644 --- a/test/extensions/filters/http/rate_limit_quota/client_test.cc +++ b/test/extensions/filters/http/rate_limit_quota/client_test.cc @@ -18,7 +18,7 @@ class RateLimitClientTest : public testing::Test { }; TEST_F(RateLimitClientTest, OpenAndCloseStream) { - EXPECT_OK(test_client.client_->startStream(test_client.stream_info_)); + EXPECT_OK(test_client.client_->startStream(&test_client.stream_info_)); EXPECT_CALL(test_client.stream_, closeStream()); EXPECT_CALL(test_client.stream_, resetStream()); test_client.client_->closeStream(); @@ -27,7 +27,7 @@ TEST_F(RateLimitClientTest, OpenAndCloseStream) { TEST_F(RateLimitClientTest, SendUsageReport) { ::envoy::service::rate_limit_quota::v3::BucketId bucket_id; TestUtility::loadFromYaml(SingleBukcetId, bucket_id); - EXPECT_OK(test_client.client_->startStream(test_client.stream_info_)); + EXPECT_OK(test_client.client_->startStream(&test_client.stream_info_)); bool end_stream = false; // Send quota usage report and ensure that we get it. EXPECT_CALL(test_client.stream_, sendMessageRaw_(_, end_stream)); @@ -39,7 +39,7 @@ TEST_F(RateLimitClientTest, SendUsageReport) { } TEST_F(RateLimitClientTest, SendRequestAndReceiveResponse) { - EXPECT_OK(test_client.client_->startStream(test_client.stream_info_)); + EXPECT_OK(test_client.client_->startStream(&test_client.stream_info_)); ASSERT_NE(test_client.stream_callbacks_, nullptr); auto empty_request_headers = Http::RequestHeaderMapImpl::create(); @@ -66,6 +66,33 @@ TEST_F(RateLimitClientTest, SendRequestAndReceiveResponse) { test_client.client_->onRemoteClose(0, ""); } +TEST_F(RateLimitClientTest, RestartStreamWhileInUse) { + ::envoy::service::rate_limit_quota::v3::BucketId bucket_id; + TestUtility::loadFromYaml(SingleBukcetId, bucket_id); + EXPECT_OK(test_client.client_->startStream(&test_client.stream_info_)); + + bool end_stream = false; + // Send quota usage report and ensure that we get it. + EXPECT_CALL(test_client.stream_, sendMessageRaw_(_, end_stream)); + const size_t bucket_id_hash = MessageUtil::hash(bucket_id); + test_client.client_->sendUsageReport(bucket_id_hash); + EXPECT_CALL(test_client.stream_, closeStream()); + EXPECT_CALL(test_client.stream_, resetStream()); + test_client.client_->closeStream(); + + // Expect the stream to reopen while trying to send the next usage report. + EXPECT_CALL(test_client.stream_, sendMessageRaw_(_, end_stream)); + test_client.client_->sendUsageReport(bucket_id_hash); + EXPECT_CALL(test_client.stream_, closeStream()); + EXPECT_CALL(test_client.stream_, resetStream()); + test_client.client_->closeStream(); + + // Expect the client to handle a restart failure. + EXPECT_CALL(*test_client.async_client_, startRaw(_, _, _, _)).WillOnce(testing::Return(nullptr)); + WAIT_FOR_LOG_CONTAINS("error", "Failed to start the stream to send reports.", + { test_client.client_->sendUsageReport(bucket_id_hash); }); +} + } // namespace } // namespace RateLimitQuota } // namespace HttpFilters diff --git a/test/extensions/filters/http/rate_limit_quota/client_test_utils.h b/test/extensions/filters/http/rate_limit_quota/client_test_utils.h index 2f19f9b48b..e6624af9b4 100644 --- a/test/extensions/filters/http/rate_limit_quota/client_test_utils.h +++ b/test/extensions/filters/http/rate_limit_quota/client_test_utils.h @@ -70,12 +70,12 @@ class RateLimitTestClient { } Grpc::RawAsyncClientSharedPtr mockCreateAsyncClient(Unused, Unused, Unused) { - auto async_client = std::make_shared(); - EXPECT_CALL(*async_client, startRaw("envoy.service.rate_limit_quota.v3.RateLimitQuotaService", - "StreamRateLimitQuotas", _, _)) - .WillOnce(Invoke(this, &RateLimitTestClient::mockStartRaw)); + async_client_ = std::make_shared(); + EXPECT_CALL(*async_client_, startRaw("envoy.service.rate_limit_quota.v3.RateLimitQuotaService", + "StreamRateLimitQuotas", _, _)) + .WillRepeatedly(Invoke(this, &RateLimitTestClient::mockStartRaw)); - return async_client; + return async_client_; } Grpc::RawAsyncStream* mockStartRaw(Unused, Unused, Grpc::RawAsyncStreamCallbacks& callbacks, @@ -97,7 +97,7 @@ class RateLimitTestClient { Grpc::RawAsyncStreamCallbacks* stream_callbacks_; Grpc::Status::GrpcStatus grpc_status_ = Grpc::Status::WellKnownGrpcStatus::Ok; RateLimitClientPtr client_; - // std::unique_ptr client_; + std::shared_ptr async_client_ = nullptr; MockRateLimitQuotaCallbacks callbacks_; bool external_inited_ = false; bool start_failed_ = false; diff --git a/test/extensions/filters/http/rate_limit_quota/integration_test.cc b/test/extensions/filters/http/rate_limit_quota/integration_test.cc index dc37a9720b..37d4a97df6 100644 --- a/test/extensions/filters/http/rate_limit_quota/integration_test.cc +++ b/test/extensions/filters/http/rate_limit_quota/integration_test.cc @@ -787,42 +787,51 @@ TEST_P(RateLimitQuotaIntegrationTest, BasicFlowPeriodicalReportWithStreamClosed) EXPECT_TRUE(response_->complete()); EXPECT_EQ(response_->headers().getStatusValue(), "200"); + // ValidMatcherConfig. + int report_interval_sec = 60; // Trigger the report periodically. for (int i = 0; i < 6; ++i) { if (i == 2) { // Close the stream. - rlqs_stream_->finishGrpcStream(Grpc::Status::Ok); + WAIT_FOR_LOG_CONTAINS("debug", "gRPC stream closed remotely with status", + { rlqs_stream_->finishGrpcStream(Grpc::Status::Canceled); }); + ASSERT_TRUE(rlqs_stream_->waitForReset()); } // Advance the time by report_interval. simTime().advanceTimeWait(std::chrono::milliseconds(report_interval_sec * 1000)); - // Only perform rlqs server check and response before stream is remotely closed. - if (i < 2) { - // Checks that the rate limit server has received the periodical reports. - ASSERT_TRUE(rlqs_stream_->waitForGrpcMessage(*dispatcher_, reports)); + if (i == 2) { + // Stream should be restarted when next required for usage reporting. + ASSERT_TRUE(rlqs_connection_->waitForNewStream(*dispatcher_, rlqs_stream_)); + rlqs_stream_->startGrpcStream(); + } - // Verify the usage report content. - ASSERT_THAT(reports.bucket_quota_usages_size(), 1); - const auto& usage = reports.bucket_quota_usages(0); - // Report only represents the usage since last report. - // In the periodical report case here, the number of request allowed and denied is 0 since no - // new requests comes in. - EXPECT_EQ(usage.num_requests_allowed(), 0); - EXPECT_EQ(usage.num_requests_denied(), 0); - // time_elapsed equals to periodical reporting interval. - EXPECT_EQ(Protobuf::util::TimeUtil::DurationToSeconds(usage.time_elapsed()), - report_interval_sec); + // Only perform rlqs server check and response before stream is remotely + // closed. Checks that the rate limit server has received the periodical + // reports. + ASSERT_TRUE(rlqs_stream_->waitForGrpcMessage(*dispatcher_, reports)); + + // Verify the usage report content. + ASSERT_THAT(reports.bucket_quota_usages_size(), 1); + const auto& usage = reports.bucket_quota_usages(0); + // Report only represents the usage since last report. + // In the periodical report case here, the number of request allowed and + // denied is 0 since no new requests comes in. + EXPECT_EQ(usage.num_requests_allowed(), 0); + EXPECT_EQ(usage.num_requests_denied(), 0); + // time_elapsed equals to periodical reporting interval. + EXPECT_EQ(Protobuf::util::TimeUtil::DurationToSeconds(usage.time_elapsed()), + report_interval_sec); - // Build the rlqs server response. - envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse rlqs_response2; - auto* bucket_action2 = rlqs_response2.add_bucket_action(); + // Build the rlqs server response. + envoy::service::rate_limit_quota::v3::RateLimitQuotaResponse rlqs_response2; + auto* bucket_action2 = rlqs_response2.add_bucket_action(); - for (const auto& [key, value] : custom_headers_cpy) { - (*bucket_action2->mutable_bucket_id()->mutable_bucket()).insert({key, value}); - } - rlqs_stream_->sendGrpcMessage(rlqs_response2); + for (const auto& [key, value] : custom_headers_cpy) { + (*bucket_action2->mutable_bucket_id()->mutable_bucket()).insert({key, value}); } + rlqs_stream_->sendGrpcMessage(rlqs_response2); } } diff --git a/test/extensions/filters/http/rate_limit_quota/mocks.h b/test/extensions/filters/http/rate_limit_quota/mocks.h index 01d374ec6c..aedd75e3e9 100644 --- a/test/extensions/filters/http/rate_limit_quota/mocks.h +++ b/test/extensions/filters/http/rate_limit_quota/mocks.h @@ -28,7 +28,7 @@ class MockRateLimitClient : public RateLimitClient { MockRateLimitClient() = default; ~MockRateLimitClient() override = default; - MOCK_METHOD(absl::Status, startStream, (const StreamInfo::StreamInfo&)); + MOCK_METHOD(absl::Status, startStream, (const StreamInfo::StreamInfo*)); MOCK_METHOD(void, closeStream, ()); MOCK_METHOD(void, sendUsageReport, (absl::optional)); From 88543c9c37f389ff6d2650bb7aae211563ab0485 Mon Sep 17 00:00:00 2001 From: Rickyp Date: Tue, 17 Sep 2024 18:39:32 -0400 Subject: [PATCH 06/28] quic: Adding QUIC listener option to reject new connections (#36070) Commit Message: Implementing reject_new_connections QUIC listener option. Additional Description: The goal is to implement a mechanism to configure the bootstrap to reject H3 traffic as early as possible in the QUIC layer. This is done by replying to the client with an empty QUIC version negotiation packet to leverage the incompatible version negotiation logic from RFC 9368. This feature is off by default. Risk Level: Low Testing: UTs Docs Changes: N/A Release Notes: added new_features/quic note --------- Signed-off-by: Ricardo Perez --- api/envoy/config/listener/v3/quic_config.proto | 7 ++++++- changelogs/current.yaml | 5 +++++ source/common/quic/active_quic_listener.cc | 11 +++++++---- source/common/quic/active_quic_listener.h | 4 +++- test/integration/quic_http_integration_test.cc | 16 ++++++++++++++++ 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/api/envoy/config/listener/v3/quic_config.proto b/api/envoy/config/listener/v3/quic_config.proto index 6ba5bbc56b..6c0a5bd201 100644 --- a/api/envoy/config/listener/v3/quic_config.proto +++ b/api/envoy/config/listener/v3/quic_config.proto @@ -25,7 +25,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: QUIC listener config] // Configuration specific to the UDP QUIC listener. -// [#next-free-field: 13] +// [#next-free-field: 14] message QuicProtocolOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.listener.QuicProtocolOptions"; @@ -94,4 +94,9 @@ message QuicProtocolOptions { // If not specified, no cmsg will be saved to QuicReceivedPacket. repeated core.v3.SocketCmsgHeaders save_cmsg_config = 12 [(validate.rules).repeated = {max_items: 1}]; + + // If true, the listener will reject connection-establishing packets at the + // QUIC layer by replying with an empty version negotiation packet to the + // client. + bool reject_new_connections = 13; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index a8f9ca6d88..e60a462af2 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -248,6 +248,11 @@ new_features: QUIC server and client support certificate compression, which can in some cases reduce the number of round trips required to setup a connection. This change temporarily disabled by setting the runtime flag ``envoy.reloadable_features.quic_support_certificate_compression`` to ``false``. +- area: quic + change: | + Added QUIC protocol option :ref:`reject_new_connections + ` to reject connection-establishing + packets at the QUIC layer. - area: tls change: | Added an extension point :ref:`custom_tls_certificate_selector diff --git a/source/common/quic/active_quic_listener.cc b/source/common/quic/active_quic_listener.cc index 002b8f5866..691c4a9c29 100644 --- a/source/common/quic/active_quic_listener.cc +++ b/source/common/quic/active_quic_listener.cc @@ -37,14 +37,17 @@ ActiveQuicListener::ActiveQuicListener( EnvoyQuicCryptoServerStreamFactoryInterface& crypto_server_stream_factory, EnvoyQuicProofSourceFactoryInterface& proof_source_factory, QuicConnectionIdGeneratorPtr&& cid_generator, QuicConnectionIdWorkerSelector worker_selector, - EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef debug_visitor_factory) + EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef debug_visitor_factory, + bool reject_new_connections) : Server::ActiveUdpListenerBase( worker_index, concurrency, parent, *listen_socket, std::make_unique( dispatcher, listen_socket, *this, dispatcher.timeSource(), listener_config.udpListenerConfig()->config().downstream_socket_config()), &listener_config), - dispatcher_(dispatcher), version_manager_(quic::CurrentSupportedHttp3Versions()), + dispatcher_(dispatcher), + version_manager_(reject_new_connections ? quic::ParsedQuicVersionVector() + : quic::CurrentSupportedHttp3Versions()), kernel_worker_routing_(kernel_worker_routing), packets_to_read_to_connection_count_ratio_(packets_to_read_to_connection_count_ratio), crypto_server_stream_factory_(crypto_server_stream_factory), @@ -264,7 +267,7 @@ ActiveQuicListenerFactory::ActiveQuicListenerFactory( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, packets_to_read_to_connection_count_ratio, DEFAULT_PACKETS_TO_READ_PER_CONNECTION)), receive_ecn_(Runtime::runtimeFeatureEnabled("envoy.reloadable_features.quic_receive_ecn")), - context_(context) { + context_(context), reject_new_connections_(config.reject_new_connections()) { const int64_t idle_network_timeout_ms = config.has_idle_timeout() ? DurationUtil::durationToMilliseconds(config.idle_timeout()) : 300000; @@ -434,7 +437,7 @@ ActiveQuicListenerFactory::createActiveQuicListener( listener_config, quic_config, kernel_worker_routing, enabled, quic_stat_names, packets_to_read_to_connection_count_ratio, receive_ecn_, crypto_server_stream_factory, proof_source_factory, std::move(cid_generator), worker_selector_, - connection_debug_visitor_factory_); + connection_debug_visitor_factory_, reject_new_connections_); } } // namespace Quic diff --git a/source/common/quic/active_quic_listener.h b/source/common/quic/active_quic_listener.h index e0f62c0298..0ec62ac7df 100644 --- a/source/common/quic/active_quic_listener.h +++ b/source/common/quic/active_quic_listener.h @@ -41,7 +41,8 @@ class ActiveQuicListener : public Envoy::Server::ActiveUdpListenerBase, EnvoyQuicProofSourceFactoryInterface& proof_source_factory, QuicConnectionIdGeneratorPtr&& cid_generator, QuicConnectionIdWorkerSelector worker_selector, - EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef debug_visitor_factory); + EnvoyQuicConnectionDebugVisitorFactoryInterfaceOptRef debug_visitor_factory, + bool reject_new_connections = false); ~ActiveQuicListener() override; @@ -159,6 +160,7 @@ class ActiveQuicListenerFactory : public Network::ActiveUdpListenerFactory, QuicConnectionIdWorkerSelector worker_selector_; bool kernel_worker_routing_{}; Server::Configuration::ServerFactoryContext& context_; + bool reject_new_connections_{}; static bool disable_kernel_bpf_packet_routing_for_test_; }; diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index b2dc7b4fa1..65251998c9 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -2380,6 +2380,22 @@ TEST_P(QuicHttpIntegrationTest, SendDisableActiveMigration) { ASSERT_TRUE(response->complete()); } +TEST_P(QuicHttpIntegrationTest, RejectTraffic) { + config_helper_.addConfigModifier([=](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + bootstrap.mutable_static_resources() + ->mutable_listeners(0) + ->mutable_udp_listener_config() + ->mutable_quic_options() + ->set_reject_new_connections(true); + }); + + initialize(); + codec_client_ = makeRawHttpConnection(makeClientConnection(lookupPort("http")), absl::nullopt); + EXPECT_TRUE(codec_client_->disconnected()); + EXPECT_EQ(quic::QUIC_INVALID_VERSION, + static_cast(codec_client_->connection())->error()); +} + // Validate that the transport parameter is not sent when `send_disable_active_migration` is // unset. TEST_P(QuicHttpIntegrationTest, UnsetSendDisableActiveMigration) { From c9ae398d91040a076085c52fe33b0c77572cf71f Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 18 Sep 2024 01:03:53 -0400 Subject: [PATCH 07/28] syscalls: minor refactor, adding coverage (#36075) minor refactor no longer fast-failing if the system can't create sockets. Risk Level: low Testing: added tests, manually tested fd failure Docs Changes: n/a Release Notes: n/a --------- Signed-off-by: Fredy Wijaya Signed-off-by: Alyssa Wilk Co-authored-by: Fredy Wijaya --- source/common/api/posix/os_sys_calls_impl.cc | 13 ++----------- test/common/api/os_sys_calls_test.cc | 15 +++++++++++++++ test/per_file_coverage.sh | 2 -- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/source/common/api/posix/os_sys_calls_impl.cc b/source/common/api/posix/os_sys_calls_impl.cc index 3fab0afc1b..ad078c7f4d 100644 --- a/source/common/api/posix/os_sys_calls_impl.cc +++ b/source/common/api/posix/os_sys_calls_impl.cc @@ -109,9 +109,6 @@ bool OsSysCallsImpl::supportsUdpGro() const { #else static const bool is_supported = [] { int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, IPPROTO_UDP); - if (fd < 0) { - return false; - } int val = 1; bool result = (0 == ::setsockopt(fd, IPPROTO_UDP, UDP_GRO, &val, sizeof(val))); ::close(fd); @@ -127,9 +124,6 @@ bool OsSysCallsImpl::supportsUdpGso() const { #else static const bool is_supported = [] { int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, IPPROTO_UDP); - if (fd < 0) { - return false; - } int optval; socklen_t optlen = sizeof(optval); bool result = (0 <= ::getsockopt(fd, IPPROTO_UDP, UDP_SEGMENT, &optval, &optlen)); @@ -160,9 +154,6 @@ bool OsSysCallsImpl::supportsIpTransparent(Network::Address::IpVersion ip_versio static constexpr auto transparent_supported = [](int family) { auto opt_tp = family == AF_INET ? ENVOY_SOCKET_IP_TRANSPARENT : ENVOY_SOCKET_IPV6_TRANSPARENT; int fd = ::socket(family, SOCK_DGRAM | SOCK_NONBLOCK, IPPROTO_UDP); - if (fd < 0) { - return false; - } int val = 1; bool result = (0 == ::setsockopt(fd, opt_tp.level(), opt_tp.option(), &val, sizeof(val))); ::close(fd); @@ -348,9 +339,9 @@ SysCallBoolResult OsSysCallsImpl::socketTcpInfo([[maybe_unused]] os_fd_t sockfd, tcp_info->tcpi_snd_cwnd = unix_tcp_info.tcpi_snd_cwnd * mss; } return {!SOCKET_FAILURE(result), !SOCKET_FAILURE(result) ? 0 : errno}; -#endif - +#else return {false, EOPNOTSUPP}; +#endif } bool OsSysCallsImpl::supportsGetifaddrs() const { return true; } diff --git a/test/common/api/os_sys_calls_test.cc b/test/common/api/os_sys_calls_test.cc index 126e109409..96e4d35e95 100644 --- a/test/common/api/os_sys_calls_test.cc +++ b/test/common/api/os_sys_calls_test.cc @@ -68,4 +68,19 @@ TEST(OsSyscallsTest, OpenPwritePreadFstatCloseStatUnlink) { TestEnvironment::removePath(path); } +TEST(OsSyscallsTest, SupportsIpTransparent) { + bool supported = Api::OsSysCallsSingleton::get().supportsIpTransparent( + TestEnvironment::getIpVersionsForTest()[0]); + EXPECT_FALSE(supported); +} + +TEST(OsSyscallsTest, SupportsMptcp) { + bool supported = Api::OsSysCallsSingleton::get().supportsMptcp(); + EXPECT_TRUE(supported); +} + +TEST(OsSyscallsTest, IoCtlInvalidFd) { + EXPECT_NE(0, Api::OsSysCallsSingleton::get().ioctl(0, 0, nullptr, 0, nullptr, 0, nullptr).errno_); +} + } // namespace Envoy diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index eae1d69683..c5bc35cf78 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -4,8 +4,6 @@ # for existing directories with low coverage. declare -a KNOWN_LOW_COVERAGE=( "source/common:96.2" -"source/common/api:84.5" # flaky due to posix: be careful adjusting -"source/common/api/posix:83.8" # flaky (accept failover non-deterministic): be careful adjusting "source/common/common/posix:96.2" # flaky due to posix: be careful adjusting "source/common/config:96.1" "source/common/crypto:95.5" From 7fb927d10a67aa6ca6737a003547ac8a707df673 Mon Sep 17 00:00:00 2001 From: "dependency-envoy[bot]" <148525496+dependency-envoy[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:21:43 +0000 Subject: [PATCH 08/28] deps: Bump `proxy_wasm_rust_sdk` -> 0.2.2 (#35323) Fix #35321 also update `rules_rust` -> 0.48.0 Fix #35291 Signed-off-by: dependency-envoy[bot] <148525496+dependency-envoy[bot]@users.noreply.github.com> Signed-off-by: Ryan Northey --- .bazelrc | 1 + .../cargo/remote/BUILD.protobuf-2.24.1.bazel | 2 +- bazel/repository_locations.bzl | 12 +++++----- .../extensions/bootstrap/wasm/test_data/BUILD | 2 +- .../filters/http/wasm/test_data/BUILD | 22 +++++++++---------- .../filters/network/wasm/test_data/BUILD | 8 +++---- 6 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.bazelrc b/.bazelrc index 10c6b7439f..9bfd858fe7 100644 --- a/.bazelrc +++ b/.bazelrc @@ -25,6 +25,7 @@ build --copt=-DABSL_MIN_LOG_LEVEL=4 build --define envoy_mobile_listener=enabled build --experimental_repository_downloader_retries=2 build --enable_platform_specific_config +build --incompatible_merge_fixed_and_default_shell_env # Pass CC, CXX and LLVM_CONFIG variables from the environment. # We assume they have stable values, so this won't cause action cache misses. diff --git a/bazel/external/cargo/remote/BUILD.protobuf-2.24.1.bazel b/bazel/external/cargo/remote/BUILD.protobuf-2.24.1.bazel index 9917db62f5..b818daadb7 100644 --- a/bazel/external/cargo/remote/BUILD.protobuf-2.24.1.bazel +++ b/bazel/external/cargo/remote/BUILD.protobuf-2.24.1.bazel @@ -33,7 +33,7 @@ licenses([ # buildifier: disable=out-of-order-load # buildifier: disable=load-on-top load( - "@rules_rust//cargo:cargo_build_script.bzl", + "@rules_rust//cargo:defs.bzl", "cargo_build_script", ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index d382a58b13..d7c5e6e3fe 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1424,12 +1424,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "WebAssembly for Proxies (Rust SDK)", project_desc = "WebAssembly for Proxies (Rust SDK)", project_url = "https://github.com/proxy-wasm/proxy-wasm-rust-sdk", - version = "0.2.1", - sha256 = "23f3f2d8c4c8069a2e72693b350d7442b7722d334f73169eea78804ff70cde20", + version = "0.2.2", + sha256 = "3d9e8f39f0356016c8ae6c74c0224eae1b44168be0ddf79e387d918a8f2cb4c6", strip_prefix = "proxy-wasm-rust-sdk-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-rust-sdk/archive/v{version}.tar.gz"], use_category = ["test_only"], - release_date = "2022-11-22", + release_date = "2024-07-21", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/proxy-wasm/proxy-wasm-rust-sdk/blob/v{version}/LICENSE", @@ -1452,9 +1452,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Bazel rust rules", project_desc = "Bazel rust rules (used by Wasm)", project_url = "https://github.com/bazelbuild/rules_rust", - version = "0.35.0", + version = "0.48.0", strip_prefix = "rules_rust-{version}", - sha256 = "3120c7aa3a146dfe6be8d5f23f4cf10af7d0f74a5aed8b94a818f88643bd24c3", + sha256 = "a4b8ede7723088dff1e909632c4282e51ddbe0e44c38eea013ee0f12d348b1c7", urls = ["https://github.com/bazelbuild/rules_rust/archive/{version}.tar.gz"], use_category = [ "controlplane", @@ -1463,7 +1463,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( ], implied_untracked_deps = ["rules_cc"], extensions = ["envoy.wasm.runtime.wasmtime"], - release_date = "2023-12-27", + release_date = "2024-07-19", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/bazelbuild/rules_rust/blob/{version}/LICENSE.txt", diff --git a/test/extensions/bootstrap/wasm/test_data/BUILD b/test/extensions/bootstrap/wasm/test_data/BUILD index e0003964ed..53f36ba5ab 100644 --- a/test/extensions/bootstrap/wasm/test_data/BUILD +++ b/test/extensions/bootstrap/wasm/test_data/BUILD @@ -15,7 +15,7 @@ wasm_rust_binary( wasi = True, deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) diff --git a/test/extensions/filters/http/wasm/test_data/BUILD b/test/extensions/filters/http/wasm/test_data/BUILD index 3cebdbc4fc..dc89c15852 100644 --- a/test/extensions/filters/http/wasm/test_data/BUILD +++ b/test/extensions/filters/http/wasm/test_data/BUILD @@ -15,7 +15,7 @@ wasm_rust_binary( srcs = ["async_call_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -24,7 +24,7 @@ wasm_rust_binary( srcs = ["body_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -33,7 +33,7 @@ wasm_rust_binary( srcs = ["close_stream_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -43,7 +43,7 @@ wasm_rust_binary( deps = [ "//bazel/external/cargo:protobuf", "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -53,7 +53,7 @@ wasm_rust_binary( deps = [ "//bazel/external/cargo:protobuf", "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -63,7 +63,7 @@ wasm_rust_binary( wasi = True, deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -72,7 +72,7 @@ wasm_rust_binary( srcs = ["metadata_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -81,7 +81,7 @@ wasm_rust_binary( srcs = ["panic_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -90,7 +90,7 @@ wasm_rust_binary( srcs = ["resume_call_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -99,7 +99,7 @@ wasm_rust_binary( srcs = ["shared_data_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -108,7 +108,7 @@ wasm_rust_binary( srcs = ["shared_queue_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) diff --git a/test/extensions/filters/network/wasm/test_data/BUILD b/test/extensions/filters/network/wasm/test_data/BUILD index 7095acc432..5767f8ba51 100644 --- a/test/extensions/filters/network/wasm/test_data/BUILD +++ b/test/extensions/filters/network/wasm/test_data/BUILD @@ -14,7 +14,7 @@ wasm_rust_binary( srcs = ["close_stream_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -23,7 +23,7 @@ wasm_rust_binary( srcs = ["logging_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -32,7 +32,7 @@ wasm_rust_binary( srcs = ["panic_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) @@ -41,7 +41,7 @@ wasm_rust_binary( srcs = ["resume_call_rust.rs"], deps = [ "@proxy_wasm_rust_sdk//:proxy_wasm", - "@proxy_wasm_rust_sdk//bazel/cargo:log", + "@proxy_wasm_rust_sdk//bazel/cargo/remote:log", ], ) From 9e9986b4b56a174814bdc2d7ca2bf8c4973c0841 Mon Sep 17 00:00:00 2001 From: "dependency-envoy[bot]" <148525496+dependency-envoy[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:01:48 +0100 Subject: [PATCH 09/28] deps: Bump `bazel_features` -> 1.17.0 (#36195) Fix #36138 Signed-off-by: dependency-envoy[bot] <148525496+dependency-envoy[bot]@users.noreply.github.com> --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index d7c5e6e3fe..94f70cfb36 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -33,11 +33,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Bazel features", project_desc = "Support Bazel feature detection from starlark", project_url = "https://github.com/bazel-contrib/bazel_features", - version = "1.15.0", - sha256 = "ba1282c1aa1d1fffdcf994ab32131d7c7551a9bc960fbf05f42d55a1b930cbfb", + version = "1.17.0", + sha256 = "bdc12fcbe6076180d835c9dd5b3685d509966191760a0eb10b276025fcb76158", urls = ["https://github.com/bazel-contrib/bazel_features/releases/download/v{version}/bazel_features-v{version}.tar.gz"], strip_prefix = "bazel_features-{version}", - release_date = "2024-08-09", + release_date = "2024-09-13", use_category = ["build"], license = "Apache-2.0", license_url = "https://github.com/bazel-contrib/bazel_features/blob/v{version}/LICENSE", From ead1c021777a33fea5b9da7be9f66c78118d8ac1 Mon Sep 17 00:00:00 2001 From: "Adi (Suissa) Peleg" Date: Wed, 18 Sep 2024 11:53:13 -0400 Subject: [PATCH 10/28] upstream-filter: update default upstream filter type (#36186) Prior to this the debug logs contain the following: ``` [source/common/http/filter_chain_helper.h:160] config: Failed to convert protobuf message to JSON string: INVALID_ARGUMENT: @type must contain at least one / and a nonempty host; got: envoy.extensions.filters.http.upstream_codec.v3.UpstreamCodec ``` After this PR they contain: ``` [source/common/http/filter_chain_helper.h:160] config: {"@type":"type.googleapis.com/envoy.extensions.filters.http.upstream_codec.v3.UpstreamCodec"} ``` Risk Level: low - seems to be impacting debug logs only Testing: N/A Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A Signed-off-by: Adi Suissa-Peleg --- source/common/upstream/upstream_impl.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 805ceb849c..a336aa2689 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1388,7 +1388,8 @@ ClusterInfoImpl::ClusterInfoImpl( if (http_filters.empty()) { auto* codec_filter = http_filters.Add(); codec_filter->set_name("envoy.filters.http.upstream_codec"); - codec_filter->mutable_typed_config()->set_type_url(upstream_codec_type_url); + codec_filter->mutable_typed_config()->set_type_url( + absl::StrCat("type.googleapis.com/", upstream_codec_type_url)); } else { const auto last_type_url = Config::Utility::getFactoryType(http_filters[http_filters.size() - 1].typed_config()); From 291b194449779fb5544b8f24af88f9acba154f51 Mon Sep 17 00:00:00 2001 From: Raven Black Date: Wed, 18 Sep 2024 11:55:21 -0400 Subject: [PATCH 11/28] Refactor async files library to use dispatcher for synchronization (#36063) Using mutexes in the async file library was a premature optimization that made it significantly more complicated and error-prone to use from the client side. The benefit was the ability to perform multiple file operations sequentially without handing control back to the calling thread, but the cost in complexity was not worth it. This change is a hefty simplification, the main benefit of which is, in this model, cancellation is simple - previously calling cancel couldn't guarantee anything so the client object would still have to handle the case of a callback potentially being called after the object's own destruction. In this simplified model, calling cancel from the originating thread, before the callback has been called, guarantees that the callback will not be called. Similar simplification may be possible in `cache_filter`, but it's not quite as clear; cache_filter currently uses `dispatcher->post` to ensure thread locality, which is now guaranteed by the cache API so that usage is unnecessary, but it also uses `dispatcher->post` to ensure that callbacks are not called during the current function context (so that, e.g. `continueDecoding()` isn't potentially called before `decodeHeaders` returns `StopAllIterationAndWatermark`, as would happen with the simple in-memory cache which calls the callback immediately). Since this is already a giant PR, and it already works as a simplification with `cache_filter` unchanged, I'm electing to defer a simplifying pass over `cache_filter` to a followup PR. Risk Level: Small, only WIP filters use the library, and all existing tests (with appropriate modifications) still pass - the only test removed was one testing the "not handing control back to the caller" sequential action behavior, which now doesn't exist. (And there are apparently some crash-bugs in the existing stuff, so there's some negative risk in that this might fix a bug!) Testing: Existing coverage. Docs Changes: Yes, code-only. Release Notes: n/a Platform Specific Features: n/a --------- Signed-off-by: Raven Black --- source/extensions/common/async_files/BUILD | 3 +- .../extensions/common/async_files/README.md | 54 +-- .../common/async_files/async_file_action.cc | 26 -- .../common/async_files/async_file_action.h | 64 +-- .../async_files/async_file_context_base.cc | 5 +- .../async_files/async_file_context_base.h | 4 +- .../async_file_context_thread_pool.cc | 104 +++-- .../async_file_context_thread_pool.h | 28 +- .../common/async_files/async_file_handle.h | 27 +- .../common/async_files/async_file_manager.cc | 31 -- .../common/async_files/async_file_manager.h | 67 +-- .../async_file_manager_thread_pool.cc | 203 ++++++--- .../async_file_manager_thread_pool.h | 31 +- .../filters/http/file_system_buffer/filter.cc | 74 ++-- .../filters/http/file_system_buffer/filter.h | 7 +- .../http/file_system_buffer/fragment.cc | 71 ++-- .../http/file_system_buffer/fragment.h | 9 +- .../file_system_http_cache.cc | 158 +++---- .../file_system_http_cache.h | 12 +- .../file_system_http_cache/insert_context.cc | 276 ++++++------ .../file_system_http_cache/insert_context.h | 66 ++- .../file_system_http_cache/lookup_context.cc | 269 ++++++------ .../file_system_http_cache/lookup_context.h | 31 +- .../async_file_handle_thread_pool_test.cc | 258 ++++++------ .../async_file_manager_thread_pool_test.cc | 253 +++++------ ...ile_manager_thread_pool_with_mocks_test.cc | 233 ++++++----- test/extensions/common/async_files/mocks.cc | 119 +++--- test/extensions/common/async_files/mocks.h | 87 ++-- .../http_cache_implementation_test_common.cc | 215 +++++----- .../http_cache_implementation_test_common.h | 28 +- .../http/file_system_buffer/filter_test.cc | 41 +- .../http/file_system_buffer/fragment_test.cc | 156 ++++--- .../file_system_http_cache_test.cc | 392 ++++++++++++------ 33 files changed, 1803 insertions(+), 1599 deletions(-) delete mode 100644 source/extensions/common/async_files/async_file_action.cc delete mode 100644 source/extensions/common/async_files/async_file_manager.cc diff --git a/source/extensions/common/async_files/BUILD b/source/extensions/common/async_files/BUILD index 8b9a420829..0858edabe4 100644 --- a/source/extensions/common/async_files/BUILD +++ b/source/extensions/common/async_files/BUILD @@ -11,7 +11,6 @@ envoy_extension_package() envoy_cc_library( name = "async_files_base", srcs = [ - "async_file_action.cc", "async_file_context_base.cc", ], hdrs = [ @@ -22,6 +21,7 @@ envoy_cc_library( ], deps = [ ":status_after_file_error", + "//envoy/event:dispatcher_interface", "//source/common/buffer:buffer_lib", "//source/common/common:utility_lib", "@com_google_absl//absl/base", @@ -53,7 +53,6 @@ envoy_cc_library( envoy_cc_library( name = "async_files", srcs = [ - "async_file_manager.cc", "async_file_manager_factory.cc", ], hdrs = [ diff --git a/source/extensions/common/async_files/README.md b/source/extensions/common/async_files/README.md index 822e0827da..ecbd7f42a2 100644 --- a/source/extensions/common/async_files/README.md +++ b/source/extensions/common/async_files/README.md @@ -3,62 +3,26 @@ An `AsyncFileManager` should be a singleton or similarly long-lived scope. It represents a thread pool for performing file operations asynchronously. -`AsyncFileManager` can create `AsyncFileHandle`s via `createAnonymousFile` or `openExistingFile`, -can postpone queuing file actions using `whenReady`, and can delete files via `unlink`. +`AsyncFileManager` can create `AsyncFileHandle`s via `createAnonymousFile` or `openExistingFile`, can stat a file by name with `stat`, and can delete files via `unlink`. # AsyncFileHandle An `AsyncFileHandle` represents a context in which asynchronous file operations can be performed. It is associated with at most one file at a time. -Each action on an AsyncFileHandle is effectively an "enqueue" action, in that it places the action in the manager's execution queue, it does not immediately perform the requested action. Actions on an `AsyncFileHandle` can be *chained*, by enqueuing another action during the callback from a previous action, e.g. - -``` -manager->createAnonymousFile("/tmp", [](absl::StatusOr opened) { - if (!opened.ok()) { - std::cout << "oh no, an error: " << opened.status() << std::endl; - return; - } - auto handle = opened.value(); - handle->write(someBuffer, 0, [handle](absl::StatusOr written) { - if (!written.ok()) { - std::cout << "oh no, an error: " << written.status() << std::endl; - return; - } - std::cout << "wrote " << written.value() << " bytes" << std::endl; - handle->close([](absl::Status closed) { - if (!closed.ok()) { - std::cout << "oh no, an error: " << closed << std::endl; - } - }).IgnoreError(); // A returned error only occurs if the file handle was closed. - }).IgnoreError(); // A returned error only occurs if the file handle was closed. -}); -``` - -Will open an unnamed file, write 5 bytes, and close it. (This is just for explanatory purposes, in practice you would most likely want the callbacks to call something on `this` rather than nesting lambdas!) - -Chaining actions, as opposed to enqueuing, passing the result to a main thread, and from there enqueuing again, will not yield the thread in a thread-pool based implementation. An advantage of this is that, for example, if 5 workers all wanted to write a 100kb file at the same moment, with unchained requests in a one-thread threadpool the sequence would most likely resemble - -``` -OPEN-OPEN-OPEN-OPEN-OPEN-WRITE-WRITE-WRITE-WRITE-WRITE-CLOSE-CLOSE-CLOSE-CLOSE-CLOSE -``` - -Versus with appropriately chained requests in a one-thread threadpool the sequence would be guaranteed to be - -``` -OPEN-WRITE-CLOSE-OPEN-WRITE-CLOSE-OPEN-WRITE-CLOSE-OPEN-WRITE-CLOSE-OPEN-WRITE-CLOSE -``` - -Expand this concept to 100+ files all asking to be written at once and you can immediately see the advantages of chaining; not having the resource issues of many files open at the same time, more localized access, etc. +Each action on an AsyncFileHandle is effectively an "enqueue" action, in that it places the action in the manager's execution queue, it does not immediately perform the requested action. ## cancellation -Each action function returns a cancellation function which can be called to remove an action from the queue and prevent the callback from being called. If the execution is already in progress, it may be undone (e.g. a file open operation will close the file if it is opening when cancel is called). The cancel function will block if the callback is already in progress when cancel is called, until the callback completes. This should not be a long block, as callbacks should be short (see callbacks below). - -As such, a client should ensure that the cleanup order is consistent - if a callback captures a file handle, the client should clean up that file handle (if present) *after* calling cancel, in case the file was opened during the call to cancel. +Each action function returns a cancellation function which can be called to remove an action from the queue and prevent the callback from being called. If the execution is already in progress, it may be undone (e.g. a file open operation will close the file if it is opening when cancel is called). The cancel function must only be called from the same thread as the +dispatcher that was provided to the original request, to ensure that cancellation and callback +cannot be happening concurrently. ## callbacks -The callbacks passed to `AsyncFileHandle` and `AsyncFileManager` are scheduled in a thread or thread pool belonging to the AsyncFileManager - therefore they should be doing minimal work, not blocking (for more than a trivial data-guard lock), and return promptly. If any significant work or blocking is required, the result of the previous action should be passed from the callback to another thread (via some dispatcher or other queuing mechanism) so the manager's thread can continue performing file operations for other clients. +The callbacks passed to `AsyncFileHandle` and `AsyncFileManager` functions are called from +the thread associated with the provided `dispatcher`, if the action was not cancelled first. + +The implementation of `AsyncFileManager` ensures that a `cancel` call from the thread associated with the dispatcher, if called prior to the callback's execution, is guaranteed to prevent the callback from being called, there is no race. ## Possible actions diff --git a/source/extensions/common/async_files/async_file_action.cc b/source/extensions/common/async_files/async_file_action.cc deleted file mode 100644 index 9687039cbe..0000000000 --- a/source/extensions/common/async_files/async_file_action.cc +++ /dev/null @@ -1,26 +0,0 @@ -#include "source/extensions/common/async_files/async_file_action.h" - -#include - -namespace Envoy { -namespace Extensions { -namespace Common { -namespace AsyncFiles { - -void AsyncFileAction::cancel() { - auto previousState = state_.exchange(State::Cancelled); - if (previousState == State::InCallback) { - // A gentle spin-lock. This situation should be rare, and callbacks are - // supposed to be quick, so we don't need a real lock here. - while (state_.load() != State::Done) { - std::this_thread::yield(); - } - } else if (previousState == State::Done) { - state_.store(State::Done); - } -} - -} // namespace AsyncFiles -} // namespace Common -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/common/async_files/async_file_action.h b/source/extensions/common/async_files/async_file_action.h index ea8bfed063..6afa7c809f 100644 --- a/source/extensions/common/async_files/async_file_action.h +++ b/source/extensions/common/async_files/async_file_action.h @@ -19,33 +19,28 @@ namespace AsyncFiles { // * If the action is already executing, CancelFunction causes the removal of any resource-consuming // return value (e.g. file handles), and prevents the callback. // * If the action is still just queued, CancelFunction prevents its execution. -using CancelFunction = std::function; +using CancelFunction = absl::AnyInvocable; // Actions to be passed to asyncFileManager->enqueue. class AsyncFileAction { public: virtual ~AsyncFileAction() = default; - // Cancel the action, as much as possible. - // - // If the action has not been started, it will become a no-op. - // - // If the action has started, onCancelledBeforeCallback will be called, - // and the callback will not. - // - // If the callback is already being called, cancel will block until the - // callback has completed. - // - // If the action is already complete, cancel does nothing. - void cancel(); - - // Performs the action represented by this instance, and calls the callback - // on completion or on error. + // Performs the action represented by this instance, and captures the + // result. virtual void execute() PURE; -protected: - enum class State { Queued, Executing, InCallback, Done, Cancelled }; - std::atomic state_{State::Queued}; + // Calls the captured callback with the captured result. + virtual void onComplete() PURE; + + // Performs any action to undo side-effects of the execution if the callback + // has not yet been called (e.g. closing a file that was just opened, removing + // a hard-link that was just created). + // Not necessary for things that don't make persistent resources, + // e.g. cancelling a write does not have to undo the write. + virtual void onCancelledBeforeCallback() {} + virtual bool hasActionIfCancelledBeforeCallback() const { return false; } + virtual bool executesEvenIfCancelled() const { return false; } }; // All concrete AsyncFileActions are a subclass of AsyncFileActionWithResult. @@ -59,38 +54,19 @@ class AsyncFileAction { // the result should be passed to another thread for handling. template class AsyncFileActionWithResult : public AsyncFileAction { public: - explicit AsyncFileActionWithResult(std::function on_complete) - : on_complete_(on_complete) {} + explicit AsyncFileActionWithResult(absl::AnyInvocable on_complete) + : on_complete_(std::move(on_complete)) {} - void execute() final { - State expected = State::Queued; - if (!state_.compare_exchange_strong(expected, State::Executing)) { - ASSERT(expected == State::Cancelled); - return; - } - expected = State::Executing; - T result = executeImpl(); - if (!state_.compare_exchange_strong(expected, State::InCallback)) { - ASSERT(expected == State::Cancelled); - onCancelledBeforeCallback(std::move(result)); - return; - } - on_complete_(std::move(result)); - state_.store(State::Done); - } + void execute() final { result_ = executeImpl(); } + void onComplete() final { std::move(on_complete_)(std::move(result_.value())); } protected: - // Performs any action to undo side-effects of the execution if the callback - // has not yet been called (e.g. closing a file that was just opened). - // Not necessary for things that don't make persistent resources, - // e.g. cancelling a write does not have to undo the write. - virtual void onCancelledBeforeCallback(T){}; - + absl::optional result_; // Implementation of the actual action. virtual T executeImpl() PURE; private: - std::function on_complete_; + absl::AnyInvocable on_complete_; }; } // namespace AsyncFiles diff --git a/source/extensions/common/async_files/async_file_context_base.cc b/source/extensions/common/async_files/async_file_context_base.cc index 20c7fc9948..070d710224 100644 --- a/source/extensions/common/async_files/async_file_context_base.cc +++ b/source/extensions/common/async_files/async_file_context_base.cc @@ -17,8 +17,9 @@ namespace AsyncFiles { AsyncFileContextBase::AsyncFileContextBase(AsyncFileManager& manager) : manager_(manager) {} -CancelFunction AsyncFileContextBase::enqueue(std::shared_ptr action) { - return manager_.enqueue(std::move(action)); +CancelFunction AsyncFileContextBase::enqueue(Event::Dispatcher* dispatcher, + std::unique_ptr action) { + return manager_.enqueue(dispatcher, std::move(action)); } } // namespace AsyncFiles diff --git a/source/extensions/common/async_files/async_file_context_base.h b/source/extensions/common/async_files/async_file_context_base.h index 37614b450d..b01652b62c 100644 --- a/source/extensions/common/async_files/async_file_context_base.h +++ b/source/extensions/common/async_files/async_file_context_base.h @@ -2,6 +2,8 @@ #include +#include "envoy/event/dispatcher.h" + #include "source/extensions/common/async_files/async_file_handle.h" #include "absl/status/statusor.h" @@ -23,7 +25,7 @@ class AsyncFileContextBase : public AsyncFileContext { protected: // Queue up an action with the AsyncFileManager. - CancelFunction enqueue(std::shared_ptr action); + CancelFunction enqueue(Event::Dispatcher* dispatcher, std::unique_ptr action); explicit AsyncFileContextBase(AsyncFileManager& manager); diff --git a/source/extensions/common/async_files/async_file_context_thread_pool.cc b/source/extensions/common/async_files/async_file_context_thread_pool.cc index 43a07ef051..5391559ff9 100644 --- a/source/extensions/common/async_files/async_file_context_thread_pool.cc +++ b/source/extensions/common/async_files/async_file_context_thread_pool.cc @@ -21,8 +21,9 @@ namespace { template class AsyncFileActionThreadPool : public AsyncFileActionWithResult { public: - explicit AsyncFileActionThreadPool(AsyncFileHandle handle, std::function on_complete) - : AsyncFileActionWithResult(on_complete), handle_(std::move(handle)) {} + explicit AsyncFileActionThreadPool(AsyncFileHandle handle, + absl::AnyInvocable on_complete) + : AsyncFileActionWithResult(std::move(on_complete)), handle_(std::move(handle)) {} protected: int& fileDescriptor() { return context()->fileDescriptor(); } @@ -39,8 +40,9 @@ template class AsyncFileActionThreadPool : public AsyncFileActionWi class ActionStat : public AsyncFileActionThreadPool> { public: - ActionStat(AsyncFileHandle handle, std::function)> on_complete) - : AsyncFileActionThreadPool>(handle, on_complete) {} + ActionStat(AsyncFileHandle handle, + absl::AnyInvocable)> on_complete) + : AsyncFileActionThreadPool>(handle, std::move(on_complete)) {} absl::StatusOr executeImpl() override { ASSERT(fileDescriptor() != -1); @@ -56,8 +58,9 @@ class ActionStat : public AsyncFileActionThreadPool> class ActionCreateHardLink : public AsyncFileActionThreadPool { public: ActionCreateHardLink(AsyncFileHandle handle, absl::string_view filename, - std::function on_complete) - : AsyncFileActionThreadPool(handle, on_complete), filename_(filename) {} + absl::AnyInvocable on_complete) + : AsyncFileActionThreadPool(handle, std::move(on_complete)), + filename_(filename) {} absl::Status executeImpl() override { ASSERT(fileDescriptor() != -1); @@ -70,11 +73,12 @@ class ActionCreateHardLink : public AsyncFileActionThreadPool { return absl::OkStatus(); } - void onCancelledBeforeCallback(absl::Status result) override { - if (result.ok()) { + void onCancelledBeforeCallback() override { + if (result_.value().ok()) { posix().unlink(filename_.c_str()); } } + bool hasActionIfCancelledBeforeCallback() const override { return true; } private: const std::string filename_; @@ -85,8 +89,9 @@ class ActionCloseFile : public AsyncFileActionThreadPool { // Here we take a copy of the AsyncFileContext's file descriptor, because the close function // sets the AsyncFileContext's file descriptor to -1. This way there will be no race of trying // to use the handle again while the close is in flight. - explicit ActionCloseFile(AsyncFileHandle handle, std::function on_complete) - : AsyncFileActionThreadPool(handle, on_complete), + explicit ActionCloseFile(AsyncFileHandle handle, + absl::AnyInvocable on_complete) + : AsyncFileActionThreadPool(handle, std::move(on_complete)), file_descriptor_(fileDescriptor()) {} absl::Status executeImpl() override { @@ -97,6 +102,8 @@ class ActionCloseFile : public AsyncFileActionThreadPool { return absl::OkStatus(); } + bool executesEvenIfCancelled() const override { return true; } + private: const int file_descriptor_; }; @@ -104,8 +111,9 @@ class ActionCloseFile : public AsyncFileActionThreadPool { class ActionReadFile : public AsyncFileActionThreadPool> { public: ActionReadFile(AsyncFileHandle handle, off_t offset, size_t length, - std::function)> on_complete) - : AsyncFileActionThreadPool>(handle, on_complete), + absl::AnyInvocable)> on_complete) + : AsyncFileActionThreadPool>(handle, + std::move(on_complete)), offset_(offset), length_(length) {} absl::StatusOr executeImpl() override { @@ -133,8 +141,9 @@ class ActionReadFile : public AsyncFileActionThreadPool> { public: ActionWriteFile(AsyncFileHandle handle, Buffer::Instance& contents, off_t offset, - std::function)> on_complete) - : AsyncFileActionThreadPool>(handle, on_complete), offset_(offset) { + absl::AnyInvocable)> on_complete) + : AsyncFileActionThreadPool>(handle, std::move(on_complete)), + offset_(offset) { contents_.move(contents); } @@ -166,8 +175,9 @@ class ActionWriteFile : public AsyncFileActionThreadPool> class ActionDuplicateFile : public AsyncFileActionThreadPool> { public: ActionDuplicateFile(AsyncFileHandle handle, - std::function)> on_complete) - : AsyncFileActionThreadPool>(handle, on_complete) {} + absl::AnyInvocable)> on_complete) + : AsyncFileActionThreadPool>(handle, std::move(on_complete)) { + } absl::StatusOr executeImpl() override { ASSERT(fileDescriptor() != -1); @@ -178,61 +188,69 @@ class ActionDuplicateFile : public AsyncFileActionThreadPool(context()->manager(), newfd.return_value_); } - void onCancelledBeforeCallback(absl::StatusOr result) override { - if (result.ok()) { - result.value()->close([](absl::Status) {}).IgnoreError(); + void onCancelledBeforeCallback() override { + if (result_.value().ok()) { + result_.value().value()->close(nullptr, [](absl::Status) {}).IgnoreError(); } } + bool hasActionIfCancelledBeforeCallback() const override { return true; } }; } // namespace -absl::StatusOr -AsyncFileContextThreadPool::stat(std::function)> on_complete) { - return checkFileAndEnqueue(std::make_shared(handle(), std::move(on_complete))); +absl::StatusOr AsyncFileContextThreadPool::stat( + Event::Dispatcher* dispatcher, + absl::AnyInvocable)> on_complete) { + return checkFileAndEnqueue(dispatcher, + std::make_unique(handle(), std::move(on_complete))); } absl::StatusOr -AsyncFileContextThreadPool::createHardLink(absl::string_view filename, - std::function on_complete) { - return checkFileAndEnqueue( - std::make_shared(handle(), filename, std::move(on_complete))); +AsyncFileContextThreadPool::createHardLink(Event::Dispatcher* dispatcher, + absl::string_view filename, + absl::AnyInvocable on_complete) { + return checkFileAndEnqueue(dispatcher, std::make_unique( + handle(), filename, std::move(on_complete))); } -absl::Status AsyncFileContextThreadPool::close(std::function on_complete) { - auto status = - checkFileAndEnqueue(std::make_shared(handle(), std::move(on_complete))) - .status(); +absl::StatusOr +AsyncFileContextThreadPool::close(Event::Dispatcher* dispatcher, + absl::AnyInvocable on_complete) { + auto ret = checkFileAndEnqueue( + dispatcher, std::make_unique(handle(), std::move(on_complete))); fileDescriptor() = -1; - return status; + return ret; } absl::StatusOr AsyncFileContextThreadPool::read( - off_t offset, size_t length, - std::function)> on_complete) { - return checkFileAndEnqueue( - std::make_shared(handle(), offset, length, std::move(on_complete))); + Event::Dispatcher* dispatcher, off_t offset, size_t length, + absl::AnyInvocable)> on_complete) { + return checkFileAndEnqueue(dispatcher, std::make_unique(handle(), offset, length, + std::move(on_complete))); } absl::StatusOr -AsyncFileContextThreadPool::write(Buffer::Instance& contents, off_t offset, - std::function)> on_complete) { - return checkFileAndEnqueue( - std::make_shared(handle(), contents, offset, std::move(on_complete))); +AsyncFileContextThreadPool::write(Event::Dispatcher* dispatcher, Buffer::Instance& contents, + off_t offset, + absl::AnyInvocable)> on_complete) { + return checkFileAndEnqueue(dispatcher, std::make_unique( + handle(), contents, offset, std::move(on_complete))); } absl::StatusOr AsyncFileContextThreadPool::duplicate( - std::function)> on_complete) { + Event::Dispatcher* dispatcher, + absl::AnyInvocable)> on_complete) { return checkFileAndEnqueue( - std::make_shared(handle(), std::move(on_complete))); + dispatcher, std::make_unique(handle(), std::move(on_complete))); } absl::StatusOr -AsyncFileContextThreadPool::checkFileAndEnqueue(std::shared_ptr action) { +AsyncFileContextThreadPool::checkFileAndEnqueue(Event::Dispatcher* dispatcher, + std::unique_ptr action) { if (fileDescriptor() == -1) { return absl::FailedPreconditionError("file was already closed"); } - return enqueue(action); + return enqueue(dispatcher, std::move(action)); } AsyncFileContextThreadPool::AsyncFileContextThreadPool(AsyncFileManager& manager, int fd) diff --git a/source/extensions/common/async_files/async_file_context_thread_pool.h b/source/extensions/common/async_files/async_file_context_thread_pool.h index 8c14d75be8..6f344375fc 100644 --- a/source/extensions/common/async_files/async_file_context_thread_pool.h +++ b/source/extensions/common/async_files/async_file_context_thread_pool.h @@ -21,27 +21,35 @@ class AsyncFileContextThreadPool final : public AsyncFileContextBase { public: explicit AsyncFileContextThreadPool(AsyncFileManager& manager, int fd); + // CancelFunction should not be called during or after the callback. + // CancelFunction should only be called from the same thread that created + // the context. + // The callback will be dispatched to the same thread that created the context. absl::StatusOr - stat(std::function)> on_complete) override; + stat(Event::Dispatcher* dispatcher, + absl::AnyInvocable)> on_complete) override; absl::StatusOr - createHardLink(absl::string_view filename, - std::function on_complete) override; - absl::Status close(std::function on_complete) override; + createHardLink(Event::Dispatcher* dispatcher, absl::string_view filename, + absl::AnyInvocable on_complete) override; + absl::StatusOr close(Event::Dispatcher* dispatcher, + absl::AnyInvocable on_complete) override; absl::StatusOr - read(off_t offset, size_t length, - std::function)> on_complete) override; + read(Event::Dispatcher* dispatcher, off_t offset, size_t length, + absl::AnyInvocable)> on_complete) override; absl::StatusOr - write(Buffer::Instance& contents, off_t offset, - std::function)> on_complete) override; + write(Event::Dispatcher* dispatcher, Buffer::Instance& contents, off_t offset, + absl::AnyInvocable)> on_complete) override; absl::StatusOr - duplicate(std::function)> on_complete) override; + duplicate(Event::Dispatcher* dispatcher, + absl::AnyInvocable)> on_complete) override; int& fileDescriptor() { return file_descriptor_; } ~AsyncFileContextThreadPool() override; protected: - absl::StatusOr checkFileAndEnqueue(std::shared_ptr action); + absl::StatusOr checkFileAndEnqueue(Event::Dispatcher* dispatcher, + std::unique_ptr action); int file_descriptor_; }; diff --git a/source/extensions/common/async_files/async_file_handle.h b/source/extensions/common/async_files/async_file_handle.h index a8e54a4166..e52885dc2c 100644 --- a/source/extensions/common/async_files/async_file_handle.h +++ b/source/extensions/common/async_files/async_file_handle.h @@ -3,6 +3,8 @@ #include #include +#include "envoy/event/dispatcher.h" + #include "source/common/buffer/buffer_impl.h" #include "source/extensions/common/async_files/async_file_action.h" @@ -19,14 +21,16 @@ class AsyncFileContext : public std::enable_shared_from_this { public: // Gets a stat struct for the file. virtual absl::StatusOr - stat(std::function)> on_complete) PURE; + stat(Event::Dispatcher* dispatcher, + absl::AnyInvocable)> on_complete) PURE; // Action to hard link the file that is currently open. Typically for use in tandem with // createAnonymousFile to turn that file into a named file after finishing writing its contents. // // If cancelled before the callback is called but after creating the file, unlinks the file. virtual absl::StatusOr - createHardLink(absl::string_view filename, std::function on_complete) PURE; + createHardLink(Event::Dispatcher* dispatcher, absl::string_view filename, + absl::AnyInvocable on_complete) PURE; // Enqueues an action to close the currently open file. // It is an error to use an AsyncFileContext after calling close. @@ -35,18 +39,18 @@ class AsyncFileContext : public std::enable_shared_from_this { // Note that because an AsyncFileHandle is a shared_ptr, it is okay to // call close during the handle's owner's destructor - that will queue the close // event, which will keep the handle alive until after the close operation - // is completed. (But be careful that the on_complete callback doesn't require any - // context from the destroyed owner!) - // close cannot be cancelled. - virtual absl::Status close(std::function on_complete) PURE; + // is completed. + // Cancelling close can abort the callback, but the close action will always complete. + virtual absl::StatusOr + close(Event::Dispatcher* dispatcher, absl::AnyInvocable on_complete) PURE; // Enqueues an action to read from the currently open file, at position offset, up to the number // of bytes specified by length. The size of the buffer passed to on_complete informs you if less // than the requested amount was read. It is an error to read on an AsyncFileContext that does not // have a file open. There must not already be an action queued for this handle. virtual absl::StatusOr - read(off_t offset, size_t length, - std::function)> on_complete) PURE; + read(Event::Dispatcher* dispatcher, off_t offset, size_t length, + absl::AnyInvocable)> on_complete) PURE; // Enqueues an action to write to the currently open file, at position offset, the bytes contained // by contents. It is an error to call write on an AsyncFileContext that does not have a file @@ -58,15 +62,16 @@ class AsyncFileContext : public std::enable_shared_from_this { // on_complete is called with the number of bytes written on success. // There must not already be an action queued for this handle. virtual absl::StatusOr - write(Buffer::Instance& contents, off_t offset, - std::function)> on_complete) PURE; + write(Event::Dispatcher* dispatcher, Buffer::Instance& contents, off_t offset, + absl::AnyInvocable)> on_complete) PURE; // Creates a new AsyncFileHandle referencing the same file. // Note that a file handle duplicated in this way shares positioning and permissions // with the original. Since AsyncFileContext functions are all position-explicit, this should not // matter. virtual absl::StatusOr duplicate( - std::function>)> on_complete) PURE; + Event::Dispatcher* dispatcher, + absl::AnyInvocable>)> on_complete) PURE; protected: virtual ~AsyncFileContext() = default; diff --git a/source/extensions/common/async_files/async_file_manager.cc b/source/extensions/common/async_files/async_file_manager.cc deleted file mode 100644 index 2de25a67dc..0000000000 --- a/source/extensions/common/async_files/async_file_manager.cc +++ /dev/null @@ -1,31 +0,0 @@ -#include "source/extensions/common/async_files/async_file_manager.h" - -#include -#include - -#include "source/common/common/macros.h" -#include "source/extensions/common/async_files/async_file_action.h" - -namespace Envoy { -namespace Extensions { -namespace Common { -namespace AsyncFiles { - -namespace { -class ActionWhenReady : public AsyncFileActionWithResult { -public: - explicit ActionWhenReady(std::function on_complete) - : AsyncFileActionWithResult(on_complete) {} - - absl::Status executeImpl() override { return absl::OkStatus(); } -}; -} // namespace - -CancelFunction AsyncFileManager::whenReady(std::function on_complete) { - return enqueue(std::make_shared(std::move(on_complete))); -} - -} // namespace AsyncFiles -} // namespace Common -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/common/async_files/async_file_manager.h b/source/extensions/common/async_files/async_file_manager.h index d515d49c63..c9f0c2c250 100644 --- a/source/extensions/common/async_files/async_file_manager.h +++ b/source/extensions/common/async_files/async_file_manager.h @@ -2,6 +2,8 @@ #include +#include "envoy/event/dispatcher.h" + #include "source/extensions/common/async_files/async_file_action.h" #include "source/extensions/common/async_files/async_file_handle.h" @@ -34,8 +36,8 @@ class AsyncFileManager { // Returns a cancellation function, which aborts the operation (and closes // the file if opened) unless the callback has already been called. virtual CancelFunction - createAnonymousFile(absl::string_view path, - std::function)> on_complete) PURE; + createAnonymousFile(Event::Dispatcher* dispatcher, absl::string_view path, + absl::AnyInvocable)> on_complete) PURE; // A mode for opening existing files. enum class Mode { ReadOnly, WriteOnly, ReadWrite }; @@ -46,53 +48,56 @@ class AsyncFileManager { // Returns a cancellation function, which aborts the operation (and closes // the file if opened) unless the callback has already been called. virtual CancelFunction - openExistingFile(absl::string_view filename, Mode mode, - std::function)> on_complete) PURE; + openExistingFile(Event::Dispatcher* dispatcher, absl::string_view filename, Mode mode, + absl::AnyInvocable)> on_complete) PURE; // Action to stat a file. // on_complete receives a stat structure on success, or an error on failure. // // Returns a cancellation function, which aborts the operation // unless the callback has already been called. - virtual CancelFunction stat(absl::string_view filename, - std::function)> on_complete) PURE; + virtual CancelFunction + stat(Event::Dispatcher* dispatcher, absl::string_view filename, + absl::AnyInvocable)> on_complete) PURE; // Action to delete a named file. // on_complete receives OK on success, or an error on failure. // // Returns a cancellation function, which aborts the operation // unless it has already been performed. - virtual CancelFunction unlink(absl::string_view filename, - std::function on_complete) PURE; - - // whenReady can be used to only perform an action when the caller hits the - // front of the thread pool's queue - this can be used to defer requesting - // a file action until it could actually take place. For example, if you're - // offloading data from memory to disk temporarily, if you queue the write - // immediately then the filesystem thread owns the data until the write - // completes, which may be blocked by heavy traffic, and it turns out you - // want the data back before then - you can't get it back, you have to wait - // for the write to complete and then read it back. - // - // If you used whenReady, you could keep the data belonging to the client - // until it's actually the client's turn to do disk access. When whenReady's - // callback is called, if you request the write at that time the performance - // will be almost identical to if you had requested the write earlier, but - // you have the opportunity to change your mind and do something different - // in the meantime. - // - // The cost of using whenReady is that it requires the client to be lock - // controlled (since the callback occurs in a different thread than the thread - // the state belongs to), versus simpler unchained operations can use queue - // based actions and not worry about ownership. - CancelFunction whenReady(std::function on_complete); + virtual CancelFunction unlink(Event::Dispatcher* dispatcher, absl::string_view filename, + absl::AnyInvocable on_complete) PURE; // Return a string description of the configuration of the manager. // (This is mostly to facilitate testing.) virtual std::string describe() const PURE; + // To facilitate testing, blocks until all queued actions have been performed and + // callbacks posted. + // This does not guarantee that the callbacks have been executed, only that they + // have been sent to the dispatcher. + virtual void waitForIdle() PURE; + +protected: + class QueuedAction { + public: + QueuedAction(std::unique_ptr action, Event::Dispatcher* dispatcher) + : action_(std::move(action)), dispatcher_(dispatcher), + state_(std::make_shared>(State::Queued)) {} + QueuedAction() = default; + std::unique_ptr action_; + Event::Dispatcher* dispatcher_ = nullptr; + enum class State { Queued, Executing, InCallback, Done, Cancelled }; + std::shared_ptr> state_; + }; + private: - virtual CancelFunction enqueue(const std::shared_ptr context) PURE; + // Puts an action in the queue for execution. + virtual CancelFunction enqueue(Event::Dispatcher* dispatcher, + std::unique_ptr action) PURE; + // Puts an action in the queue for its `onCancelledBeforeCallback` function to be + // called. + virtual void postCancelledActionForCleanup(std::unique_ptr action) PURE; friend class AsyncFileContextBase; friend class AsyncFileManagerTest; diff --git a/source/extensions/common/async_files/async_file_manager_thread_pool.cc b/source/extensions/common/async_files/async_file_manager_thread_pool.cc index 1dfd7b84e4..dd6c0a3a86 100644 --- a/source/extensions/common/async_files/async_file_manager_thread_pool.cc +++ b/source/extensions/common/async_files/async_file_manager_thread_pool.cc @@ -15,17 +15,6 @@ namespace Extensions { namespace Common { namespace AsyncFiles { -namespace { -// ThreadNextAction is per worker thread; if enqueue is called from a callback -// the action goes directly into ThreadNextAction, otherwise it goes into the -// queue and is eventually pulled out into ThreadNextAction by a worker thread. -thread_local std::shared_ptr ThreadNextAction; - -// ThreadIsWorker is set to true for worker threads, and will be false -// for all other threads. -thread_local bool ThreadIsWorker = false; -} // namespace - AsyncFileManagerThreadPool::AsyncFileManagerThreadPool( const envoy::extensions::common::async_files::v3::AsyncFileManagerConfig& config, Api::OsSysCalls& posix) @@ -50,6 +39,7 @@ AsyncFileManagerThreadPool::~AsyncFileManagerThreadPool() ABSL_LOCKS_EXCLUDED(qu absl::MutexLock lock(&queue_mutex_); terminate_ = true; } + // This destructor will be blocked by this loop until all queued file actions are complete. while (!thread_pool_.empty()) { thread_pool_.back().join(); thread_pool_.pop_back(); @@ -60,46 +50,123 @@ std::string AsyncFileManagerThreadPool::describe() const { return absl::StrCat("thread_pool_size = ", thread_pool_.size()); } -std::function AsyncFileManagerThreadPool::enqueue(std::shared_ptr action) { - auto cancel_func = [action]() { action->cancel(); }; - // If an action is being enqueued from within a callback, we don't have to actually queue it, - // we can just set it as the thread's next action - this acts to chain the actions without - // yielding to another file. - if (ThreadIsWorker) { - ASSERT(!ThreadNextAction); // only do one file action per callback. - ThreadNextAction = std::move(action); - return cancel_func; - } +void AsyncFileManagerThreadPool::waitForIdle() { + const auto condition = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(queue_mutex_) { + return active_workers_ == 0 && queue_.empty() && cleanup_queue_.empty(); + }; absl::MutexLock lock(&queue_mutex_); - queue_.push(std::move(action)); + queue_mutex_.Await(absl::Condition(&condition)); +} + +absl::AnyInvocable +AsyncFileManagerThreadPool::enqueue(Event::Dispatcher* dispatcher, + std::unique_ptr action) { + QueuedAction entry{std::move(action), dispatcher}; + auto cancel_func = [dispatcher, state = entry.state_]() { + ASSERT(dispatcher == nullptr || dispatcher->isThreadSafe()); + state->store(QueuedAction::State::Cancelled); + }; + absl::MutexLock lock(&queue_mutex_); + queue_.push(std::move(entry)); return cancel_func; } +void AsyncFileManagerThreadPool::postCancelledActionForCleanup( + std::unique_ptr action) { + absl::MutexLock lock(&queue_mutex_); + cleanup_queue_.push(std::move(action)); +} + +void AsyncFileManagerThreadPool::executeAction(QueuedAction&& queued_action) { + using State = QueuedAction::State; + State expected = State::Queued; + std::shared_ptr> state = std::move(queued_action.state_); + std::unique_ptr action = std::move(queued_action.action_); + if (!state->compare_exchange_strong(expected, State::Executing)) { + ASSERT(expected == State::Cancelled); + if (action->executesEvenIfCancelled()) { + action->execute(); + } + return; + } + action->execute(); + expected = State::Executing; + if (!state->compare_exchange_strong(expected, State::InCallback)) { + ASSERT(expected == State::Cancelled); + action->onCancelledBeforeCallback(); + return; + } + if (queued_action.dispatcher_ == nullptr) { + // No need to bother arranging the callback, because a dispatcher was not provided. + return; + } + // If it is necessary to explicitly undo an action on cancel then the lambda will need a + // pointer to this manager that is guaranteed to outlive the lambda, in order to be able + // to perform that cancel operation on a thread belonging to the file manager. + // So capture a shared_ptr if necessary, but, to avoid unnecessary shared_ptr wrangling, + // leave it empty if the action doesn't have an associated cancel operation. + std::shared_ptr manager; + if (action->hasActionIfCancelledBeforeCallback()) { + manager = shared_from_this(); + } + queued_action.dispatcher_->post([manager = std::move(manager), action = std::move(action), + state = std::move(state)]() mutable { + // This callback runs on the caller's thread. + State expected = State::InCallback; + if (state->compare_exchange_strong(expected, State::Done)) { + // Action was not cancelled; run the captured callback on the caller's thread. + action->onComplete(); + return; + } + ASSERT(expected == State::Cancelled); + if (manager == nullptr) { + // Action had a "do nothing" cancellation so we don't need to post a cleanup action. + return; + } + // If an action with side-effects was cancelled after being posted, its + // side-effects need to be undone as the caller can no longer receive the + // returned context. That undo action will need to be done on one of the + // file manager's threads, as it is file related, so post it to the thread pool. + manager->postCancelledActionForCleanup(std::move(action)); + }); +} + void AsyncFileManagerThreadPool::worker() { - ThreadIsWorker = true; + const auto condition = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(queue_mutex_) { + return !queue_.empty() || !cleanup_queue_.empty() || terminate_; + }; + { + absl::MutexLock lock(&queue_mutex_); + active_workers_++; + } while (true) { - const auto condition = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(queue_mutex_) { - return !queue_.empty() || terminate_; - }; + QueuedAction action; + std::unique_ptr cleanup_action; { absl::MutexLock lock(&queue_mutex_); + active_workers_--; queue_mutex_.Await(absl::Condition(&condition)); - if (terminate_) { + if (terminate_ && queue_.empty() && cleanup_queue_.empty()) { return; } - ThreadNextAction = std::move(queue_.front()); - queue_.pop(); + active_workers_++; + if (!queue_.empty()) { + action = std::move(queue_.front()); + queue_.pop(); + } + if (!cleanup_queue_.empty()) { + cleanup_action = std::move(cleanup_queue_.front()); + cleanup_queue_.pop(); + } + } + if (action.action_ != nullptr) { + executeAction(std::move(action)); + action.action_ = nullptr; + } + if (cleanup_action != nullptr) { + std::move(cleanup_action)->onCancelledBeforeCallback(); + cleanup_action = nullptr; } - resolveActions(); - } -} - -void AsyncFileManagerThreadPool::resolveActions() { - while (ThreadNextAction) { - // Move the action out of ThreadNextAction so that its callback can enqueue - // a different ThreadNextAction without self-destructing. - std::shared_ptr action = std::move(ThreadNextAction); - action->execute(); } } @@ -108,15 +175,17 @@ namespace { class ActionWithFileResult : public AsyncFileActionWithResult> { public: ActionWithFileResult(AsyncFileManagerThreadPool& manager, - std::function)> on_complete) - : AsyncFileActionWithResult(on_complete), manager_(manager) {} + absl::AnyInvocable)> on_complete) + : AsyncFileActionWithResult(std::move(on_complete)), manager_(manager) {} protected: - void onCancelledBeforeCallback(absl::StatusOr result) override { - if (result.ok()) { - result.value()->close([](absl::Status) {}).IgnoreError(); + void onCancelledBeforeCallback() override { + if (result_.value().ok()) { + result_.value().value()->close(nullptr, [](absl::Status) {}).IgnoreError(); } } + bool hasActionIfCancelledBeforeCallback() const override { return true; } + AsyncFileManagerThreadPool& manager_; Api::OsSysCalls& posix() { return manager_.posix(); } }; @@ -124,8 +193,8 @@ class ActionWithFileResult : public AsyncFileActionWithResult)> on_complete) - : ActionWithFileResult(manager, on_complete), path_(path) {} + absl::AnyInvocable)> on_complete) + : ActionWithFileResult(manager, std::move(on_complete)), path_(path) {} absl::StatusOr executeImpl() override { Api::SysCallIntResult open_result; @@ -187,8 +256,8 @@ class ActionOpenExistingFile : public ActionWithFileResult { public: ActionOpenExistingFile(AsyncFileManagerThreadPool& manager, absl::string_view filename, AsyncFileManager::Mode mode, - std::function)> on_complete) - : ActionWithFileResult(manager, on_complete), filename_(filename), mode_(mode) {} + absl::AnyInvocable)> on_complete) + : ActionWithFileResult(manager, std::move(on_complete)), filename_(filename), mode_(mode) {} absl::StatusOr executeImpl() override { auto open_result = posix().open(filename_.c_str(), openFlags()); @@ -217,8 +286,8 @@ class ActionOpenExistingFile : public ActionWithFileResult { class ActionStat : public AsyncFileActionWithResult> { public: ActionStat(Api::OsSysCalls& posix, absl::string_view filename, - std::function)> on_complete) - : AsyncFileActionWithResult(on_complete), posix_(posix), filename_(filename) {} + absl::AnyInvocable)> on_complete) + : AsyncFileActionWithResult(std::move(on_complete)), posix_(posix), filename_(filename) {} absl::StatusOr executeImpl() override { struct stat ret; @@ -237,8 +306,8 @@ class ActionStat : public AsyncFileActionWithResult> class ActionUnlink : public AsyncFileActionWithResult { public: ActionUnlink(Api::OsSysCalls& posix, absl::string_view filename, - std::function on_complete) - : AsyncFileActionWithResult(on_complete), posix_(posix), filename_(filename) {} + absl::AnyInvocable on_complete) + : AsyncFileActionWithResult(std::move(on_complete)), posix_(posix), filename_(filename) {} absl::Status executeImpl() override { Api::SysCallIntResult unlink_result = posix_.unlink(filename_.c_str()); @@ -256,25 +325,31 @@ class ActionUnlink : public AsyncFileActionWithResult { } // namespace CancelFunction AsyncFileManagerThreadPool::createAnonymousFile( - absl::string_view path, std::function)> on_complete) { - return enqueue(std::make_shared(*this, path, on_complete)); + Event::Dispatcher* dispatcher, absl::string_view path, + absl::AnyInvocable)> on_complete) { + return enqueue(dispatcher, + std::make_unique(*this, path, std::move(on_complete))); } CancelFunction AsyncFileManagerThreadPool::openExistingFile( - absl::string_view filename, Mode mode, - std::function)> on_complete) { - return enqueue(std::make_shared(*this, filename, mode, on_complete)); + Event::Dispatcher* dispatcher, absl::string_view filename, Mode mode, + absl::AnyInvocable)> on_complete) { + return enqueue(dispatcher, std::make_unique(*this, filename, mode, + std::move(on_complete))); } -CancelFunction -AsyncFileManagerThreadPool::stat(absl::string_view filename, - std::function)> on_complete) { - return enqueue(std::make_shared(posix(), filename, on_complete)); +CancelFunction AsyncFileManagerThreadPool::stat( + Event::Dispatcher* dispatcher, absl::string_view filename, + absl::AnyInvocable)> on_complete) { + return enqueue(dispatcher, + std::make_unique(posix(), filename, std::move(on_complete))); } -CancelFunction AsyncFileManagerThreadPool::unlink(absl::string_view filename, - std::function on_complete) { - return enqueue(std::make_shared(posix(), filename, on_complete)); +CancelFunction +AsyncFileManagerThreadPool::unlink(Event::Dispatcher* dispatcher, absl::string_view filename, + absl::AnyInvocable on_complete) { + return enqueue(dispatcher, + std::make_unique(posix(), filename, std::move(on_complete))); } } // namespace AsyncFiles diff --git a/source/extensions/common/async_files/async_file_manager_thread_pool.h b/source/extensions/common/async_files/async_file_manager_thread_pool.h index 2a82202384..81c927eda7 100644 --- a/source/extensions/common/async_files/async_file_manager_thread_pool.h +++ b/source/extensions/common/async_files/async_file_manager_thread_pool.h @@ -24,23 +24,25 @@ namespace AsyncFiles { // performed the previous action in the chain immediately performs the newly chained // action. class AsyncFileManagerThreadPool : public AsyncFileManager, + public std::enable_shared_from_this, protected Logger::Loggable { public: explicit AsyncFileManagerThreadPool( const envoy::extensions::common::async_files::v3::AsyncFileManagerConfig& config, Api::OsSysCalls& posix); ~AsyncFileManagerThreadPool() ABSL_LOCKS_EXCLUDED(queue_mutex_) override; + CancelFunction createAnonymousFile( + Event::Dispatcher* dispatcher, absl::string_view path, + absl::AnyInvocable)> on_complete) override; CancelFunction - createAnonymousFile(absl::string_view path, - std::function)> on_complete) override; - CancelFunction - openExistingFile(absl::string_view filename, Mode mode, - std::function)> on_complete) override; - CancelFunction stat(absl::string_view filename, - std::function)> on_complete) override; - CancelFunction unlink(absl::string_view filename, - std::function on_complete) override; + openExistingFile(Event::Dispatcher* dispatcher, absl::string_view filename, Mode mode, + absl::AnyInvocable)> on_complete) override; + CancelFunction stat(Event::Dispatcher* dispatcher, absl::string_view filename, + absl::AnyInvocable)> on_complete) override; + CancelFunction unlink(Event::Dispatcher* dispatcher, absl::string_view filename, + absl::AnyInvocable on_complete) override; std::string describe() const override; + void waitForIdle() override; Api::OsSysCalls& posix() const { return posix_; } #ifdef O_TMPFILE @@ -56,13 +58,18 @@ class AsyncFileManagerThreadPool : public AsyncFileManager, #endif // O_TMPFILE private: - std::function enqueue(std::shared_ptr action) + absl::AnyInvocable enqueue(Event::Dispatcher* dispatcher, + std::unique_ptr action) + ABSL_LOCKS_EXCLUDED(queue_mutex_) override; + void postCancelledActionForCleanup(std::unique_ptr action) ABSL_LOCKS_EXCLUDED(queue_mutex_) override; void worker() ABSL_LOCKS_EXCLUDED(queue_mutex_); - void resolveActions(); absl::Mutex queue_mutex_; - std::queue> queue_ ABSL_GUARDED_BY(queue_mutex_); + void executeAction(QueuedAction&& action); + std::queue queue_ ABSL_GUARDED_BY(queue_mutex_); + int active_workers_ ABSL_GUARDED_BY(queue_mutex_) = 0; + std::queue> cleanup_queue_ ABSL_GUARDED_BY(queue_mutex_); bool terminate_ ABSL_GUARDED_BY(queue_mutex_) = false; std::vector thread_pool_; diff --git a/source/extensions/filters/http/file_system_buffer/filter.cc b/source/extensions/filters/http/file_system_buffer/filter.cc index 56764975b0..fbb4420a7d 100644 --- a/source/extensions/filters/http/file_system_buffer/filter.cc +++ b/source/extensions/filters/http/file_system_buffer/filter.cc @@ -283,7 +283,7 @@ bool FileSystemBufferFilter::maybeOutputResponse() { void BufferedStreamState::close() { if (async_file_handle_) { - auto queued = async_file_handle_->close([](absl::Status) {}); + auto queued = async_file_handle_->close(nullptr, [](absl::Status) {}); ASSERT(queued.ok()); } } @@ -375,11 +375,11 @@ bool FileSystemBufferFilter::maybeStorage(BufferedStreamState& state, state.storage_used_ -= size; state.memory_used_ += size; ENVOY_STREAM_LOG(debug, "retrieving buffer fragment (size={}) from storage", callbacks, size); - auto queued = - (**earliest_storage_fragment) - .fromStorage(state.async_file_handle_, getSafeDispatch(), getOnFileActionCompleted()); + auto queued = (**earliest_storage_fragment) + .fromStorage(state.async_file_handle_, request_callbacks_->dispatcher(), + getOnFileActionCompleted()); ASSERT(queued.ok()); - cancel_in_flight_async_action_ = queued.value(); + cancel_in_flight_async_action_ = std::move(queued.value()); return true; } } @@ -389,30 +389,19 @@ bool FileSystemBufferFilter::maybeStorage(BufferedStreamState& state, if (!state.async_file_handle_) { // File isn't open yet - open it and then check again if we still need to store data. ENVOY_STREAM_LOG(debug, "memory buffer exceeded - creating buffer file", callbacks); - // We can't use getSafeDispatcher here because we need to close the file if the filter - // was deleted before the callback, not just do nothing. + // The callback won't be called if the filter was destroyed, and if the file was + // racily created it will be closed. cancel_in_flight_async_action_ = config_->asyncFileManager().createAnonymousFile( - config_->storageBufferPath(), - [this, is_destroyed = is_destroyed_, dispatcher = &request_callbacks_->dispatcher(), - &state](absl::StatusOr file_handle) { - dispatcher->post([this, is_destroyed, &state, file_handle]() { - if (*is_destroyed) { - // If we opened a file but the filter went away in the meantime, close the file - // to avoid leaving a dangling file handle. - if (file_handle.ok()) { - file_handle.value()->close([](absl::Status) {}).IgnoreError(); - } - return; - } - if (!file_handle.ok()) { - filterError(fmt::format("{} failed to create buffer file: {}", filterName(), - file_handle.status().ToString())); - return; - } - state.async_file_handle_ = std::move(file_handle.value()); - cancel_in_flight_async_action_ = nullptr; - onStateChange(); - }); + &request_callbacks_->dispatcher(), config_->storageBufferPath(), + [this, &state](absl::StatusOr file_handle) { + if (!file_handle.ok()) { + filterError(fmt::format("{} failed to create buffer file: {}", filterName(), + file_handle.status().ToString())); + return; + } + state.async_file_handle_ = std::move(file_handle.value()); + cancel_in_flight_async_action_ = nullptr; + onStateChange(); }); return true; } @@ -424,18 +413,19 @@ bool FileSystemBufferFilter::maybeStorage(BufferedStreamState& state, state.storage_consumed_ += size; state.memory_used_ -= size; ENVOY_STREAM_LOG(debug, "sending buffer fragment (size={}) to storage", callbacks, size); - auto to_storage = fragment->toStorage(state.async_file_handle_, state.storage_offset_, - getSafeDispatch(), getOnFileActionCompleted()); + auto to_storage = + fragment->toStorage(state.async_file_handle_, state.storage_offset_, + request_callbacks_->dispatcher(), getOnFileActionCompleted()); ASSERT(to_storage.ok()); - cancel_in_flight_async_action_ = to_storage.value(); + cancel_in_flight_async_action_ = std::move(to_storage.value()); state.storage_offset_ += size; return true; } return false; } -std::function FileSystemBufferFilter::getOnFileActionCompleted() { - // This callback is only run via getSafeDispatch, so is safe to capture 'this' - +absl::AnyInvocable FileSystemBufferFilter::getOnFileActionCompleted() { + // This callback is aborted in onDestroy, so is safe to capture 'this' - // it won't be called if the filter has been deleted. return [this](absl::Status status) { cancel_in_flight_async_action_ = nullptr; @@ -447,19 +437,17 @@ std::function FileSystemBufferFilter::getOnFileActionComplet }; } -std::function)> FileSystemBufferFilter::getSafeDispatch() { - return [is_destroyed = is_destroyed_, - dispatcher = &request_callbacks_->dispatcher()](std::function callback) { - dispatcher->post([is_destroyed, callback = std::move(callback)]() { - if (!*is_destroyed) { - callback(); - } - }); - }; +void FileSystemBufferFilter::safeDispatch(absl::AnyInvocable fn) { + request_callbacks_->dispatcher().post( + [is_destroyed = is_destroyed_, fn = std::move(fn)]() mutable { + if (!*is_destroyed) { + std::move(fn)(); + } + }); } void FileSystemBufferFilter::dispatchStateChanged() { - getSafeDispatch()([this]() { onStateChange(); }); + safeDispatch([this]() { onStateChange(); }); } } // namespace FileSystemBuffer diff --git a/source/extensions/filters/http/file_system_buffer/filter.h b/source/extensions/filters/http/file_system_buffer/filter.h index d53feb0794..be452c41d6 100644 --- a/source/extensions/filters/http/file_system_buffer/filter.h +++ b/source/extensions/filters/http/file_system_buffer/filter.h @@ -102,14 +102,15 @@ class FileSystemBufferFilter : public Http::StreamFilter, // These operations are asynchronous; the impacted buffer fragments are in an unusable state until // the operation completes. bool maybeStorage(BufferedStreamState& state, Http::StreamFilterCallbacks& callbacks); - std::function getOnFileActionCompleted(); + absl::AnyInvocable getOnFileActionCompleted(); // Called if an unrecoverable error occurs in the filter (e.g. a file operation fails). Internal // server error. void filterError(absl::string_view err); - // Returns a safe dispatch function that aborts if the filter has been destroyed. - std::function)> getSafeDispatch(); + // Dispatch a callback wrapped such that it is not called if the filter has been destroyed + // by the time it pops off the dispatch queue. + void safeDispatch(absl::AnyInvocable fn); // Queue an onStateChange in the dispatcher. This is used to get the next piece of work back // into the Envoy thread from an AsyncFiles thread, or to queue work that may not be allowed diff --git a/source/extensions/filters/http/file_system_buffer/fragment.cc b/source/extensions/filters/http/file_system_buffer/fragment.cc index d2da3bbd1c..bebde1364e 100644 --- a/source/extensions/filters/http/file_system_buffer/fragment.cc +++ b/source/extensions/filters/http/file_system_buffer/fragment.cc @@ -35,60 +35,53 @@ std::unique_ptr Fragment::extract() { std::unique_ptr MemoryFragment::extract() { return std::move(buffer_); } -absl::StatusOr -Fragment::toStorage(AsyncFileHandle file, off_t offset, - std::function)> dispatch, - std::function on_done) { +absl::StatusOr Fragment::toStorage(AsyncFileHandle file, off_t offset, + Event::Dispatcher& dispatcher, + absl::AnyInvocable on_done) { ASSERT(isMemory()); auto data = absl::get(data_).extract(); data_.emplace(); + // This callback is only called if the filter was not destroyed in the meantime, + // so it is safe to use `this`. return file->write( - *data, offset, - [this, dispatch = std::move(dispatch), size = size_, on_done = std::move(on_done), - offset](absl::StatusOr result) { - // size is captured because we can't safely use 'this' until we're in the dispatch callback. + &dispatcher, *data, offset, + [this, on_done = std::move(on_done), offset](absl::StatusOr result) mutable { if (!result.ok()) { - dispatch([status = result.status(), on_done = std::move(on_done)]() { on_done(status); }); - } else if (result.value() != size) { + std::move(on_done)(result.status()); + } else if (result.value() != size_) { auto status = absl::AbortedError( - fmt::format("buffer write wrote {} bytes, wanted {}", result.value(), size)); - dispatch( - [on_done = std::move(on_done), status = std::move(status)]() { on_done(status); }); + fmt::format("buffer write wrote {} bytes, wanted {}", result.value(), size_)); + std::move(on_done)(status); } else { - dispatch([this, status = result.status(), offset, on_done = std::move(on_done)] { - data_.emplace(offset); - on_done(absl::OkStatus()); - }); + data_.emplace(offset); + std::move(on_done)(absl::OkStatus()); } }); } absl::StatusOr -Fragment::fromStorage(AsyncFileHandle file, std::function)> dispatch, - std::function on_done) { +Fragment::fromStorage(AsyncFileHandle file, Event::Dispatcher& dispatcher, + absl::AnyInvocable on_done) { ASSERT(isStorage()); off_t offset = absl::get(data_).offset(); data_.emplace(); - return file->read( - offset, size_, - [this, dispatch = std::move(dispatch), size = size_, - on_done = std::move(on_done)](absl::StatusOr> result) { - // size is captured because we can't safely use 'this' until we're in the dispatch callback. - if (!result.ok()) { - dispatch([on_done = std::move(on_done), status = result.status()]() { on_done(status); }); - } else if (result.value()->length() != size) { - auto status = absl::AbortedError( - fmt::format("buffer read got {} bytes, wanted {}", result.value()->length(), size)); - dispatch( - [on_done = std::move(on_done), status = std::move(status)]() { on_done(status); }); - } else { - auto buffer = std::shared_ptr(std::move(result.value())); - dispatch([this, on_done = std::move(on_done), buffer = std::move(buffer)]() { - data_.emplace(*buffer); - on_done(absl::OkStatus()); - }); - } - }); + // This callback is only called if the filter was not destroyed in the meantime, + // so it is safe to use `this`. + return file->read(&dispatcher, offset, size_, + [this, on_done = std::move(on_done)]( + absl::StatusOr> result) mutable { + if (!result.ok()) { + std::move(on_done)(result.status()); + } else if (result.value()->length() != size_) { + auto status = + absl::AbortedError(fmt::format("buffer read got {} bytes, wanted {}", + result.value()->length(), size_)); + std::move(on_done)(status); + } else { + data_.emplace(*result.value()); + std::move(on_done)(absl::OkStatus()); + } + }); } } // namespace FileSystemBuffer diff --git a/source/extensions/filters/http/file_system_buffer/fragment.h b/source/extensions/filters/http/file_system_buffer/fragment.h index 2e6efe2704..f38667da8e 100644 --- a/source/extensions/filters/http/file_system_buffer/fragment.h +++ b/source/extensions/filters/http/file_system_buffer/fragment.h @@ -70,8 +70,8 @@ class Fragment { // When called from a filter, the dispatcher function must abort without calling the // callback if the filter or fragment has been destroyed. absl::StatusOr toStorage(AsyncFileHandle file, off_t offset, - std::function)> dispatch, - std::function on_done); + Event::Dispatcher& dispatcher, + absl::AnyInvocable on_done); // Starts the transition for this fragment from storage to memory. // @@ -79,9 +79,8 @@ class Fragment { // // When called from a filter, the dispatcher function must abort without calling the // callback if the filter or fragment has been destroyed. - absl::StatusOr fromStorage(AsyncFileHandle file, - std::function)> dispatch, - std::function on_done); + absl::StatusOr fromStorage(AsyncFileHandle file, Event::Dispatcher& dispatcher, + absl::AnyInvocable on_done); // Moves the buffer from a memory instance to the returned value and resets the fragment to // size 0. diff --git a/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.cc b/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.cc index 687a0e40d8..1e5a49c6c6 100644 --- a/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.cc +++ b/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.cc @@ -25,7 +25,7 @@ const size_t FileSystemHttpCache::max_update_headers_copy_chunk_size_ = 128 * 10 const CacheStats& FileSystemHttpCache::stats() const { return shared_->stats_; } const ConfigProto& FileSystemHttpCache::config() const { return shared_->config_; } -void FileSystemHttpCache::writeVaryNodeToDisk(const Key& key, +void FileSystemHttpCache::writeVaryNodeToDisk(Event::Dispatcher& dispatcher, const Key& key, const Http::ResponseHeaderMap& response_headers, std::shared_ptr cleanup) { auto vary_values = VaryHeaderUtils::getVaryValues(response_headers); @@ -35,8 +35,9 @@ void FileSystemHttpCache::writeVaryNodeToDisk(const Key& key, h->set_value(absl::StrJoin(vary_values, ",")); std::string filename = absl::StrCat(cachePath(), generateFilename(key)); async_file_manager_->createAnonymousFile( - cachePath(), [headers, filename = std::move(filename), - cleanup](absl::StatusOr open_result) { + &dispatcher, cachePath(), + [headers, filename = std::move(filename), cleanup, + dispatcher = &dispatcher](absl::StatusOr open_result) { if (!open_result.ok()) { ENVOY_LOG(warn, "writing vary node, failed to createAnonymousFile: {}", open_result.status()); @@ -51,20 +52,20 @@ void FileSystemHttpCache::writeVaryNodeToDisk(const Key& key, buf2.add(buf); size_t sz = buf2.length(); auto queued = file_handle->write( - buf2, 0, - [file_handle, cleanup, sz, + dispatcher, buf2, 0, + [dispatcher, file_handle, cleanup, sz, filename = std::move(filename)](absl::StatusOr write_result) { if (!write_result.ok() || write_result.value() != sz) { ENVOY_LOG(warn, "writing vary node, failed to write: {}", write_result.status()); - file_handle->close([](absl::Status) {}).IgnoreError(); + file_handle->close(nullptr, [](absl::Status) {}).IgnoreError(); return; } auto queued = file_handle->createHardLink( - filename, [cleanup, file_handle](absl::Status link_result) { + dispatcher, filename, [cleanup, file_handle](absl::Status link_result) { if (!link_result.ok()) { ENVOY_LOG(warn, "writing vary node, failed to link: {}", link_result); } - file_handle->close([](absl::Status) {}).IgnoreError(); + file_handle->close(nullptr, [](absl::Status) {}).IgnoreError(); }); ASSERT(queued.ok()); }); @@ -114,35 +115,37 @@ FileSystemHttpCache::makeVaryKey(const Key& base, const VaryAllowList& vary_allo return vary_key; } -LookupContextPtr FileSystemHttpCache::makeLookupContext(LookupRequest&& lookup, - Http::StreamDecoderFilterCallbacks&) { - return std::make_unique(*this, std::move(lookup)); +LookupContextPtr +FileSystemHttpCache::makeLookupContext(LookupRequest&& lookup, + Http::StreamDecoderFilterCallbacks& callbacks) { + return std::make_unique(callbacks.dispatcher(), *this, std::move(lookup)); } // Helper class to reduce the lambda depth of updateHeaders. class HeaderUpdateContext : public Logger::Loggable { public: - HeaderUpdateContext(const FileSystemHttpCache& cache, const Key& key, - std::shared_ptr cleanup, + HeaderUpdateContext(Event::Dispatcher& dispatcher, const FileSystemHttpCache& cache, + const Key& key, std::shared_ptr cleanup, const Http::ResponseHeaderMap& response_headers, const ResponseMetadata& metadata, std::function on_complete) - : filepath_(absl::StrCat(cache.cachePath(), cache.generateFilename(key))), + : dispatcher_(dispatcher), + filepath_(absl::StrCat(cache.cachePath(), cache.generateFilename(key))), cache_path_(cache.cachePath()), cleanup_(cleanup), async_file_manager_(cache.asyncFileManager()), response_headers_(Http::createHeaderMap(response_headers)), response_metadata_(metadata), on_complete_(on_complete) {} void begin(std::shared_ptr ctx) { - async_file_manager_->openExistingFile(filepath_, - Common::AsyncFiles::AsyncFileManager::Mode::ReadOnly, - [ctx, this](absl::StatusOr open_result) { - if (!open_result.ok()) { - fail("failed to open", open_result.status()); - return; - } - read_handle_ = std::move(open_result.value()); - unlinkOriginal(ctx); - }); + async_file_manager_->openExistingFile( + dispatcher(), filepath_, Common::AsyncFiles::AsyncFileManager::Mode::ReadOnly, + [ctx = std::move(ctx), this](absl::StatusOr open_result) { + if (!open_result.ok()) { + fail("failed to open", open_result.status()); + return; + } + read_handle_ = std::move(open_result.value()); + unlinkOriginal(std::move(ctx)); + }); } ~HeaderUpdateContext() { @@ -150,44 +153,46 @@ class HeaderUpdateContext : public Logger::Loggable { // write_handle_ can only be set if read_handle_ is set, so this ordering is safe. if (read_handle_) { read_handle_ - ->close([write_handle = write_handle_](absl::Status) { - if (write_handle) { - write_handle->close([](absl::Status) {}).IgnoreError(); - } - }) + ->close(dispatcher(), + [write_handle = write_handle_](absl::Status) { + if (write_handle) { + write_handle->close(nullptr, [](absl::Status) {}).IgnoreError(); + } + }) .IgnoreError(); } } private: void unlinkOriginal(std::shared_ptr ctx) { - async_file_manager_->unlink(filepath_, [ctx, this](absl::Status unlink_result) { - if (!unlink_result.ok()) { - ENVOY_LOG(warn, "file_system_http_cache: {} for update cache file {}: {}", "unlink failed", - filepath_, unlink_result); - // But keep going, because unlink might have failed because the file was already - // deleted after we opened it. Worth a try to replace it! - } - readHeaderBlock(ctx); - }); + async_file_manager_->unlink( + dispatcher(), filepath_, [ctx = std::move(ctx), this](absl::Status unlink_result) { + if (!unlink_result.ok()) { + ENVOY_LOG(warn, "file_system_http_cache: {} for update cache file {}: {}", + "unlink failed", filepath_, unlink_result); + // But keep going, because unlink might have failed because the file was already + // deleted after we opened it. Worth a try to replace it! + } + readHeaderBlock(std::move(ctx)); + }); } void readHeaderBlock(std::shared_ptr ctx) { auto queued = read_handle_->read( - 0, CacheFileFixedBlock::size(), - [ctx, this](absl::StatusOr read_result) { + dispatcher(), 0, CacheFileFixedBlock::size(), + [ctx = std::move(ctx), this](absl::StatusOr read_result) { if (!read_result.ok() || read_result.value()->length() != CacheFileFixedBlock::size()) { fail("failed to read header block", read_result.status()); return; } header_block_.populateFromStringView(read_result.value()->toString()); - readHeaders(ctx); + readHeaders(std::move(ctx)); }); ASSERT(queued.ok()); } void readHeaders(std::shared_ptr ctx) { auto queued = read_handle_->read( - header_block_.offsetToHeaders(), header_block_.headerSize(), - [ctx, this](absl::StatusOr read_result) { + dispatcher(), header_block_.offsetToHeaders(), header_block_.headerSize(), + [ctx = std::move(ctx), this](absl::StatusOr read_result) { if (!read_result.ok() || read_result.value()->length() != header_block_.headerSize()) { fail("failed to read headers", read_result.status()); return; @@ -207,19 +212,20 @@ class HeaderUpdateContext : public Logger::Loggable { size_t new_header_size = headerProtoSize(header_proto_); header_size_difference_ = header_block_.headerSize() - new_header_size; header_block_.setHeadersSize(new_header_size); - startWriting(ctx); + startWriting(std::move(ctx)); }); ASSERT(queued.ok()); } void startWriting(std::shared_ptr ctx) { async_file_manager_->createAnonymousFile( - cache_path_, [ctx, this](absl::StatusOr create_result) { + dispatcher(), cache_path_, + [ctx = std::move(ctx), this](absl::StatusOr create_result) { if (!create_result.ok()) { fail("failed to open new cache file", create_result.status()); return; } write_handle_ = std::move(create_result.value()); - writeHeaderBlockAndHeaders(ctx); + writeHeaderBlockAndHeaders(std::move(ctx)); }); } void writeHeaderBlockAndHeaders(std::shared_ptr ctx) { @@ -227,13 +233,14 @@ class HeaderUpdateContext : public Logger::Loggable { header_block_.serializeToBuffer(buf); buf.add(bufferFromProto(header_proto_)); auto sz = buf.length(); - auto queued = - write_handle_->write(buf, 0, [ctx, sz, this](absl::StatusOr write_result) { + auto queued = write_handle_->write( + dispatcher(), buf, 0, + [ctx = std::move(ctx), sz, this](absl::StatusOr write_result) { if (!write_result.ok() || write_result.value() != sz) { fail("failed to write header block and headers", write_result.status()); return; } - copyBodyAndTrailers(ctx, header_block_.offsetToBody()); + copyBodyAndTrailers(std::move(ctx), header_block_.offsetToBody()); }); ASSERT(queued.ok()); } @@ -245,33 +252,34 @@ class HeaderUpdateContext : public Logger::Loggable { } sz = std::min(sz, FileSystemHttpCache::max_update_headers_copy_chunk_size_); auto queued = read_handle_->read( - offset + header_size_difference_, sz, - [ctx, offset, sz, this](absl::StatusOr read_result) { + dispatcher(), offset + header_size_difference_, sz, + [ctx = std::move(ctx), offset, sz, this](absl::StatusOr read_result) { if (!read_result.ok() || read_result.value()->length() != sz) { fail("failed to read body chunk", read_result.status()); return; } - auto queued = - write_handle_->write(*read_result.value(), offset, - [ctx, offset, sz, this](absl::StatusOr write_result) { - if (!write_result.ok() || write_result.value() != sz) { - fail("failed to write body chunk", write_result.status()); - return; - } - copyBodyAndTrailers(ctx, offset + sz); - }); + auto queued = write_handle_->write( + dispatcher(), *read_result.value(), offset, + [ctx = std::move(ctx), offset, sz, this](absl::StatusOr write_result) { + if (!write_result.ok() || write_result.value() != sz) { + fail("failed to write body chunk", write_result.status()); + return; + } + copyBodyAndTrailers(std::move(ctx), offset + sz); + }); ASSERT(queued.ok()); }); ASSERT(queued.ok()); } void linkNewFile(std::shared_ptr ctx) { - auto queued = write_handle_->createHardLink(filepath_, [ctx, this](absl::Status link_result) { - if (!link_result.ok()) { - fail("failed to link new cache file", link_result); - return; - } - on_complete_(true); - }); + auto queued = write_handle_->createHardLink( + dispatcher(), filepath_, [ctx = std::move(ctx), this](absl::Status link_result) { + if (!link_result.ok()) { + fail("failed to link new cache file", link_result); + return; + } + on_complete_(true); + }); ASSERT(queued.ok()); } void fail(absl::string_view msg, absl::Status status) { @@ -279,6 +287,8 @@ class HeaderUpdateContext : public Logger::Loggable { status); on_complete_(false); } + Event::Dispatcher* dispatcher() { return &dispatcher_; } + Event::Dispatcher& dispatcher_; std::string filepath_; std::string cache_path_; std::shared_ptr cleanup_; @@ -293,17 +303,19 @@ class HeaderUpdateContext : public Logger::Loggable { std::function on_complete_; }; -void FileSystemHttpCache::updateHeaders(const LookupContext& lookup_context, +void FileSystemHttpCache::updateHeaders(const LookupContext& base_lookup_context, const Http::ResponseHeaderMap& response_headers, const ResponseMetadata& metadata, std::function on_complete) { - const Key& key = dynamic_cast(lookup_context).key(); + const FileLookupContext& lookup_context = + dynamic_cast(base_lookup_context); + const Key& key = lookup_context.key(); auto cleanup = maybeStartWritingEntry(key); if (!cleanup) { return; } - auto ctx = std::make_shared(*this, key, cleanup, response_headers, metadata, - on_complete); + auto ctx = std::make_shared( + *lookup_context.dispatcher(), *this, key, cleanup, response_headers, metadata, on_complete); ctx->begin(ctx); } @@ -326,10 +338,10 @@ std::shared_ptr FileSystemHttpCache::maybeStartWritingEntry(const Key& } std::shared_ptr -FileSystemHttpCache::setCacheEntryToVary(const Key& key, +FileSystemHttpCache::setCacheEntryToVary(Event::Dispatcher& dispatcher, const Key& key, const Http::ResponseHeaderMap& response_headers, const Key& varied_key, std::shared_ptr cleanup) { - writeVaryNodeToDisk(key, response_headers, cleanup); + writeVaryNodeToDisk(dispatcher, key, response_headers, cleanup); return maybeStartWritingEntry(varied_key); } diff --git a/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.h b/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.h index 2c32d110ec..9e81bf0410 100644 --- a/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.h +++ b/source/extensions/http/cache/file_system_http_cache/file_system_http_cache.h @@ -137,9 +137,9 @@ class FileSystemHttpCache : public HttpCache, * if varied_key was added. */ ABSL_MUST_USE_RESULT std::shared_ptr - setCacheEntryToVary(const Key& key, const Http::ResponseHeaderMap& response_headers, - const Key& varied_key, std::shared_ptr cleanup) - ABSL_LOCKS_EXCLUDED(cache_mu_); + setCacheEntryToVary(Event::Dispatcher& dispatcher, const Key& key, + const Http::ResponseHeaderMap& response_headers, const Key& varied_key, + std::shared_ptr cleanup) ABSL_LOCKS_EXCLUDED(cache_mu_); /** * Returns the extension name. @@ -187,6 +187,9 @@ class FileSystemHttpCache : public HttpCache, using PostEvictionCallback = std::function; + // Waits for all queued actions to be completed. + void drainAsyncFileActionsForTest() { async_file_manager_->waitForIdle(); }; + private: /** * Writes a vary node to disk for the given key. A vary node in the cache consists of @@ -196,7 +199,8 @@ class FileSystemHttpCache : public HttpCache, * be extracted. * @param cleanup the cleanup operation to be performed when the write completes. */ - void writeVaryNodeToDisk(const Key& key, const Http::ResponseHeaderMap& response_headers, + void writeVaryNodeToDisk(Event::Dispatcher& dispatcher, const Key& key, + const Http::ResponseHeaderMap& response_headers, std::shared_ptr cleanup); // A shared_ptr to keep the cache singleton alive as long as any of its caches are in use. diff --git a/source/extensions/http/cache/file_system_http_cache/insert_context.cc b/source/extensions/http/cache/file_system_http_cache/insert_context.cc index 53c066b9b0..98e0565773 100644 --- a/source/extensions/http/cache/file_system_http_cache/insert_context.cc +++ b/source/extensions/http/cache/file_system_http_cache/insert_context.cc @@ -31,7 +31,7 @@ FileInsertContext::FileInsertContext(std::shared_ptr cache, void FileInsertContext::insertHeaders(const Http::ResponseHeaderMap& response_headers, const ResponseMetadata& metadata, InsertCallback insert_complete, bool end_stream) { - absl::MutexLock lock(&mu_); + ASSERT(dispatcher()->isThreadSafe()); callback_in_flight_ = insert_complete; const VaryAllowList& vary_allow_list = lookup_context_->lookup().varyAllowList(); const Http::RequestHeaderMap& request_headers = lookup_context_->lookup().requestHeaders(); @@ -47,7 +47,8 @@ void FileInsertContext::insertHeaders(const Http::ResponseHeaderMap& response_he cancelInsert(); return; } - cleanup_ = cache_->setCacheEntryToVary(old_key, response_headers, key_, cleanup_); + cleanup_ = + cache_->setCacheEntryToVary(*dispatcher(), old_key, response_headers, key_, cleanup_); } else { cleanup_ = cache_->maybeStartWritingEntry(key_); } @@ -56,74 +57,98 @@ void FileInsertContext::insertHeaders(const Http::ResponseHeaderMap& response_he cancelInsert(); return; } - auto header_proto = makeCacheFileHeaderProto(key_, response_headers, metadata); - // Open the file. + cache_file_header_proto_ = makeCacheFileHeaderProto(key_, response_headers, metadata); + end_stream_after_headers_ = end_stream; + on_insert_complete_ = std::move(insert_complete); + createFile(); +} + +void FileInsertContext::createFile() { + ASSERT(dispatcher()->isThreadSafe()); + ASSERT(!cancel_action_in_flight_); + ASSERT(callback_in_flight_ != nullptr); cancel_action_in_flight_ = cache_->asyncFileManager()->createAnonymousFile( - cache_->cachePath(), [this, end_stream, header_proto, - insert_complete](absl::StatusOr open_result) { - absl::MutexLock lock(&mu_); + dispatcher(), cache_->cachePath(), [this](absl::StatusOr open_result) { cancel_action_in_flight_ = nullptr; if (!open_result.ok()) { cancelInsert("failed to create anonymous file"); return; } file_handle_ = std::move(open_result.value()); - Buffer::OwnedImpl unset_header; - header_block_.serializeToBuffer(unset_header); - // Write an empty header block. - auto queued = file_handle_->write( - unset_header, 0, [this, end_stream, header_proto](absl::StatusOr write_result) { - absl::MutexLock lock(&mu_); - cancel_action_in_flight_ = nullptr; - if (!write_result.ok() || write_result.value() != CacheFileFixedBlock::size()) { - cancelInsert(writeFailureMessage("empty header block", write_result, - CacheFileFixedBlock::size())); - return; - } - auto buf = bufferFromProto(header_proto); - auto sz = buf.length(); - auto queued = file_handle_->write( - buf, header_block_.offsetToHeaders(), - [this, end_stream, sz](absl::StatusOr write_result) { - absl::MutexLock lock(&mu_); - cancel_action_in_flight_ = nullptr; - if (!write_result.ok() || write_result.value() != sz) { - cancelInsert(writeFailureMessage("headers", write_result, sz)); - return; - } - header_block_.setHeadersSize(sz); - if (end_stream) { - commit(callback_in_flight_); - return; - } - auto cb = callback_in_flight_; - callback_in_flight_ = nullptr; - cb(true); - }); - ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); - }); - ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); + writeEmptyHeaderBlock(); }); } +void FileInsertContext::writeEmptyHeaderBlock() { + ASSERT(dispatcher()->isThreadSafe()); + ASSERT(!cancel_action_in_flight_); + ASSERT(callback_in_flight_ != nullptr); + Buffer::OwnedImpl unset_header; + header_block_.serializeToBuffer(unset_header); + // Write an empty header block. + auto queued = file_handle_->write( + dispatcher(), unset_header, 0, [this](absl::StatusOr write_result) { + cancel_action_in_flight_ = nullptr; + if (!write_result.ok() || write_result.value() != CacheFileFixedBlock::size()) { + cancelInsert( + writeFailureMessage("empty header block", write_result, CacheFileFixedBlock::size())); + return; + } + writeHeaderProto(); + }); + ASSERT(queued.ok(), queued.status().ToString()); + cancel_action_in_flight_ = std::move(queued.value()); +} + +void FileInsertContext::succeedCurrentAction() { + ASSERT(!cancel_action_in_flight_); + ASSERT(callback_in_flight_ != nullptr); + auto cb = std::move(callback_in_flight_); + callback_in_flight_ = nullptr; + cb(true); +} + +void FileInsertContext::writeHeaderProto() { + ASSERT(dispatcher()->isThreadSafe()); + ASSERT(!cancel_action_in_flight_); + ASSERT(callback_in_flight_ != nullptr); + auto buf = bufferFromProto(cache_file_header_proto_); + auto sz = buf.length(); + auto queued = + file_handle_->write(dispatcher(), buf, header_block_.offsetToHeaders(), + [this, sz](absl::StatusOr write_result) { + cancel_action_in_flight_ = nullptr; + if (!write_result.ok() || write_result.value() != sz) { + cancelInsert(writeFailureMessage("headers", write_result, sz)); + return; + } + header_block_.setHeadersSize(sz); + if (end_stream_after_headers_) { + commit(); + return; + } + succeedCurrentAction(); + }); + ASSERT(queued.ok(), queued.status().ToString()); + cancel_action_in_flight_ = std::move(queued.value()); +} + void FileInsertContext::insertBody(const Buffer::Instance& fragment, InsertCallback ready_for_next_fragment, bool end_stream) { - absl::MutexLock lock(&mu_); + ASSERT(dispatcher()->isThreadSafe()); + ASSERT(!cancel_action_in_flight_, "should be no actions in flight when receiving new data"); + ASSERT(!callback_in_flight_); if (!cleanup_) { // Already cancelled, do nothing, return failure. ready_for_next_fragment(false); return; } - ASSERT(!cancel_action_in_flight_, "should be no actions in flight when receiving new data"); callback_in_flight_ = ready_for_next_fragment; size_t sz = fragment.length(); Buffer::OwnedImpl consumable_fragment(fragment); auto queued = file_handle_->write( - consumable_fragment, header_block_.offsetToBody() + header_block_.bodySize(), + dispatcher(), consumable_fragment, header_block_.offsetToBody() + header_block_.bodySize(), [this, sz, end_stream](absl::StatusOr write_result) { - absl::MutexLock lock(&mu_); cancel_action_in_flight_ = nullptr; if (!write_result.ok() || write_result.value() != sz) { cancelInsert(writeFailureMessage("body chunk", write_result, sz)); @@ -131,116 +156,121 @@ void FileInsertContext::insertBody(const Buffer::Instance& fragment, } header_block_.setBodySize(header_block_.bodySize() + sz); if (end_stream) { - commit(callback_in_flight_); + commit(); } else { - auto cb = callback_in_flight_; - callback_in_flight_ = nullptr; - cb(true); + succeedCurrentAction(); } }); ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); + cancel_action_in_flight_ = std::move(queued.value()); } void FileInsertContext::insertTrailers(const Http::ResponseTrailerMap& trailers, InsertCallback insert_complete) { - absl::MutexLock lock(&mu_); + ASSERT(dispatcher()->isThreadSafe()); + ASSERT(!cancel_action_in_flight_, "should be no actions in flight when receiving new data"); + ASSERT(!callback_in_flight_); if (!cleanup_) { // Already cancelled, do nothing, return failure. insert_complete(false); return; } - ASSERT(!cancel_action_in_flight_, "should be no actions in flight when receiving new data"); callback_in_flight_ = insert_complete; CacheFileTrailer file_trailer = makeCacheFileTrailerProto(trailers); Buffer::OwnedImpl consumable_buffer = bufferFromProto(file_trailer); size_t sz = consumable_buffer.length(); auto queued = - file_handle_->write(consumable_buffer, header_block_.offsetToTrailers(), + file_handle_->write(dispatcher(), consumable_buffer, header_block_.offsetToTrailers(), [this, sz](absl::StatusOr write_result) { - absl::MutexLock lock(&mu_); cancel_action_in_flight_ = nullptr; if (!write_result.ok() || write_result.value() != sz) { cancelInsert(writeFailureMessage("trailer chunk", write_result, sz)); return; } header_block_.setTrailersSize(sz); - commit(callback_in_flight_); + commit(); }); ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); + cancel_action_in_flight_ = std::move(queued.value()); } -void FileInsertContext::onDestroy() { - absl::MutexLock lock(&mu_); - cancelInsert("InsertContext destroyed prematurely"); -} +void FileInsertContext::onDestroy() { cancelInsert("InsertContext destroyed prematurely"); } -void FileInsertContext::commit(InsertCallback callback) { - mu_.AssertHeld(); +void FileInsertContext::commit() { + ASSERT(dispatcher()->isThreadSafe()); + ASSERT(!cancel_action_in_flight_); + ASSERT(callback_in_flight_ != nullptr); // Write the file header block now that we know the sizes of the pieces. Buffer::OwnedImpl block_buffer; - callback_in_flight_ = callback; header_block_.serializeToBuffer(block_buffer); - auto queued = file_handle_->write(block_buffer, 0, [this](absl::StatusOr write_result) { - absl::MutexLock lock(&mu_); - cancel_action_in_flight_ = nullptr; - if (!write_result.ok() || write_result.value() != CacheFileFixedBlock::size()) { - cancelInsert(writeFailureMessage("header block", write_result, CacheFileFixedBlock::size())); - return; - } - // Unlink any existing cache entry with this filename. - cancel_action_in_flight_ = cache_->asyncFileManager()->stat( - absl::StrCat(cache_->cachePath(), cache_->generateFilename(key_)), - [this](absl::StatusOr stat_result) { - absl::MutexLock lock(&mu_); - cancel_action_in_flight_ = nullptr; - size_t file_size = 0; - if (stat_result.ok()) { - file_size = stat_result.value().st_size; - } - cancel_action_in_flight_ = cache_->asyncFileManager()->unlink( - absl::StrCat(cache_->cachePath(), cache_->generateFilename(key_)), - [this, file_size](absl::Status unlink_result) { - if (unlink_result.ok()) { - cache_->trackFileRemoved(file_size); - } - // We can ignore failure of unlink - the file may or may not have previously - // existed. - absl::MutexLock lock(&mu_); - cancel_action_in_flight_ = nullptr; - // Link the file to its filename. - auto queued = file_handle_->createHardLink( - absl::StrCat(cache_->cachePath(), cache_->generateFilename(key_)), - [this](absl::Status link_result) { - absl::MutexLock lock(&mu_); - cancel_action_in_flight_ = nullptr; - if (!link_result.ok()) { - cancelInsert(absl::StrCat("failed to link file (", link_result.ToString(), - "): ", cache_->cachePath(), - cache_->generateFilename(key_))); - return; - } - ENVOY_LOG(debug, "created cache file {}", cache_->generateFilename(key_)); - callback_in_flight_(true); - callback_in_flight_ = nullptr; - uint64_t file_size = - header_block_.offsetToTrailers() + header_block_.trailerSize(); - cache_->trackFileAdded(file_size); - // By clearing cleanup before destructor, we prevent logging an error. - cleanup_ = nullptr; - }); - ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); - }); - }); - }); + auto queued = file_handle_->write( + dispatcher(), block_buffer, 0, [this](absl::StatusOr write_result) { + cancel_action_in_flight_ = nullptr; + if (!write_result.ok() || write_result.value() != CacheFileFixedBlock::size()) { + cancelInsert( + writeFailureMessage("header block", write_result, CacheFileFixedBlock::size())); + return; + } + commitMeasureExisting(); + }); ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); + cancel_action_in_flight_ = std::move(queued.value()); +} + +std::string FileInsertContext::pathAndFilename() { + return absl::StrCat(cache_->cachePath(), cache_->generateFilename(key_)); +} + +void FileInsertContext::commitMeasureExisting() { + ASSERT(!cancel_action_in_flight_); + ASSERT(callback_in_flight_ != nullptr); + cancel_action_in_flight_ = cache_->asyncFileManager()->stat( + dispatcher(), pathAndFilename(), [this](absl::StatusOr stat_result) { + cancel_action_in_flight_ = nullptr; + if (stat_result.ok()) { + commitUnlinkExisting(stat_result.value().st_size); + } else { + commitUnlinkExisting(0); + } + }); +} + +void FileInsertContext::commitUnlinkExisting(size_t file_size) { + ASSERT(!cancel_action_in_flight_); + ASSERT(callback_in_flight_ != nullptr); + cancel_action_in_flight_ = cache_->asyncFileManager()->unlink( + dispatcher(), pathAndFilename(), [this, file_size](absl::Status unlink_result) { + cancel_action_in_flight_ = nullptr; + if (unlink_result.ok()) { + cache_->trackFileRemoved(file_size); + } + commitCreateHardLink(); + }); +} + +void FileInsertContext::commitCreateHardLink() { + ASSERT(!cancel_action_in_flight_); + ASSERT(callback_in_flight_ != nullptr); + auto queued = file_handle_->createHardLink( + dispatcher(), pathAndFilename(), [this](absl::Status link_result) { + cancel_action_in_flight_ = nullptr; + if (!link_result.ok()) { + cancelInsert(absl::StrCat("failed to link file (", link_result.ToString(), + "): ", pathAndFilename())); + return; + } + ENVOY_LOG(debug, "created cache file {}", cache_->generateFilename(key_)); + succeedCurrentAction(); + uint64_t file_size = header_block_.offsetToTrailers() + header_block_.trailerSize(); + cache_->trackFileAdded(file_size); + // By clearing cleanup before destructor, we prevent logging an error. + cleanup_ = nullptr; + }); + ASSERT(queued.ok(), queued.status().ToString()); + cancel_action_in_flight_ = std::move(queued.value()); } void FileInsertContext::cancelInsert(absl::string_view error) { - mu_.AssertHeld(); if (cancel_action_in_flight_) { cancel_action_in_flight_(); cancel_action_in_flight_ = nullptr; @@ -256,12 +286,14 @@ void FileInsertContext::cancelInsert(absl::string_view error) { } } if (file_handle_) { - auto close_status = file_handle_->close([](absl::Status) {}); + auto close_status = file_handle_->close(nullptr, [](absl::Status) {}); ASSERT(close_status.ok()); file_handle_ = nullptr; } } +Event::Dispatcher* FileInsertContext::dispatcher() const { return lookup_context_->dispatcher(); } + } // namespace FileSystemHttpCache } // namespace Cache } // namespace HttpFilters diff --git a/source/extensions/http/cache/file_system_http_cache/insert_context.h b/source/extensions/http/cache/file_system_http_cache/insert_context.h index 71e6f4d77b..1f8e666533 100644 --- a/source/extensions/http/cache/file_system_http_cache/insert_context.h +++ b/source/extensions/http/cache/file_system_http_cache/insert_context.h @@ -48,15 +48,55 @@ class FileInsertContext : public InsertContext, public Logger::Loggable lookup_context_; Key key_; std::shared_ptr cache_; - absl::Mutex mu_; // guards file operations - std::shared_ptr cleanup_ ABSL_GUARDED_BY(mu_); - AsyncFileHandle file_handle_ ABSL_GUARDED_BY(mu_); - std::function callback_in_flight_ ABSL_GUARDED_BY(mu_); - CancelFunction cancel_action_in_flight_ ABSL_GUARDED_BY(mu_); - CacheFileFixedBlock header_block_ ABSL_GUARDED_BY(mu_); + std::shared_ptr cleanup_; + AsyncFileHandle file_handle_; + absl::AnyInvocable callback_in_flight_; + CancelFunction cancel_action_in_flight_; + CacheFileFixedBlock header_block_; /** * If seen_end_stream_ is not true (i.e. InsertContext has not yet delivered the @@ -69,19 +109,7 @@ class FileInsertContext : public InsertContext, public Logger::LoggableopenExistingFile( - filepath(), Common::AsyncFiles::AsyncFileManager::Mode::ReadOnly, - [this, cb](absl::StatusOr open_result) { - absl::MutexLock lock(&mu_); + dispatcher(), filepath(), Common::AsyncFiles::AsyncFileManager::Mode::ReadOnly, + [this](absl::StatusOr open_result) { cancel_action_in_flight_ = nullptr; if (!open_result.ok()) { - cache_.stats().cache_miss_.inc(); - cb(LookupResult{}, /* end_stream (ignored) = */ false); - return; + return doCacheMiss(); } ASSERT(!file_handle_); file_handle_ = std::move(open_result.value()); - auto queued = file_handle_->read( - 0, CacheFileFixedBlock::size(), - [this, cb](absl::StatusOr read_result) { - absl::MutexLock lock(&mu_); - cancel_action_in_flight_ = nullptr; - if (!read_result.ok() || - read_result.value()->length() != CacheFileFixedBlock::size()) { - invalidateCacheEntry(); - cache_.stats().cache_miss_.inc(); - cb(LookupResult{}, /* end_stream (ignored) = */ false); - return; - } - header_block_.populateFromStringView(read_result.value()->toString()); - if (!header_block_.isValid()) { - invalidateCacheEntry(); - cache_.stats().cache_miss_.inc(); - cb(LookupResult{}, /* end_stream (ignored) = */ false); - return; - } - auto queued = file_handle_->read( - header_block_.offsetToHeaders(), header_block_.headerSize(), - [this, cb](absl::StatusOr read_result) { - absl::MutexLock lock(&mu_); - cancel_action_in_flight_ = nullptr; - if (!read_result.ok() || - read_result.value()->length() != header_block_.headerSize()) { - invalidateCacheEntry(); - cache_.stats().cache_miss_.inc(); - cb(LookupResult{}, /* end_stream (ignored) = */ false); - return; - } - auto header_proto = makeCacheFileHeaderProto(*read_result.value()); - if (header_proto.headers_size() == 1 && - header_proto.headers().at(0).key() == "vary") { - auto maybe_vary_key = cache_.makeVaryKey( - key_, lookup().varyAllowList(), - absl::StrSplit(header_proto.headers().at(0).value(), ','), - lookup().requestHeaders()); - if (!maybe_vary_key.has_value()) { - cache_.stats().cache_miss_.inc(); - cb(LookupResult{}, /* end_stream (ignored) = */ false); - return; - } - key_ = maybe_vary_key.value(); - auto fh = std::move(file_handle_); - file_handle_ = nullptr; - // It should be possible to cancel close, to make this safe. - // (it should still close the file, but cancel the callback.) - auto queued = fh->close([this, cb](absl::Status) { - absl::MutexLock lock(&mu_); - // Restart getHeaders with the new key. - return getHeadersWithLock(cb); - }); - ASSERT(queued.ok(), queued.ToString()); - return; - } - cache_.stats().cache_hit_.inc(); - cb(lookup().makeLookupResult(headersFromHeaderProto(header_proto), - metadataFromHeaderProto(header_proto), - header_block_.bodySize()), - /* end_stream = */ header_block_.trailerSize() == 0 && - header_block_.bodySize() == 0); - }); - ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); - }); - ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); + getHeaderBlockFromFile(); + }); +} + +void FileLookupContext::doCacheMiss() { + cache_.stats().cache_miss_.inc(); + std::move(lookup_headers_callback_)(LookupResult{}, /* end_stream (ignored) = */ false); + lookup_headers_callback_ = nullptr; +} + +void FileLookupContext::doCacheEntryInvalid() { + invalidateCacheEntry(); + doCacheMiss(); +} + +void FileLookupContext::getHeaderBlockFromFile() { + ASSERT(dispatcher()->isThreadSafe()); + auto queued = file_handle_->read( + dispatcher(), 0, CacheFileFixedBlock::size(), + [this](absl::StatusOr read_result) { + ASSERT(dispatcher()->isThreadSafe()); + cancel_action_in_flight_ = nullptr; + if (!read_result.ok() || read_result.value()->length() != CacheFileFixedBlock::size()) { + return doCacheEntryInvalid(); + } + header_block_.populateFromStringView(read_result.value()->toString()); + if (!header_block_.isValid()) { + return doCacheEntryInvalid(); + } + getHeadersFromFile(); + }); + ASSERT(queued.ok(), queued.status().ToString()); + cancel_action_in_flight_ = std::move(queued.value()); +} + +void FileLookupContext::getHeadersFromFile() { + ASSERT(dispatcher()->isThreadSafe()); + auto queued = file_handle_->read( + dispatcher(), header_block_.offsetToHeaders(), header_block_.headerSize(), + [this](absl::StatusOr read_result) { + ASSERT(dispatcher()->isThreadSafe()); + cancel_action_in_flight_ = nullptr; + if (!read_result.ok() || read_result.value()->length() != header_block_.headerSize()) { + return doCacheEntryInvalid(); + } + auto header_proto = makeCacheFileHeaderProto(*read_result.value()); + if (header_proto.headers_size() == 1 && header_proto.headers().at(0).key() == "vary") { + auto maybe_vary_key = cache_.makeVaryKey( + key_, lookup().varyAllowList(), + absl::StrSplit(header_proto.headers().at(0).value(), ','), lookup().requestHeaders()); + if (!maybe_vary_key.has_value()) { + return doCacheMiss(); + } + key_ = maybe_vary_key.value(); + return closeFileAndGetHeadersAgainWithNewVaryKey(); + } + cache_.stats().cache_hit_.inc(); + std::move(lookup_headers_callback_)( + lookup().makeLookupResult(headersFromHeaderProto(header_proto), + metadataFromHeaderProto(header_proto), + header_block_.bodySize()), + /* end_stream = */ header_block_.trailerSize() == 0 && header_block_.bodySize() == 0); }); + ASSERT(queued.ok(), queued.status().ToString()); + cancel_action_in_flight_ = std::move(queued.value()); +} + +void FileLookupContext::closeFileAndGetHeadersAgainWithNewVaryKey() { + ASSERT(dispatcher()->isThreadSafe()); + auto queued = file_handle_->close(dispatcher(), [this](absl::Status) { + ASSERT(dispatcher()->isThreadSafe()); + file_handle_ = nullptr; + // Restart with the new key. + return tryOpenCacheFile(); + }); + ASSERT(queued.ok(), queued.status().ToString()); + cancel_action_in_flight_ = std::move(queued.value()); } void FileLookupContext::invalidateCacheEntry() { + ASSERT(dispatcher()->isThreadSafe()); + // We don't capture the cancel action here because we want these operations to continue even + // if the filter was destroyed in the meantime. For the same reason, we must not capture 'this'. cache_.asyncFileManager()->stat( - filepath(), [file = filepath(), - cache = cache_.shared_from_this()](absl::StatusOr stat_result) { + dispatcher(), filepath(), + [file = filepath(), cache = cache_.shared_from_this(), + dispatcher = dispatcher()](absl::StatusOr stat_result) { + ASSERT(dispatcher->isThreadSafe()); size_t file_size = 0; if (stat_result.ok()) { file_size = stat_result.value().st_size; } - cache->asyncFileManager()->unlink(file, [cache, file_size](absl::Status unlink_result) { - if (unlink_result.ok()) { - cache->trackFileRemoved(file_size); - } - }); + cache->asyncFileManager()->unlink(dispatcher, file, + [cache, file_size](absl::Status unlink_result) { + if (unlink_result.ok()) { + cache->trackFileRemoved(file_size); + } + }); }); } void FileLookupContext::getBody(const AdjustedByteRange& range, LookupBodyCallback&& cb) { - absl::MutexLock lock(&mu_); + ASSERT(dispatcher()->isThreadSafe()); + ASSERT(cb); ASSERT(!cancel_action_in_flight_); + ASSERT(file_handle_); auto queued = file_handle_->read( - header_block_.offsetToBody() + range.begin(), range.length(), - [this, cb, range](absl::StatusOr read_result) { - absl::MutexLock lock(&mu_); + dispatcher(), header_block_.offsetToBody() + range.begin(), range.length(), + [this, cb = std::move(cb), range](absl::StatusOr read_result) { + ASSERT(dispatcher()->isThreadSafe()); cancel_action_in_flight_ = nullptr; if (!read_result.ok() || read_result.value()->length() != range.length()) { invalidateCacheEntry(); // Calling callback with nullptr fails the request. - cb(nullptr, /* end_stream (ignored) = */ false); + std::move(cb)(nullptr, /* end_stream (ignored) = */ false); return; } - cb(std::move(read_result.value()), - /* end_stream = */ range.end() == header_block_.bodySize() && - header_block_.trailerSize() == 0); + std::move(cb)(std::move(read_result.value()), + /* end_stream = */ range.end() == header_block_.bodySize() && + header_block_.trailerSize() == 0); }); ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); + cancel_action_in_flight_ = std::move(queued.value()); } void FileLookupContext::getTrailers(LookupTrailersCallback&& cb) { + ASSERT(dispatcher()->isThreadSafe()); ASSERT(cb); - absl::MutexLock lock(&mu_); ASSERT(!cancel_action_in_flight_); - auto queued = file_handle_->read(header_block_.offsetToTrailers(), header_block_.trailerSize(), - [this, cb](absl::StatusOr read_result) { - absl::MutexLock lock(&mu_); - cancel_action_in_flight_ = nullptr; - if (!read_result.ok() || read_result.value()->length() != - header_block_.trailerSize()) { - invalidateCacheEntry(); - // There is no failure response for getTrailers, so we just - // say there were no trailers in the event of this failure. - cb(Http::ResponseTrailerMapImpl::create()); - return; - } - CacheFileTrailer trailer; - trailer.ParseFromString(read_result.value()->toString()); - cb(trailersFromTrailerProto(trailer)); - }); + ASSERT(file_handle_); + auto queued = file_handle_->read( + dispatcher(), header_block_.offsetToTrailers(), header_block_.trailerSize(), + [this, cb = std::move(cb)](absl::StatusOr read_result) { + ASSERT(dispatcher()->isThreadSafe()); + cancel_action_in_flight_ = nullptr; + if (!read_result.ok() || read_result.value()->length() != header_block_.trailerSize()) { + invalidateCacheEntry(); + // There is no failure response for getTrailers, so we just + // say there were no trailers in the event of this failure. + std::move(cb)(Http::ResponseTrailerMapImpl::create()); + return; + } + CacheFileTrailer trailer; + trailer.ParseFromString(read_result.value()->toString()); + std::move(cb)(trailersFromTrailerProto(trailer)); + }); ASSERT(queued.ok(), queued.status().ToString()); - cancel_action_in_flight_ = queued.value(); + cancel_action_in_flight_ = std::move(queued.value()); } void FileLookupContext::onDestroy() { - CancelFunction cancel; - { - absl::MutexLock lock(&mu_); - cancel = std::move(cancel_action_in_flight_); + if (cancel_action_in_flight_) { + std::move(cancel_action_in_flight_)(); cancel_action_in_flight_ = nullptr; } - while (cancel) { - // We mustn't hold the lock while calling cancel, as it can potentially wait for - // a callback to complete, and the callback might take the lock. - cancel(); - { - // It's possible that while calling cancel, another action was started - if - // that happened, we must cancel that one too! - absl::MutexLock lock(&mu_); - cancel = std::move(cancel_action_in_flight_); - cancel_action_in_flight_ = nullptr; - } - } - { - absl::MutexLock lock(&mu_); - if (file_handle_) { - auto status = file_handle_->close([](absl::Status) {}); - ASSERT(status.ok(), status.ToString()); - file_handle_ = nullptr; - } + if (file_handle_) { + auto status = file_handle_->close(nullptr, [](absl::Status) {}); + ASSERT(status.ok(), status.status().ToString()); + file_handle_ = nullptr; } } diff --git a/source/extensions/http/cache/file_system_http_cache/lookup_context.h b/source/extensions/http/cache/file_system_http_cache/lookup_context.h index db4668798f..f2454fba5b 100644 --- a/source/extensions/http/cache/file_system_http_cache/lookup_context.h +++ b/source/extensions/http/cache/file_system_http_cache/lookup_context.h @@ -19,8 +19,9 @@ using Envoy::Extensions::Common::AsyncFiles::CancelFunction; class FileLookupContext : public LookupContext { public: - FileLookupContext(FileSystemHttpCache& cache, LookupRequest&& lookup) - : cache_(cache), key_(lookup.key()), lookup_(std::move(lookup)) {} + FileLookupContext(Event::Dispatcher& dispatcher, FileSystemHttpCache& cache, + LookupRequest&& lookup) + : dispatcher_(dispatcher), cache_(cache), key_(lookup.key()), lookup_(std::move(lookup)) {} // From LookupContext void getHeaders(LookupHeadersCallback&& cb) final; @@ -34,28 +35,34 @@ class FileLookupContext : public LookupContext { const LookupRequest& lookup() const { return lookup_; } const Key& key() const { return key_; } bool workInProgress() const; + Event::Dispatcher* dispatcher() const { return &dispatcher_; } private: - void getHeadersWithLock(LookupHeadersCallback cb) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + void tryOpenCacheFile(); + void doCacheMiss(); + void doCacheEntryInvalid(); + void getHeaderBlockFromFile(); + void getHeadersFromFile(); + void closeFileAndGetHeadersAgainWithNewVaryKey(); // In the event that the cache failed to retrieve, remove the cache entry from the // cache so we don't keep repeating the same failure. - void invalidateCacheEntry() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + void invalidateCacheEntry(); - std::string filepath() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + std::string filepath(); + + Event::Dispatcher& dispatcher_; // We can safely use a reference here, because the shared_ptr to a cache is guaranteed to outlive // all filters that use it. FileSystemHttpCache& cache_; - // File actions may be initiated in the file thread or the filter thread, and cancelled or - // completed from either, therefore must be guarded by a mutex. - absl::Mutex mu_; - AsyncFileHandle file_handle_ ABSL_GUARDED_BY(mu_); - CancelFunction cancel_action_in_flight_ ABSL_GUARDED_BY(mu_); - CacheFileFixedBlock header_block_ ABSL_GUARDED_BY(mu_); - Key key_ ABSL_GUARDED_BY(mu_); + AsyncFileHandle file_handle_; + CancelFunction cancel_action_in_flight_; + CacheFileFixedBlock header_block_; + Key key_; + LookupHeadersCallback lookup_headers_callback_; const LookupRequest lookup_; }; diff --git a/test/extensions/common/async_files/async_file_handle_thread_pool_test.cc b/test/extensions/common/async_files/async_file_handle_thread_pool_test.cc index b4e63ed44d..668ae5b98d 100644 --- a/test/extensions/common/async_files/async_file_handle_thread_pool_test.cc +++ b/test/extensions/common/async_files/async_file_handle_thread_pool_test.cc @@ -34,38 +34,48 @@ using ::testing::StrictMock; class AsyncFileHandleHelpers { public: + void resolveFileActions() { + manager_->waitForIdle(); + dispatcher_->run(Event::Dispatcher::RunType::Block); + } void close(AsyncFileHandle& handle) { - std::promise close_result; - EXPECT_OK(handle->close([&](absl::Status status) { close_result.set_value(status); })); - EXPECT_OK(close_result.get_future().get()); + absl::Status close_result; + EXPECT_OK( + handle->close(dispatcher_.get(), [&](absl::Status status) { close_result = status; })); + resolveFileActions(); + EXPECT_OK(close_result); } AsyncFileHandle createAnonymousFile() { - std::promise create_result; - manager_->createAnonymousFile(tmpdir_, [&](absl::StatusOr result) { - create_result.set_value(result.value()); - }); - return create_result.get_future().get(); + AsyncFileHandle create_result; + manager_->createAnonymousFile( + dispatcher_.get(), tmpdir_, + [&](absl::StatusOr result) { create_result = result.value(); }); + resolveFileActions(); + return create_result; } AsyncFileHandle openExistingFile(absl::string_view filename, AsyncFileManager::Mode mode) { - std::promise open_result; - manager_->openExistingFile(filename, mode, [&](absl::StatusOr result) { - open_result.set_value(result.value()); - }); - return open_result.get_future().get(); + AsyncFileHandle open_result; + manager_->openExistingFile( + dispatcher_.get(), filename, mode, + [&](absl::StatusOr result) { open_result = result.value(); }); + resolveFileActions(); + return open_result; } const char* test_tmpdir = std::getenv("TEST_TMPDIR"); std::string tmpdir_ = test_tmpdir ? test_tmpdir : "/tmp"; - std::unique_ptr singleton_manager_; - std::shared_ptr factory_; + std::unique_ptr singleton_manager_ = + std::make_unique(); + std::shared_ptr factory_ = + AsyncFileManagerFactory::singleton(singleton_manager_.get()); std::shared_ptr manager_; + Api::ApiPtr api_ = Api::createApiForTest(); + Event::DispatcherPtr dispatcher_ = api_->allocateDispatcher("test_thread"); }; class AsyncFileHandleTest : public testing::Test, public AsyncFileHandleHelpers { public: void SetUp() override { - singleton_manager_ = std::make_unique(); - factory_ = AsyncFileManagerFactory::singleton(singleton_manager_.get()); envoy::extensions::common::async_files::v3::AsyncFileManagerConfig config; config.mutable_thread_pool()->set_thread_count(1); manager_ = factory_->getAsyncFileManager(config); @@ -77,8 +87,6 @@ class AsyncFileHandleWithMockPosixTest : public testing::Test, public AsyncFileH void SetUp() override { EXPECT_CALL(mock_posix_file_operations_, supportsAllPosixFileOperations()) .WillRepeatedly(Return(true)); - singleton_manager_ = std::make_unique(); - factory_ = AsyncFileManagerFactory::singleton(singleton_manager_.get()); envoy::extensions::common::async_files::v3::AsyncFileManagerConfig config; config.mutable_thread_pool()->set_thread_count(1); manager_ = factory_->getAsyncFileManager(config, &mock_posix_file_operations_); @@ -103,49 +111,41 @@ TEST_F(AsyncFileHandleTest, WriteReadClose) { absl::StatusOr write_status, second_write_status; absl::StatusOr read_status, second_read_status; Buffer::OwnedImpl hello("hello"); - std::promise close_status; - EXPECT_OK(handle->write(hello, 0, [&](absl::StatusOr status) { + ASSERT_OK(handle->write(dispatcher_.get(), hello, 0, [&](absl::StatusOr status) { write_status = std::move(status); - // Make sure writing at an offset works - Buffer::OwnedImpl two_chars("p!"); - EXPECT_OK(handle->write(two_chars, 3, [&](absl::StatusOr status) { - second_write_status = std::move(status); - EXPECT_OK(handle->read(0, 5, [&](absl::StatusOr status) { - read_status = std::move(status); - // Verify reading at an offset. - EXPECT_OK(handle->read(2, 3, [&](absl::StatusOr status) { - second_read_status = std::move(status); - EXPECT_OK(handle->close( - [&](absl::Status status) { close_status.set_value(std::move(status)); })); - })); - })); - })); })); - ASSERT_OK(close_status.get_future().get()); - // The first write was 5 characters. + resolveFileActions(); EXPECT_THAT(write_status, IsOkAndHolds(5U)); - - // The second write was 2 characters. + Buffer::OwnedImpl two_chars("p!"); + ASSERT_OK(handle->write(dispatcher_.get(), two_chars, 3, [&](absl::StatusOr status) { + second_write_status = std::move(status); + })); + resolveFileActions(); EXPECT_THAT(second_write_status, IsOkAndHolds(2U)); - - // This should be "hello" from the first write, with the last two characters replaced with "p!" - // from the second write. - EXPECT_OK(read_status); + ASSERT_OK(handle->read(dispatcher_.get(), 0, 5, [&](absl::StatusOr status) { + read_status = std::move(status); + })); + resolveFileActions(); + ASSERT_OK(read_status); EXPECT_THAT(*read_status.value(), BufferStringEqual("help!")); - - // Second read should have three characters in it. - EXPECT_OK(second_read_status); + ASSERT_OK(handle->read(dispatcher_.get(), 2, 3, [&](absl::StatusOr status) { + second_read_status = std::move(status); + })); + resolveFileActions(); + ASSERT_OK(second_read_status); EXPECT_THAT(*second_read_status.value(), BufferStringEqual("lp!")); + close(handle); } TEST_F(AsyncFileHandleTest, LinkCreatesNamedFile) { auto handle = createAnonymousFile(); - std::promise> write_status_promise; + absl::StatusOr write_status; // Write "hello" to the anonymous file. Buffer::OwnedImpl data("hello"); - EXPECT_OK(handle->write( - data, 0, [&](absl::StatusOr status) { write_status_promise.set_value(status); })); - absl::StatusOr write_status = write_status_promise.get_future().get(); + EXPECT_OK(handle->write(dispatcher_.get(), data, 0, [&](absl::StatusOr status) { + write_status = std::move(status); + })); + resolveFileActions(); ASSERT_THAT(write_status, IsOkAndHolds(5U)); char filename[1024]; snprintf(filename, sizeof(filename), "%s/async_link_test.XXXXXX", tmpdir_.c_str()); @@ -160,12 +160,13 @@ TEST_F(AsyncFileHandleTest, LinkCreatesNamedFile) { posix.unlink(filename); // Link the anonymous file into our tmp file name. - std::promise link_status; + absl::Status link_status = absl::InternalError("not set"); std::cout << "Linking as " << filename << std::endl; - EXPECT_OK(handle->createHardLink(std::string(filename), - [&](absl::Status status) { link_status.set_value(status); })); - ASSERT_OK(link_status.get_future().get()); + EXPECT_OK(handle->createHardLink(dispatcher_.get(), std::string(filename), + [&](absl::Status status) { link_status = status; })); + resolveFileActions(); + ASSERT_OK(link_status); // Read the contents of the linked file back, raw. char fileContents[6]; fileContents[5] = '\0'; @@ -183,11 +184,10 @@ TEST_F(AsyncFileHandleTest, LinkCreatesNamedFile) { TEST_F(AsyncFileHandleTest, LinkReturnsErrorIfLinkFails) { auto handle = createAnonymousFile(); - std::promise link_status_promise; - EXPECT_OK(handle->createHardLink("/some/path/that/does/not/exist", [&](absl::Status status) { - link_status_promise.set_value(status); - })); - absl::Status link_status = link_status_promise.get_future().get(); + absl::Status link_status = absl::InternalError("not set"); + EXPECT_OK(handle->createHardLink(dispatcher_.get(), "/some/path/that/does/not/exist", + [&](absl::Status status) { link_status = std::move(status); })); + resolveFileActions(); ASSERT_EQ(absl::StatusCode::kNotFound, link_status.code()) << link_status; close(handle); } @@ -219,11 +219,11 @@ TEST_F(AsyncFileHandleTest, OpenExistingWriteOnlyFailsOnRead) { TestTmpFile tmpfile(tmpdir_); auto handle = openExistingFile(tmpfile.name(), AsyncFileManager::Mode::WriteOnly); - std::promise> read_status_promise; - EXPECT_OK(handle->read(0, 5, [&](absl::StatusOr status) { - read_status_promise.set_value(std::move(status)); + absl::StatusOr read_status; + EXPECT_OK(handle->read(dispatcher_.get(), 0, 5, [&](absl::StatusOr status) { + read_status = std::move(status); })); - absl::StatusOr read_status = read_status_promise.get_future().get(); + resolveFileActions(); ASSERT_EQ(absl::StatusCode::kFailedPrecondition, read_status.status().code()) << read_status.status(); close(handle); @@ -234,11 +234,14 @@ TEST_F(AsyncFileHandleTest, OpenExistingWriteOnlyCanWrite) { TestTmpFile tmpfile(tmpdir_); auto handle = openExistingFile(tmpfile.name(), AsyncFileManager::Mode::WriteOnly); - std::promise> write_status; + absl::StatusOr write_status; Buffer::OwnedImpl buf("nine char"); - EXPECT_OK(handle->write( - buf, 0, [&](absl::StatusOr status) { write_status.set_value(std::move(status)); })); - ASSERT_EQ(9, write_status.get_future().get().value()); + EXPECT_OK(handle->write(dispatcher_.get(), buf, 0, [&](absl::StatusOr status) { + write_status = std::move(status); + })); + resolveFileActions(); + ASSERT_OK(write_status); + EXPECT_EQ(9, write_status.value()); close(handle); } @@ -247,12 +250,12 @@ TEST_F(AsyncFileHandleTest, OpenExistingReadOnlyFailsOnWrite) { TestTmpFile tmpfile(tmpdir_); auto handle = openExistingFile(tmpfile.name(), AsyncFileManager::Mode::ReadOnly); - std::promise> write_status_promise; + absl::StatusOr write_status; Buffer::OwnedImpl buf("hello"); - EXPECT_OK(handle->write(buf, 0, [&](absl::StatusOr status) { - write_status_promise.set_value(std::move(status)); + EXPECT_OK(handle->write(dispatcher_.get(), buf, 0, [&](absl::StatusOr status) { + write_status = std::move(status); })); - auto write_status = write_status_promise.get_future().get(); + resolveFileActions(); ASSERT_EQ(absl::StatusCode::kFailedPrecondition, write_status.status().code()) << write_status.status(); close(handle); @@ -263,11 +266,13 @@ TEST_F(AsyncFileHandleTest, OpenExistingReadOnlyCanRead) { TestTmpFile tmpfile(tmpdir_); auto handle = openExistingFile(tmpfile.name(), AsyncFileManager::Mode::ReadOnly); - std::promise> read_status; - EXPECT_OK(handle->read(0, 5, [&](absl::StatusOr status) { - read_status.set_value(std::move(status)); + absl::StatusOr read_status; + EXPECT_OK(handle->read(dispatcher_.get(), 0, 5, [&](absl::StatusOr status) { + read_status = std::move(status); })); - ASSERT_EQ("hello", read_status.get_future().get().value()->toString()); + resolveFileActions(); + ASSERT_OK(read_status); + ASSERT_EQ("hello", read_status.value()->toString()); close(handle); } @@ -276,38 +281,40 @@ TEST_F(AsyncFileHandleTest, OpenExistingReadWriteCanReadAndWrite) { TestTmpFile tmpfile(tmpdir_); auto handle = openExistingFile(tmpfile.name(), AsyncFileManager::Mode::ReadWrite); - std::promise> write_status_promise; + absl::StatusOr write_status; Buffer::OwnedImpl buf("p me!"); - EXPECT_OK(handle->write(buf, 3, [&](absl::StatusOr status) { - write_status_promise.set_value(std::move(status)); + EXPECT_OK(handle->write(dispatcher_.get(), buf, 3, [&](absl::StatusOr status) { + write_status = std::move(status); })); - auto write_status = write_status_promise.get_future().get(); + resolveFileActions(); ASSERT_THAT(write_status, IsOkAndHolds(5U)); - std::promise> read_status_promise; - EXPECT_OK(handle->read(0, 8, [&](absl::StatusOr status) { - read_status_promise.set_value(std::move(status)); + absl::StatusOr read_status; + EXPECT_OK(handle->read(dispatcher_.get(), 0, 8, [&](absl::StatusOr status) { + read_status = std::move(status); })); - auto read_status = read_status_promise.get_future().get(); - EXPECT_OK(read_status); + resolveFileActions(); + ASSERT_OK(read_status); EXPECT_THAT(*read_status.value(), BufferStringEqual("help me!")); close(handle); } TEST_F(AsyncFileHandleTest, DuplicateCreatesIndependentHandle) { auto handle = createAnonymousFile(); - std::promise> duplicate_status_promise; - EXPECT_OK(handle->duplicate( - [&](absl::StatusOr status) { duplicate_status_promise.set_value(status); })); - auto duplicate_status = duplicate_status_promise.get_future().get(); + absl::StatusOr duplicate_status; + EXPECT_OK(handle->duplicate(dispatcher_.get(), [&](absl::StatusOr status) { + duplicate_status = std::move(status); + })); + resolveFileActions(); ASSERT_OK(duplicate_status); AsyncFileHandle dup_file = std::move(duplicate_status.value()); // Close the original file. close(handle); - std::promise> write_status_promise; + absl::StatusOr write_status; Buffer::OwnedImpl buf("hello"); - EXPECT_OK(dup_file->write( - buf, 0, [&](absl::StatusOr result) { write_status_promise.set_value(result); })); - auto write_status = write_status_promise.get_future().get(); + EXPECT_OK(dup_file->write(dispatcher_.get(), buf, 0, [&](absl::StatusOr result) { + write_status = std::move(result); + })); + resolveFileActions(); // writing to the duplicate file should still work. EXPECT_THAT(write_status, IsOkAndHolds(5U)); close(dup_file); @@ -320,11 +327,11 @@ TEST_F(AsyncFileHandleWithMockPosixTest, PartialReadReturnsPartialResult) { memcpy(buf, "hel", 3); return Api::SysCallSizeResult{3, 0}; }); - std::promise> read_status_promise; - EXPECT_OK(handle->read(0, 5, [&](absl::StatusOr status) { - read_status_promise.set_value(std::move(status.value())); + absl::StatusOr read_status; + EXPECT_OK(handle->read(dispatcher_.get(), 0, 5, [&](absl::StatusOr status) { + read_status = std::move(status.value()); })); - auto read_status = read_status_promise.get_future().get(); + resolveFileActions(); EXPECT_OK(read_status); EXPECT_THAT(*read_status.value(), BufferStringEqual("hel")); close(handle); @@ -344,11 +351,11 @@ TEST_F(AsyncFileHandleWithMockPosixTest, PartialWriteRetries) { .WillOnce(Return(Api::SysCallSizeResult{3, 0})); EXPECT_CALL(mock_posix_file_operations_, pwrite(_, IsMemoryMatching("lo"), 2, 3)) .WillOnce(Return(Api::SysCallSizeResult{2, 0})); - std::promise> write_status_promise; - EXPECT_OK(handle->write(write_value, 0, [&](absl::StatusOr status) { - write_status_promise.set_value(std::move(status.value())); + absl::StatusOr write_status; + EXPECT_OK(handle->write(dispatcher_.get(), write_value, 0, [&](absl::StatusOr status) { + write_status = std::move(status.value()); })); - auto write_status = write_status_promise.get_future().get(); + resolveFileActions(); EXPECT_THAT(write_status, IsOkAndHolds(5U)); close(handle); } @@ -361,9 +368,8 @@ TEST_F(AsyncFileHandleWithMockPosixTest, CancellingDuplicateInProgressClosesTheF finishing_dup.get_future().wait(); return Api::SysCallSocketResult{4242, 0}; }); - auto cancel_dup = handle->duplicate([](absl::StatusOr) { - // Callback is not called if we cancel (already validated in manager tests) - // so this is unimportant. + auto cancel_dup = handle->duplicate(dispatcher_.get(), [](absl::StatusOr) { + FAIL() << "cancelled callback should not be called"; }); entering_dup.get_future().wait(); cancel_dup.value()(); @@ -386,8 +392,8 @@ TEST_F(AsyncFileHandleWithMockPosixTest, CancellingCreateHardLinkInProgressRemov finishing_hardlink.get_future().wait(); return Api::SysCallIntResult{0, 0}; }); - auto cancel_hardlink = handle->createHardLink(filename, [](absl::Status) { - // Callback is not called if we cancel, so this is unimportant. + auto cancel_hardlink = handle->createHardLink(dispatcher_.get(), filename, [](absl::Status) { + FAIL() << "cancelled callback should not be called"; }); entering_hardlink.get_future().wait(); cancel_hardlink.value()(); @@ -410,8 +416,8 @@ TEST_F(AsyncFileHandleWithMockPosixTest, CancellingFailedCreateHardLinkInProgres finishing_hardlink.get_future().wait(); return Api::SysCallIntResult{-1, EBADF}; }); - auto cancel_hardlink = handle->createHardLink(filename, [](absl::Status) { - // Callback is not called if we cancel, so this is unimportant. + auto cancel_hardlink = handle->createHardLink(dispatcher_.get(), filename, [](absl::Status) { + FAIL() << "cancelled callback should not be called"; }); entering_hardlink.get_future().wait(); cancel_hardlink.value()(); @@ -429,11 +435,11 @@ TEST_F(AsyncFileHandleWithMockPosixTest, StatSuccessReturnsPopulatedStatStruct) *buffer = expected_stat; return Api::SysCallIntResult{0, 0}; }); - std::promise> fstat_status_promise; - EXPECT_OK(handle->stat([&](absl::StatusOr status) { - fstat_status_promise.set_value(std::move(status)); + absl::StatusOr fstat_status; + EXPECT_OK(handle->stat(dispatcher_.get(), [&](absl::StatusOr status) { + fstat_status = std::move(status); })); - auto fstat_status = fstat_status_promise.get_future().get(); + resolveFileActions(); EXPECT_THAT(fstat_status, IsOkAndHolds(testing::Field(&stat::st_size, expected_stat.st_size))); close(handle); } @@ -442,11 +448,11 @@ TEST_F(AsyncFileHandleWithMockPosixTest, StatFailureReportsError) { auto handle = createAnonymousFile(); EXPECT_CALL(mock_posix_file_operations_, fstat(_, _)) .WillOnce(Return(Api::SysCallIntResult{-1, EBADF})); - std::promise> fstat_status_promise; - EXPECT_OK(handle->stat([&](absl::StatusOr status) { - fstat_status_promise.set_value(std::move(status)); + absl::StatusOr fstat_status; + EXPECT_OK(handle->stat(dispatcher_.get(), [&](absl::StatusOr status) { + fstat_status = std::move(status); })); - auto fstat_status = fstat_status_promise.get_future().get(); + resolveFileActions(); EXPECT_THAT(fstat_status, StatusIs(absl::StatusCode::kFailedPrecondition)); close(handle); } @@ -455,9 +461,10 @@ TEST_F(AsyncFileHandleWithMockPosixTest, CloseFailureReportsError) { auto handle = createAnonymousFile(); EXPECT_CALL(mock_posix_file_operations_, close(1)) .WillOnce(Return(Api::SysCallIntResult{-1, EBADF})); - std::promise close_status_promise; - EXPECT_OK(handle->close([&](absl::Status status) { close_status_promise.set_value(status); })); - auto close_status = close_status_promise.get_future().get(); + absl::Status close_status; + EXPECT_OK(handle->close(dispatcher_.get(), + [&](absl::Status status) { close_status = std::move(status); })); + resolveFileActions(); EXPECT_EQ(absl::StatusCode::kFailedPrecondition, close_status.code()) << close_status; } @@ -465,19 +472,20 @@ TEST_F(AsyncFileHandleWithMockPosixTest, DuplicateFailureReportsError) { auto handle = createAnonymousFile(); EXPECT_CALL(mock_posix_file_operations_, duplicate(_)) .WillOnce(Return(Api::SysCallIntResult{-1, EBADF})); - std::promise> dup_status_promise; - EXPECT_OK(handle->duplicate( - [&](absl::StatusOr status) { dup_status_promise.set_value(status); })); - auto dup_status = dup_status_promise.get_future().get(); + absl::StatusOr dup_status; + EXPECT_OK(handle->duplicate(dispatcher_.get(), [&](absl::StatusOr status) { + dup_status = std::move(status); + })); + resolveFileActions(); EXPECT_THAT(dup_status, StatusIs(absl::StatusCode::kFailedPrecondition)); close(handle); } TEST_F(AsyncFileHandleWithMockPosixTest, EnqueuingActionAfterCloseReturnsError) { auto handle = createAnonymousFile(); - EXPECT_OK(handle->close([](absl::Status) {})); - auto failed_status = handle->close([](absl::Status) {}); - EXPECT_EQ(absl::StatusCode::kFailedPrecondition, failed_status.code()) << failed_status; + EXPECT_OK(handle->close(dispatcher_.get(), [](absl::Status) {})); + auto failed_status = handle->close(dispatcher_.get(), [](absl::Status) {}); + EXPECT_THAT(failed_status, StatusIs(absl::StatusCode::kFailedPrecondition)); } } // namespace AsyncFiles diff --git a/test/extensions/common/async_files/async_file_manager_thread_pool_test.cc b/test/extensions/common/async_files/async_file_manager_thread_pool_test.cc index d0c262ddf9..7956b61db2 100644 --- a/test/extensions/common/async_files/async_file_manager_thread_pool_test.cc +++ b/test/extensions/common/async_files/async_file_manager_thread_pool_test.cc @@ -38,82 +38,29 @@ enum class BlockerState { class AsyncFileActionBlockedUntilReleased : public AsyncFileActionWithResult { public: - explicit AsyncFileActionBlockedUntilReleased(std::atomic& state_out) - : AsyncFileActionWithResult([this](bool result) { onComplete(result); }), - state_out_(state_out) { - absl::MutexLock lock(&blocking_mutex_); - state_out_.store(BlockerState::Start); - } - void setState(BlockerState state) ABSL_EXCLUSIVE_LOCKS_REQUIRED(blocking_mutex_) { - stage_ = state; - state_out_.store(state); - } + using AsyncFileActionWithResult::AsyncFileActionWithResult; bool executeImpl() final { - absl::MutexLock lock(&blocking_mutex_); - ASSERT(stage_ == BlockerState::Start); - setState(BlockerState::BlockingDuringExecution); - auto condition = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(blocking_mutex_) { - return stage_ == BlockerState::UnblockedExecution; - }; - blocking_mutex_.Await(absl::Condition(&condition)); - setState(BlockerState::ExecutionFinished); - return true; - } - void onComplete(bool result ABSL_ATTRIBUTE_UNUSED) { - absl::MutexLock lock(&blocking_mutex_); - setState(BlockerState::BlockingDuringCallback); - auto condition = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(blocking_mutex_) { - return stage_ == BlockerState::UnblockedCallback; - }; - blocking_mutex_.Await(absl::Condition(&condition)); + executing_.set_value(); + bool ret = continue_executing_.get_future().wait_for(std::chrono::seconds(1)) == + std::future_status::ready; + return ret; } - bool waitUntilExecutionBlocked() ABSL_LOCKS_EXCLUDED(blocking_mutex_) { - absl::MutexLock lock(&blocking_mutex_); - auto condition = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(blocking_mutex_) { - return stage_ == BlockerState::BlockingDuringExecution; - }; - return blocking_mutex_.AwaitWithTimeout(absl::Condition(&condition), absl::Seconds(1)); + bool waitUntilExecutionBlocked() { + return executing_future_.wait_for(std::chrono::seconds(1)) == std::future_status::ready; } - bool waitUntilCallbackBlocked() ABSL_LOCKS_EXCLUDED(blocking_mutex_) { - absl::MutexLock lock(&blocking_mutex_); - auto condition = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(blocking_mutex_) { - return stage_ == BlockerState::BlockingDuringCallback; - }; - return blocking_mutex_.AwaitWithTimeout(absl::Condition(&condition), absl::Seconds(1)); + bool isStarted() { + return executing_future_.wait_for(std::chrono::milliseconds(1)) == std::future_status::ready; } - bool unblockExecution() ABSL_LOCKS_EXCLUDED(blocking_mutex_) { - absl::MutexLock lock(&blocking_mutex_); - if (stage_ != BlockerState::BlockingDuringExecution) { - return false; - } - setState(BlockerState::UnblockedExecution); + bool unblockExecution() { + continue_executing_.set_value(); return true; } - bool unblockCallback() ABSL_LOCKS_EXCLUDED(blocking_mutex_) { - absl::MutexLock lock(&blocking_mutex_); - if (stage_ != BlockerState::BlockingDuringCallback) { - return false; - } - setState(BlockerState::UnblockedCallback); - return true; - } - bool isStarted() ABSL_LOCKS_EXCLUDED(blocking_mutex_) { - absl::MutexLock lock(&blocking_mutex_); - auto condition = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(blocking_mutex_) { - return stage_ != BlockerState::Start; - }; - // Very short timeout because we can be expecting to fail this. - return blocking_mutex_.AwaitWithTimeout(absl::Condition(&condition), absl::Milliseconds(30)); - } - bool doWholeFlow() ABSL_LOCKS_EXCLUDED(blocking_mutex_) { - return waitUntilExecutionBlocked() && unblockExecution() && waitUntilCallbackBlocked() && - unblockCallback(); - } + bool doWholeFlow() { return waitUntilExecutionBlocked() && unblockExecution(); } private: - absl::Mutex blocking_mutex_; - BlockerState stage_ ABSL_GUARDED_BY(blocking_mutex_) = BlockerState::Start; - std::atomic& state_out_; + std::promise executing_; + std::future executing_future_ = executing_.get_future(); + std::promise continue_executing_; }; class AsyncFileManagerTest : public testing::Test { @@ -123,6 +70,11 @@ class AsyncFileManagerTest : public testing::Test { factory_ = AsyncFileManagerFactory::singleton(singleton_manager_.get()); } + void resolveFileActions() { + manager_->waitForIdle(); + dispatcher_->run(Event::Dispatcher::RunType::Block); + } + protected: std::unique_ptr singleton_manager_; std::shared_ptr factory_; @@ -132,14 +84,16 @@ class AsyncFileManagerTest : public testing::Test { std::shared_ptr manager_; AsyncFileActionBlockedUntilReleased* blocker_[3]; - std::atomic blocker_last_state_[3]; + std::vector blocker_callback_result_ = std::vector(3); // returns the cancellation function. - std::function enqueueBlocker(int index) { - auto blocker = - std::make_shared(blocker_last_state_[index]); + CancelFunction enqueueBlocker(int index) { + auto blocker = std::make_unique( + [this, index](bool result) { blocker_callback_result_[index] = result; }); blocker_[index] = blocker.get(); - return manager_->enqueue(std::move(blocker)); + return manager_->enqueue(dispatcher_.get(), std::move(blocker)); } + Api::ApiPtr api_ = Api::createApiForTest(); + Event::DispatcherPtr dispatcher_ = api_->allocateDispatcher("test_thread"); }; TEST_F(AsyncFileManagerTest, WorksWithThreadPoolSizeZero) { @@ -151,6 +105,7 @@ TEST_F(AsyncFileManagerTest, WorksWithThreadPoolSizeZero) { EXPECT_THAT(manager_->describe(), testing::ContainsRegex("thread_pool_size = [1-9]\\d*")); enqueueBlocker(0); EXPECT_TRUE(blocker_[0]->doWholeFlow()); + resolveFileActions(); factory_.reset(); } @@ -168,9 +123,9 @@ TEST_F(AsyncFileManagerTest, ThreadsBlockAppropriately) { EXPECT_FALSE(blocker_[2]->isStarted()); ASSERT_TRUE(blocker_[0]->doWholeFlow()); // When one of the workers finishes, the third action should be able to start. - EXPECT_TRUE(blocker_[2]->isStarted()); - EXPECT_TRUE(blocker_[1]->doWholeFlow()); EXPECT_TRUE(blocker_[2]->doWholeFlow()); + EXPECT_TRUE(blocker_[1]->doWholeFlow()); + resolveFileActions(); factory_.reset(); } @@ -184,19 +139,23 @@ class AsyncFileManagerSingleThreadTest : public AsyncFileManagerTest { manager_ = factory->getAsyncFileManager(config); } -private: +protected: std::unique_ptr singleton_manager_; }; -TEST_F(AsyncFileManagerSingleThreadTest, AbortingDuringExecutionCancelsTheCallback) { - auto cancelBlocker0 = enqueueBlocker(0); +TEST_F(AsyncFileManagerSingleThreadTest, CancellingDuringExecutionCancelsTheCallback) { + EXPECT_FALSE(blocker_callback_result_[0]); + CancelFunction cancelBlocker0 = enqueueBlocker(0); + EXPECT_FALSE(blocker_callback_result_[0]); ASSERT_TRUE(blocker_[0]->waitUntilExecutionBlocked()); enqueueBlocker(1); ASSERT_FALSE(blocker_[1]->isStarted()); cancelBlocker0(); blocker_[0]->unblockExecution(); ASSERT_TRUE(blocker_[1]->doWholeFlow()); - EXPECT_EQ(BlockerState::ExecutionFinished, blocker_last_state_[0].load()); + resolveFileActions(); + EXPECT_FALSE(blocker_callback_result_[0]); + EXPECT_TRUE(blocker_callback_result_[1]); } TEST_F(AsyncFileManagerSingleThreadTest, AbortingBeforeExecutionCancelsTheExecution) { @@ -208,66 +167,42 @@ TEST_F(AsyncFileManagerSingleThreadTest, AbortingBeforeExecutionCancelsTheExecut // Blocker 1 should never start, having been cancelled before it // was popped from the queue. We can't check its internal value because // it should also have been deleted, so we can only check its output state. - EXPECT_EQ(BlockerState::Start, blocker_last_state_[1].load()); -} - -TEST_F(AsyncFileManagerSingleThreadTest, AbortingDuringCallbackBlocksUntilCallbackCompletes) { - auto cancel = enqueueBlocker(0); - blocker_[0]->waitUntilExecutionBlocked(); - blocker_[0]->unblockExecution(); - blocker_[0]->waitUntilCallbackBlocked(); - std::atomic delayed_action_occurred; - std::thread callback_unblocker([&] { - // Using future::wait_for because lint forbids us from sleeping in - // real-time, but here we're forcing a race to go a specific way, using - // real-time because there's no other practical option here. - std::promise pauser; - pauser.get_future().wait_for(std::chrono::milliseconds(50)); - delayed_action_occurred.store(true); - blocker_[0]->unblockCallback(); - }); - cancel(); - EXPECT_TRUE(delayed_action_occurred.load()); - EXPECT_EQ(BlockerState::UnblockedCallback, blocker_last_state_[0].load()); - callback_unblocker.join(); + resolveFileActions(); + EXPECT_TRUE(blocker_callback_result_[0]); + EXPECT_FALSE(blocker_callback_result_[1]); } TEST_F(AsyncFileManagerSingleThreadTest, AbortingAfterCallbackHasNoObservableEffect) { auto cancel = enqueueBlocker(0); EXPECT_TRUE(blocker_[0]->doWholeFlow()); + resolveFileActions(); cancel(); - EXPECT_EQ(BlockerState::UnblockedCallback, blocker_last_state_[0].load()); } -template class WaitForResult { -public: - std::function callback() { - return [this](T result) { saveResult(result); }; - } - void saveResult(T result) { result_.set_value(std::move(result)); } - T getResult() { return result_.get_future().get(); } - -private: - std::promise result_; -}; - TEST_F(AsyncFileManagerSingleThreadTest, CreateAnonymousFileWorks) { - WaitForResult> handle_blocker; - manager_->createAnonymousFile(tmpdir_, handle_blocker.callback()); - AsyncFileHandle handle = handle_blocker.getResult().value(); + AsyncFileHandle handle; + manager_->createAnonymousFile(dispatcher_.get(), tmpdir_, [&](absl::StatusOr h) { + handle = std::move(h.value()); + }); + resolveFileActions(); // Open a second one, to ensure we get two distinct files // (and for coverage, because the second one doesn't use the once_flag path) - WaitForResult> second_handle_blocker; - manager_->createAnonymousFile(tmpdir_, second_handle_blocker.callback()); - AsyncFileHandle second_handle = second_handle_blocker.getResult().value(); - WaitForResult close_blocker; - EXPECT_OK(handle->close(close_blocker.callback())); - absl::Status status = close_blocker.getResult(); - EXPECT_OK(status); - WaitForResult second_close_blocker; - EXPECT_OK(second_handle->close(second_close_blocker.callback())); - status = second_close_blocker.getResult(); - EXPECT_OK(status); + AsyncFileHandle second_handle; + manager_->createAnonymousFile(dispatcher_.get(), tmpdir_, [&](absl::StatusOr h) { + second_handle = std::move(h.value()); + }); + resolveFileActions(); + EXPECT_THAT(handle, testing::NotNull()); + EXPECT_THAT(second_handle, testing::NotNull()); + EXPECT_NE(handle, second_handle); + absl::Status close_result = absl::InternalError("not set"); + EXPECT_OK(handle->close(dispatcher_.get(), [&](absl::Status s) { close_result = std::move(s); })); + absl::Status second_close_result = absl::InternalError("not set"); + EXPECT_OK(second_handle->close(dispatcher_.get(), + [&](absl::Status s) { second_close_result = std::move(s); })); + resolveFileActions(); + EXPECT_OK(close_result); + EXPECT_OK(second_close_result); } TEST_F(AsyncFileManagerSingleThreadTest, OpenExistingFileStatAndUnlinkWork) { @@ -276,47 +211,53 @@ TEST_F(AsyncFileManagerSingleThreadTest, OpenExistingFileStatAndUnlinkWork) { Api::OsSysCalls& posix = Api::OsSysCallsSingleton().get(); auto fd = posix.mkstemp(filename); posix.close(fd.return_value_); - WaitForResult> handle_blocker; - manager_->openExistingFile(filename, AsyncFileManager::Mode::ReadWrite, - handle_blocker.callback()); - AsyncFileHandle handle = handle_blocker.getResult().value(); - WaitForResult close_blocker; - EXPECT_OK(handle->close(close_blocker.callback())); - absl::Status status = close_blocker.getResult(); - EXPECT_OK(status); - WaitForResult> stat_blocker; - manager_->stat(filename, stat_blocker.callback()); - absl::StatusOr stat_result = stat_blocker.getResult(); + AsyncFileHandle handle; + manager_->openExistingFile( + dispatcher_.get(), filename, AsyncFileManager::Mode::ReadWrite, + [&](absl::StatusOr h) { handle = std::move(h.value()); }); + resolveFileActions(); + absl::Status close_result; + EXPECT_OK(handle->close(dispatcher_.get(), [&](absl::Status s) { close_result = std::move(s); })); + EXPECT_OK(close_result); + absl::StatusOr stat_result; + manager_->stat(dispatcher_.get(), filename, + [&](absl::StatusOr result) { stat_result = std::move(result); }); + resolveFileActions(); EXPECT_OK(stat_result); EXPECT_EQ(0, stat_result.value().st_size); - WaitForResult unlink_blocker; - manager_->unlink(filename, unlink_blocker.callback()); - status = unlink_blocker.getResult(); - EXPECT_OK(status); + absl::Status unlink_result = absl::InternalError("not set"); + manager_->unlink(dispatcher_.get(), filename, + [&](absl::Status s) { unlink_result = std::move(s); }); + resolveFileActions(); + EXPECT_OK(unlink_result); + // Make sure unlink deleted the file. struct stat s; EXPECT_EQ(-1, stat(filename, &s)); } TEST_F(AsyncFileManagerSingleThreadTest, OpenExistingFileFailsForNonexistent) { - WaitForResult> handle_blocker; - manager_->openExistingFile(absl::StrCat(tmpdir_, "/nonexistent_file"), - AsyncFileManager::Mode::ReadWrite, handle_blocker.callback()); - absl::Status status = handle_blocker.getResult().status(); - EXPECT_THAT(status, HasStatusCode(absl::StatusCode::kNotFound)); + absl::StatusOr handle_result; + manager_->openExistingFile(dispatcher_.get(), absl::StrCat(tmpdir_, "/nonexistent_file"), + AsyncFileManager::Mode::ReadWrite, + [&](absl::StatusOr r) { handle_result = r; }); + resolveFileActions(); + EXPECT_THAT(handle_result, HasStatusCode(absl::StatusCode::kNotFound)); } TEST_F(AsyncFileManagerSingleThreadTest, StatFailsForNonexistent) { - WaitForResult> stat_blocker; - manager_->stat(absl::StrCat(tmpdir_, "/nonexistent_file"), stat_blocker.callback()); - absl::StatusOr result = stat_blocker.getResult(); - EXPECT_THAT(result, HasStatusCode(absl::StatusCode::kNotFound)); + absl::StatusOr stat_result; + manager_->stat(dispatcher_.get(), absl::StrCat(tmpdir_, "/nonexistent_file"), + [&](absl::StatusOr r) { stat_result = std::move(r); }); + resolveFileActions(); + EXPECT_THAT(stat_result, HasStatusCode(absl::StatusCode::kNotFound)); } TEST_F(AsyncFileManagerSingleThreadTest, UnlinkFailsForNonexistent) { - WaitForResult handle_blocker; - manager_->unlink(absl::StrCat(tmpdir_, "/nonexistent_file"), handle_blocker.callback()); - absl::Status status = handle_blocker.getResult(); - EXPECT_THAT(status, HasStatusCode(absl::StatusCode::kNotFound)); + absl::Status unlink_result; + manager_->unlink(dispatcher_.get(), absl::StrCat(tmpdir_, "/nonexistent_file"), + [&](absl::Status s) { unlink_result = std::move(s); }); + resolveFileActions(); + EXPECT_THAT(unlink_result, HasStatusCode(absl::StatusCode::kNotFound)); } } // namespace AsyncFiles diff --git a/test/extensions/common/async_files/async_file_manager_thread_pool_with_mocks_test.cc b/test/extensions/common/async_files/async_file_manager_thread_pool_with_mocks_test.cc index 03a854da6b..8e05843621 100644 --- a/test/extensions/common/async_files/async_file_manager_thread_pool_with_mocks_test.cc +++ b/test/extensions/common/async_files/async_file_manager_thread_pool_with_mocks_test.cc @@ -46,68 +46,48 @@ class AsyncFileManagerWithMockFilesTest : public ::testing::Test { manager_ = factory_->getAsyncFileManager(config, &mock_posix_file_operations_); } + void resolveFileActions(int cycles = 1) { + for (int i = 0; i < cycles; i++) { + manager_->waitForIdle(); + dispatcher_->run(Event::Dispatcher::RunType::Block); + } + } + protected: std::unique_ptr singleton_manager_; StrictMock mock_posix_file_operations_; std::shared_ptr factory_; std::shared_ptr manager_; static constexpr absl::string_view tmpdir_{"/mocktmp"}; + Api::ApiPtr api_ = Api::createApiForTest(); + Event::DispatcherPtr dispatcher_ = api_->allocateDispatcher("test_thread"); }; -TEST_F(AsyncFileManagerWithMockFilesTest, ChainedOperationsWorkAndSkipQueue) { - int fd = 1; - std::promise write_blocker; - EXPECT_CALL(mock_posix_file_operations_, open(Eq(tmpdir_), O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR)) - .WillOnce(Return(Api::SysCallIntResult{fd, 0})); - EXPECT_CALL(mock_posix_file_operations_, pwrite(fd, _, 5, 0)) - .WillOnce([&write_blocker](int, const void*, size_t, off_t) { - write_blocker.get_future().wait(); - return Api::SysCallSizeResult{5, 0}; - }); - // Chain open/write/close. Write will block because of the mock expectation. - manager_->createAnonymousFile(tmpdir_, [&](absl::StatusOr result) { - AsyncFileHandle handle = result.value(); - Buffer::OwnedImpl buf("hello"); - EXPECT_OK(handle->write(buf, 0, [handle](absl::StatusOr result) { - EXPECT_THAT(result, IsOkAndHolds(5U)); - EXPECT_OK(handle->close([](absl::Status result) { EXPECT_OK(result); })); - })); - }); - // Separately queue another action. - std::promise did_second_action; - manager_->whenReady([&](absl::Status) { did_second_action.set_value(); }); - auto second_action_future = did_second_action.get_future(); - // Ensure that the second action didn't get a turn while the file operations are still blocking. - EXPECT_EQ(std::future_status::timeout, - second_action_future.wait_for(std::chrono::milliseconds(1))); - EXPECT_CALL(mock_posix_file_operations_, close(fd)).WillOnce(Return(Api::SysCallIntResult{0, 0})); - // Unblock the write. - write_blocker.set_value(); - // Ensure that the second action does get a turn after the file operations completed. - EXPECT_EQ(std::future_status::ready, second_action_future.wait_for(std::chrono::seconds(1))); -} - TEST_F(AsyncFileManagerWithMockFilesTest, CancellingAQueuedActionPreventsItFromExecuting) { std::promise ready; - // Add a blocking action so we can guarantee cancel is called before unlink begins. - manager_->whenReady([&](absl::Status) { ready.get_future().wait(); }); + EXPECT_CALL(mock_posix_file_operations_, stat(_, _)).WillOnce([&](const char*, struct stat*) { + ready.get_future().wait(); + return Api::SysCallIntResult{0, 0}; + }); + // Queue a blocking action so we can guarantee cancel is called before unlink begins. + manager_->stat(dispatcher_.get(), tmpdir_, [&](absl::StatusOr) {}); // Ensure that unlink doesn't get called. EXPECT_CALL(mock_posix_file_operations_, unlink(_)).Times(0); - auto cancel_unlink = manager_->unlink("irrelevant", [](absl::Status) {}); + auto cancel_unlink = manager_->unlink(dispatcher_.get(), "irrelevant", [](absl::Status) { + FAIL() << "canceled action should not call callback"; + }); cancel_unlink(); - std::promise done; - // Add a notifying action so we can ensure that the unlink action was passed by the time - // the test ends. - manager_->whenReady([&](absl::Status) { done.set_value(); }); ready.set_value(); - done.get_future().wait(); + resolveFileActions(); } TEST_F(AsyncFileManagerWithMockFilesTest, CancellingACompletedActionDoesNothingImportant) { - std::promise ready; - auto cancel = manager_->whenReady([&](absl::Status) { ready.set_value(); }); - ready.get_future().wait(); - std::this_thread::yield(); + EXPECT_CALL(mock_posix_file_operations_, stat(_, _)); + auto cancel = + manager_->stat(dispatcher_.get(), "irrelevant", [&](absl::StatusOr) {}); + resolveFileActions(); + // This is to make sure cancel() doesn't end up with a dangling pointer or something + // when the action is resolved. cancel(); } @@ -122,14 +102,14 @@ TEST_F(AsyncFileManagerWithMockFilesTest, allow_open_to_finish.get_future().wait(); return Api::SysCallIntResult{fd, 0}; }); - std::atomic callback_was_called{false}; - // Queue opening the file, record if the callback was called (it shouldn't be). + EXPECT_CALL(mock_posix_file_operations_, stat(_, _)); auto cancelOpen = manager_->createAnonymousFile( - tmpdir_, - [&callback_was_called](absl::StatusOr) { callback_was_called.store(true); }); + dispatcher_.get(), tmpdir_, + [&](absl::StatusOr) { FAIL() << "callback should not be called"; }); // Separately queue another action. - std::promise did_second_action; - manager_->whenReady([&](absl::Status) { did_second_action.set_value(true); }); + bool did_second_action = false; + manager_->stat(dispatcher_.get(), "", + [&](absl::StatusOr) { did_second_action = true; }); // Wait for the open operation to be entered. wait_for_open_to_be_executing.get_future().wait(); // Cancel the open request (but too late to actually stop it!) @@ -138,12 +118,10 @@ TEST_F(AsyncFileManagerWithMockFilesTest, EXPECT_CALL(mock_posix_file_operations_, close(fd)).WillOnce(Return(Api::SysCallIntResult{0, 0})); // Allow the open operation to complete. allow_open_to_finish.set_value(); + resolveFileActions(); // Ensure that the second action does get a turn after the file operation relinquishes the thread. // (This also ensures that the file operation reached the end, so the file should be closed now.) - ASSERT_EQ(std::future_status::ready, - did_second_action.get_future().wait_for(std::chrono::milliseconds(100))); - // Ensure the callback for the open operation was *not* called, because it was cancelled. - EXPECT_FALSE(callback_was_called.load()); + EXPECT_TRUE(did_second_action); } TEST_F(AsyncFileManagerWithMockFilesTest, @@ -157,14 +135,14 @@ TEST_F(AsyncFileManagerWithMockFilesTest, allow_open_to_finish.get_future().wait(); return Api::SysCallIntResult{fd, 0}; }); - std::atomic callback_was_called{false}; - // Queue opening the file, record if the callback was called (it shouldn't be). + EXPECT_CALL(mock_posix_file_operations_, stat(_, _)); auto cancelOpen = manager_->openExistingFile( - filename, AsyncFileManager::Mode::ReadWrite, - [&callback_was_called](absl::StatusOr) { callback_was_called.store(true); }); + dispatcher_.get(), filename, AsyncFileManager::Mode::ReadWrite, + [&](absl::StatusOr) { FAIL() << "callback should not be called"; }); // Separately queue another action. - std::promise did_second_action; - manager_->whenReady([&](absl::Status) { did_second_action.set_value(true); }); + bool did_second_action; + manager_->stat(dispatcher_.get(), "irrelevant", + [&](absl::StatusOr) { did_second_action = true; }); // Wait for the open operation to be entered. wait_for_open_to_be_executing.get_future().wait(); // Cancel the open request (but too late to actually stop it!) @@ -173,12 +151,57 @@ TEST_F(AsyncFileManagerWithMockFilesTest, EXPECT_CALL(mock_posix_file_operations_, close(fd)).WillOnce(Return(Api::SysCallIntResult{0, 0})); // Allow the open operation to complete. allow_open_to_finish.set_value(); + resolveFileActions(); // Ensure that the second action does get a turn after the file operation relinquishes the thread. // (This also ensures that the file operation reached the end, so the file should be closed now.) - ASSERT_EQ(std::future_status::ready, - did_second_action.get_future().wait_for(std::chrono::milliseconds(100))); - // Ensure the callback for the open operation was *not* called, because it was cancelled. - EXPECT_FALSE(callback_was_called.load()); + EXPECT_TRUE(did_second_action); +} + +TEST_F(AsyncFileManagerWithMockFilesTest, CloseActionExecutesEvenIfCancelled) { + int fd = 1; + // First do a successful open so we have a file handle. + EXPECT_CALL(mock_posix_file_operations_, open(Eq(tmpdir_), O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR)) + .WillOnce(Return(Api::SysCallIntResult{fd, 0})); + AsyncFileHandle handle; + manager_->createAnonymousFile( + dispatcher_.get(), tmpdir_, + [&](absl::StatusOr result) { handle = std::move(result.value()); }); + resolveFileActions(); + ASSERT_THAT(handle, testing::NotNull()); + EXPECT_CALL(mock_posix_file_operations_, close(fd)); + CancelFunction cancel = handle->close(nullptr, [](absl::Status) {}).value(); + cancel(); + resolveFileActions(); +} + +TEST_F(AsyncFileManagerWithMockFilesTest, CancellingBeforeCallbackUndoesActionsWithSideEffects) { + int fd = 1; + // First do a successful open so we have a file handle. + EXPECT_CALL(mock_posix_file_operations_, open(Eq(tmpdir_), O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR)) + .WillOnce(Return(Api::SysCallIntResult{fd, 0})); + AsyncFileHandle handle; + manager_->createAnonymousFile( + dispatcher_.get(), tmpdir_, + [&](absl::StatusOr result) { handle = std::move(result.value()); }); + resolveFileActions(); + ASSERT_THAT(handle, testing::NotNull()); + EXPECT_CALL(mock_posix_file_operations_, linkat(fd, _, _, Eq(tmpdir_), _)); + CancelFunction cancel_hard_link = + handle + ->createHardLink( + dispatcher_.get(), tmpdir_, + [&](absl::Status) { FAIL() << "callback should not be called in this test"; }) + .value(); + // wait for the manager to post to the dispatcher, but don't consume it yet. + manager_->waitForIdle(); + // cancel while it's in the dispatcher queue. + cancel_hard_link(); + // Cancellation should remove the link because the callback was not executed. + EXPECT_CALL(mock_posix_file_operations_, unlink(Eq(tmpdir_))); + resolveFileActions(); + EXPECT_CALL(mock_posix_file_operations_, close(fd)).WillOnce(Return(Api::SysCallIntResult{0, 0})); + ASSERT_OK(handle->close(nullptr, [](absl::Status) {})); + resolveFileActions(); } TEST_F(AsyncFileManagerWithMockFilesTest, OpenFailureInCreateAnonymousReturnsAnError) { @@ -188,22 +211,23 @@ TEST_F(AsyncFileManagerWithMockFilesTest, OpenFailureInCreateAnonymousReturnsAnE EXPECT_CALL(mock_posix_file_operations_, open(Eq(tmpdir_), O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR)) .WillOnce(Return(Api::SysCallIntResult{fd, 0})); EXPECT_CALL(mock_posix_file_operations_, close(fd)).WillOnce(Return(Api::SysCallIntResult{0, 0})); - std::promise first_open_was_called; - manager_->createAnonymousFile(tmpdir_, [&](absl::StatusOr result) { - EXPECT_OK(result.value()->close([](absl::Status) {})); - first_open_was_called.set_value(); - }); - // We have to synchronize on this to avoid racily adding a different matching expectation for the - // first 'open'. - first_open_was_called.get_future().wait(); + AsyncFileHandle handle; + manager_->createAnonymousFile( + dispatcher_.get(), tmpdir_, + [&](absl::StatusOr result) { handle = std::move(result.value()); }); + resolveFileActions(); + ASSERT_THAT(handle, testing::NotNull()); + EXPECT_OK(handle->close(nullptr, [](absl::Status) {})); + resolveFileActions(); EXPECT_CALL(mock_posix_file_operations_, open(Eq(tmpdir_), O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR)) .WillOnce(Return(Api::SysCallIntResult{-1, EMFILE})); // Capture the result of the second open call, to verify that the error code came through. - std::promise captured_result; - manager_->createAnonymousFile(tmpdir_, [&](absl::StatusOr result) { - captured_result.set_value(result.status()); - }); - EXPECT_EQ(absl::StatusCode::kResourceExhausted, captured_result.get_future().get().code()); + absl::Status captured_result; + manager_->createAnonymousFile( + dispatcher_.get(), tmpdir_, + [&](absl::StatusOr result) { captured_result = result.status(); }); + resolveFileActions(); + EXPECT_EQ(absl::StatusCode::kResourceExhausted, captured_result.code()) << captured_result; } TEST_F(AsyncFileManagerWithMockFilesTest, CreateAnonymousFallbackMkstempReturnsAnErrorOnFailure) { @@ -212,11 +236,12 @@ TEST_F(AsyncFileManagerWithMockFilesTest, CreateAnonymousFallbackMkstempReturnsA EXPECT_CALL(mock_posix_file_operations_, mkstemp(Eq(std::string(tmpdir_) + "/buffer.XXXXXX"))) .WillOnce(Return(Api::SysCallIntResult{-1, EMFILE})); // Capture the result of the open call, to verify that the error code came through. - std::promise captured_result; - manager_->createAnonymousFile(tmpdir_, [&](absl::StatusOr result) { - captured_result.set_value(result.status()); - }); - EXPECT_EQ(absl::StatusCode::kResourceExhausted, captured_result.get_future().get().code()); + absl::Status captured_result; + manager_->createAnonymousFile( + dispatcher_.get(), tmpdir_, + [&](absl::StatusOr result) { captured_result = result.status(); }); + resolveFileActions(); + EXPECT_EQ(absl::StatusCode::kResourceExhausted, captured_result.code()) << captured_result; } TEST_F(AsyncFileManagerWithMockFilesTest, CreateAnonymousFallbackReturnsAnErrorIfPathTooLong) { @@ -228,11 +253,12 @@ TEST_F(AsyncFileManagerWithMockFilesTest, CreateAnonymousFallbackReturnsAnErrorI open(Eq(too_long_tmpdir), O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR)) .WillOnce(Return(Api::SysCallIntResult{-1, EBADF})); // Capture the result of the open call, to verify that the error code came through. - std::promise captured_result; - manager_->createAnonymousFile(too_long_tmpdir, [&](absl::StatusOr result) { - captured_result.set_value(result.status()); - }); - EXPECT_EQ(absl::StatusCode::kInvalidArgument, captured_result.get_future().get().code()); + absl::Status captured_result; + manager_->createAnonymousFile( + dispatcher_.get(), too_long_tmpdir, + [&](absl::StatusOr result) { captured_result = result.status(); }); + resolveFileActions(); + EXPECT_EQ(absl::StatusCode::kInvalidArgument, captured_result.code()) << captured_result; } TEST_F(AsyncFileManagerWithMockFilesTest, @@ -255,11 +281,12 @@ TEST_F(AsyncFileManagerWithMockFilesTest, EXPECT_CALL(mock_posix_file_operations_, unlink(Eq(absl::StrCat(tmpdir_, "/buffer.ABCDEF")))) .WillOnce(Return(Api::SysCallIntResult{0, 0})); } - std::promise captured_result; - manager_->createAnonymousFile(tmpdir_, [&](absl::StatusOr result) { - captured_result.set_value(result.status()); - }); - EXPECT_EQ(absl::StatusCode::kUnimplemented, captured_result.get_future().get().code()); + absl::Status captured_result; + manager_->createAnonymousFile( + dispatcher_.get(), tmpdir_, + [&](absl::StatusOr result) { captured_result = result.status(); }); + resolveFileActions(); + EXPECT_EQ(absl::StatusCode::kUnimplemented, captured_result.code()) << captured_result; } TEST_F(AsyncFileManagerWithMockFilesTest, @@ -280,15 +307,17 @@ TEST_F(AsyncFileManagerWithMockFilesTest, EXPECT_CALL(mock_posix_file_operations_, close(fd)) .WillOnce(Return(Api::SysCallIntResult{0, 0})); } - std::promise callback_complete; - manager_->createAnonymousFile(tmpdir_, [&](absl::StatusOr result) { - EXPECT_OK(result); - EXPECT_OK(result.value()->close([&](absl::Status result) { - EXPECT_OK(result); - callback_complete.set_value(); - })); - }); - callback_complete.get_future().wait(); + bool callbacks_complete = false; + manager_->createAnonymousFile( + dispatcher_.get(), tmpdir_, [&](absl::StatusOr result) { + EXPECT_OK(result); + EXPECT_OK(result.value()->close(dispatcher_.get(), [&](absl::Status result) { + EXPECT_OK(result); + callbacks_complete = true; + })); + }); + resolveFileActions(2); + EXPECT_TRUE(callbacks_complete); } } // namespace AsyncFiles diff --git a/test/extensions/common/async_files/mocks.cc b/test/extensions/common/async_files/mocks.cc index 347f61af0a..69e60d6d97 100644 --- a/test/extensions/common/async_files/mocks.cc +++ b/test/extensions/common/async_files/mocks.cc @@ -11,69 +11,88 @@ using ::testing::_; MockAsyncFileContext::MockAsyncFileContext(std::shared_ptr manager) : manager_(manager) { - ON_CALL(*this, stat(_)) - .WillByDefault([this](std::function)> on_complete) { - return manager_->enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); + ON_CALL(*this, stat(_, _)) + .WillByDefault([this](Event::Dispatcher* dispatcher, + absl::AnyInvocable)> on_complete) { + return manager_->enqueue(dispatcher, + std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); }); - ON_CALL(*this, createHardLink(_, _)) - .WillByDefault([this](absl::string_view, std::function on_complete) { - return manager_->enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); + ON_CALL(*this, createHardLink(_, _, _)) + .WillByDefault([this](Event::Dispatcher* dispatcher, absl::string_view, + absl::AnyInvocable on_complete) { + return manager_->enqueue(dispatcher, + std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); }); - EXPECT_CALL(*this, close(_)).WillOnce([this](std::function on_complete) { - manager_->enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); - return absl::OkStatus(); - }); - ON_CALL(*this, read(_, _, _)) - .WillByDefault([this](off_t, size_t, - std::function)> on_complete) { - return manager_->enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); + EXPECT_CALL(*this, close(_, _)) + .WillOnce([this](Event::Dispatcher* dispatcher, + absl::AnyInvocable on_complete) mutable { + return manager_->enqueue(dispatcher, + std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); }); - ON_CALL(*this, write(_, _, _)) - .WillByDefault([this](Buffer::Instance&, off_t, - std::function)> on_complete) { - return manager_->enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); + ON_CALL(*this, read(_, _, _, _)) + .WillByDefault( + [this](Event::Dispatcher* dispatcher, off_t, size_t, + absl::AnyInvocable)> on_complete) { + return manager_->enqueue(dispatcher, + std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); + }); + ON_CALL(*this, write(_, _, _, _)) + .WillByDefault([this](Event::Dispatcher* dispatcher, Buffer::Instance&, off_t, + absl::AnyInvocable)> on_complete) { + return manager_->enqueue(dispatcher, + std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); }); - ON_CALL(*this, duplicate(_)) + ON_CALL(*this, duplicate(_, _)) .WillByDefault( - [this]( - std::function>)> on_complete) { - return manager_->enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); + [this](Event::Dispatcher* dispatcher, + absl::AnyInvocable>)> + on_complete) { + return manager_->enqueue(dispatcher, + std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); }); }; MockAsyncFileManager::MockAsyncFileManager() { - ON_CALL(*this, enqueue(_)).WillByDefault([this](const std::shared_ptr action) { - queue_.push_back(std::dynamic_pointer_cast(action)); - return [this]() { mockCancel(); }; - }); - ON_CALL(*this, stat(_, _)) + ON_CALL(*this, enqueue(_, _)) .WillByDefault( - [this](absl::string_view, std::function)> on_complete) { - return enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); + [this](Event::Dispatcher* dispatcher, std::unique_ptr action) { + auto entry = QueuedAction{std::move(action), dispatcher}; + auto cancel_func = [this, state = entry.state_]() { + state->store(QueuedAction::State::Cancelled); + mockCancel(); + }; + queue_.push(std::move(entry)); + return cancel_func; }); - ON_CALL(*this, createAnonymousFile(_, _)) - .WillByDefault([this](absl::string_view, - std::function)> on_complete) { - return enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); + ON_CALL(*this, stat(_, _, _)) + .WillByDefault([this](Event::Dispatcher* dispatcher, absl::string_view, + absl::AnyInvocable)> on_complete) { + return enqueue(dispatcher, std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); + }); + ON_CALL(*this, createAnonymousFile(_, _, _)) + .WillByDefault([this](Event::Dispatcher* dispatcher, absl::string_view, + absl::AnyInvocable)> on_complete) { + return enqueue(dispatcher, std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); }); - ON_CALL(*this, openExistingFile(_, _, _)) - .WillByDefault([this](absl::string_view, Mode, - std::function)> on_complete) { - return enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); + ON_CALL(*this, openExistingFile(_, _, _, _)) + .WillByDefault([this](Event::Dispatcher* dispatcher, absl::string_view, Mode, + absl::AnyInvocable)> on_complete) { + return enqueue(dispatcher, std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); }); - ON_CALL(*this, unlink(_, _)) - .WillByDefault([this](absl::string_view, std::function on_complete) { - return enqueue( - std::shared_ptr(new TypedMockAsyncFileAction(on_complete))); + ON_CALL(*this, unlink(_, _, _)) + .WillByDefault([this](Event::Dispatcher* dispatcher, absl::string_view, + absl::AnyInvocable on_complete) { + return enqueue(dispatcher, std::unique_ptr( + new TypedMockAsyncFileAction(std::move(on_complete)))); }); } diff --git a/test/extensions/common/async_files/mocks.h b/test/extensions/common/async_files/mocks.h index f5f62d6fae..0c890fe95e 100644 --- a/test/extensions/common/async_files/mocks.h +++ b/test/extensions/common/async_files/mocks.h @@ -1,8 +1,11 @@ #pragma once +#include + #include "source/extensions/common/async_files/async_file_handle.h" #include "source/extensions/common/async_files/async_file_manager.h" #include "source/extensions/common/async_files/async_file_manager_factory.h" +#include "source/extensions/common/async_files/async_file_manager_thread_pool.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -23,18 +26,23 @@ class MockAsyncFileContext : public Extensions::Common::AsyncFiles::AsyncFileCon // These can be consumed by calling MockAsyncFileManager::nextActionCompletes // with the desired parameter to the on_complete callback. MOCK_METHOD(absl::StatusOr, stat, - (std::function)> on_complete)); + (Event::Dispatcher * dispatcher, + absl::AnyInvocable)> on_complete)); MOCK_METHOD(absl::StatusOr, createHardLink, - (absl::string_view filename, std::function on_complete)); - MOCK_METHOD(absl::Status, close, (std::function on_complete)); + (Event::Dispatcher * dispatcher, absl::string_view filename, + absl::AnyInvocable on_complete)); + MOCK_METHOD(absl::StatusOr, close, + (Event::Dispatcher * dispatcher, absl::AnyInvocable on_complete)); MOCK_METHOD(absl::StatusOr, read, - (off_t offset, size_t length, - std::function)> on_complete)); + (Event::Dispatcher * dispatcher, off_t offset, size_t length, + absl::AnyInvocable)> on_complete)); MOCK_METHOD(absl::StatusOr, write, - (Buffer::Instance & contents, off_t offset, - std::function)> on_complete)); - MOCK_METHOD(absl::StatusOr, duplicate, - (std::function>)> on_complete)); + (Event::Dispatcher * dispatcher, Buffer::Instance& contents, off_t offset, + absl::AnyInvocable)> on_complete)); + MOCK_METHOD( + absl::StatusOr, duplicate, + (Event::Dispatcher * dispatcher, + absl::AnyInvocable>)> on_complete)); private: std::shared_ptr manager_; @@ -50,7 +58,8 @@ class MockAsyncFileAction : public AsyncFileAction { template class TypedMockAsyncFileAction : public MockAsyncFileAction { public: - explicit TypedMockAsyncFileAction(T on_complete) : on_complete_(on_complete) {} + explicit TypedMockAsyncFileAction(T on_complete) : on_complete_(std::move(on_complete)) {} + void onComplete() override {} T on_complete_; std::string describe() const override { return typeid(T).name(); } }; @@ -60,17 +69,20 @@ class MockAsyncFileManager : public Extensions::Common::AsyncFiles::AsyncFileMan MockAsyncFileManager(); // The default behavior of the methods that would enqueue an action is to enqueue a mock action. MOCK_METHOD(CancelFunction, createAnonymousFile, - (absl::string_view path, - std::function)> on_complete)); + (Event::Dispatcher * dispatcher, absl::string_view path, + absl::AnyInvocable)> on_complete)); MOCK_METHOD(CancelFunction, openExistingFile, - (absl::string_view filename, Mode mode, - std::function)> on_complete)); + (Event::Dispatcher * dispatcher, absl::string_view filename, Mode mode, + absl::AnyInvocable)> on_complete)); MOCK_METHOD(CancelFunction, stat, - (absl::string_view filename, - std::function)> on_complete)); + (Event::Dispatcher * dispatcher, absl::string_view filename, + absl::AnyInvocable)> on_complete)); MOCK_METHOD(CancelFunction, unlink, - (absl::string_view filename, std::function on_complete)); + (Event::Dispatcher * dispatcher, absl::string_view filename, + absl::AnyInvocable on_complete)); MOCK_METHOD(std::string, describe, (), (const)); + MOCK_METHOD(void, waitForIdle, ()); + MOCK_METHOD(void, postCancelledActionForCleanup, (std::unique_ptr action)); // mockCancel is called any time any action queued by mock is cancelled. It isn't overriding // a function from the real class, it's just used for verification. @@ -83,19 +95,29 @@ class MockAsyncFileManager : public Extensions::Common::AsyncFiles::AsyncFileMan // not just a Status or a T. template void nextActionCompletes(T result) { ASSERT_FALSE(queue_.empty()); + auto entry = std::move(queue_.front()); + queue_.pop(); auto action = - std::dynamic_pointer_cast>>(queue_.front()); - ASSERT_TRUE(action.get() != nullptr) - << "mismatched type for nextActionCompletes: action is " << queue_.front()->describe() + dynamic_cast>*>(entry.action_.get()); + ASSERT_TRUE(action != nullptr) + << "mismatched type for nextActionCompletes: action is " << action->describe() << ", nextActionCompletes was given " << typeid(T).name(); - queue_.pop_front(); - action->on_complete_(std::move(result)); + if (entry.dispatcher_) { + entry.dispatcher_->post([action = std::move(entry.action_), state = std::move(entry.state_), + result = std::move(result)]() mutable { + if (state->load() != QueuedAction::State::Cancelled) { + dynamic_cast>*>(action.get()) + ->on_complete_(std::move(result)); + } + }); + } } - std::deque> queue_; + std::queue queue_; private: - MOCK_METHOD(CancelFunction, enqueue, (const std::shared_ptr action)); + MOCK_METHOD(CancelFunction, enqueue, + (Event::Dispatcher * dispatcher, std::unique_ptr action)); friend class MockAsyncFileContext; }; @@ -107,15 +129,16 @@ class MockAsyncFileManagerFactory : public Extensions::Common::AsyncFiles::Async }; // Add deduction guides for comping with the ctad-maybe-unsupported warning -TypedMockAsyncFileAction(std::function) - ->TypedMockAsyncFileAction>; -TypedMockAsyncFileAction(std::function)>) - ->TypedMockAsyncFileAction)>>; -TypedMockAsyncFileAction(std::function)>) - ->TypedMockAsyncFileAction)>>; -TypedMockAsyncFileAction(std::function>)>) +TypedMockAsyncFileAction(absl::AnyInvocable) + ->TypedMockAsyncFileAction>; +TypedMockAsyncFileAction(absl::AnyInvocable)>) + ->TypedMockAsyncFileAction)>>; +TypedMockAsyncFileAction(absl::AnyInvocable)>) + ->TypedMockAsyncFileAction)>>; +TypedMockAsyncFileAction( + absl::AnyInvocable>)>) ->TypedMockAsyncFileAction< - std::function>)>>; + absl::AnyInvocable>)>>; } // namespace AsyncFiles } // namespace Common diff --git a/test/extensions/filters/http/cache/http_cache_implementation_test_common.cc b/test/extensions/filters/http/cache/http_cache_implementation_test_common.cc index e1db6c9d44..c5df0fa728 100644 --- a/test/extensions/filters/http/cache/http_cache_implementation_test_common.cc +++ b/test/extensions/filters/http/cache/http_cache_implementation_test_common.cc @@ -40,6 +40,17 @@ MATCHER(IsOk, "") { return arg.ok(); } } // namespace +void HttpCacheTestDelegate::pumpDispatcher() { + // There may be multiple steps in a cache operation going back and forth with work + // on a cache's thread and work on the filter's thread. So drain both things up to + // 10 times each. This number is arbitrary and could be increased if necessary for + // a cache implementation. + for (int i = 0; i < 10; i++) { + beforePumpingDispatcher(); + dispatcher().run(Event::Dispatcher::RunType::Block); + } +} + HttpCacheImplementationTest::HttpCacheImplementationTest() : delegate_(GetParam()()), vary_allow_list_(getConfig().allowed_vary_headers(), factory_context_) { @@ -47,11 +58,9 @@ HttpCacheImplementationTest::HttpCacheImplementationTest() request_headers_.setHost("example.com"); request_headers_.setScheme("https"); request_headers_.setCopy(Http::CustomHeaders::get().CacheControl, "max-age=3600"); - - EXPECT_CALL(dispatcher_, post(_)).Times(AnyNumber()); - EXPECT_CALL(dispatcher_, isThreadSafe()).Times(AnyNumber()); - - delegate_->setUp(dispatcher_); + ON_CALL(encoder_callbacks_, dispatcher()).WillByDefault(testing::ReturnRef(dispatcher())); + ON_CALL(decoder_callbacks_, dispatcher()).WillByDefault(testing::ReturnRef(dispatcher())); + delegate_->setUp(); } HttpCacheImplementationTest::~HttpCacheImplementationTest() { @@ -64,155 +73,134 @@ bool HttpCacheImplementationTest::updateHeaders( absl::string_view request_path, const Http::TestResponseHeaderMapImpl& response_headers, const ResponseMetadata& metadata) { LookupContextPtr lookup_context = lookup(request_path); - auto update_promise = std::make_shared>(); + bool captured_result = false; + bool seen_result = false; cache()->updateHeaders(*lookup_context, response_headers, metadata, - [update_promise](bool result) { update_promise->set_value(result); }); - auto update_future = update_promise->get_future(); - if (std::future_status::ready != update_future.wait_for(std::chrono::seconds(5))) { - EXPECT_TRUE(false) << "timed out in updateHeaders " << request_path; - return false; - } - return update_future.get(); + [&captured_result, &seen_result](bool result) { + captured_result = result; + seen_result = true; + }); + pumpDispatcher(); + EXPECT_TRUE(seen_result); + return captured_result; } LookupContextPtr HttpCacheImplementationTest::lookup(absl::string_view request_path) { LookupRequest request = makeLookupRequest(request_path); LookupContextPtr context = cache()->makeLookupContext(std::move(request), decoder_callbacks_); - auto headers_promise = std::make_shared>>(); - context->getHeaders([headers_promise](LookupResult&& result, bool end_stream) { - headers_promise->set_value(std::make_pair(std::move(result), end_stream)); + bool seen_result = false; + context->getHeaders([this, &seen_result](LookupResult&& result, bool end_stream) { + lookup_result_ = std::move(result); + lookup_end_stream_after_headers_ = end_stream; + seen_result = true; }); - auto headers_future = headers_promise->get_future(); - if (std::future_status::ready == headers_future.wait_for(std::chrono::seconds(5))) { - auto result_pair = headers_future.get(); - lookup_result_ = std::move(result_pair.first); - lookup_end_stream_after_headers_ = result_pair.second; - } else { - EXPECT_TRUE(false) << "timed out in lookup " << request_path; - } - + pumpDispatcher(); + EXPECT_TRUE(seen_result); return context; } absl::Status HttpCacheImplementationTest::insert(LookupContextPtr lookup, const Http::TestResponseHeaderMapImpl& headers, - const absl::string_view body, - std::chrono::milliseconds timeout) { - return insert(std::move(lookup), headers, body, absl::nullopt, timeout); + const absl::string_view body) { + return insert(std::move(lookup), headers, body, absl::nullopt); } absl::Status HttpCacheImplementationTest::insert( LookupContextPtr lookup, const Http::TestResponseHeaderMapImpl& headers, - const absl::string_view body, const absl::optional trailers, - std::chrono::milliseconds timeout) { + const absl::string_view body, const absl::optional trailers) { // For responses with body, we must wait for insertBody's callback before // calling insertTrailers or completing. Note, in a multipart body test this // would need to check for the callback having been called for *every* body part, // but since the test only uses single-part bodies, inserting trailers or // completing in direct response to the callback works. - std::shared_ptr> insert_promise; - auto make_insert_callback = [&insert_promise]() { - insert_promise = std::make_shared>(); - return [insert_promise](bool success_ready_for_more) { - insert_promise->set_value(success_ready_for_more); - }; - }; - auto wait_for_insert = [&insert_promise, timeout](absl::string_view fn) { - auto future = insert_promise->get_future(); - auto result = future.wait_for(timeout); - if (result == std::future_status::timeout) { - return absl::DeadlineExceededError(absl::StrCat("Timed out waiting for ", fn)); - } - if (!future.get()) { - return absl::UnknownError(absl::StrCat("Insert was aborted by cache in ", fn)); - } - return absl::OkStatus(); - }; - + bool inserted_headers = false; + bool inserted_body = false; + bool inserted_trailers = false; InsertContextPtr inserter = cache()->makeInsertContext(std::move(lookup), encoder_callbacks_); absl::Cleanup destroy_inserter{[&inserter] { inserter->onDestroy(); }}; const ResponseMetadata metadata{time_system_.systemTime()}; bool headers_end_stream = body.empty() && !trailers.has_value(); - inserter->insertHeaders(headers, metadata, make_insert_callback(), headers_end_stream); - auto status = wait_for_insert("insertHeaders()"); - if (!status.ok()) { - return status; + inserter->insertHeaders( + headers, metadata, [&inserted_headers](bool result) { inserted_headers = result; }, + headers_end_stream); + pumpDispatcher(); + if (!inserted_headers) { + return absl::InternalError("headers were not inserted"); } if (headers_end_stream) { return absl::OkStatus(); } if (!body.empty()) { - inserter->insertBody(Buffer::OwnedImpl(body), make_insert_callback(), - /*end_stream=*/!trailers.has_value()); - auto status = wait_for_insert("insertBody()"); - if (!status.ok()) { - return status; + inserter->insertBody( + Buffer::OwnedImpl(body), [&inserted_body](bool result) { inserted_body = result; }, + /*end_stream=*/!trailers.has_value()); + pumpDispatcher(); + if (!inserted_body) { + return absl::InternalError("body was not inserted"); } } - - if (trailers.has_value()) { - inserter->insertTrailers(trailers.value(), make_insert_callback()); - auto status = wait_for_insert("insertTrailers()"); - if (!status.ok()) { - return status; - } + if (!trailers.has_value()) { + return absl::OkStatus(); + } + inserter->insertTrailers(trailers.value(), + [&inserted_trailers](bool result) { inserted_trailers = result; }); + pumpDispatcher(); + if (!inserted_trailers) { + return absl::InternalError("trailers were not inserted"); } return absl::OkStatus(); } absl::Status HttpCacheImplementationTest::insert(absl::string_view request_path, const Http::TestResponseHeaderMapImpl& headers, - const absl::string_view body, - std::chrono::milliseconds timeout) { - return insert(lookup(request_path), headers, body, timeout); + const absl::string_view body) { + return insert(lookup(request_path), headers, body); } std::pair HttpCacheImplementationTest::getHeaders(LookupContext& context) { - Http::ResponseHeaderMapPtr response_headers_ptr; - auto headers_promise = - std::make_shared>>(); - context.getHeaders([headers_promise](LookupResult&& lookup_result, bool end_stream) { + std::pair returned_pair; + bool seen_result = false; + context.getHeaders([&returned_pair, &seen_result](LookupResult&& lookup_result, bool end_stream) { EXPECT_NE(lookup_result.cache_entry_status_, CacheEntryStatus::Unusable); EXPECT_NE(lookup_result.headers_, nullptr); - headers_promise->set_value(std::make_pair(std::move(lookup_result.headers_), end_stream)); + returned_pair.first = std::move(lookup_result.headers_); + returned_pair.second = end_stream; + seen_result = true; }); - auto future = headers_promise->get_future(); - EXPECT_EQ(std::future_status::ready, future.wait_for(std::chrono::seconds(5))); - return future.get(); + pumpDispatcher(); + EXPECT_TRUE(seen_result); + return returned_pair; } std::pair HttpCacheImplementationTest::getBody(LookupContext& context, uint64_t start, uint64_t end) { AdjustedByteRange range(start, end); - auto body_promise = std::make_shared>>(); - context.getBody(range, [body_promise](Buffer::InstancePtr&& data, bool end_stream) { - EXPECT_NE(data, nullptr); - body_promise->set_value(data ? std::make_pair(data->toString(), end_stream) - : std::make_pair("", end_stream)); - }); - auto future = body_promise->get_future(); - EXPECT_EQ(std::future_status::ready, future.wait_for(std::chrono::seconds(5))); - return future.get(); + std::pair returned_pair; + bool seen_result = false; + context.getBody(range, + [&returned_pair, &seen_result](Buffer::InstancePtr&& data, bool end_stream) { + EXPECT_NE(data, nullptr); + returned_pair = std::make_pair(data->toString(), end_stream); + seen_result = true; + }); + pumpDispatcher(); + EXPECT_TRUE(seen_result); + return returned_pair; } Http::TestResponseTrailerMapImpl HttpCacheImplementationTest::getTrailers(LookupContext& context) { - auto trailers_promise = - std::make_shared>>(); - context.getTrailers([trailers_promise](Http::ResponseTrailerMapPtr&& data) { + Http::ResponseTrailerMapPtr trailers; + context.getTrailers([&trailers](Http::ResponseTrailerMapPtr&& data) { if (data) { - trailers_promise->set_value(std::move(data)); + trailers = std::move(data); } }); - auto future = trailers_promise->get_future(); - EXPECT_EQ(std::future_status::ready, future.wait_for(std::chrono::seconds(5))); - Http::TestResponseTrailerMapImpl trailers; - if (std::future_status::ready == future.wait_for(std::chrono::seconds(5))) { - trailers = *future.get(); - } - return trailers; + pumpDispatcher(); + EXPECT_THAT(trailers, testing::NotNull()); + return *trailers; } LookupRequest HttpCacheImplementationTest::makeLookupRequest(absl::string_view request_path) { @@ -423,30 +411,23 @@ TEST_P(HttpCacheImplementationTest, StreamingPut) { InsertContextPtr inserter = cache()->makeInsertContext(lookup(request_path), encoder_callbacks_); absl::Cleanup destroy_inserter{[&inserter] { inserter->onDestroy(); }}; ResponseMetadata metadata{time_system_.systemTime()}; - std::promise insert_headers_promise; - std::promise insert_body1_promise; - std::promise insert_body2_promise; + bool inserted_headers = false; + bool inserted_body1 = false; + bool inserted_body2 = false; inserter->insertHeaders( - response_headers, metadata, - [&insert_headers_promise](bool ready) { insert_headers_promise.set_value(ready); }, false); - auto insert_future = insert_headers_promise.get_future(); - ASSERT_EQ(std::future_status::ready, insert_future.wait_for(std::chrono::seconds(5))) - << "timed out waiting for inserts to complete"; - ASSERT_TRUE(insert_future.get()); + response_headers, metadata, [&inserted_headers](bool ready) { inserted_headers = ready; }, + false); + pumpDispatcher(); + ASSERT_TRUE(inserted_headers); inserter->insertBody( - Buffer::OwnedImpl("Hello, "), - [&insert_body1_promise](bool ready) { insert_body1_promise.set_value(ready); }, false); - insert_future = insert_body1_promise.get_future(); - ASSERT_EQ(std::future_status::ready, insert_future.wait_for(std::chrono::seconds(5))) - << "timed out waiting for inserts to complete"; - ASSERT_TRUE(insert_future.get()); + Buffer::OwnedImpl("Hello, "), [&inserted_body1](bool ready) { inserted_body1 = ready; }, + false); + pumpDispatcher(); + ASSERT_TRUE(inserted_body1); inserter->insertBody( - Buffer::OwnedImpl("World!"), - [&insert_body2_promise](bool ready) { insert_body2_promise.set_value(ready); }, true); - insert_future = insert_body2_promise.get_future(); - ASSERT_EQ(std::future_status::ready, insert_future.wait_for(std::chrono::seconds(5))) - << "timed out waiting for inserts to complete"; - ASSERT_TRUE(insert_future.get()); + Buffer::OwnedImpl("World!"), [&inserted_body2](bool ready) { inserted_body2 = ready; }, true); + pumpDispatcher(); + ASSERT_TRUE(inserted_body2); LookupContextPtr name_lookup = lookup(request_path); ASSERT_EQ(CacheEntryStatus::Ok, lookup_result_.cache_entry_status_); diff --git a/test/extensions/filters/http/cache/http_cache_implementation_test_common.h b/test/extensions/filters/http/cache/http_cache_implementation_test_common.h index d172f95fab..497bfcad81 100644 --- a/test/extensions/filters/http/cache/http_cache_implementation_test_common.h +++ b/test/extensions/filters/http/cache/http_cache_implementation_test_common.h @@ -28,8 +28,9 @@ class HttpCacheTestDelegate { public: virtual ~HttpCacheTestDelegate() = default; - virtual void setUp(Event::MockDispatcher& dispatcher) { dispatcher_ = &dispatcher; } + virtual void setUp() {} virtual void tearDown() {} + virtual std::shared_ptr cache() = 0; // Specifies whether or not the cache supports validating stale cache entries @@ -38,14 +39,21 @@ class HttpCacheTestDelegate { // RequiresValidation. virtual bool validationEnabled() const = 0; - Event::MockDispatcher& dispatcher() { return *dispatcher_; } + // May be overridden to, for example, also drain other threads into the dispatcher + // before draining the dispatcher. + virtual void beforePumpingDispatcher(){}; + void pumpDispatcher(); + + Event::Dispatcher& dispatcher() { return *dispatcher_; } private: - Event::MockDispatcher* dispatcher_ = nullptr; + Api::ApiPtr api_ = Api::createApiForTest(); + Event::DispatcherPtr dispatcher_ = api_->allocateDispatcher("test_thread"); }; class HttpCacheImplementationTest - : public testing::TestWithParam()>> { + : public Event::TestUsingSimulatedTime, + public testing::TestWithParam()>> { public: static constexpr absl::Duration kLastValidUpdateMinInterval = absl::Seconds(10); @@ -55,21 +63,19 @@ class HttpCacheImplementationTest std::shared_ptr cache() const { return delegate_->cache(); } bool validationEnabled() const { return delegate_->validationEnabled(); } + void pumpDispatcher() { delegate_->pumpDispatcher(); } LookupContextPtr lookup(absl::string_view request_path); absl::Status insert(LookupContextPtr lookup, const Http::TestResponseHeaderMapImpl& headers, - const absl::string_view body, - std::chrono::milliseconds timeout = std::chrono::seconds(1)); + const absl::string_view body); virtual absl::Status insert(LookupContextPtr lookup, const Http::TestResponseHeaderMapImpl& headers, const absl::string_view body, - const absl::optional trailers, - std::chrono::milliseconds timeout = std::chrono::seconds(1)); + const absl::optional trailers); absl::Status insert(absl::string_view request_path, - const Http::TestResponseHeaderMapImpl& headers, const absl::string_view body, - std::chrono::milliseconds timeout = std::chrono::seconds(1)); + const Http::TestResponseHeaderMapImpl& headers, const absl::string_view body); // Returns the headers and a bool for end_stream. std::pair getHeaders(LookupContext& context); @@ -99,7 +105,7 @@ class HttpCacheImplementationTest bool lookup_end_stream_after_headers_; Http::TestRequestHeaderMapImpl request_headers_; Event::SimulatedTimeSystem time_system_; - Event::MockDispatcher dispatcher_; + Event::Dispatcher& dispatcher() { return delegate_->dispatcher(); } DateFormatter formatter_{"%a, %d %b %Y %H:%M:%S GMT"}; NiceMock decoder_callbacks_; NiceMock encoder_callbacks_; diff --git a/test/extensions/filters/http/file_system_buffer/filter_test.cc b/test/extensions/filters/http/file_system_buffer/filter_test.cc index 78541b92b3..cf3bd20171 100644 --- a/test/extensions/filters/http/file_system_buffer/filter_test.cc +++ b/test/extensions/filters/http/file_system_buffer/filter_test.cc @@ -23,6 +23,7 @@ using Extensions::Common::AsyncFiles::MockAsyncFileManager; using Extensions::Common::AsyncFiles::MockAsyncFileManagerFactory; using ::testing::NiceMock; using ::testing::Return; +using ::testing::ReturnRef; class FileSystemBufferFilterTest : public testing::Test { public: @@ -42,8 +43,9 @@ class FileSystemBufferFilterTest : public testing::Test { protected: void expectAsyncFileCreated() { - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); } + void pumpDispatcher() { dispatcher_->run(Event::Dispatcher::RunType::Block); } // A file is only opened by the filter if it's going to chain a write immediately afterwards, // so we combine completing the open and expecting the write, which allows for simplified control // of the handle. @@ -56,13 +58,13 @@ class FileSystemBufferFilterTest : public testing::Test { } void expectWriteWithPosition(MockAsyncFileHandle handle, absl::string_view content, off_t offset) { - EXPECT_CALL(*handle, write(BufferStringEqual(std::string(content)), offset, _)); + EXPECT_CALL(*handle, write(_, BufferStringEqual(std::string(content)), offset, _)); } void completeWriteOfSize(size_t length) { mock_async_file_manager_->nextActionCompletes(absl::StatusOr{length}); } void expectRead(MockAsyncFileHandle handle, off_t offset, size_t size) { - EXPECT_CALL(*handle, read(offset, size, _)); + EXPECT_CALL(*handle, read(_, offset, size, _)); } void completeRead(absl::string_view content) { mock_async_file_manager_->nextActionCompletes(absl::StatusOr>{ @@ -86,6 +88,10 @@ class FileSystemBufferFilterTest : public testing::Test { proto_config); } + void useDeferredDispatcher() { + EXPECT_CALL(decoder_callbacks_, dispatcher()).WillRepeatedly(ReturnRef(*dispatcher_)); + } + void createFilterFromYaml(absl::string_view yaml) { auto config = configFromYaml(yaml); filter_ = std::make_shared(config); @@ -105,14 +111,6 @@ class FileSystemBufferFilterTest : public testing::Test { ASSERT_FALSE(continued_encoding_); continued_encoding_ = true; }); - // Using EXPECT_CALL rather than ON_CALL because this one was set up in Envoy code and - // isn't a NiceMock. Using EXPECT reduces log noise. - // For simplicity's sake the dispatcher just runs the function immediately - // - since the mock is capturing callbacks, we're effectively triggering - // the dispatcher when we resolve the callback. - EXPECT_CALL(decoder_callbacks_.dispatcher_, post(_)).WillRepeatedly([](Event::PostCb fn) { - fn(); - }); filter_->setDecoderFilterCallbacks(decoder_callbacks_); filter_->setEncoderFilterCallbacks(encoder_callbacks_); EXPECT_CALL(decoder_callbacks_, onDecoderFilterAboveWriteBufferHighWatermark()) @@ -154,6 +152,8 @@ class FileSystemBufferFilterTest : public testing::Test { bool continued_encoding_ = false; int request_source_watermark_ = 0; int response_source_watermark_ = 0; + Api::ApiPtr api_ = Api::createApiForTest(); + Event::DispatcherPtr dispatcher_ = api_->allocateDispatcher("test_thread"); LogLevelSetter log_level_setter_ = LogLevelSetter(ENVOY_SPDLOG_LEVEL(debug)); }; @@ -738,26 +738,23 @@ TEST_F(FileSystemBufferFilterTest, FilterDestroyedWhileFileActionIsInDispatcherI behavior: fully_buffer: {} )"); + useDeferredDispatcher(); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, false)); + pumpDispatcher(); expectAsyncFileCreated(); Buffer::OwnedImpl request_body{"12345678901234567890"}; EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(request_body, false)); + pumpDispatcher(); completeCreateFileAndExpectWrite("1234567890"); - Event::PostCb intercepted_dispatcher_callback; - // Our default mock dispatcher behavior calls the callback immediately - here we intercept - // one so we can call it after a 'realistic' delay during which something else happened to - // destroy the filter. - EXPECT_CALL(decoder_callbacks_.dispatcher_, post(_)) - .WillOnce([&intercepted_dispatcher_callback](Event::PostCb fn) { - intercepted_dispatcher_callback = std::move(fn); - }); + pumpDispatcher(); + // Action completes in file system manager. Don't consume the dispatched callback yet. completeWriteOfSize(10); + // Action is cancelled before callback; async file manager prevents the callback after this. + EXPECT_CALL(*mock_async_file_manager_, mockCancel()); destroyFilter(); - // Callback called from dispatcher after filter was destroyed and its pointer invalidated, - // should not cause a crash. - intercepted_dispatcher_callback(); + pumpDispatcher(); } TEST_F(FileSystemBufferFilterTest, MergesRouteConfig) { diff --git a/test/extensions/filters/http/file_system_buffer/fragment_test.cc b/test/extensions/filters/http/file_system_buffer/fragment_test.cc index 662173729c..59e1644fca 100644 --- a/test/extensions/filters/http/file_system_buffer/fragment_test.cc +++ b/test/extensions/filters/http/file_system_buffer/fragment_test.cc @@ -22,33 +22,33 @@ using StatusHelpers::HasStatusMessage; using ::testing::_; using ::testing::Eq; using ::testing::HasSubstr; +using ::testing::MockFunction; using ::testing::StrictMock; -std::function storageSuccessCallback() { - return [](absl::Status status) { ASSERT_OK(status); }; -} - -template -std::function storageFailureCallback(MatcherT matcher) { - return [matcher](absl::Status status) { EXPECT_THAT(status, matcher); }; -} - -void dispatchImmediately(std::function callback) { callback(); } - class FileSystemBufferFilterFragmentTest : public ::testing::Test { public: + void resolveFileActions() { dispatcher_->run(Event::Dispatcher::RunType::Block); } + protected: MockAsyncFileHandle handle_ = std::make_shared>(); void moveFragmentToStorage(Fragment* frag) { - EXPECT_CALL(*handle_, write(_, _, _)) - .WillOnce( - [frag](Buffer::Instance&, off_t, std::function)> callback) { - callback(frag->size()); - return []() {}; - }); - EXPECT_OK(frag->toStorage(handle_, 123, &dispatchImmediately, storageSuccessCallback())); + EXPECT_CALL(*handle_, write(_, _, _, _)) + .WillOnce([frag](Event::Dispatcher* dispatcher, Buffer::Instance&, off_t, + absl::AnyInvocable)> callback) { + dispatcher->post([frag, callback = std::move(callback)]() mutable { + std::move(callback)(frag->size()); + }); + return []() {}; + }); + MockFunction callback; + EXPECT_OK(frag->toStorage(handle_, 123, *dispatcher_, callback.AsStdFunction())); + EXPECT_CALL(callback, Call(absl::OkStatus())); + resolveFileActions(); } + + Api::ApiPtr api_ = Api::createApiForTest(); + Event::DispatcherPtr dispatcher_ = api_->allocateDispatcher("test_thread"); }; TEST_F(FileSystemBufferFilterFragmentTest, CreatesAndExtractsWithoutCopying) { @@ -77,39 +77,42 @@ TEST_F(FileSystemBufferFilterFragmentTest, CreatesFragmentFromPartialBufferAndCo TEST_F(FileSystemBufferFilterFragmentTest, WritesAndReadsBack) { Buffer::OwnedImpl input("hello"); Fragment frag(input); - std::function)> captured_write_callback; - EXPECT_CALL(*handle_, write(BufferStringEqual("hello"), 123, _)) - .WillOnce([&captured_write_callback](Buffer::Instance&, off_t, - std::function)> callback) { - captured_write_callback = std::move(callback); + EXPECT_CALL(*handle_, write(_, BufferStringEqual("hello"), 123, _)) + .WillOnce([](Event::Dispatcher* dispatcher, Buffer::Instance&, off_t, + absl::AnyInvocable)> callback) { + dispatcher->post([callback = std::move(callback)]() mutable { std::move(callback)(5); }); return []() {}; }); // Request the fragment be moved to storage. - EXPECT_OK(frag.toStorage(handle_, 123, &dispatchImmediately, storageSuccessCallback())); + MockFunction write_callback; + EXPECT_OK(frag.toStorage(handle_, 123, *dispatcher_, write_callback.AsStdFunction())); // Before the file confirms written, the state should be neither in memory nor in storage. EXPECT_FALSE(frag.isMemory()); EXPECT_FALSE(frag.isStorage()); // Fake the file thread confirming 5 bytes were written. - captured_write_callback(5); + EXPECT_CALL(write_callback, Call(absl::OkStatus())); + resolveFileActions(); // Now the fragment should be tagged as being in storage. EXPECT_TRUE(frag.isStorage()); EXPECT_FALSE(frag.isMemory()); - std::function>)> captured_read_callback; - EXPECT_CALL(*handle_, read(123, 5, _)) + EXPECT_CALL(*handle_, read(_, 123, 5, _)) .WillOnce( - [&captured_read_callback]( - off_t, size_t, - std::function>)> callback) { - captured_read_callback = std::move(callback); + [](Event::Dispatcher* dispatcher, off_t, size_t, + absl::AnyInvocable>)> callback) { + dispatcher->post([callback = std::move(callback)]() mutable { + std::move(callback)(std::make_unique("hello")); + }); return []() {}; }); // Request the fragment be moved from storage. - EXPECT_OK(frag.fromStorage(handle_, &dispatchImmediately, storageSuccessCallback())); + MockFunction read_callback; + EXPECT_OK(frag.fromStorage(handle_, *dispatcher_, read_callback.AsStdFunction())); // Before the file confirms read, the state should be neither in memory nor storage. EXPECT_FALSE(frag.isMemory()); EXPECT_FALSE(frag.isStorage()); // Fake the file thread completing read. - captured_read_callback(std::make_unique("hello")); + EXPECT_CALL(read_callback, Call(absl::OkStatus())); + resolveFileActions(); // Now the fragment should be tagged as being in memory. EXPECT_TRUE(frag.isMemory()); EXPECT_FALSE(frag.isStorage()); @@ -121,86 +124,81 @@ TEST_F(FileSystemBufferFilterFragmentTest, WritesAndReadsBack) { TEST_F(FileSystemBufferFilterFragmentTest, ReturnsErrorOnWriteError) { Buffer::OwnedImpl input("hello"); Fragment frag(input); - auto write_error = absl::UnknownError("write error"); - std::function)> captured_write_callback; - EXPECT_CALL(*handle_, write(BufferStringEqual("hello"), 123, _)) - .WillOnce([&captured_write_callback](Buffer::Instance&, off_t, - std::function)> callback) { - captured_write_callback = std::move(callback); + EXPECT_CALL(*handle_, write(_, BufferStringEqual("hello"), 123, _)) + .WillOnce([](Event::Dispatcher* dispatcher, Buffer::Instance&, off_t, + absl::AnyInvocable)> callback) { + dispatcher->post([callback = std::move(callback)]() mutable { + std::move(callback)(absl::UnknownError("write error")); + }); return []() {}; }); // Request the fragment be moved to storage. - EXPECT_OK( - frag.toStorage(handle_, 123, &dispatchImmediately, storageFailureCallback(Eq(write_error)))); - - // Fake file system declares a write error. This should - // provoke the expected error in the callback above. - captured_write_callback(write_error); + MockFunction callback; + EXPECT_OK(frag.toStorage(handle_, 123, *dispatcher_, callback.AsStdFunction())); + EXPECT_CALL(callback, Call(Eq(absl::UnknownError("write error")))); + resolveFileActions(); } TEST_F(FileSystemBufferFilterFragmentTest, ReturnsErrorOnWriteIncomplete) { Buffer::OwnedImpl input("hello"); Fragment frag(input); - std::function)> captured_write_callback; - EXPECT_CALL(*handle_, write(BufferStringEqual("hello"), 123, _)) - .WillOnce([&captured_write_callback](Buffer::Instance&, off_t, - std::function)> callback) { - captured_write_callback = std::move(callback); + EXPECT_CALL(*handle_, write(_, BufferStringEqual("hello"), 123, _)) + .WillOnce([](Event::Dispatcher* dispatcher, Buffer::Instance&, off_t, + absl::AnyInvocable)> callback) { + dispatcher->post([callback = std::move(callback)]() mutable { std::move(callback)(2); }); return []() {}; }); // Request the fragment be moved to storage. - EXPECT_OK(frag.toStorage( - handle_, 123, &dispatchImmediately, - storageFailureCallback(HasStatusMessage(HasSubstr("wrote 2 bytes, wanted 5"))))); - + MockFunction callback; + EXPECT_OK(frag.toStorage(handle_, 123, *dispatcher_, callback.AsStdFunction())); // Fake file says it wrote 2 bytes when the fragment was of size 5 - this should // provoke the expected error in the callback above. - captured_write_callback(2); + EXPECT_CALL(callback, Call(HasStatusMessage(HasSubstr("wrote 2 bytes, wanted 5")))); + resolveFileActions(); } TEST_F(FileSystemBufferFilterFragmentTest, ReturnsErrorOnReadError) { Buffer::OwnedImpl input("hello"); Fragment frag(input); moveFragmentToStorage(&frag); - auto read_error = absl::UnknownError("read error"); - std::function>)> captured_read_callback; - EXPECT_CALL(*handle_, read(123, 5, _)) + EXPECT_CALL(*handle_, read(_, 123, 5, _)) .WillOnce( - [&captured_read_callback]( - off_t, size_t, - std::function>)> callback) { - captured_read_callback = std::move(callback); + [](Event::Dispatcher* dispatcher, off_t, size_t, + absl::AnyInvocable>)> callback) { + dispatcher->post([callback = std::move(callback)]() mutable { + std::move(callback)(absl::UnknownError("read error")); + }); return []() {}; }); // Request the fragment be moved from storage. - EXPECT_OK( - frag.fromStorage(handle_, &dispatchImmediately, storageFailureCallback(Eq(read_error)))); - // Fake file system declares a read error. This should - // provoke the expected error in the callback above. - captured_read_callback(read_error); + MockFunction callback; + EXPECT_OK(frag.fromStorage(handle_, *dispatcher_, callback.AsStdFunction())); + EXPECT_CALL(callback, Call(Eq(absl::UnknownError("read error")))); + resolveFileActions(); } TEST_F(FileSystemBufferFilterFragmentTest, ReturnsErrorOnReadIncomplete) { Buffer::OwnedImpl input("hello"); Fragment frag(input); moveFragmentToStorage(&frag); - std::function>)> captured_read_callback; - EXPECT_CALL(*handle_, read(123, 5, _)) + absl::AnyInvocable>)> + captured_read_callback; + EXPECT_CALL(*handle_, read(_, 123, 5, _)) .WillOnce( - [&captured_read_callback]( - off_t, size_t, - std::function>)> callback) { - captured_read_callback = std::move(callback); + [](Event::Dispatcher* dispatcher, off_t, size_t, + absl::AnyInvocable>)> callback) { + dispatcher->post([callback = std::move(callback)]() mutable { + std::move(callback)(std::make_unique("he")); + }); return []() {}; }); + MockFunction callback; // Request the fragment be moved from storage. - EXPECT_OK(frag.fromStorage( - handle_, &dispatchImmediately, - storageFailureCallback(HasStatusMessage(HasSubstr("read got 2 bytes, wanted 5"))))); - - // Fake file system declares a read error. This should + EXPECT_OK(frag.fromStorage(handle_, *dispatcher_, callback.AsStdFunction())); + // Mock file system declares a read error. This should // provoke the expected error in the callback above. - captured_read_callback(std::make_unique("he")); + EXPECT_CALL(callback, Call(HasStatusMessage(HasSubstr("read got 2 bytes, wanted 5")))); + resolveFileActions(); } } // namespace FileSystemBuffer diff --git a/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc b/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc index 208163fb91..29c87ea43a 100644 --- a/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc +++ b/test/extensions/http/cache/file_system_http_cache/file_system_http_cache_test.cc @@ -335,6 +335,8 @@ class FileSystemHttpCacheTestWithMockFiles : public FileSystemHttpCacheTest { return ret; } + void pumpDispatcher() { dispatcher_->run(Event::Dispatcher::RunType::Block); } + protected: ::testing::NiceMock mock_singleton_manager_; std::shared_ptr mock_async_file_manager_factory_ = @@ -364,6 +366,8 @@ class FileSystemHttpCacheTestWithMockFiles : public FileSystemHttpCacheTest { std::function expect_true_callback_; size_t headers_size_; size_t trailers_size_; + Api::ApiPtr api_ = Api::createApiForTest(); + Event::DispatcherPtr dispatcher_ = api_->allocateDispatcher("test_thread"); }; TEST_F(FileSystemHttpCacheTestWithMockFiles, WriteVaryNodeFailingToCreateFileJustAborts) { @@ -375,19 +379,21 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, WriteVaryNodeFailingToCreateFileJus {"cache-control", "public,max-age=3600"}, {"vary", "accept"}}; // one file created for the vary node, one for the actual write. - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)).Times(2); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)).Times(2); inserter->insertHeaders( response_headers, metadata_, [&](bool result) { EXPECT_FALSE(result); }, true); // File handle for the vary node. // (This is the failure under test, we expect write to *not* be called.) mock_async_file_manager_->nextActionCompletes( absl::StatusOr{absl::UnknownError("create failure for vary node")}); + pumpDispatcher(); // Fail to create file for the cache entry node. // (This provokes the false callback to insertHeaders.) mock_async_file_manager_->nextActionCompletes( absl::StatusOr{absl::UnknownError("open failure")}); + pumpDispatcher(); // File handle was not used and is expected to be closed. - EXPECT_OK(mock_async_file_handle_->close([](absl::Status) {})); + EXPECT_OK(mock_async_file_handle_->close(nullptr, [](absl::Status) {})); } TEST_F(FileSystemHttpCacheTestWithMockFiles, WriteVaryNodeFailingToWriteJustClosesTheFile) { @@ -399,28 +405,31 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, WriteVaryNodeFailingToWriteJustClos {"cache-control", "public,max-age=3600"}, {"vary", "accept"}}; // one file created for the vary node, one for the actual write. - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)).Times(2); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)).Times(2); inserter->insertHeaders( response_headers, metadata_, [&](bool result) { EXPECT_FALSE(result); }, true); - EXPECT_CALL(*mock_async_file_handle_, write(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, write(_, _, _, _)); // File handle for the vary node. // (This triggers the expected write call.) mock_async_file_manager_->nextActionCompletes( absl::StatusOr{mock_async_file_handle_}); + pumpDispatcher(); // Fail to create file for the cache entry node. // (This provokes the false callback to insertHeaders.) mock_async_file_manager_->nextActionCompletes( absl::StatusOr{absl::UnknownError("open failure")}); + pumpDispatcher(); // Fail to write for the vary node. mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("write failure"))); + pumpDispatcher(); } TEST_F(FileSystemHttpCacheTestWithMockFiles, LookupDuringAnotherInsertPreventsInserts) { auto inserter = testInserter(); absl::Cleanup destroy_inserter{[&inserter]() { inserter->onDestroy(); }}; // First inserter will try to create a file. - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); inserter->insertHeaders( response_headers_, metadata_, [&](bool result) { EXPECT_FALSE(result); }, false); @@ -429,12 +438,13 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, LookupDuringAnotherInsertPreventsIn // Allow the first inserter to complete after the second lookup was made. mock_async_file_manager_->nextActionCompletes( absl::StatusOr{absl::UnknownError("intentionally failed to open file")}); + pumpDispatcher(); inserter2->insertHeaders(response_headers_, metadata_, expect_false_callback_, false); inserter2->insertBody(Buffer::OwnedImpl("boop"), expect_false_callback_, false); inserter2->insertTrailers(response_trailers_, expect_false_callback_); EXPECT_EQ(false_callbacks_called_, 3); // The file handle didn't actually get used in this test, but is expected to be closed. - EXPECT_OK(mock_async_file_handle_->close([](absl::Status) {})); + EXPECT_OK(mock_async_file_handle_->close(nullptr, [](absl::Status) {})); } TEST_F(FileSystemHttpCacheTestWithMockFiles, DuplicateInsertWhileInsertInProgressIsPrevented) { @@ -443,29 +453,31 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, DuplicateInsertWhileInsertInProgres auto inserter2 = testInserter(); absl::Cleanup destroy_inserter2{[&inserter2]() { inserter2->onDestroy(); }}; // First inserter will try to create a file. - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); inserter->insertHeaders(response_headers_, metadata_, expect_false_callback_, false); inserter2->insertHeaders(response_headers_, metadata_, expect_false_callback_, false); // Allow the first inserter to complete after the second insert was called. mock_async_file_manager_->nextActionCompletes( absl::StatusOr{absl::UnknownError("intentionally failed to open file")}); + pumpDispatcher(); inserter2->insertBody(Buffer::OwnedImpl("boop"), expect_false_callback_, false); inserter2->insertTrailers(response_trailers_, expect_false_callback_); EXPECT_EQ(false_callbacks_called_, 4); // The file handle didn't actually get used in this test, but is expected to be closed. - EXPECT_OK(mock_async_file_handle_->close([](absl::Status) {})); + EXPECT_OK(mock_async_file_handle_->close(nullptr, [](absl::Status) {})); } TEST_F(FileSystemHttpCacheTestWithMockFiles, FailedOpenForReadReturnsMiss) { auto lookup = testLookupContext(); absl::Cleanup destroy_lookup([&lookup]() { lookup->onDestroy(); }); LookupResult result; - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); lookup->getHeaders([&](LookupResult&& r, bool /*end_stream*/) { result = std::move(r); }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("Intentionally failed to open file"))); + pumpDispatcher(); // File handle didn't get used but is expected to be closed. - EXPECT_OK(mock_async_file_handle_->close([](absl::Status) {})); + EXPECT_OK(mock_async_file_handle_->close(nullptr, [](absl::Status) {})); EXPECT_EQ(result.cache_entry_status_, CacheEntryStatus::Unusable); EXPECT_EQ(cache_->stats().cache_miss_.value(), 1); EXPECT_EQ(cache_->stats().cache_hit_.value(), 0); @@ -480,21 +492,25 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, FailedReadOfHeaderBlockInvalidatesT auto lookup = testLookupContext(); absl::Cleanup destroy_lookup([&lookup]() { lookup->onDestroy(); }); LookupResult result; - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); lookup->getHeaders([&](LookupResult&& r, bool /*end_stream*/) { result = std::move(r); }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); - EXPECT_CALL(*mock_async_file_manager_, stat(_, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); + pumpDispatcher(); + EXPECT_CALL(*mock_async_file_manager_, stat(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("intentional failure to read"))); + pumpDispatcher(); struct stat stat_result = {}; stat_result.st_size = 12345; // stat mock_async_file_manager_->nextActionCompletes(absl::StatusOr{stat_result}); + pumpDispatcher(); // unlink mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); EXPECT_EQ(result.cache_entry_status_, CacheEntryStatus::Unusable); waitForEvictionThreadIdle(); // Should have deducted the size of the file that got deleted. Since we started at 2 * 12345, @@ -519,19 +535,23 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, ReadWithInvalidHeaderBlockInvalidat auto lookup = testLookupContext(); absl::Cleanup destroy_lookup([&lookup]() { lookup->onDestroy(); }); LookupResult result; - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); lookup->getHeaders([&](LookupResult&& r, bool /*end_stream*/) { result = std::move(r); }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); - EXPECT_CALL(*mock_async_file_manager_, stat(_, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); + pumpDispatcher(); + EXPECT_CALL(*mock_async_file_manager_, stat(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(invalidHeaderBlock())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr{ absl::UnknownError("intentionally failed to stat, for coverage")}); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::UnknownError("intentionally failed to unlink, for coverage")); + pumpDispatcher(); EXPECT_EQ(result.cache_entry_status_, CacheEntryStatus::Unusable); } @@ -539,22 +559,27 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, FailedReadOfHeaderProtoInvalidatesT auto lookup = testLookupContext(); absl::Cleanup destroy_lookup([&lookup]() { lookup->onDestroy(); }); LookupResult result; - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); lookup->getHeaders([&](LookupResult&& r, bool /*end_stream*/) { result = std::move(r); }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(0))); - EXPECT_CALL(*mock_async_file_manager_, stat(_, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); + pumpDispatcher(); + EXPECT_CALL(*mock_async_file_manager_, stat(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("intentional failure to read"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr{ absl::UnknownError("intentionally failed to stat, for coverage")}); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::UnknownError("intentionally failed to unlink, for coverage")); + pumpDispatcher(); EXPECT_EQ(result.cache_entry_status_, CacheEntryStatus::Unusable); } @@ -563,34 +588,40 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, FailedReadOfBodyInvalidatesTheCache absl::Cleanup destroy_lookup([&lookup]() { lookup->onDestroy(); }); LookupResult result; bool end_stream_after_headers = true; // initialized wrong to ensure it's set. - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); lookup->getHeaders([&](LookupResult&& r, bool es) { result = std::move(r); end_stream_after_headers = es; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(0))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBuffer())); + pumpDispatcher(); // result should be populated. EXPECT_NE(result.cache_entry_status_, CacheEntryStatus::Unusable); EXPECT_FALSE(end_stream_after_headers); - EXPECT_CALL(*mock_async_file_handle_, read(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, _, _, _)); lookup->getBody(AdjustedByteRange(0, 8), [&](Buffer::InstancePtr body, bool /*end_stream*/) { EXPECT_EQ(body.get(), nullptr); }); - EXPECT_CALL(*mock_async_file_manager_, stat(_, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); + EXPECT_CALL(*mock_async_file_manager_, stat(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("intentional failure to read"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr{ absl::UnknownError("intentionally failed to stat, for coverage")}); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::UnknownError("intentionally failed to unlink, for coverage")); + pumpDispatcher(); } TEST_F(FileSystemHttpCacheTestWithMockFiles, FailedReadOfTrailersInvalidatesTheCacheEntry) { @@ -598,230 +629,268 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, FailedReadOfTrailersInvalidatesTheC absl::Cleanup destroy_lookup([&lookup]() { lookup->onDestroy(); }); LookupResult result; bool end_stream_after_headers = true; - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); lookup->getHeaders([&](LookupResult&& r, bool es) { result = std::move(r); end_stream_after_headers = es; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(0))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBuffer())); + pumpDispatcher(); // result should be populated. EXPECT_NE(result.cache_entry_status_, CacheEntryStatus::Unusable); EXPECT_FALSE(end_stream_after_headers); - EXPECT_CALL(*mock_async_file_handle_, read(_, 8, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, _, 8, _)); lookup->getBody(AdjustedByteRange(0, 8), [&](Buffer::InstancePtr body, bool end_stream) { EXPECT_EQ(body->toString(), "beepbeep"); EXPECT_FALSE(end_stream); }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(std::make_unique("beepbeep"))); - EXPECT_CALL(*mock_async_file_handle_, read(_, _, _)); + pumpDispatcher(); + EXPECT_CALL(*mock_async_file_handle_, read(_, _, _, _)); // No point validating that the trailers are empty since that's not even particularly // desirable behavior - it's a quirk of the filter that we can't properly signify an error. lookup->getTrailers([&](Http::ResponseTrailerMapPtr) {}); - EXPECT_CALL(*mock_async_file_manager_, stat(_, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); + EXPECT_CALL(*mock_async_file_manager_, stat(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); mock_async_file_manager_->nextActionCompletes(absl::StatusOr( absl::UnknownError("intentional failure to read trailers"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr{ absl::UnknownError("intentionally failed to stat, for coverage")}); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::UnknownError("intentionally failed to unlink, for coverage")); + pumpDispatcher(); } TEST_F(FileSystemHttpCacheTestWithMockFiles, ReadWithMultipleBlocksWorksCorrectly) { trailers_size_ = 0; auto lookup = testLookupContext(); LookupResult result; - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); EXPECT_CALL(*mock_async_file_handle_, - read(CacheFileFixedBlock::offsetToHeaders(), headers_size_, _)); + read(_, CacheFileFixedBlock::offsetToHeaders(), headers_size_, _)); lookup->getHeaders([&](LookupResult&& r, bool end_stream) { result = std::move(r); EXPECT_FALSE(end_stream) << "in headers"; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(8))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBuffer())); + pumpDispatcher(); EXPECT_CALL(*mock_async_file_handle_, - read(CacheFileFixedBlock::offsetToHeaders() + headers_size_, 4, _)); + read(_, CacheFileFixedBlock::offsetToHeaders() + headers_size_, 4, _)); EXPECT_CALL(*mock_async_file_handle_, - read(CacheFileFixedBlock::offsetToHeaders() + headers_size_ + 4, 4, _)); + read(_, CacheFileFixedBlock::offsetToHeaders() + headers_size_ + 4, 4, _)); lookup->getBody(AdjustedByteRange(0, 4), [&](Buffer::InstancePtr body, bool end_stream) { EXPECT_EQ(body->toString(), "beep"); EXPECT_FALSE(end_stream) << "in body part 1"; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(std::make_unique("beep"))); + pumpDispatcher(); lookup->getBody(AdjustedByteRange(4, 8), [&](Buffer::InstancePtr body, bool end_stream) { EXPECT_EQ(body->toString(), "boop"); EXPECT_TRUE(end_stream) << "in body part 2"; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(std::make_unique("boop"))); + pumpDispatcher(); // While we're here, incidentally test the behavior of aborting a lookup in progress // while no file actions are in flight. lookup->onDestroy(); lookup.reset(); // There should be a file-close in the queue. mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); } TEST_F(FileSystemHttpCacheTestWithMockFiles, DestroyingALookupWithFileActionInFlightCancelsAction) { auto lookup = testLookupContext(); absl::Cleanup destroy_lookup([&lookup]() { lookup->onDestroy(); }); LookupResult result; - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, mockCancel()).WillOnce([this]() { - mock_async_file_manager_->queue_.pop_front(); - }); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, mockCancel()); lookup->getHeaders([&](LookupResult&& r, bool /*end_stream*/) { result = std::move(r); }); // File wasn't used in this test but is expected to be closed. - EXPECT_OK(mock_async_file_handle_->close([](absl::Status) {})); + EXPECT_OK(mock_async_file_handle_->close(nullptr, [](absl::Status) {})); } TEST_F(FileSystemHttpCacheTestWithMockFiles, DestroyingInsertContextWithFileActionInFlightCancelsAction) { auto inserter = testInserter(); absl::Cleanup destroy_inserter([&inserter]() { inserter->onDestroy(); }); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*mock_async_file_manager_, mockCancel()).WillOnce([this]() { - mock_async_file_manager_->queue_.pop_front(); - }); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, mockCancel()); inserter->insertHeaders(response_headers_, metadata_, expect_false_callback_, false); // File wasn't used in this test but is expected to be closed. - EXPECT_OK(mock_async_file_handle_->close([](absl::Status) {})); + EXPECT_OK(mock_async_file_handle_->close(nullptr, [](absl::Status) {})); } TEST_F(FileSystemHttpCacheTestWithMockFiles, InsertAbortsOnFailureToWriteEmptyHeaderBlock) { auto inserter = testInserter(); absl::Cleanup destroy_inserter([&inserter]() { inserter->onDestroy(); }); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*mock_async_file_handle_, write(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, write(_, _, _, _)); inserter->insertHeaders(response_headers_, metadata_, expect_false_callback_, false); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr( absl::UnknownError("intentionally failed write to empty header block"))); + pumpDispatcher(); EXPECT_EQ(false_callbacks_called_, 1); } TEST_F(FileSystemHttpCacheTestWithMockFiles, InsertAbortsOnFailureToWriteHeaderChunk) { auto inserter = testInserter(); absl::Cleanup destroy_inserter([&inserter]() { inserter->onDestroy(); }); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*mock_async_file_handle_, write(_, _, _)).Times(2); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, write(_, _, _, _)).Times(2); inserter->insertHeaders(response_headers_, metadata_, expect_false_callback_, false); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(CacheFileFixedBlock::size())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("intentionally failed write of header chunk"))); + pumpDispatcher(); EXPECT_EQ(false_callbacks_called_, 1); } TEST_F(FileSystemHttpCacheTestWithMockFiles, InsertAbortsOnFailureToWriteBodyChunk) { auto inserter = testInserter(); absl::Cleanup destroy_inserter([&inserter]() { inserter->onDestroy(); }); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*mock_async_file_handle_, write(_, _, _)).Times(3); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, write(_, _, _, _)).Times(3); inserter->insertHeaders(response_headers_, metadata_, expect_true_callback_, false); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(CacheFileFixedBlock::size())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(headers_size_)); + pumpDispatcher(); EXPECT_EQ(true_callbacks_called_, 1); inserter->insertBody(Buffer::OwnedImpl("woop"), expect_false_callback_, false); // Intentionally undersized write of body chunk. mock_async_file_manager_->nextActionCompletes(absl::StatusOr(1)); + pumpDispatcher(); EXPECT_EQ(false_callbacks_called_, 1); } TEST_F(FileSystemHttpCacheTestWithMockFiles, InsertAbortsOnFailureToWriteTrailerChunk) { auto inserter = testInserter(); absl::Cleanup destroy_inserter([&inserter]() { inserter->onDestroy(); }); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*mock_async_file_handle_, write(_, _, _)).Times(4); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, write(_, _, _, _)).Times(4); inserter->insertHeaders(response_headers_, metadata_, expect_true_callback_, false); const absl::string_view body = "woop"; mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(CacheFileFixedBlock::size())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(headers_size_)); + pumpDispatcher(); EXPECT_EQ(true_callbacks_called_, 1); inserter->insertBody(Buffer::OwnedImpl(body), expect_true_callback_, false); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(body.size())); + pumpDispatcher(); EXPECT_EQ(true_callbacks_called_, 2); inserter->insertTrailers(response_trailers_, expect_false_callback_); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("intentionally failed write of trailer chunk"))); + pumpDispatcher(); EXPECT_EQ(false_callbacks_called_, 1); } TEST_F(FileSystemHttpCacheTestWithMockFiles, InsertAbortsOnFailureToWriteUpdatedHeaderBlock) { auto inserter = testInserter(); absl::Cleanup destroy_inserter([&inserter]() { inserter->onDestroy(); }); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*mock_async_file_handle_, write(_, _, _)).Times(5); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, write(_, _, _, _)).Times(5); inserter->insertHeaders(response_headers_, metadata_, expect_true_callback_, false); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(CacheFileFixedBlock::size())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(headers_size_)); + pumpDispatcher(); EXPECT_EQ(true_callbacks_called_, 1); const absl::string_view body = "woop"; inserter->insertBody(Buffer::OwnedImpl(body), expect_true_callback_, false); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(body.size())); + pumpDispatcher(); EXPECT_EQ(true_callbacks_called_, 2); inserter->insertTrailers(response_trailers_, expect_false_callback_); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(trailers_size_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr( absl::UnknownError("intentionally failed write of updated header block"))); + pumpDispatcher(); EXPECT_EQ(false_callbacks_called_, 1); } TEST_F(FileSystemHttpCacheTestWithMockFiles, InsertAbortsOnFailureToLinkFile) { auto inserter = testInserter(); absl::Cleanup destroy_inserter([&inserter]() { inserter->onDestroy(); }); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*mock_async_file_handle_, write(_, _, _)).Times(5); - EXPECT_CALL(*mock_async_file_manager_, stat(_, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, createHardLink(_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, write(_, _, _, _)).Times(5); + EXPECT_CALL(*mock_async_file_manager_, stat(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, createHardLink(_, _, _)); inserter->insertHeaders(response_headers_, metadata_, expect_true_callback_, false); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(CacheFileFixedBlock::size())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(headers_size_)); + pumpDispatcher(); EXPECT_EQ(true_callbacks_called_, 1); const absl::string_view body = "woop"; inserter->insertBody(Buffer::OwnedImpl(body), expect_true_callback_, false); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(body.size())); + pumpDispatcher(); EXPECT_EQ(true_callbacks_called_, 2); inserter->insertTrailers(response_trailers_, expect_false_callback_); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(trailers_size_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(CacheFileFixedBlock::size())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr{ absl::UnknownError("intentionally failed to stat, for coverage")}); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::UnknownError("intentionally failed to link cache file")); + pumpDispatcher(); EXPECT_EQ(false_callbacks_called_, 1); } @@ -834,16 +903,17 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsIfFileOpenFailed {"cache-control", "public,max-age=3600"}, }; auto lookup_context = testLookupContext(); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("Intentionally failed to open file"))); + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); // File is not used in this test, but is expected to be closed. - EXPECT_OK(mock_async_file_handle_->close([](absl::Status) {})); + EXPECT_OK(mock_async_file_handle_->close(nullptr, [](absl::Status) {})); } TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersKeepsTryingIfUnlinkOriginalFails) { @@ -855,19 +925,23 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersKeepsTryingIfUnlinkOri {"cache-control", "public,max-age=3600"}, }; auto lookup_context = testLookupContext(); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::UnknownError("Intentionally failed to unlink")); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr( absl::UnknownError("Intentionally failed to read header block"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close read handle + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); } @@ -881,21 +955,26 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsIfReadHeadersFai {"cache-control", "public,max-age=3600"}, }; auto lookup_context = testLookupContext(); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(0))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr( absl::UnknownError("Intentionally failed to read headers block"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close read handle + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); } @@ -918,22 +997,27 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsIfReadHeadersFin {"cache-control", "public,max-age=3600"}, }; auto lookup_context = testLookupContext(); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); EXPECT_CALL(*mock_async_file_handle_, - read(CacheFileFixedBlock::size(), vary_headers_buffer->length(), _)); + read(_, CacheFileFixedBlock::size(), vary_headers_buffer->length(), _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(std::move(vary_block_buffer))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(std::move(vary_headers_buffer))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close read handle + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); } @@ -947,24 +1031,30 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsIfOpenForWriteFa {"cache-control", "public,max-age=3600"}, }; auto lookup_context = testLookupContext(); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(0))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBuffer())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr( absl::UnknownError("Intentionally failed to create file for write"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close read handle + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); } @@ -980,27 +1070,35 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsIfWriteHeaderBlo auto lookup_context = testLookupContext(); MockAsyncFileHandle write_handle = std::make_shared>(mock_async_file_manager_); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*write_handle, write(_, 0, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*write_handle, write(_, _, 0, _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(0))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBuffer())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(write_handle)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("Intentionally failed to write header block"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close read handle + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close write handle + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); } @@ -1019,31 +1117,40 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsIfReadBodyFails) auto lookup_context = testLookupContext(); MockAsyncFileHandle write_handle = std::make_shared>(mock_async_file_manager_); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*write_handle, write(_, 0, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*write_handle, write(_, _, 0, _)); EXPECT_CALL(*mock_async_file_handle_, - read(CacheFileFixedBlock::size() + headers_size_, body_size + trailers_size_, _)); + read(_, CacheFileFixedBlock::size() + headers_size_, body_size + trailers_size_, _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(body_size))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBuffer())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(write_handle)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(updated_headers_size + CacheFileFixedBlock::size())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("intentionally failed body read"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close read handle + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close write handle + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); } @@ -1062,36 +1169,46 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsIfWriteBodyFails auto lookup_context = testLookupContext(); MockAsyncFileHandle write_handle = std::make_shared>(mock_async_file_manager_); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*write_handle, write(_, 0, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*write_handle, write(_, _, 0, _)); EXPECT_CALL(*mock_async_file_handle_, - read(CacheFileFixedBlock::size() + headers_size_, body_size + trailers_size_, _)); - EXPECT_CALL(*write_handle, write(_, CacheFileFixedBlock::size() + updated_headers_size, _)); + read(_, CacheFileFixedBlock::size() + headers_size_, body_size + trailers_size_, _)); + EXPECT_CALL(*write_handle, write(_, _, CacheFileFixedBlock::size() + updated_headers_size, _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(body_size))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBuffer())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(write_handle)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(updated_headers_size + CacheFileFixedBlock::size())); + pumpDispatcher(); std::string body_and_trailers; body_and_trailers.resize(body_size + trailers_size_); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(std::make_unique(body_and_trailers))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("intentionally failed body write"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close read handle + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close write handle + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); } @@ -1110,19 +1227,20 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersCopiesInChunksIfBodySi auto lookup_context = testLookupContext(); MockAsyncFileHandle write_handle = std::make_shared>(mock_async_file_manager_); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*write_handle, write(_, 0, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*write_handle, write(_, _, 0, _)); EXPECT_CALL(*mock_async_file_handle_, - read(CacheFileFixedBlock::size() + headers_size_, + read(_, CacheFileFixedBlock::size() + headers_size_, FileSystemHttpCache::max_update_headers_copy_chunk_size_, _)); - EXPECT_CALL(*write_handle, write(_, CacheFileFixedBlock::size() + updated_headers_size, _)); + EXPECT_CALL(*write_handle, write(_, _, CacheFileFixedBlock::size() + updated_headers_size, _)); EXPECT_CALL( *mock_async_file_handle_, - read(CacheFileFixedBlock::size() + headers_size_ + + read(_, + CacheFileFixedBlock::size() + headers_size_ + FileSystemHttpCache::max_update_headers_copy_chunk_size_, body_size + trailers_size_ - FileSystemHttpCache::max_update_headers_copy_chunk_size_, _)); @@ -1131,24 +1249,35 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersCopiesInChunksIfBodySi [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(body_size))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBuffer())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(write_handle)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(updated_headers_size + CacheFileFixedBlock::size())); + pumpDispatcher(); std::string body_chunk; body_chunk.resize(FileSystemHttpCache::max_update_headers_copy_chunk_size_); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(std::make_unique(body_chunk))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(FileSystemHttpCache::max_update_headers_copy_chunk_size_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr( absl::UnknownError("intentionally failed second body read"))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close read handle + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close write handle + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); } @@ -1167,37 +1296,48 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsIfLinkFails) { auto lookup_context = testLookupContext(); MockAsyncFileHandle write_handle = std::make_shared>(mock_async_file_manager_); - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); - EXPECT_CALL(*mock_async_file_manager_, unlink(_, _)); - EXPECT_CALL(*mock_async_file_handle_, read(0, CacheFileFixedBlock::size(), _)); - EXPECT_CALL(*mock_async_file_handle_, read(CacheFileFixedBlock::size(), headers_size_, _)); - EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _)); - EXPECT_CALL(*write_handle, write(_, 0, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); + EXPECT_CALL(*mock_async_file_manager_, unlink(_, _, _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, 0, CacheFileFixedBlock::size(), _)); + EXPECT_CALL(*mock_async_file_handle_, read(_, CacheFileFixedBlock::size(), headers_size_, _)); + EXPECT_CALL(*mock_async_file_manager_, createAnonymousFile(_, _, _)); + EXPECT_CALL(*write_handle, write(_, _, 0, _)); EXPECT_CALL(*mock_async_file_handle_, - read(CacheFileFixedBlock::size() + headers_size_, body_size + trailers_size_, _)); - EXPECT_CALL(*write_handle, write(_, CacheFileFixedBlock::size() + updated_headers_size, _)); - EXPECT_CALL(*write_handle, createHardLink(_, _)); + read(_, CacheFileFixedBlock::size() + headers_size_, body_size + trailers_size_, _)); + EXPECT_CALL(*write_handle, write(_, _, CacheFileFixedBlock::size() + updated_headers_size, _)); + EXPECT_CALL(*write_handle, createHardLink(_, _, _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(mock_async_file_handle_)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBlock(body_size))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(testHeaderBuffer())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(write_handle)); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(updated_headers_size + CacheFileFixedBlock::size())); + pumpDispatcher(); std::string body_and_trailers; body_and_trailers.resize(body_size + trailers_size_); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(std::make_unique(body_and_trailers))); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::StatusOr(body_and_trailers.size())); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::UnknownError("intentionally failed to link")); + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close read handle + pumpDispatcher(); mock_async_file_manager_->nextActionCompletes(absl::OkStatus()); // close write handle + pumpDispatcher(); lookup_context->onDestroy(); EXPECT_FALSE(update_success); } @@ -1210,7 +1350,7 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsEarlyIfCacheEntr {"x-whatever", "updated"}, {"cache-control", "public,max-age=3600"}, }; - EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _)); + EXPECT_CALL(*mock_async_file_manager_, openExistingFile(_, _, _, _)); bool update_success; cache_->updateHeaders(*lookup_context, response_headers, {time_system_.systemTime()}, [&update_success](bool success) { update_success = success; }); @@ -1221,10 +1361,11 @@ TEST_F(FileSystemHttpCacheTestWithMockFiles, UpdateHeadersAbortsEarlyIfCacheEntr [&update_success](bool success) { update_success = success; }); mock_async_file_manager_->nextActionCompletes( absl::StatusOr(absl::UnknownError("intentionally failed to open file"))); + pumpDispatcher(); lookup_context->onDestroy(); lookup_context_2->onDestroy(); // The file handle didn't actually get used in this test, but is expected to be closed. - EXPECT_OK(mock_async_file_handle_->close([](absl::Status) {})); + EXPECT_OK(mock_async_file_handle_->close(nullptr, [](absl::Status) {})); } // For the standard cache tests from http_cache_implementation_test_common.cc @@ -1236,6 +1377,7 @@ class FileSystemHttpCacheTestDelegate : public HttpCacheTestDelegate, FileSystemHttpCacheTestDelegate() { initCache(); } std::shared_ptr cache() override { return cache_; } bool validationEnabled() const override { return true; } + void beforePumpingDispatcher() override { cache_->drainAsyncFileActionsForTest(); } }; // For the standard cache tests from http_cache_implementation_test_common.cc From 975a005b5722be0ad81dadcf97964038514c45f3 Mon Sep 17 00:00:00 2001 From: "dependency-envoy[bot]" <148525496+dependency-envoy[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 17:50:41 +0100 Subject: [PATCH 12/28] deps: Bump `rules_rust` -> 0.50.1 (#36194) Fix #36087 Signed-off-by: dependency-envoy[bot] <148525496+dependency-envoy[bot]@users.noreply.github.com> Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 94f70cfb36..3ed519854c 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1452,9 +1452,9 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Bazel rust rules", project_desc = "Bazel rust rules (used by Wasm)", project_url = "https://github.com/bazelbuild/rules_rust", - version = "0.48.0", + version = "0.50.1", strip_prefix = "rules_rust-{version}", - sha256 = "a4b8ede7723088dff1e909632c4282e51ddbe0e44c38eea013ee0f12d348b1c7", + sha256 = "ddfc0210b19498086d09c458672ef2a6fb7790103dbb1b2da73c54677c330ed1", urls = ["https://github.com/bazelbuild/rules_rust/archive/{version}.tar.gz"], use_category = [ "controlplane", @@ -1463,7 +1463,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( ], implied_untracked_deps = ["rules_cc"], extensions = ["envoy.wasm.runtime.wasmtime"], - release_date = "2024-07-19", + release_date = "2024-09-11", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/bazelbuild/rules_rust/blob/{version}/LICENSE.txt", From c6aa25c0dfc17a8f8df4fd52fc7665e1799aa142 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 18 Sep 2024 17:05:34 -0400 Subject: [PATCH 13/28] tooling: narrowing exception checks (#36185) I reviewed a PR adding exceptions to a previously exception free path without realizing it. Narrowing the checks so we're more likely to catch this in CI. Risk Level: n/a (tooling only) Testing: ci Docs Changes: n/a Release Notes: n/a part of https://github.com/envoyproxy/envoy/issues/27412 Signed-off-by: Alyssa Wilk --- tools/code_format/config.yaml | 86 +++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index bbc5d3f81d..5b1d73eda6 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -160,36 +160,86 @@ paths: # Extensions can exempt entire directories but new extensions # points should ideally use StatusOr - source/extensions/access_loggers - - source/extensions/bootstrap - - source/extensions/clusters + - source/extensions/bootstrap/wasm/config.cc + - source/extensions/clusters/eds/ + - source/extensions/clusters/logical_dns + - source/extensions/clusters/original_dst + - source/extensions/clusters/redis + - source/extensions/clusters/static + - source/extensions/clusters/strict_dns - source/extensions/common - - source/extensions/config + - source/extensions/config/validators/minimum_clusters/minimum_clusters_validator.cc - source/extensions/config_subscription - - source/extensions/compression/zstd/common - - source/extensions/early_data - - source/extensions/filters + - source/extensions/compression/zstd/common/dictionary_manager.h + - source/extensions/filters/http/adaptive_concurrency/controller + - source/extensions/filters/http/admission_control + - source/extensions/filters/http/aws_lambda + - source/extensions/filters/http/aws_request_signing + - source/extensions/filters/http/bandwidth_limit + - source/extensions/filters/http/basic_auth + - source/extensions/filters/http/cache + - source/extensions/filters/http/cdn_loop + - source/extensions/filters/http/common + - source/extensions/filters/http/composite + - source/extensions/filters/http/ext_authz + - source/extensions/filters/http/ext_proc + - source/extensions/filters/http/file_system_buffer + - source/extensions/filters/http/gcp_authn + - source/extensions/filters/http/grpc_field_extraction + - source/extensions/filters/http/grpc_json_transcoder + - source/extensions/filters/http/grpc_stats + - source/extensions/filters/http/header_mutation + - source/extensions/filters/http/header_to_metadata + - source/extensions/filters/http/health_check + - source/extensions/filters/http/ip_tagging + - source/extensions/filters/http/json_to_metadata + - source/extensions/filters/http/jwt_authn + - source/extensions/filters/http/local_ratelimit + - source/extensions/filters/http/oauth2 + - source/extensions/filters/http/file_system_buffer + - source/extensions/filters/http/header_to_metadata + - source/extensions/filters/http/on_demand + - source/extensions/filters/http/json_to_metadata + - source/extensions/filters/http/json_to_metadata + - source/extensions/filters/http/thrift_to_metadata + - source/extensions/filters/http/lua + - source/extensions/filters/http/proto_message_extraction + - source/extensions/filters/http/rate_limit_quota + - source/extensions/filters/http/ratelimit + - source/extensions/filters/http/oauth2 + - source/extensions/filters/http/wasm + - source/extensions/filters/network + - source/extensions/filters/common + - source/extensions/filters/udp + - source/extensions/filters/listener - source/extensions/formatter - - source/extensions/geoip_providers + - source/extensions/geoip_providers/maxmind/geoip_provider.cc - source/extensions/grpc_credentials - - source/extensions/health_check + - source/extensions/health_check/event_sinks/file/file_sink_impl.h - source/extensions/health_checkers - - source/extensions/http - - source/extensions/io_socket + - source/extensions/http/cache/file_system_http_cache/config.cc + - source/extensions/http/custom_response + - source/extensions/http/early_header_mutation + - source/extensions/http/injected_credentials + - source/extensions/http/original_ip_detection + - source/extensions/http/stateful_session + - source/extensions/io_socket/user_space - source/extensions/key_value - - source/extensions/listener_managers - - source/extensions/load_balancing_policies + - source/extensions/load_balancing_policies/maglev + - source/extensions/load_balancing_policies/ring_hash + - source/extensions/load_balancing_policies/subset - source/extensions/matching - - source/extensions/network - - source/extensions/path + - source/extensions/network/dns_resolver/cares/ - source/extensions/quic/server_preferred_address - source/extensions/rate_limit_descriptors - source/extensions/resource_monitors - - source/extensions/retry - - source/extensions/router + - source/extensions/router/cluster_specifiers/lua/lua_cluster_specifier.cc - source/extensions/stat_sinks - - source/extensions/string_matcher + - source/extensions/string_matcher/lua/match.cc - source/extensions/tracers - - source/extensions/transport_sockets + - source/extensions/transport_sockets/internal_upstream + - source/extensions/transport_sockets/tls/cert_validator + - source/extensions/transport_sockets/tcp_stats/config.cc # Only one C++ file should instantiate grpc_init grpc_init: From a2d9eba9188e14d6238e779755b1f2bec73463d6 Mon Sep 17 00:00:00 2001 From: Adam Kotwasinski Date: Wed, 18 Sep 2024 16:03:15 -0700 Subject: [PATCH 14/28] kafka: upgrade to 3.8, add support for more requests (#36166) Commit Message: kafka: upgrade to 3.8, add support for more requests Additional Description: upgrade kafka dependency to 3.8, add necessary parsing code and deserializer to process new stuff present in 3.8 (nullable struct) Risk Level: low Testing: automated suite + manual with [envoy-kafka-tests](https://github.com/adamkotwasinski/envoy-kafka-tests/pull/13) Docs Changes: readme updates due to version bump Release Notes: n/a Platform Specific Features: n/a --------- Signed-off-by: Adam Kotwasinski --- bazel/repository_locations.bzl | 12 +- .../network/source/protocol/generator.py | 69 +++++++----- .../filters/network/source/serialization.h | 104 ++++++++++++++++++ .../network/test/serialization_test.cc | 16 +++ .../network_filters/kafka_broker_filter.rst | 5 +- .../network_filters/kafka_mesh_filter.rst | 2 +- 6 files changed, 172 insertions(+), 36 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 3ed519854c..6752ec066c 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1333,13 +1333,13 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Kafka (source)", project_desc = "Open-source distributed event streaming platform", project_url = "https://kafka.apache.org", - version = "3.5.1", - sha256 = "9715589a02148fb21bc80d79f29763dbd371457bedcbbeab3db4f5c7fdd2d29c", + version = "3.8.0", + sha256 = "8761a0c22738201d3049f11f78c8e6c0f201203ba799157e498ef7eb04c259f3", strip_prefix = "kafka-{version}/clients/src/main/resources/common/message", urls = ["https://github.com/apache/kafka/archive/{version}.zip"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.network.kafka_broker", "envoy.filters.network.kafka_mesh"], - release_date = "2023-07-14", + release_date = "2024-07-23", cpe = "cpe:2.3:a:apache:kafka:*", license = "Apache-2.0", license_url = "https://github.com/apache/kafka/blob/{version}/LICENSE", @@ -1363,11 +1363,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Kafka (server binary)", project_desc = "Open-source distributed event streaming platform", project_url = "https://kafka.apache.org", - version = "3.5.1", - sha256 = "f7b74d544023f2c0ec52a179de59975cb64e34ea03650d829328b407b560e4da", + version = "3.8.0", + sha256 = "e0297cc6fdb09ef9d9905751b25d2b629c17528f8629b60561eeff87ce29099c", strip_prefix = "kafka_2.13-{version}", urls = ["https://archive.apache.org/dist/kafka/{version}/kafka_2.13-{version}.tgz"], - release_date = "2023-07-21", + release_date = "2024-07-23", use_category = ["test_only"], ), proxy_wasm_cpp_sdk = dict( diff --git a/contrib/kafka/filters/network/source/protocol/generator.py b/contrib/kafka/filters/network/source/protocol/generator.py index 31762b1757..656856219d 100755 --- a/contrib/kafka/filters/network/source/protocol/generator.py +++ b/contrib/kafka/filters/network/source/protocol/generator.py @@ -153,9 +153,8 @@ def parse_messages(self, input_files): amended = re.sub(r'-2147483648', 'INT32_MIN', without_empty_newlines) message_spec = json.loads(amended) api_key = message_spec['apiKey'] - # (adam.kotwasinski) Higher API keys in the future versions of Kafka need - # some more changes to parse. - if api_key < 68 or api_key == 69: + # (adam.kotwasinski) Telemetry is not supported for now. + if api_key not in [71, 72]: message = self.parse_top_level_element(message_spec) messages.append(message) except Exception as e: @@ -224,8 +223,9 @@ def parse_complex_type(self, type_name, field_spec, versions): fields.append(child) # Some structures share the same name, use request/response as prefix. - if cpp_name in ['EntityData', 'EntryData', 'PartitionData', 'PartitionSnapshot', - 'SnapshotId', 'TopicData', 'TopicPartitions', 'TopicSnapshot']: + if cpp_name in ['Cursor', 'DirectoryData', 'EntityData', 'EntryData', 'PartitionData', + 'PartitionSnapshot', 'SnapshotId', 'TopicData', 'TopicPartitions', + 'TopicSnapshot']: cpp_name = self.type.capitalize() + type_name # Some of the types repeat multiple times (e.g. AlterableConfig). @@ -370,9 +370,9 @@ def example_value(self): class FieldSpec: """ - Represents a field present in a structure (request, or child structure thereof). - Contains name, type, and versions when it is used (nullable or not). - """ + Represents a field present in a structure (request, or child structure thereof). + Contains name, type, and versions when it is used (nullable or not). + """ def __init__(self, name, type, version_usage, version_usage_as_nullable): import re @@ -387,10 +387,10 @@ def is_nullable(self): def is_nullable_in_version(self, version): """ - Whether the field is nullable in given version. - Fields can be non-nullable in earlier versions. - See https://github.com/apache/kafka/tree/2.2.0-rc0/clients/src/main/resources/common/message#nullable-fields - """ + Whether the field is nullable in given version. + Fields can be non-nullable in earlier versions. + See https://github.com/apache/kafka/tree/3.8.0/clients/src/main/resources/common/message#nullable-fields + """ return version in self.version_usage_as_nullable def used_in_version(self, version): @@ -428,13 +428,21 @@ def example_value_for_test(self, version): def deserializer_name_in_version(self, version, compact): if self.is_nullable_in_version(version): - return 'Nullable%s' % self.type.deserializer_name_in_version(version, compact) + underlying_deserializer = self.type.deserializer_name_in_version(version, compact) + # Handles KAFKA-14425 - structs (complex types) can now be nullable. + if isinstance(self.type, Complex): + return 'NullableStructDeserializer<%s>' % underlying_deserializer + else: + return 'Nullable%s' % underlying_deserializer else: return self.type.deserializer_name_in_version(version, compact) def is_printable(self): return self.type.is_printable() + def __str__(self): + return '%s(%s)' % (self.name, self.type) + class TypeSpecification: @@ -471,10 +479,10 @@ def is_printable(self): class Array(TypeSpecification): """ - Represents array complex type. - To use instance of this type, it is necessary to declare structures required by self.underlying - (e.g. to use Array, we need to have `struct Foo {...}`). - """ + Represents array complex type. + To use instance of this type, it is necessary to declare structures required by self.underlying + (e.g. to use Array, we need to have `struct Foo {...}`). + """ def __init__(self, underlying): self.underlying = underlying @@ -505,6 +513,9 @@ def example_value_for_test(self, version): def is_printable(self): return self.underlying.is_printable() + def __str__(self): + return self.name + class Primitive(TypeSpecification): """ @@ -643,6 +654,9 @@ def example_value_for_test(self, version): def is_printable(self): return self.name not in ['Bytes'] + def __str__(self): + return self.name + class FieldSerializationSpec(): @@ -679,9 +693,9 @@ def register_flexible_versions(self, flexible_versions): def compute_declaration_chain(self): """ - Computes all dependencies, what means all non-primitive types used by this type. - They need to be declared before this struct is declared. - """ + Computes all dependencies, what means all non-primitive types used by this type. + They need to be declared before this struct is declared. + """ result = [] for field in self.fields: field_dependencies = field.type.compute_declaration_chain() @@ -700,10 +714,10 @@ def get_extra(self, key): def compute_constructors(self): """ - Field lists for different versions may not differ (as Kafka can bump version without any - changes). But constructors need to be unique, so we need to remove duplicates if the signatures - match. - """ + Field lists for different versions may not differ (as Kafka can bump version without any + changes). But constructors need to be unique, so we need to remove duplicates + if the signatures match. + """ signature_to_constructor = {} for field_list in self.compute_field_lists(): signature = field_list.constructor_signature() @@ -724,8 +738,8 @@ def compute_constructors(self): def compute_field_lists(self): """ - Return field lists representing each of structure versions. - """ + Return field lists representing each of structure versions. + """ field_lists = [] for version in self.versions: field_list = FieldList(version, version in self.flexible_versions, self.fields) @@ -772,6 +786,9 @@ def example_value_for_test(self, version): def is_printable(self): return True + def __str__(self): + return self.name + class RenderingHelper: """ diff --git a/contrib/kafka/filters/network/source/serialization.h b/contrib/kafka/filters/network/source/serialization.h index 902fbd9c00..f2ea6a7545 100644 --- a/contrib/kafka/filters/network/source/serialization.h +++ b/contrib/kafka/filters/network/source/serialization.h @@ -918,6 +918,72 @@ class NullableCompactArrayDeserializer bool ready_{false}; }; +/** + * Nullable objects are sent as single byte and following data. + * Reference: https://issues.apache.org/jira/browse/KAFKA-14425 + */ +template +class NullableStructDeserializer + : public Deserializer> { +public: + using ResponseType = absl::optional; + + uint32_t feed(absl::string_view& data) override { + + if (data.empty()) { + return 0; + } + + uint32_t bytes_read = 0; + + if (!marker_consumed_) { + // Read marker byte from input. + int8_t marker; + safeMemcpy(&marker, data.data()); + data = {data.data() + 1, data.size() - 1}; + bytes_read += 1; + marker_consumed_ = true; + + if (marker >= 0) { + data_buf_ = absl::make_optional(DeserializerType()); + } else { + return bytes_read; + } + } + + if (data_buf_) { + bytes_read += data_buf_->feed(data); + } + + return bytes_read; + } + + bool ready() const override { + if (marker_consumed_) { + if (data_buf_) { + return data_buf_->ready(); + } else { + return true; // It's an empty optional. + } + } else { + return false; + } + } + + ResponseType get() const override { + if (data_buf_) { + const typename ResponseType::value_type deserialized_form = data_buf_->get(); + return absl::make_optional(deserialized_form); + } else { + return absl::nullopt; + } + } + +private: + bool marker_consumed_{false}; + absl::optional data_buf_; // Present if marker was consumed and was 0 or more. +}; + /** * Kafka UUID is basically two longs, so we are going to keep model them the same way. * Reference: @@ -996,6 +1062,12 @@ class EncodingContext { */ template uint32_t computeSize(const NullableArray& arg) const; + /** + * Compute size of given nullable object, if it were to be encoded. + * @return serialized size of argument. + */ + template uint32_t computeSize(const absl::optional& arg) const; + /** * Compute size of given reference, if it were to be compactly encoded. * @return serialized size of argument. @@ -1032,6 +1104,12 @@ class EncodingContext { */ template uint32_t encode(const NullableArray& arg, Buffer::Instance& dst); + /** + * Encode given nullable object in a buffer. + * @return bytes written + */ + template uint32_t encode(const absl::optional& arg, Buffer::Instance& dst); + /** * Compactly encode given reference in a buffer. * @return bytes written. @@ -1135,6 +1213,15 @@ inline uint32_t EncodingContext::computeSize(const NullableArray& arg) const return arg ? computeSize(*arg) : sizeof(int32_t); } +/** + * Template overload for nullable T. + * The size of nullable object is 1 (for market byte) and the size of real object (if any). + */ +template +inline uint32_t EncodingContext::computeSize(const absl::optional& arg) const { + return 1 + (arg ? computeSize(*arg) : 0); +} + /** * Template overload for Uuid. */ @@ -1388,6 +1475,23 @@ uint32_t EncodingContext::encode(const NullableArray& arg, Buffer::Instance& } } +/** + * Encode nullable object as marker byte (1 if present, -1 otherwise), then if object is present, + * have it serialise itself. + */ +template +uint32_t EncodingContext::encode(const absl::optional& arg, Buffer::Instance& dst) { + if (arg) { + const int8_t marker = 1; + encode(marker, dst); + const uint32_t written = encode(*arg, dst); + return 1 + written; + } else { + const int8_t marker = -1; + return encode(marker, dst); + } +} + /** * Template overload for Uuid. */ diff --git a/contrib/kafka/filters/network/test/serialization_test.cc b/contrib/kafka/filters/network/test/serialization_test.cc index 59aa0e567b..bbb80c3bd8 100644 --- a/contrib/kafka/filters/network/test/serialization_test.cc +++ b/contrib/kafka/filters/network/test/serialization_test.cc @@ -41,6 +41,8 @@ TEST_EmptyDeserializerShouldNotBeReady(BytesDeserializer); TEST_EmptyDeserializerShouldNotBeReady(CompactBytesDeserializer); TEST_EmptyDeserializerShouldNotBeReady(NullableBytesDeserializer); TEST_EmptyDeserializerShouldNotBeReady(NullableCompactBytesDeserializer); +using ExampleNullableStructDeserializer = NullableStructDeserializer; +TEST_EmptyDeserializerShouldNotBeReady(ExampleNullableStructDeserializer); TEST_EmptyDeserializerShouldNotBeReady(UuidDeserializer); TEST(ArrayDeserializer, EmptyBufferShouldNotBeReady) { @@ -544,6 +546,20 @@ TEST(NullableCompactArrayDeserializer, ShouldConsumeCorrectAmountOfDataForLargeI NullableCompactArrayDeserializer>(value); } +// Nullable struct. + +using ExampleNSD = NullableStructDeserializer; + +TEST(NullableStructDeserializer, ShouldConsumeCorrectAmountOfData) { + const ExampleNSD::ResponseType value = {42}; + serializeThenDeserializeAndCheckEquality(value); +} + +TEST(NullableStructDeserializer, ShouldConsumeNullStruct) { + const ExampleNSD::ResponseType value = absl::nullopt; + serializeThenDeserializeAndCheckEquality(value); +} + // UUID. TEST(UuidDeserializer, ShouldDeserialize) { diff --git a/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst b/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst index b8d0302796..2025137c17 100644 --- a/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst +++ b/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst @@ -5,9 +5,8 @@ Kafka Broker filter The Apache Kafka broker filter decodes the client protocol for `Apache Kafka `_, both the requests and responses in the payload. -The message versions in `Kafka 3.5.1 `_ -are supported (apart from ConsumerGroupHeartbeat - what means consumers configured with -``group.protocol=consumer``). +The message versions in `Kafka 3.8.0 `_ +are supported. By default the filter attempts not to influence the communication between client and brokers, so the messages that could not be decoded (due to Kafka client or broker running a newer version than diff --git a/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst b/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst index 4729473489..6da435ad83 100644 --- a/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst +++ b/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst @@ -12,7 +12,7 @@ clients. The requests received by this filter instance can be forwarded to one of multiple clusters, depending on the configured forwarding rules. -Corresponding message versions from Kafka 3.5.1 are supported. +Corresponding message versions from Kafka 3.8.0 are supported. * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.network.kafka_mesh.v3alpha.KafkaMesh``. * :ref:`v3 API reference ` From 596beacc826b4d0d1ec57c112ce965a070a9d92c Mon Sep 17 00:00:00 2001 From: phlax Date: Thu, 19 Sep 2024 00:04:56 +0100 Subject: [PATCH 15/28] python/deps: Update `cryptography` to resolve vulnerabilities (#36180) Signed-off-by: Ryan Northey --- tools/base/requirements.in | 3 +- tools/base/requirements.txt | 68 +++++++++++++++++-------------------- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 608947efed..0cd07a89d4 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -10,7 +10,7 @@ clang-format==14.0.6 clang-tidy==14.0.6 colorama coloredlogs -cryptography>=42.0.0 +cryptography>=43.0.1 dependatool>=0.2.2 envoy.base.utils>=0.5.0 envoy.code.check>=0.5.12 @@ -36,6 +36,7 @@ ply # `MessageFactory class is deprecated. Please use GetMessageClass() instead of MessageFactory.GetPrototype` protobuf<5.29.0 pygithub +pyopenssl>=24.2.1 pyreadline pyyaml setuptools diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 354b0da3c7..7d8dd8884d 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -458,39 +458,34 @@ coloredlogs==15.0.1 \ crcmod==1.7 \ --hash=sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e # via gsutil -cryptography==42.0.8 \ - --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ - --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ - --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ - --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ - --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ - --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ - --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ - --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ - --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ - --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ - --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ - --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ - --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ - --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ - --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ - --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ - --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ - --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ - --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ - --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ - --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ - --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ - --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ - --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ - --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ - --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ - --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ - --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ - --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ - --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ - --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ - --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e +cryptography==43.0.1 \ + --hash=sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494 \ + --hash=sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806 \ + --hash=sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d \ + --hash=sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062 \ + --hash=sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2 \ + --hash=sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4 \ + --hash=sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1 \ + --hash=sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85 \ + --hash=sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84 \ + --hash=sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042 \ + --hash=sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d \ + --hash=sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962 \ + --hash=sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2 \ + --hash=sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa \ + --hash=sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d \ + --hash=sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365 \ + --hash=sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96 \ + --hash=sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47 \ + --hash=sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d \ + --hash=sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d \ + --hash=sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c \ + --hash=sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb \ + --hash=sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277 \ + --hash=sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172 \ + --hash=sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034 \ + --hash=sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a \ + --hash=sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289 # via # -r requirements.in # aioquic @@ -1152,10 +1147,11 @@ pynacl==1.5.0 \ --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \ --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543 # via pygithub -pyopenssl==24.1.0 \ - --hash=sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad \ - --hash=sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f +pyopenssl==24.2.1 \ + --hash=sha256:4247f0dbe3748d560dcbb2ff3ea01af0f9a1a001ef5f7c4c647956ed8cbf0e95 \ + --hash=sha256:967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d # via + # -r requirements.in # aioquic # gcs-oauth2-boto-plugin # gsutil From 6069f86c529d9ee3712cdd5f7cb738d3f4fb1acf Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Wed, 18 Sep 2024 21:50:39 -0400 Subject: [PATCH 16/28] Using drop_overload category in EDS to report drop_overload stats in LRS (#36047) Using drop_overload category in EDS to report drop_overload stats in LRS. This is a follow up PR to support drop_overload load report service: https://github.com/envoyproxy/envoy/pull/31384 Currently it is reporting with a fixed category "drop_overload". This PR changes it into the category passed in by cluster or EDS policy configuration. --------- Signed-off-by: Yanjun Xiang --- changelogs/current.yaml | 5 ++++ envoy/upstream/thread_local_cluster.h | 10 ++++++++ envoy/upstream/upstream.h | 10 ++++++++ .../common/upstream/cluster_manager_impl.cc | 25 +++++++++++-------- source/common/upstream/cluster_manager_impl.h | 13 +++++++--- .../upstream/health_discovery_service.h | 6 ++++- source/common/upstream/load_stats_reporter.cc | 2 +- source/common/upstream/upstream_impl.cc | 1 + source/common/upstream/upstream_impl.h | 3 +++ source/server/admin/config_dump_handler.cc | 2 +- test/common/upstream/upstream_impl_test.cc | 4 ++- test/mocks/upstream/cluster.cc | 6 +++++ test/mocks/upstream/cluster.h | 3 +++ test/mocks/upstream/thread_local_cluster.cc | 5 ++++ test/mocks/upstream/thread_local_cluster.h | 2 ++ test/server/admin/config_dump_handler_test.cc | 3 ++- 16 files changed, 82 insertions(+), 18 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index e60a462af2..bfc2be2fc5 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -128,6 +128,11 @@ bug_fixes: the number of requests per I/O cycle is configured and an HTTP decoder filter that pauses filter chain is present. This behavior can be reverted by setting the runtime guard ``envoy.reloadable_features.use_filter_manager_state_for_downstream_end_stream`` to false. +- area: upstream + change: | + Fixed a bug using hard coded drop category when reporting drop_overload stats to the load report service. + It is changed to use drop category that is set in + :ref:`category `. - area: proxy_filter change: | Fixed a bug in the ``CONNECT`` implementation that would cause the ``CONNECT`` request created to be invalid when the diff --git a/envoy/upstream/thread_local_cluster.h b/envoy/upstream/thread_local_cluster.h index cb97bc55ef..69a42751c0 100644 --- a/envoy/upstream/thread_local_cluster.h +++ b/envoy/upstream/thread_local_cluster.h @@ -157,10 +157,20 @@ class ThreadLocalCluster { */ virtual UnitFloat dropOverload() const PURE; + /** + * @return the thread local cluster drop_category configuration. + */ + virtual const std::string& dropCategory() const PURE; + /** * Set up the drop_overload value for the thread local cluster. */ virtual void setDropOverload(UnitFloat drop_overload) PURE; + + /** + * Set up the drop_category value for the thread local cluster. + */ + virtual void setDropCategory(absl::string_view drop_category) PURE; }; using ThreadLocalClusterOptRef = absl::optional>; diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h index a4c6c2b889..e307de9471 100644 --- a/envoy/upstream/upstream.h +++ b/envoy/upstream/upstream.h @@ -1315,10 +1315,20 @@ class Cluster { */ virtual UnitFloat dropOverload() const PURE; + /** + * @return the cluster drop_category_ configuration. + */ + virtual const std::string& dropCategory() const PURE; + /** * Set up the drop_overload value for the cluster. */ virtual void setDropOverload(UnitFloat drop_overload) PURE; + + /** + * Set up the drop_category value for the thread local cluster. + */ + virtual void setDropCategory(absl::string_view drop_category) PURE; }; using ClusterSharedPtr = std::shared_ptr; diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index fcdef2ad85..c65cdb5f15 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -1236,15 +1236,18 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_ pending_cluster_creations_.erase(cm_cluster.cluster().info()->name()); const UnitFloat drop_overload = cm_cluster.cluster().dropOverload(); + const std::string drop_category = cm_cluster.cluster().dropCategory(); // Populate the cluster initialization object based on this update. ClusterInitializationObjectConstSharedPtr cluster_initialization_object = - addOrUpdateClusterInitializationObjectIfSupported( - params, cm_cluster.cluster().info(), load_balancer_factory, host_map, drop_overload); + addOrUpdateClusterInitializationObjectIfSupported(params, cm_cluster.cluster().info(), + load_balancer_factory, host_map, + drop_overload, drop_category); tls_.runOnAllThreads([info = cm_cluster.cluster().info(), params = std::move(params), add_or_update_cluster, load_balancer_factory, map = std::move(host_map), cluster_initialization_object = std::move(cluster_initialization_object), - drop_overload](OptRef cluster_manager) { + drop_overload, drop_category = std::move(drop_category)]( + OptRef cluster_manager) { ASSERT(cluster_manager.has_value(), "Expected the ThreadLocalClusterManager to be set during ClusterManagerImpl creation."); @@ -1302,6 +1305,7 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_ if (cluster_manager->thread_local_clusters_[info->name()]) { cluster_manager->thread_local_clusters_[info->name()]->setDropOverload(drop_overload); + cluster_manager->thread_local_clusters_[info->name()]->setDropCategory(drop_category); } for (const auto& per_priority : params.per_priority_update_params_) { cluster_manager->updateClusterMembership( @@ -1338,7 +1342,7 @@ ClusterManagerImpl::ClusterInitializationObjectConstSharedPtr ClusterManagerImpl::addOrUpdateClusterInitializationObjectIfSupported( const ThreadLocalClusterUpdateParams& params, ClusterInfoConstSharedPtr cluster_info, LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, - UnitFloat drop_overload) { + UnitFloat drop_overload, absl::string_view drop_category) { if (!deferralIsSupportedForCluster(cluster_info)) { return nullptr; } @@ -1369,13 +1373,13 @@ ClusterManagerImpl::addOrUpdateClusterInitializationObjectIfSupported( entry->second->per_priority_state_, params, std::move(cluster_info), load_balancer_factory == nullptr ? entry->second->load_balancer_factory_ : load_balancer_factory, - map, drop_overload); + map, drop_overload, drop_category); cluster_initialization_map_[cluster_name] = new_initialization_object; return new_initialization_object; } else { // We need to create a fresh Cluster Initialization Object. auto new_initialization_object = std::make_shared( - params, std::move(cluster_info), load_balancer_factory, map, drop_overload); + params, std::move(cluster_info), load_balancer_factory, map, drop_overload, drop_category); cluster_initialization_map_[cluster_name] = new_initialization_object; return new_initialization_object; } @@ -1409,6 +1413,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::initializeClusterInlineIfExis initialization_object->cross_priority_host_map_); } thread_local_clusters_[cluster]->setDropOverload(initialization_object->drop_overload_); + thread_local_clusters_[cluster]->setDropCategory(initialization_object->drop_category_); // Remove the CIO as we've initialized the cluster. thread_local_deferred_clusters_.erase(entry); @@ -1419,9 +1424,9 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::initializeClusterInlineIfExis ClusterManagerImpl::ClusterInitializationObject::ClusterInitializationObject( const ThreadLocalClusterUpdateParams& params, ClusterInfoConstSharedPtr cluster_info, LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, - UnitFloat drop_overload) + UnitFloat drop_overload, absl::string_view drop_category) : cluster_info_(std::move(cluster_info)), load_balancer_factory_(load_balancer_factory), - cross_priority_host_map_(map), drop_overload_(drop_overload) { + cross_priority_host_map_(map), drop_overload_(drop_overload), drop_category_(drop_category) { // Copy the update since the map is empty. for (const auto& update : params.per_priority_update_params_) { per_priority_state_.emplace(update.priority_, update); @@ -1432,10 +1437,10 @@ ClusterManagerImpl::ClusterInitializationObject::ClusterInitializationObject( const absl::flat_hash_map& per_priority_state, const ThreadLocalClusterUpdateParams& update_params, ClusterInfoConstSharedPtr cluster_info, LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, - UnitFloat drop_overload) + UnitFloat drop_overload, absl::string_view drop_category) : per_priority_state_(per_priority_state), cluster_info_(std::move(cluster_info)), load_balancer_factory_(load_balancer_factory), cross_priority_host_map_(map), - drop_overload_(drop_overload) { + drop_overload_(drop_overload), drop_category_(drop_category) { // Because EDS Clusters receive the entire ClusterLoadAssignment but only // provides the delta we must process the hosts_added and hosts_removed and diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index da8e2eda75..3d4bfa1656 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -427,20 +427,22 @@ class ClusterManagerImpl : public ClusterManager, ClusterInitializationObject(const ThreadLocalClusterUpdateParams& params, ClusterInfoConstSharedPtr cluster_info, LoadBalancerFactorySharedPtr load_balancer_factory, - HostMapConstSharedPtr map, UnitFloat drop_overload); + HostMapConstSharedPtr map, UnitFloat drop_overload, + absl::string_view drop_category); ClusterInitializationObject( const absl::flat_hash_map& per_priority_state, const ThreadLocalClusterUpdateParams& update_params, ClusterInfoConstSharedPtr cluster_info, LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, - UnitFloat drop_overload); + UnitFloat drop_overload, absl::string_view drop_category); absl::flat_hash_map per_priority_state_; const ClusterInfoConstSharedPtr cluster_info_; const LoadBalancerFactorySharedPtr load_balancer_factory_; const HostMapConstSharedPtr cross_priority_host_map_; UnitFloat drop_overload_{0}; + const std::string drop_category_; }; using ClusterInitializationObjectConstSharedPtr = @@ -610,7 +612,11 @@ class ClusterManagerImpl : public ClusterManager, void drainConnPools(DrainConnectionsHostPredicate predicate, ConnectionPool::DrainBehavior behavior); UnitFloat dropOverload() const override { return drop_overload_; } + const std::string& dropCategory() const override { return drop_category_; } void setDropOverload(UnitFloat drop_overload) override { drop_overload_ = drop_overload; } + void setDropCategory(absl::string_view drop_category) override { + drop_category_ = drop_category; + } private: Http::ConnectionPool::Instance* @@ -627,6 +633,7 @@ class ClusterManagerImpl : public ClusterManager, ThreadLocalClusterManagerImpl& parent_; PrioritySetImpl priority_set_; UnitFloat drop_overload_{0}; + std::string drop_category_; // Don't change the order of cluster_info_ and lb_factory_/lb_ as the the lb_factory_/lb_ // may keep a reference to the cluster_info_. @@ -889,7 +896,7 @@ class ClusterManagerImpl : public ClusterManager, ClusterInitializationObjectConstSharedPtr addOrUpdateClusterInitializationObjectIfSupported( const ThreadLocalClusterUpdateParams& params, ClusterInfoConstSharedPtr cluster_info, LoadBalancerFactorySharedPtr load_balancer_factory, HostMapConstSharedPtr map, - UnitFloat drop_overload); + UnitFloat drop_overload, absl::string_view drop_category); bool deferralIsSupportedForCluster(const ClusterInfoConstSharedPtr& info) const; diff --git a/source/common/upstream/health_discovery_service.h b/source/common/upstream/health_discovery_service.h index 2cd21b06da..f2e1d539e4 100644 --- a/source/common/upstream/health_discovery_service.h +++ b/source/common/upstream/health_discovery_service.h @@ -72,8 +72,10 @@ class HdsCluster : public Cluster, Logger::Loggable { std::vector healthCheckers() { return health_checkers_; }; std::vector hosts() { return *hosts_; }; - UnitFloat dropOverload() const override { return UnitFloat(0); } + UnitFloat dropOverload() const override { return drop_overload_; } + const std::string& dropCategory() const override { return drop_category_; } void setDropOverload(UnitFloat) override {} + void setDropCategory(absl::string_view) override {} protected: PrioritySetImpl priority_set_; @@ -99,6 +101,8 @@ class HdsCluster : public Cluster, Logger::Loggable { std::vector health_checkers_; HealthCheckerMap health_checkers_map_; TimeSource& time_source_; + UnitFloat drop_overload_{0}; + const std::string drop_category_; absl::Status updateHealthchecks( const Protobuf::RepeatedPtrField& health_checks); diff --git a/source/common/upstream/load_stats_reporter.cc b/source/common/upstream/load_stats_reporter.cc index a06d85c40d..0e64bf7e88 100644 --- a/source/common/upstream/load_stats_reporter.cc +++ b/source/common/upstream/load_stats_reporter.cc @@ -127,7 +127,7 @@ void LoadStatsReporter::sendLoadStatsRequest() { cluster.info()->loadReportStats().upstream_rq_drop_overload_.latch(); if (drop_overload_count > 0) { auto* dropped_request = cluster_stats->add_dropped_requests(); - dropped_request->set_category("drop_overload"); + dropped_request->set_category(cluster.dropCategory()); dropped_request->set_dropped_count(drop_overload_count); } diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index a336aa2689..bfee8e6b2b 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1832,6 +1832,7 @@ absl::Status ClusterImplBase::parseDropOverloadConfig( drop_ratio = std::min(drop_ratio, float(drop_ratio_runtime) / float(MAX_DROP_OVERLOAD_RUNTIME)); drop_overload_ = UnitFloat(drop_ratio); + drop_category_ = policy.drop_overloads(0).category(); return absl::OkStatus(); } diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 4ce4791a01..47d3246857 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -1200,7 +1200,9 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable callback) override; UnitFloat dropOverload() const override { return drop_overload_; } + const std::string& dropCategory() const override { return drop_category_; } void setDropOverload(UnitFloat drop_overload) override { drop_overload_ = drop_overload; } + void setDropCategory(absl::string_view drop_category) override { drop_category_ = drop_category; } protected: ClusterImplBase(const envoy::config::cluster::v3::Cluster& cluster, @@ -1271,6 +1273,7 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable 0) { auto* drop_overload = policy.add_drop_overloads(); - drop_overload->set_category("drop_overload"); + drop_overload->set_category(cluster.dropCategory()); auto* percent = drop_overload->mutable_drop_percentage(); percent->set_denominator(envoy::type::v3::FractionalPercent::MILLION); percent->set_numerator(uint32_t(value)); diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 3ef5cef66f..be63bcf58d 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -275,6 +275,7 @@ TEST_P(StrictDnsParamTest, DropOverLoadConfigTestBasicMillion) { false); auto cluster = *StrictDnsClusterImpl::create(cluster_config, factory_context, dns_resolver); EXPECT_EQ(0.000035f, cluster->dropOverload().value()); + EXPECT_EQ("test", cluster->dropCategory()); } TEST_P(StrictDnsParamTest, DropOverLoadConfigTestBasicTenThousand) { @@ -290,7 +291,7 @@ TEST_P(StrictDnsParamTest, DropOverLoadConfigTestBasicTenThousand) { load_assignment: policy: drop_overloads: - category: test + category: foo drop_percentage: numerator: 1000 denominator: TEN_THOUSAND @@ -301,6 +302,7 @@ TEST_P(StrictDnsParamTest, DropOverLoadConfigTestBasicTenThousand) { false); auto cluster = *StrictDnsClusterImpl::create(cluster_config, factory_context, dns_resolver); EXPECT_EQ(0.1f, cluster->dropOverload().value()); + EXPECT_EQ("foo", cluster->dropCategory()); } TEST_P(StrictDnsParamTest, DropOverLoadConfigTestBadDenominator) { diff --git a/test/mocks/upstream/cluster.cc b/test/mocks/upstream/cluster.cc index 39c2bd4e77..e1cb719069 100644 --- a/test/mocks/upstream/cluster.cc +++ b/test/mocks/upstream/cluster.cc @@ -8,6 +8,8 @@ namespace Upstream { using ::testing::_; using ::testing::Invoke; using ::testing::Return; +using ::testing::ReturnRef; + MockCluster::MockCluster() { ON_CALL(*this, info()).WillByDefault(Return(info_)); ON_CALL(*this, initialize(_)) @@ -16,9 +18,13 @@ MockCluster::MockCluster() { initialize_callback_ = callback; })); ON_CALL(*this, dropOverload()).WillByDefault(Return(drop_overload_)); + ON_CALL(*this, dropCategory()).WillByDefault(ReturnRef(drop_category_)); ON_CALL(*this, setDropOverload(_)).WillByDefault(Invoke([this](UnitFloat drop_overload) -> void { drop_overload_ = drop_overload; })); + ON_CALL(*this, setDropCategory(_)) + .WillByDefault(Invoke( + [this](absl::string_view drop_category) -> void { drop_category_ = drop_category; })); } MockCluster::~MockCluster() = default; diff --git a/test/mocks/upstream/cluster.h b/test/mocks/upstream/cluster.h index f2fe2bad99..cb90626596 100644 --- a/test/mocks/upstream/cluster.h +++ b/test/mocks/upstream/cluster.h @@ -26,12 +26,15 @@ class MockCluster : public Cluster { MOCK_METHOD(PrioritySet&, prioritySet, ()); MOCK_METHOD(const PrioritySet&, prioritySet, (), (const)); MOCK_METHOD(UnitFloat, dropOverload, (), (const)); + MOCK_METHOD(const std::string&, dropCategory, (), (const)); MOCK_METHOD(void, setDropOverload, (UnitFloat)); + MOCK_METHOD(void, setDropCategory, (absl::string_view)); std::shared_ptr info_{new ::testing::NiceMock()}; std::function initialize_callback_; Network::Address::InstanceConstSharedPtr source_address_; UnitFloat drop_overload_{0}; + std::string drop_category_{"drop_overload"}; }; } // namespace Upstream } // namespace Envoy diff --git a/test/mocks/upstream/thread_local_cluster.cc b/test/mocks/upstream/thread_local_cluster.cc index 2e4951d461..bc1024331b 100644 --- a/test/mocks/upstream/thread_local_cluster.cc +++ b/test/mocks/upstream/thread_local_cluster.cc @@ -19,9 +19,14 @@ MockThreadLocalCluster::MockThreadLocalCluster() { .WillByDefault(Return(Upstream::TcpPoolData([]() {}, &tcp_conn_pool_))); ON_CALL(*this, httpAsyncClient()).WillByDefault(ReturnRef(async_client_)); ON_CALL(*this, dropOverload()).WillByDefault(Return(cluster_.drop_overload_)); + ON_CALL(*this, dropCategory()).WillByDefault(ReturnRef(cluster_.drop_category_)); ON_CALL(*this, setDropOverload(_)).WillByDefault(Invoke([this](UnitFloat drop_overload) -> void { cluster_.drop_overload_ = drop_overload; })); + ON_CALL(*this, setDropCategory(_)) + .WillByDefault(Invoke([this](absl::string_view drop_category) -> void { + cluster_.drop_category_ = drop_category; + })); } MockThreadLocalCluster::~MockThreadLocalCluster() = default; diff --git a/test/mocks/upstream/thread_local_cluster.h b/test/mocks/upstream/thread_local_cluster.h index 0efb100c20..266aa94c99 100644 --- a/test/mocks/upstream/thread_local_cluster.h +++ b/test/mocks/upstream/thread_local_cluster.h @@ -40,7 +40,9 @@ class MockThreadLocalCluster : public ThreadLocalCluster { MOCK_METHOD(Tcp::AsyncTcpClientPtr, tcpAsyncClient, (LoadBalancerContext * context, Tcp::AsyncTcpClientOptionsConstSharedPtr options)); MOCK_METHOD(UnitFloat, dropOverload, (), (const)); + MOCK_METHOD(const std::string&, dropCategory, (), (const)); MOCK_METHOD(void, setDropOverload, (UnitFloat)); + MOCK_METHOD(void, setDropCategory, (absl::string_view)); NiceMock cluster_; NiceMock lb_; diff --git a/test/server/admin/config_dump_handler_test.cc b/test/server/admin/config_dump_handler_test.cc index eb2718e0c8..4606be607e 100644 --- a/test/server/admin/config_dump_handler_test.cc +++ b/test/server/admin/config_dump_handler_test.cc @@ -151,7 +151,8 @@ TEST_P(AdminInstanceTest, ConfigDumpWithEndpoint) { hostname_for_healthcheck, "tcp://1.2.3.5:90", 5, 6); // Adding drop_overload config. ON_CALL(cluster, dropOverload()).WillByDefault(Return(UnitFloat(0.00035))); - + const std::string drop_overload = "drop_overload"; + ON_CALL(cluster, dropCategory()).WillByDefault(ReturnRef(drop_overload)); Buffer::OwnedImpl response; Http::TestResponseHeaderMapImpl header_map; EXPECT_EQ(Http::Code::OK, getCallback("/config_dump?include_eds", header_map, response)); From fdfdad004bc71f6db1fab2360546362bc1c2880a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Sep 2024 08:54:37 +0100 Subject: [PATCH 17/28] build(deps): bump distroless/base-nossl-debian12 from `fb10a97` to `4cc93c5` in /ci (#36206) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ci/Dockerfile-envoy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Dockerfile-envoy b/ci/Dockerfile-envoy index 826b3e869f..134511d212 100644 --- a/ci/Dockerfile-envoy +++ b/ci/Dockerfile-envoy @@ -58,7 +58,7 @@ COPY --chown=0:0 --chmod=755 \ # STAGE: envoy-distroless -FROM gcr.io/distroless/base-nossl-debian12:nonroot@sha256:fb10a979880367004a93467d9dad87eea1af67c6adda0a0060d2e785a8c1a0e6 AS envoy-distroless +FROM gcr.io/distroless/base-nossl-debian12:nonroot@sha256:4cc93c5b247e24470905bf3cdf8285aeac176bb0e7c62ee2b748a95c0c4123b5 AS envoy-distroless EXPOSE 10000 ENTRYPOINT ["/usr/local/bin/envoy"] CMD ["-c", "/etc/envoy/envoy.yaml"] From 8d74446c8a3d219572380731bae42ab328426996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20R=2E=20Sede=C3=B1o?= Date: Thu, 19 Sep 2024 05:35:10 -0400 Subject: [PATCH 18/28] Fewer external deps (#36187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit Message: Remove the native.bind entries for the following libraries: - libcircllhist - uring - ares - xxhash - sqlparser - tclap - fmtlib - spdlog - benchmark - libsxg - yaml_cpp - event - colm - ragel - zlib - brotlienc - brotlidec - zstd - msgpack - opentelemetry_api - dd_trace_cpp - cpp2sky - json - http_parser - hessian2_codec_object_codec_lib - hessian2_codec_codec_impl - su-exec - googletest - protoc - opencensus_trace - opencensus_trace_b3 - opencensus_trace_cloud_trace_context - opencensus_trace_grpc_trace_bin - opencensus_trace_trace_context - opencensus_exporter_ocagent - opencensus_exporter_stdout - opencensus_exporter_stackdriver - opencensus_exporter_zipkin - quiche_common_platform - quiche_http2_adapter - quiche_http2_protocol - quiche_http2_test_tools - quiche_quic_platform - quiche_quic_platform_base - quiche_http2_hpack - quiche_http2_hpack_decoder - jwt_verify_lib - simple_lru_cache_lib - luajit - tcmalloc - tcmalloc_profile_marshaler - tcmalloc_malloc_extension - gperftools - librdkafka Update `external_deps` that used them to `deps` of their actual locations. Some examples: - `zlib` ā†’ `//bazel/foreign_cc:zlib` - `json` ā†’ `@com_github_nlohmann_json//:json` The bindings that remain are either used by dependencies that need to be updated or have uses in some envoy bazel rules that make removing them trickier. These are the easy ones. Risk Level: low Testing: Building envoy and running integration test suites. Existing tests pass. --------- Signed-off-by: Alejandro R SedeƱo --- bazel/EXTERNAL_DEPS.md | 4 +- bazel/envoy_build_system.bzl | 4 +- bazel/envoy_internal.bzl | 18 +- bazel/envoy_library.bzl | 26 +- bazel/envoy_pch.bzl | 2 +- bazel/envoy_test.bzl | 5 +- bazel/external/json.BUILD | 19 -- bazel/external/quiche.BUILD | 20 +- bazel/foreign_cc/BUILD | 2 +- bazel/protobuf.patch | 4 +- bazel/repositories.bzl | 237 +----------------- ci/do_ci.sh | 2 +- contrib/common/sqlutils/source/BUILD | 2 +- contrib/common/sqlutils/test/BUILD | 2 +- .../cryptomb/private_key_providers/test/BUILD | 6 +- .../matching/input_matchers/source/BUILD | 8 +- .../matching/input_matchers/test/BUILD | 4 +- .../kafka/filters/network/source/mesh/BUILD | 5 +- contrib/kafka/filters/network/test/mesh/BUILD | 6 +- .../mysql_proxy/filters/network/test/BUILD | 2 +- .../qatzstd/compressor/source/BUILD | 2 +- .../qat/private_key_providers/source/BUILD | 4 +- contrib/sxg/filters/http/source/BUILD | 2 +- distribution/binary/BUILD | 2 +- distribution/binary/compiler.bzl | 2 +- envoy/grpc/BUILD | 4 +- mobile/bazel/protobuf.patch | 4 +- source/common/common/BUILD | 18 +- source/common/compression/zstd/common/BUILD | 2 +- source/common/crypto/BUILD | 4 +- source/common/event/BUILD | 14 +- source/common/filesystem/BUILD | 4 +- source/common/grpc/BUILD | 8 +- source/common/http/BUILD | 12 +- source/common/http/http1/BUILD | 2 +- source/common/http/http2/BUILD | 12 +- source/common/io/BUILD | 2 +- source/common/json/BUILD | 4 +- source/common/orca/BUILD | 8 +- source/common/protobuf/BUILD | 20 +- source/common/quic/BUILD | 30 +-- source/common/stats/BUILD | 4 +- source/common/tls/BUILD | 28 +-- source/common/tls/cert_validator/BUILD | 4 +- source/common/upstream/BUILD | 4 +- .../extensions/access_loggers/fluentd/BUILD | 4 +- source/extensions/common/dubbo/BUILD | 6 +- source/extensions/common/wasm/BUILD | 2 +- source/extensions/common/wasm/ext/BUILD | 2 +- .../compression/brotli/compressor/BUILD | 2 +- .../compression/brotli/decompressor/BUILD | 2 +- .../extensions/compression/gzip/common/BUILD | 2 +- .../compression/gzip/compressor/BUILD | 2 +- .../compression/gzip/decompressor/BUILD | 2 +- .../extensions/compression/zstd/common/BUILD | 2 +- .../extensions/config_subscription/rest/BUILD | 4 +- source/extensions/filters/common/lua/BUILD | 4 +- .../adaptive_concurrency/controller/BUILD | 4 +- source/extensions/filters/http/common/BUILD | 4 +- .../extensions/filters/http/gcp_authn/BUILD | 6 +- .../filters/http/grpc_field_extraction/BUILD | 8 +- .../filters/http/grpc_json_transcoder/BUILD | 4 +- .../extensions/filters/http/jwt_authn/BUILD | 18 +- source/extensions/filters/http/oauth2/BUILD | 4 +- .../filters/listener/http_inspector/BUILD | 2 +- .../filters/network/dubbo_proxy/BUILD | 6 +- .../extensions/filters/udp/dns_filter/BUILD | 2 +- source/extensions/health_checkers/grpc/BUILD | 4 +- .../network/dns_resolver/cares/BUILD | 2 +- source/extensions/tracers/datadog/BUILD | 4 +- source/extensions/tracers/opencensus/BUILD | 20 +- source/extensions/tracers/opentelemetry/BUILD | 4 +- source/extensions/tracers/skywalking/BUILD | 8 +- source/extensions/tracers/xray/BUILD | 1 - .../extensions/transport_sockets/alts/BUILD | 16 +- .../transport_sockets/starttls/BUILD | 4 +- .../tls/cert_validator/spiffe/BUILD | 4 +- source/server/BUILD | 4 +- test/BUILD | 6 +- test/benchmark/BUILD | 6 +- test/common/buffer/BUILD | 4 +- test/common/common/BUILD | 22 +- test/common/crypto/BUILD | 4 +- test/common/formatter/BUILD | 4 +- test/common/grpc/BUILD | 4 +- test/common/http/BUILD | 8 +- test/common/http/http2/BUILD | 26 +- test/common/listener_manager/BUILD | 8 +- test/common/network/BUILD | 8 +- test/common/orca/BUILD | 8 +- test/common/protobuf/BUILD | 2 +- test/common/quic/BUILD | 8 +- test/common/quic/platform/BUILD | 2 +- test/common/router/BUILD | 12 +- test/common/stats/BUILD | 24 +- test/common/tls/BUILD | 6 +- test/common/upstream/BUILD | 4 +- test/dependencies/BUILD | 4 +- test/extensions/access_loggers/fluentd/BUILD | 4 +- .../access_loggers/open_telemetry/BUILD | 4 +- test/extensions/bootstrap/wasm/BUILD | 4 +- test/extensions/clusters/eds/BUILD | 4 +- test/extensions/common/wasm/BUILD | 4 +- test/extensions/filters/http/compressor/BUILD | 6 +- test/extensions/filters/http/ext_proc/BUILD | 2 - .../message_converter/BUILD | 2 - .../extraction_util/BUILD | 1 - .../filters/listener/tls_inspector/BUILD | 4 +- .../filters/network/redis_proxy/BUILD | 8 +- test/extensions/filters/udp/dns_filter/BUILD | 2 +- .../load_balancing_policies/common/BUILD | 4 +- .../load_balancing_policies/subset/BUILD | 4 +- test/extensions/tracers/datadog/BUILD | 4 +- test/extensions/tracers/opentelemetry/BUILD | 4 +- test/extensions/tracers/skywalking/BUILD | 4 +- test/extensions/transport_sockets/alts/BUILD | 8 +- test/fuzz/BUILD | 4 +- test/server/BUILD | 4 +- test/test_common/BUILD | 4 +- test/tools/router_check/BUILD | 2 +- test/tools/schema_validator/BUILD | 2 +- test/tools/wee8_compile/BUILD | 4 +- 122 files changed, 259 insertions(+), 745 deletions(-) delete mode 100644 bazel/external/json.BUILD diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md index b7919b5b06..2694e7cc05 100644 --- a/bazel/EXTERNAL_DEPS.md +++ b/bazel/EXTERNAL_DEPS.md @@ -16,7 +16,7 @@ build process. 1. Define a new Bazel repository in [`bazel/repositories.bzl`](repositories.bzl), in the `envoy_dependencies()` function. 2. Reference your new external dependency in some `envoy_cc_library` via the - `external_deps` attribute. + `deps` attribute. 3. `bazel test //test/...` ## External CMake (preferred) @@ -28,7 +28,7 @@ This is the preferred style of adding dependencies that use CMake for their buil 2. Add an `envoy_cmake` rule to [`bazel/foreign_cc/BUILD`](foreign_cc/BUILD). This will reference the source repository in step 1. 3. Reference your new external dependency in some `envoy_cc_library` via the name bound in step 1 - `external_deps` attribute. + `deps` attribute. 4. `bazel test //test/...` # Adding external dependencies to Envoy (Python) diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index f6492e18d2..5d7f8654fa 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -211,13 +211,13 @@ def envoy_proto_descriptor(name, out, srcs = [], external_deps = []): options.extend(["-I" + include_path for include_path in include_paths]) options.append("--descriptor_set_out=$@") - cmd = "$(location //external:protoc) " + " ".join(options + input_files) + cmd = "$(location @com_google_protobuf//:protoc) " + " ".join(options + input_files) native.genrule( name = name, srcs = srcs, outs = [out], cmd = cmd, - tools = ["//external:protoc"], + tools = ["@com_google_protobuf//:protoc"], ) # Dependencies on Google grpc should be wrapped with this function. diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index 45c6acb8ec..b84d93ef7a 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -166,15 +166,15 @@ def tcmalloc_external_dep(repository): repository + "//bazel:disable_tcmalloc": None, repository + "//bazel:disable_tcmalloc_on_linux_x86_64": None, repository + "//bazel:disable_tcmalloc_on_linux_aarch64": None, - repository + "//bazel:debug_tcmalloc": envoy_external_dep_path("gperftools"), - repository + "//bazel:debug_tcmalloc_on_linux_x86_64": envoy_external_dep_path("gperftools"), - repository + "//bazel:debug_tcmalloc_on_linux_aarch64": envoy_external_dep_path("gperftools"), - repository + "//bazel:gperftools_tcmalloc": envoy_external_dep_path("gperftools"), - repository + "//bazel:gperftools_tcmalloc_on_linux_x86_64": envoy_external_dep_path("gperftools"), - repository + "//bazel:gperftools_tcmalloc_on_linux_aarch64": envoy_external_dep_path("gperftools"), - repository + "//bazel:linux_x86_64": envoy_external_dep_path("tcmalloc"), - repository + "//bazel:linux_aarch64": envoy_external_dep_path("tcmalloc"), - "//conditions:default": envoy_external_dep_path("gperftools"), + repository + "//bazel:debug_tcmalloc": repository + "//bazel/foreign_cc:gperftools", + repository + "//bazel:debug_tcmalloc_on_linux_x86_64": repository + "//bazel/foreign_cc:gperftools", + repository + "//bazel:debug_tcmalloc_on_linux_aarch64": repository + "//bazel/foreign_cc:gperftools", + repository + "//bazel:gperftools_tcmalloc": repository + "//bazel/foreign_cc:gperftools", + repository + "//bazel:gperftools_tcmalloc_on_linux_x86_64": repository + "//bazel/foreign_cc:gperftools", + repository + "//bazel:gperftools_tcmalloc_on_linux_aarch64": repository + "//bazel/foreign_cc:gperftools", + repository + "//bazel:linux_x86_64": "@com_github_google_tcmalloc//tcmalloc", + repository + "//bazel:linux_aarch64": "@com_github_google_tcmalloc//tcmalloc", + "//conditions:default": repository + "//bazel/foreign_cc:gperftools", }) # Select the given values if default path normalization is on in the current build. diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index c1266770d7..d5ed602b2f 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -24,23 +24,23 @@ def tcmalloc_external_deps(repository): repository + "//bazel:disable_tcmalloc": [], repository + "//bazel:disable_tcmalloc_on_linux_x86_64": [], repository + "//bazel:disable_tcmalloc_on_linux_aarch64": [], - repository + "//bazel:debug_tcmalloc": [envoy_external_dep_path("gperftools")], - repository + "//bazel:debug_tcmalloc_on_linux_x86_64": [envoy_external_dep_path("gperftools")], - repository + "//bazel:debug_tcmalloc_on_linux_aarch64": [envoy_external_dep_path("gperftools")], - repository + "//bazel:gperftools_tcmalloc": [envoy_external_dep_path("gperftools")], - repository + "//bazel:gperftools_tcmalloc_on_linux_x86_64": [envoy_external_dep_path("gperftools")], - repository + "//bazel:gperftools_tcmalloc_on_linux_aarch64": [envoy_external_dep_path("gperftools")], + repository + "//bazel:debug_tcmalloc": [repository + "//bazel/foreign_cc:gperftools"], + repository + "//bazel:debug_tcmalloc_on_linux_x86_64": [repository + "//bazel/foreign_cc:gperftools"], + repository + "//bazel:debug_tcmalloc_on_linux_aarch64": [repository + "//bazel/foreign_cc:gperftools"], + repository + "//bazel:gperftools_tcmalloc": [repository + "//bazel/foreign_cc:gperftools"], + repository + "//bazel:gperftools_tcmalloc_on_linux_x86_64": [repository + "//bazel/foreign_cc:gperftools"], + repository + "//bazel:gperftools_tcmalloc_on_linux_aarch64": [repository + "//bazel/foreign_cc:gperftools"], repository + "//bazel:linux_x86_64": [ - envoy_external_dep_path("tcmalloc"), - envoy_external_dep_path("tcmalloc_profile_marshaler"), - envoy_external_dep_path("tcmalloc_malloc_extension"), + "@com_github_google_tcmalloc//tcmalloc", + "@com_github_google_tcmalloc//tcmalloc:profile_marshaler", + "@com_github_google_tcmalloc//tcmalloc:malloc_extension", ], repository + "//bazel:linux_aarch64": [ - envoy_external_dep_path("tcmalloc"), - envoy_external_dep_path("tcmalloc_profile_marshaler"), - envoy_external_dep_path("tcmalloc_malloc_extension"), + "@com_github_google_tcmalloc//tcmalloc", + "@com_github_google_tcmalloc//tcmalloc:profile_marshaler", + "@com_github_google_tcmalloc//tcmalloc:malloc_extension", ], - "//conditions:default": [envoy_external_dep_path("gperftools")], + "//conditions:default": [repository + "//bazel/foreign_cc:gperftools"], }) # Envoy C++ library targets that need no transformations or additional dependencies before being diff --git a/bazel/envoy_pch.bzl b/bazel/envoy_pch.bzl index 843937d8dc..1e9766b680 100644 --- a/bazel/envoy_pch.bzl +++ b/bazel/envoy_pch.bzl @@ -27,8 +27,8 @@ def envoy_pch_library( name, includes, deps, - external_deps, visibility, + external_deps = [], testonly = False, repository = ""): native.cc_library( diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index 0bdfc36c51..c9a4266603 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -40,7 +40,7 @@ def _envoy_cc_test_infrastructure_library( extra_deps = [] pch_copts = [] if disable_pch: - extra_deps = [envoy_external_dep_path("googletest")] + extra_deps = ["@com_google_googletest//:gtest"] else: extra_deps = envoy_pch_deps(repository, "//test:test_pch") pch_copts = envoy_pch_copts(repository, "//test:test_pch") @@ -175,9 +175,10 @@ def envoy_cc_test( linkopts = _envoy_test_linkopts() + linkopts, linkstatic = envoy_linkstatic(), malloc = tcmalloc_external_dep(repository), - deps = envoy_stdlib_deps() + deps + [envoy_external_dep_path(dep) for dep in external_deps + ["googletest"]] + [ + deps = envoy_stdlib_deps() + deps + [envoy_external_dep_path(dep) for dep in external_deps] + [ repository + "//test:main", repository + "//test/test_common:test_version_linkstamp", + "@com_google_googletest//:gtest", ] + envoy_pch_deps(repository, "//test:test_pch"), # from https://github.com/google/googletest/blob/6e1970e2376c14bf658eb88f655a054030353f9f/googlemock/src/gmock.cc#L51 # 2 - by default, mocks act as StrictMocks. diff --git a/bazel/external/json.BUILD b/bazel/external/json.BUILD deleted file mode 100644 index cc19af32bf..0000000000 --- a/bazel/external/json.BUILD +++ /dev/null @@ -1,19 +0,0 @@ -licenses(["notice"]) # Apache 2 - -cc_library( - name = "nlohmann_json_lib", - hdrs = glob([ - "include/nlohmann/*.hpp", - "include/nlohmann/**/*.hpp", - "include/nlohmann/*/*/*.hpp", - ]), - includes = ["external/nlohmann_json_lib"], - visibility = ["//visibility:public"], -) - -cc_library( - name = "json", - includes = ["include"], - visibility = ["//visibility:public"], - deps = [":nlohmann_json_lib"], -) diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 950f9426a6..3923966229 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -2460,10 +2460,7 @@ envoy_quic_cc_library( "quiche/quic/core/crypto/quic_compressed_certs_cache.h", "quiche/quic/core/crypto/transport_parameters.h", ], - external_deps = [ - "ssl", - "zlib", - ], + external_deps = ["ssl"], tags = [ "pg3", ], @@ -2489,6 +2486,7 @@ envoy_quic_cc_library( ":quic_core_utils_lib", ":quic_core_versions_lib", ":quic_platform", + "@envoy//bazel/foreign_cc:zlib", ], ) @@ -2502,9 +2500,6 @@ envoy_quic_cc_library( "quiche/quic/core/crypto/quic_client_session_cache.h", "quiche/quic/core/crypto/quic_crypto_client_config.h", ], - external_deps = [ - "zlib", - ], tags = [ "pg3", ], @@ -2513,6 +2508,7 @@ envoy_quic_cc_library( ":quic_core_crypto_client_proof_source_lib", ":quic_core_crypto_crypto_handshake_lib", ":quiche_common_platform_client_stats", + "@envoy//bazel/foreign_cc:zlib", ], ) @@ -2524,10 +2520,7 @@ envoy_quic_cc_library( hdrs = [ "quiche/quic/core/crypto/quic_crypto_server_config.h", ], - external_deps = [ - "ssl", - "zlib", - ], + external_deps = ["ssl"], tags = [ "pg3", ], @@ -2536,6 +2529,7 @@ envoy_quic_cc_library( ":quic_core_proto_crypto_server_config_proto_header", ":quic_core_server_id_lib", ":quic_server_crypto_tls_handshake_lib", + "@envoy//bazel/foreign_cc:zlib", ], ) @@ -2658,9 +2652,7 @@ envoy_quic_cc_library( name = "quic_core_crypto_proof_source_x509_lib", srcs = ["quiche/quic/core/crypto/proof_source_x509.cc"], hdrs = ["quiche/quic/core/crypto/proof_source_x509.h"], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ ":quic_core_crypto_certificate_view_lib", ":quic_core_crypto_crypto_handshake_lib", diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index 81f04ab3d5..60271a4925 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -310,8 +310,8 @@ envoy_cmake( deps = [ ":ares", ":nghttp2", + ":zlib", "//external:ssl", - "//external:zlib", ], ) diff --git a/bazel/protobuf.patch b/bazel/protobuf.patch index 833b4aa9b7..c07b5ce9d8 100644 --- a/bazel/protobuf.patch +++ b/bazel/protobuf.patch @@ -101,12 +101,12 @@ index e7555ee10..a93beb1c5 100644 diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel --- a/src/google/protobuf/io/BUILD.bazel +++ b/src/google/protobuf/io/BUILD.bazel -@@ -138,7 +138,7 @@ cc_library( +@@ -138,6 +138,6 @@ cc_library( "@com_google_absl//absl/log:absl_log", ] + select({ "//build_defs:config_msvc": [], - "//conditions:default": ["@zlib"], -+ "//conditions:default": ["//external:zlib"], ++ "//conditions:default": ["@envoy//bazel/foreign_cc:zlib"], }), ) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index f0fed861ee..99f1d57d6c 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -165,7 +165,6 @@ def envoy_dependencies(skip_targets = []): _com_github_nghttp2_nghttp2() _com_github_msgpack_cpp() _com_github_skyapm_cpp2sky() - _com_github_nodejs_http_parser() _com_github_alibaba_hessian2_codec() _com_github_tencent_rapidjson() _com_github_nlohmann_json() @@ -267,20 +266,12 @@ def _com_github_openhistogram_libcircllhist(): name = "com_github_openhistogram_libcircllhist", build_file = "@envoy//bazel/external:libcircllhist.BUILD", ) - native.bind( - name = "libcircllhist", - actual = "@com_github_openhistogram_libcircllhist//:libcircllhist", - ) def _com_github_axboe_liburing(): external_http_archive( name = "com_github_axboe_liburing", build_file_content = BUILD_ALL_CONTENT, ) - native.bind( - name = "uring", - actual = "@envoy//bazel/foreign_cc:liburing_linux", - ) def _com_github_bazel_buildtools(): # TODO(phlax): Add binary download @@ -294,30 +285,18 @@ def _com_github_c_ares_c_ares(): name = "com_github_c_ares_c_ares", build_file_content = BUILD_ALL_CONTENT, ) - native.bind( - name = "ares", - actual = "@envoy//bazel/foreign_cc:ares", - ) def _com_github_cyan4973_xxhash(): external_http_archive( name = "com_github_cyan4973_xxhash", build_file = "@envoy//bazel/external:xxhash.BUILD", ) - native.bind( - name = "xxhash", - actual = "@com_github_cyan4973_xxhash//:xxhash", - ) def _com_github_envoyproxy_sqlparser(): external_http_archive( name = "com_github_envoyproxy_sqlparser", build_file = "@envoy//bazel/external:sqlparser.BUILD", ) - native.bind( - name = "sqlparser", - actual = "@com_github_envoyproxy_sqlparser//:sqlparser", - ) def _com_github_mirror_tclap(): external_http_archive( @@ -325,30 +304,18 @@ def _com_github_mirror_tclap(): build_file = "@envoy//bazel/external:tclap.BUILD", patch_args = ["-p1"], ) - native.bind( - name = "tclap", - actual = "@com_github_mirror_tclap//:tclap", - ) def _com_github_fmtlib_fmt(): external_http_archive( name = "com_github_fmtlib_fmt", build_file = "@envoy//bazel/external:fmtlib.BUILD", ) - native.bind( - name = "fmtlib", - actual = "@com_github_fmtlib_fmt//:fmtlib", - ) def _com_github_gabime_spdlog(): external_http_archive( name = "com_github_gabime_spdlog", build_file = "@envoy//bazel/external:spdlog.BUILD", ) - native.bind( - name = "spdlog", - actual = "@com_github_gabime_spdlog//:spdlog", - ) def _com_github_google_benchmark(): external_http_archive( @@ -358,10 +325,6 @@ def _com_github_google_benchmark(): name = "libpfm", build_file = "@com_github_google_benchmark//tools:libpfm.BUILD.bazel", ) - native.bind( - name = "benchmark", - actual = "@com_github_google_benchmark//:benchmark", - ) def _com_github_google_libprotobuf_mutator(): external_http_archive( @@ -375,11 +338,6 @@ def _com_github_google_libsxg(): build_file_content = BUILD_ALL_CONTENT, ) - native.bind( - name = "libsxg", - actual = "@envoy//bazel/foreign_cc:libsxg", - ) - def _com_github_unicode_org_icu(): external_http_archive( name = "com_github_unicode_org_icu", @@ -437,40 +395,24 @@ def _com_github_jbeder_yaml_cpp(): external_http_archive( name = "com_github_jbeder_yaml_cpp", ) - native.bind( - name = "yaml_cpp", - actual = "@com_github_jbeder_yaml_cpp//:yaml-cpp", - ) def _com_github_libevent_libevent(): external_http_archive( name = "com_github_libevent_libevent", build_file_content = BUILD_ALL_CONTENT, ) - native.bind( - name = "event", - actual = "@envoy//bazel/foreign_cc:event", - ) def _net_colm_open_source_colm(): external_http_archive( name = "net_colm_open_source_colm", build_file_content = BUILD_ALL_CONTENT, ) - native.bind( - name = "colm", - actual = "@envoy//bazel/foreign_cc:colm", - ) def _net_colm_open_source_ragel(): external_http_archive( name = "net_colm_open_source_ragel", build_file_content = BUILD_ALL_CONTENT, ) - native.bind( - name = "ragel", - actual = "@envoy//bazel/foreign_cc:ragel", - ) def _net_zlib(): external_http_archive( @@ -480,11 +422,6 @@ def _net_zlib(): patches = ["@envoy//bazel/foreign_cc:zlib.patch"], ) - native.bind( - name = "zlib", - actual = "@envoy//bazel/foreign_cc:zlib", - ) - # Bind for grpc. native.bind( name = "madler_zlib", @@ -524,14 +461,6 @@ def _org_brotli(): external_http_archive( name = "org_brotli", ) - native.bind( - name = "brotlienc", - actual = "@org_brotli//:brotlienc", - ) - native.bind( - name = "brotlidec", - actual = "@org_brotli//:brotlidec", - ) def _com_github_facebook_zstd(): external_http_archive( @@ -539,11 +468,6 @@ def _com_github_facebook_zstd(): build_file_content = BUILD_ALL_CONTENT, ) - native.bind( - name = "zstd", - actual = "@envoy//bazel/foreign_cc:zstd", - ) - def _com_google_cel_cpp(): external_http_archive( "com_google_cel_cpp", @@ -584,10 +508,6 @@ def _com_github_msgpack_cpp(): name = "com_github_msgpack_cpp", build_file = "@envoy//bazel/external:msgpack.BUILD", ) - native.bind( - name = "msgpack", - actual = "@com_github_msgpack_cpp//:msgpack", - ) def _io_hyperscan(): external_http_archive( @@ -608,17 +528,9 @@ def _io_vectorscan(): def _io_opentelemetry_api_cpp(): external_http_archive(name = "io_opentelemetry_cpp") - native.bind( - name = "opentelemetry_api", - actual = "@io_opentelemetry_cpp//api:api", - ) def _com_github_datadog_dd_trace_cpp(): external_http_archive("com_github_datadog_dd_trace_cpp") - native.bind( - name = "dd_trace_cpp", - actual = "@com_github_datadog_dd_trace_cpp//:dd_trace_cpp", - ) def _com_github_skyapm_cpp2sky(): external_http_archive( @@ -627,10 +539,6 @@ def _com_github_skyapm_cpp2sky(): external_http_archive( name = "skywalking_data_collect_protocol", ) - native.bind( - name = "cpp2sky", - actual = "@com_github_skyapm_cpp2sky//source:cpp2sky_data_lib", - ) def _com_github_tencent_rapidjson(): external_http_archive( @@ -641,39 +549,16 @@ def _com_github_tencent_rapidjson(): def _com_github_nlohmann_json(): external_http_archive( name = "com_github_nlohmann_json", - build_file = "@envoy//bazel/external:json.BUILD", - ) - native.bind( - name = "json", - actual = "@com_github_nlohmann_json//:json", - ) - -def _com_github_nodejs_http_parser(): - native.bind( - name = "http_parser", - actual = "@envoy//bazel/external/http_parser", ) def _com_github_alibaba_hessian2_codec(): external_http_archive("com_github_alibaba_hessian2_codec") - native.bind( - name = "hessian2_codec_object_codec_lib", - actual = "@com_github_alibaba_hessian2_codec//hessian2/basic_codec:object_codec_lib", - ) - native.bind( - name = "hessian2_codec_codec_impl", - actual = "@com_github_alibaba_hessian2_codec//hessian2:codec_impl_lib", - ) def _com_github_ncopa_suexec(): external_http_archive( name = "com_github_ncopa_suexec", build_file = "@envoy//bazel/external:su-exec.BUILD", ) - native.bind( - name = "su-exec", - actual = "@com_github_ncopa_suexec//:su-exec", - ) def _com_google_googletest(): external_http_archive( @@ -681,10 +566,6 @@ def _com_google_googletest(): patches = ["@envoy//bazel:googletest.patch"], patch_args = ["-p1"], ) - native.bind( - name = "googletest", - actual = "@com_google_googletest//:gtest", - ) # TODO(jmarantz): replace the use of bind and external_deps with just # the direct Bazel path at all sites. This will make it easier to @@ -736,6 +617,7 @@ def _com_google_protobuf(): patch_args = ["-p1"], ) + # Needed by grpc, jwt_verify_lib, maybe others. native.bind( name = "protobuf", actual = "@com_google_protobuf//:protobuf", @@ -748,10 +630,6 @@ def _com_google_protobuf(): name = "protocol_compiler", actual = "@com_google_protobuf//:protoc", ) - native.bind( - name = "protoc", - actual = "@com_google_protobuf//:protoc", - ) # Needed for `bazel fetch` to work with @com_google_protobuf # https://github.com/google/protobuf/blob/v3.6.1/util/python/BUILD#L6-L9 @@ -759,6 +637,8 @@ def _com_google_protobuf(): name = "python_headers", actual = "//bazel:python_headers", ) + + # Needed by grpc until we update again. native.bind( name = "upb_base_lib", actual = "@com_google_protobuf//upb:base", @@ -788,42 +668,6 @@ def _io_opencensus_cpp(): external_http_archive( name = "io_opencensus_cpp", ) - native.bind( - name = "opencensus_trace", - actual = "@io_opencensus_cpp//opencensus/trace", - ) - native.bind( - name = "opencensus_trace_b3", - actual = "@io_opencensus_cpp//opencensus/trace:b3", - ) - native.bind( - name = "opencensus_trace_cloud_trace_context", - actual = "@io_opencensus_cpp//opencensus/trace:cloud_trace_context", - ) - native.bind( - name = "opencensus_trace_grpc_trace_bin", - actual = "@io_opencensus_cpp//opencensus/trace:grpc_trace_bin", - ) - native.bind( - name = "opencensus_trace_trace_context", - actual = "@io_opencensus_cpp//opencensus/trace:trace_context", - ) - native.bind( - name = "opencensus_exporter_ocagent", - actual = "@io_opencensus_cpp//opencensus/exporters/trace/ocagent:ocagent_exporter", - ) - native.bind( - name = "opencensus_exporter_stdout", - actual = "@io_opencensus_cpp//opencensus/exporters/trace/stdout:stdout_exporter", - ) - native.bind( - name = "opencensus_exporter_stackdriver", - actual = "@io_opencensus_cpp//opencensus/exporters/trace/stackdriver:stackdriver_exporter", - ) - native.bind( - name = "opencensus_exporter_zipkin", - actual = "@io_opencensus_cpp//opencensus/exporters/trace/zipkin:zipkin_exporter", - ) def _com_github_curl(): # The usage by AWS extensions common utilities is deprecated and will be removed by Q3 2024 after @@ -857,6 +701,8 @@ def _v8(): ], patch_args = ["-p1"], ) + + # Needed by proxy_wasm_cpp_host. native.bind( name = "wee8", actual = "@v8//:wee8", @@ -867,6 +713,8 @@ def _com_googlesource_chromium_base_trace_event_common(): name = "com_googlesource_chromium_base_trace_event_common", build_file = "@v8//:bazel/BUILD.trace_event_common", ) + + # Needed by v8. native.bind( name = "base_trace_event_common", actual = "@com_googlesource_chromium_base_trace_event_common//:trace_event_common", @@ -878,38 +726,6 @@ def _com_github_google_quiche(): patch_cmds = ["find quiche/ -type f -name \"*.bazel\" -delete"], build_file = "@envoy//bazel/external:quiche.BUILD", ) - native.bind( - name = "quiche_common_platform", - actual = "@com_github_google_quiche//:quiche_common_platform", - ) - native.bind( - name = "quiche_http2_adapter", - actual = "@com_github_google_quiche//:http2_adapter", - ) - native.bind( - name = "quiche_http2_protocol", - actual = "@com_github_google_quiche//:http2_adapter_http2_protocol", - ) - native.bind( - name = "quiche_http2_test_tools", - actual = "@com_github_google_quiche//:http2_adapter_mock_http2_visitor", - ) - native.bind( - name = "quiche_quic_platform", - actual = "@com_github_google_quiche//:quic_platform", - ) - native.bind( - name = "quiche_quic_platform_base", - actual = "@com_github_google_quiche//:quic_platform_base", - ) - native.bind( - name = "quiche_http2_hpack", - actual = "@com_github_google_quiche//:http2_hpack_hpack_lib", - ) - native.bind( - name = "quiche_http2_hpack_decoder", - actual = "@com_github_google_quiche//:http2_hpack_decoder_hpack_decoder_lib", - ) def _com_googlesource_googleurl(): external_http_archive( @@ -949,7 +765,7 @@ def _com_github_grpc_grpc(): ) native.bind( name = "cares", - actual = "//external:ares", + actual = "@envoy//bazel/foreign_cc:ares", ) native.bind( @@ -983,6 +799,7 @@ def _rules_proto_grpc(): def _re2(): external_http_archive("com_googlesource_code_re2") + # Needed by grpc. native.bind( name = "re2", actual = "@com_googlesource_code_re2//:re2", @@ -1016,16 +833,6 @@ def _emsdk(): def _com_github_google_jwt_verify(): external_http_archive("com_github_google_jwt_verify") - native.bind( - name = "jwt_verify_lib", - actual = "@com_github_google_jwt_verify//:jwt_verify_lib", - ) - - native.bind( - name = "simple_lru_cache_lib", - actual = "@com_github_google_jwt_verify//:simple_lru_cache_lib", - ) - def _com_github_luajit_luajit(): external_http_archive( name = "com_github_luajit_luajit", @@ -1035,38 +842,16 @@ def _com_github_luajit_luajit(): patch_cmds = ["chmod u+x build.py"], ) - native.bind( - name = "luajit", - actual = "@envoy//bazel/foreign_cc:luajit", - ) - def _com_github_google_tcmalloc(): external_http_archive( name = "com_github_google_tcmalloc", ) - native.bind( - name = "tcmalloc", - actual = "@com_github_google_tcmalloc//tcmalloc", - ) - native.bind( - name = "tcmalloc_profile_marshaler", - actual = "@com_github_google_tcmalloc//tcmalloc:profile_marshaler", - ) - native.bind( - name = "tcmalloc_malloc_extension", - actual = "@com_github_google_tcmalloc//tcmalloc:malloc_extension", - ) - def _com_github_gperftools_gperftools(): external_http_archive( name = "com_github_gperftools_gperftools", build_file_content = BUILD_ALL_CONTENT, ) - native.bind( - name = "gperftools", - actual = "@envoy//bazel/foreign_cc:gperftools", - ) def _com_github_wamr(): external_http_archive( @@ -1157,10 +942,6 @@ filegroup( patches = ["@envoy//bazel/foreign_cc:librdkafka.patch"], patch_args = ["-p1"], ) - native.bind( - name = "librdkafka", - actual = "@envoy//bazel/foreign_cc:librdkafka", - ) # This archive provides Kafka (and Zookeeper) binaries, that are used during Kafka integration # tests. diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 3258e8aad1..6d4295b53b 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -225,7 +225,7 @@ function bazel_binary_build() { //test/tools/router_check:router_check_tool "${CONFIG_ARGS[@]}" # Build su-exec utility - bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" external:su-exec + bazel build "${BAZEL_BUILD_OPTIONS[@]}" --remote_download_toplevel -c "${COMPILE_TYPE}" @com_github_ncopa_suexec//:su-exec cp_binary_for_image_build "${BINARY_TYPE}" "${COMPILE_TYPE}" "${EXE_NAME}" } diff --git a/contrib/common/sqlutils/source/BUILD b/contrib/common/sqlutils/source/BUILD index c100c39ae9..fb660bbee5 100644 --- a/contrib/common/sqlutils/source/BUILD +++ b/contrib/common/sqlutils/source/BUILD @@ -12,8 +12,8 @@ envoy_cc_library( name = "sqlutils_lib", srcs = ["sqlutils.cc"], hdrs = ["sqlutils.h"], - external_deps = ["sqlparser"], deps = [ "//source/common/protobuf:utility_lib", + "@com_github_envoyproxy_sqlparser//:sqlparser", ], ) diff --git a/contrib/common/sqlutils/test/BUILD b/contrib/common/sqlutils/test/BUILD index e89cb0138b..087e44dcdb 100644 --- a/contrib/common/sqlutils/test/BUILD +++ b/contrib/common/sqlutils/test/BUILD @@ -13,8 +13,8 @@ envoy_cc_test( srcs = [ "sqlutils_test.cc", ], - external_deps = ["sqlparser"], deps = [ "//contrib/common/sqlutils/source:sqlutils_lib", + "@com_github_envoyproxy_sqlparser//:sqlparser", ], ) diff --git a/contrib/cryptomb/private_key_providers/test/BUILD b/contrib/cryptomb/private_key_providers/test/BUILD index 2940695756..777cc49f89 100644 --- a/contrib/cryptomb/private_key_providers/test/BUILD +++ b/contrib/cryptomb/private_key_providers/test/BUILD @@ -87,13 +87,11 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "speed_test", srcs = ["speed_test.cc"], - external_deps = [ - "benchmark", - "ssl", - ], + external_deps = ["ssl"], deps = [ "//contrib/cryptomb/private_key_providers/source:ipp_crypto_wrapper_lib", "//source/common/common:assert_lib", "//source/common/common:utility_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/contrib/hyperscan/matching/input_matchers/source/BUILD b/contrib/hyperscan/matching/input_matchers/source/BUILD index ee3ce33489..8f00c2bdfa 100644 --- a/contrib/hyperscan/matching/input_matchers/source/BUILD +++ b/contrib/hyperscan/matching/input_matchers/source/BUILD @@ -5,10 +5,6 @@ load( "envoy_cmake", "envoy_contrib_package", ) -load( - "//bazel:envoy_internal.bzl", - "envoy_external_dep_path", -) load( "//contrib:all_contrib_extensions.bzl", "envoy_contrib_linux_aarch64_constraints", @@ -38,7 +34,7 @@ envoy_cmake( tags = ["skip_on_windows"], target_compatible_with = envoy_contrib_linux_x86_64_constraints(), deps = [ - envoy_external_dep_path("ragel"), + "//bazel/foreign_cc:ragel", ], ) @@ -64,7 +60,7 @@ envoy_cmake( tags = ["skip_on_windows"], target_compatible_with = envoy_contrib_linux_aarch64_constraints(), deps = [ - envoy_external_dep_path("ragel"), + "//bazel/foreign_cc:ragel", ], ) diff --git a/contrib/hyperscan/matching/input_matchers/test/BUILD b/contrib/hyperscan/matching/input_matchers/test/BUILD index aa1ebb140f..3be84e8b8c 100644 --- a/contrib/hyperscan/matching/input_matchers/test/BUILD +++ b/contrib/hyperscan/matching/input_matchers/test/BUILD @@ -39,10 +39,10 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "hyperscan_speed_test", srcs = ["hyperscan_speed_test.cc"], - external_deps = ["benchmark"], deps = [ "//source/common/common:assert_lib", "//source/common/common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@com_googlesource_code_re2//:re2", ] + select({ "//bazel:linux_x86_64": [ @@ -57,13 +57,13 @@ envoy_cc_benchmark_binary( envoy_cc_benchmark_binary( name = "matcher_speed_test", srcs = ["matcher_speed_test.cc"], - external_deps = ["benchmark"], deps = [ "//source/common/common:assert_lib", "//source/common/common:regex_lib", "//source/common/common:utility_lib", "//source/common/thread_local:thread_local_lib", "//test/mocks/event:event_mocks", + "@com_github_google_benchmark//:benchmark", ] + select({ "//bazel:linux_x86_64": [ "//contrib/hyperscan/matching/input_matchers/source:hyperscan_matcher_lib", diff --git a/contrib/kafka/filters/network/source/mesh/BUILD b/contrib/kafka/filters/network/source/mesh/BUILD index b52c41fa5f..69f6e35ff2 100644 --- a/contrib/kafka/filters/network/source/mesh/BUILD +++ b/contrib/kafka/filters/network/source/mesh/BUILD @@ -4,7 +4,6 @@ load( "envoy_cc_library", "envoy_contrib_package", ) -load("//bazel:envoy_internal.bzl", "envoy_external_dep_path") licenses(["notice"]) # Apache 2 @@ -253,9 +252,9 @@ envoy_cc_library( ], tags = ["skip_on_windows"], deps = [ + "//bazel/foreign_cc:librdkafka", "//envoy/common:pure_lib", "@com_google_absl//absl/strings", - envoy_external_dep_path("librdkafka"), ], ) @@ -270,7 +269,7 @@ envoy_cc_library( tags = ["skip_on_windows"], deps = [ ":librdkafka_utils_lib", - envoy_external_dep_path("librdkafka"), + "//bazel/foreign_cc:librdkafka", "//source/common/common:macros", ], ) diff --git a/contrib/kafka/filters/network/test/mesh/BUILD b/contrib/kafka/filters/network/test/mesh/BUILD index 06a76c4afe..184e5623c6 100644 --- a/contrib/kafka/filters/network/test/mesh/BUILD +++ b/contrib/kafka/filters/network/test/mesh/BUILD @@ -4,10 +4,6 @@ load( "envoy_cc_test_library", "envoy_contrib_package", ) -load( - "//bazel:envoy_internal.bzl", - "envoy_external_dep_path", -) licenses(["notice"]) # Apache 2 @@ -104,7 +100,7 @@ envoy_cc_test_library( hdrs = ["kafka_mocks.h"], tags = ["skip_on_windows"], deps = [ - envoy_external_dep_path("librdkafka"), + "//bazel/foreign_cc:librdkafka", "//contrib/kafka/filters/network/source/mesh:librdkafka_utils_lib", ], ) diff --git a/contrib/mysql_proxy/filters/network/test/BUILD b/contrib/mysql_proxy/filters/network/test/BUILD index 1a7e65a434..bb6bfecf23 100644 --- a/contrib/mysql_proxy/filters/network/test/BUILD +++ b/contrib/mysql_proxy/filters/network/test/BUILD @@ -109,7 +109,6 @@ envoy_cc_test( "mysql_command_test.cc", ], data = ["mysql_test_config.yaml"], - external_deps = ["sqlparser"], deps = [ ":mysql_test_utils_lib", "//contrib/mysql_proxy/filters/network/source:codec_lib", @@ -117,5 +116,6 @@ envoy_cc_test( "//source/common/tcp_proxy", "//source/extensions/filters/network/tcp_proxy:config", "//test/integration:integration_lib", + "@com_github_envoyproxy_sqlparser//:sqlparser", ], ) diff --git a/contrib/qat/compression/qatzstd/compressor/source/BUILD b/contrib/qat/compression/qatzstd/compressor/source/BUILD index 960fa2b9b5..f1d7caa3a3 100644 --- a/contrib/qat/compression/qatzstd/compressor/source/BUILD +++ b/contrib/qat/compression/qatzstd/compressor/source/BUILD @@ -34,8 +34,8 @@ make( "install", ], deps = [ + "//bazel/foreign_cc:zstd", "//contrib/qat:qatlib", - "//external:zstd", ], ) diff --git a/contrib/qat/private_key_providers/source/BUILD b/contrib/qat/private_key_providers/source/BUILD index e6dfb8ac26..76757e972f 100644 --- a/contrib/qat/private_key_providers/source/BUILD +++ b/contrib/qat/private_key_providers/source/BUILD @@ -32,9 +32,7 @@ envoy_cc_library( "qat.h", "qat_private_key_provider.h", ], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ ":libqat_wrapper_lib", "//envoy/api:api_interface", diff --git a/contrib/sxg/filters/http/source/BUILD b/contrib/sxg/filters/http/source/BUILD index 1d3b5b9231..83a1f04817 100644 --- a/contrib/sxg/filters/http/source/BUILD +++ b/contrib/sxg/filters/http/source/BUILD @@ -21,7 +21,6 @@ envoy_cc_library( "filter.h", "filter_config.h", ], - external_deps = ["libsxg"], deps = [ "//envoy/server:filter_config_interface", "//source/common/config:datasource_lib", @@ -31,6 +30,7 @@ envoy_cc_library( "//source/common/secret:secret_provider_impl_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", "@envoy_api//contrib/envoy/extensions/filters/http/sxg/v3alpha:pkg_cc_proto", + "//bazel/foreign_cc:libsxg", # use boringssl alias to select fips vs non-fips version. "//bazel:boringssl", ], diff --git a/distribution/binary/BUILD b/distribution/binary/BUILD index 5d610f0d07..4f72887530 100644 --- a/distribution/binary/BUILD +++ b/distribution/binary/BUILD @@ -18,7 +18,7 @@ bundled( "//distribution:envoy-dwarf": "dbg/envoy.dwp", "//distribution:envoy-contrib-binary": "dbg/envoy-contrib", "//distribution:envoy-contrib-dwarf": "dbg/envoy-contrib.dwp", - "//external:su-exec": "utils/su-exec", + "@com_github_ncopa_suexec//:su-exec": "utils/su-exec", }, ) diff --git a/distribution/binary/compiler.bzl b/distribution/binary/compiler.bzl index 1506be1260..aa4a287371 100644 --- a/distribution/binary/compiler.bzl +++ b/distribution/binary/compiler.bzl @@ -32,7 +32,7 @@ # targets = { # "//distribution:envoy-binary": "envoy", # "//distribution:envoy-contrib-binary": "envoy-contrib", -# "//external:su-exec": "utils/su-exec", +# "@com_github_ncopa_suexec//:su-exec": "utils/su-exec", # }, # ) # diff --git a/envoy/grpc/BUILD b/envoy/grpc/BUILD index 47b9631194..50ae15b205 100644 --- a/envoy/grpc/BUILD +++ b/envoy/grpc/BUILD @@ -46,9 +46,7 @@ envoy_cc_library( envoy_cc_library( name = "google_grpc_creds_interface", hdrs = ["google_grpc_creds.h"], - external_deps = [ - "grpc", - ], + external_deps = ["grpc"], deps = [ "//envoy/api:api_interface", "//envoy/config:typed_config_interface", diff --git a/mobile/bazel/protobuf.patch b/mobile/bazel/protobuf.patch index e786c7ebe1..d3bb997483 100644 --- a/mobile/bazel/protobuf.patch +++ b/mobile/bazel/protobuf.patch @@ -12,12 +12,12 @@ diff --git a/BUILD b/BUILD index efc3d8e7f..746ad4851 100644 --- a/BUILD +++ b/BUILD -@@ -24,7 +24,7 @@ config_setting( +@@ -24,4 +24,4 @@ config_setting( # ZLIB configuration ################################################################################ -ZLIB_DEPS = ["@zlib//:zlib"] -+ZLIB_DEPS = ["//external:zlib"] ++ZLIB_DEPS = ["@envoy//bazel/foreign_cc:zlib"] ################################################################################ # Protobuf Runtime Library diff --git a/source/common/common/BUILD b/source/common/common/BUILD index 415b71d4c2..e2f26a09a2 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -127,11 +127,9 @@ envoy_cc_library( envoy_basic_cc_library( name = "fmt_lib", hdrs = ["fmt.h"], - external_deps = [ - "fmtlib", - ], deps = [ "//envoy/common:base_includes", + "@com_github_fmtlib_fmt//:fmtlib", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", ], @@ -141,11 +139,11 @@ envoy_cc_library( name = "hash_lib", srcs = ["hash.cc"], hdrs = ["hash.h"], - external_deps = ["xxhash"], deps = [ ":macros", ":safe_memcpy_lib", "//envoy/common:base_includes", + "@com_github_cyan4973_xxhash//:xxhash", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", ], @@ -167,14 +165,12 @@ envoy_cc_library( name = "key_value_store_lib", srcs = ["key_value_store_base.cc"], hdrs = ["key_value_store_base.h"], - external_deps = [ - "quiche_quic_platform", - ], deps = [ "//envoy/common:key_value_store_interface", "//envoy/event:dispatcher_interface", "//envoy/filesystem:filesystem_interface", "//source/common/config:ttl_lib", + "@com_github_google_quiche//:quic_platform", "@com_github_google_quiche//:quiche_common_lib", "@com_google_absl//absl/cleanup", ], @@ -229,10 +225,8 @@ envoy_cc_library( name = "base_logger_lib", srcs = ["base_logger.cc"], hdrs = ["base_logger.h"], - external_deps = [ - "spdlog", - ], deps = [ + "@com_github_gabime_spdlog//:spdlog", "@com_google_absl//absl/strings", ], ) @@ -585,9 +579,6 @@ envoy_cc_library( envoy_pch_library( name = "common_pch", - external_deps = [ - "spdlog", - ], includes = [ "envoy/config/bootstrap/v3/bootstrap.pb.h", "envoy/config/cluster/v3/cluster.pb.h", @@ -598,6 +589,7 @@ envoy_pch_library( "envoy/service/discovery/v3/discovery.pb.h", "spdlog/sinks/android_sink.h", "spdlog/spdlog.h", + "@com_github_gabime_spdlog//:spdlog", ], visibility = ["//visibility:public"], deps = [ diff --git a/source/common/compression/zstd/common/BUILD b/source/common/compression/zstd/common/BUILD index e9ecaa5338..523ef652ef 100644 --- a/source/common/compression/zstd/common/BUILD +++ b/source/common/compression/zstd/common/BUILD @@ -12,8 +12,8 @@ envoy_cc_library( name = "zstd_base_lib", srcs = ["base.cc"], hdrs = ["base.h"], - external_deps = ["zstd"], deps = [ + "//bazel/foreign_cc:zstd", "//source/common/buffer:buffer_lib", ], ) diff --git a/source/common/crypto/BUILD b/source/common/crypto/BUILD index 1baf0903f6..3f52f93003 100644 --- a/source/common/crypto/BUILD +++ b/source/common/crypto/BUILD @@ -19,9 +19,7 @@ envoy_cc_library( "utility.h", "utility_impl.h", ], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ "//envoy/buffer:buffer_interface", "//envoy/common/crypto:crypto_interface", diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 659d8e76b0..0bcc005674 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -88,8 +88,8 @@ envoy_cc_library( name = "event_impl_base_lib", srcs = ["event_impl_base.cc"], hdrs = ["event_impl_base.h"], - external_deps = [ - "event", + deps = [ + "//bazel/foreign_cc:event", ], ) @@ -133,10 +133,8 @@ envoy_cc_library( name = "libevent_lib", srcs = ["libevent.cc"], hdrs = ["libevent.h"], - external_deps = [ - "event", - ], deps = [ + "//bazel/foreign_cc:event", "//source/common/common:assert_lib", "//source/common/common:c_smart_ptr_lib", ], @@ -146,11 +144,11 @@ envoy_cc_library( name = "libevent_scheduler_lib", srcs = ["libevent_scheduler.cc"], hdrs = ["libevent_scheduler.h"], - external_deps = ["event"], deps = [ ":libevent_lib", ":schedulable_cb_lib", ":timer_lib", + "//bazel/foreign_cc:event", "//envoy/event:dispatcher_interface", "//envoy/event:timer_interface", "//source/common/common:assert_lib", @@ -161,10 +159,10 @@ envoy_cc_library( name = "schedulable_cb_lib", srcs = ["schedulable_cb_impl.cc"], hdrs = ["schedulable_cb_impl.h"], - external_deps = ["event"], deps = [ ":event_impl_base_lib", ":libevent_lib", + "//bazel/foreign_cc:event", "//envoy/event:schedulable_cb_interface", ], ) @@ -173,10 +171,10 @@ envoy_cc_library( name = "timer_lib", srcs = ["timer_impl.cc"], hdrs = ["timer_impl.h"], - external_deps = ["event"], deps = [ ":event_impl_base_lib", ":libevent_lib", + "//bazel/foreign_cc:event", "//envoy/event:timer_interface", "//source/common/common:scope_tracker", "//source/common/common:utility_lib", diff --git a/source/common/filesystem/BUILD b/source/common/filesystem/BUILD index 65f8e0c87b..783882d695 100644 --- a/source/common/filesystem/BUILD +++ b/source/common/filesystem/BUILD @@ -106,9 +106,6 @@ envoy_cc_library( "inotify/watcher_impl.h", ], }), - external_deps = [ - "event", - ], include_prefix = "source/common/filesystem", strip_include_prefix = select({ "//bazel:apple": "kqueue", @@ -116,6 +113,7 @@ envoy_cc_library( "//conditions:default": "inotify", }), deps = [ + "//bazel/foreign_cc:event", "//envoy/api:api_interface", "//envoy/event:dispatcher_interface", "//source/common/buffer:buffer_lib", diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index 3e183c19e2..200d365aec 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -133,9 +133,7 @@ envoy_cc_library( name = "google_grpc_utils_lib", srcs = ["google_grpc_utils.cc"], hdrs = ["google_grpc_utils.h"], - external_deps = [ - "grpc", - ], + external_deps = ["grpc"], deps = [ ":google_grpc_creds_lib", "//envoy/api:api_interface", @@ -166,9 +164,7 @@ envoy_cc_library( name = "google_async_client_lib", srcs = ["google_async_client_impl.cc"], hdrs = ["google_async_client_impl.h"], - external_deps = [ - "grpc", - ], + external_deps = ["grpc"], deps = [ ":context_lib", ":google_grpc_context_lib", diff --git a/source/common/http/BUILD b/source/common/http/BUILD index c9d337388d..f0dc9190b8 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -240,7 +240,6 @@ envoy_cc_library( "http_server_properties_cache_impl.h", "http_server_properties_cache_manager_impl.h", ], - external_deps = ["quiche_quic_platform"], deps = [ ":http3_status_tracker_impl_lib", "//envoy/common:time_interface", @@ -253,6 +252,7 @@ envoy_cc_library( "//source/common/common:logger_lib", "//source/common/config:utility_lib", "@com_github_google_quiche//:http2_core_alt_svc_wire_format_lib", + "@com_github_google_quiche//:quic_platform", "@envoy_api//envoy/config/common/key_value/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], @@ -519,10 +519,6 @@ envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], hdrs = ["utility.h"], - external_deps = [ - "http_parser", - "quiche_http2_protocol", - ], deps = [ ":character_set_validation_lib", ":exception_lib", @@ -530,6 +526,7 @@ envoy_cc_library( ":headers_lib", ":http_option_limits_lib", ":message_lib", + "//bazel/external/http_parser", "//envoy/common:regex_interface", "//envoy/http:codes_interface", "//envoy/http:filter_interface", @@ -545,6 +542,7 @@ envoy_cc_library( "//source/common/network:utility_lib", "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_features_lib", + "@com_github_google_quiche//:http2_adapter_http2_protocol", "@com_google_absl//absl/container:node_hash_set", "@com_google_absl//absl/types:optional", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", @@ -556,9 +554,6 @@ envoy_cc_library( name = "header_utility_lib", srcs = ["header_utility.cc"], hdrs = ["header_utility.h"], - external_deps = [ - "quiche_http2_adapter", - ], deps = [ ":character_set_validation_lib", ":header_map_lib", @@ -574,6 +569,7 @@ envoy_cc_library( "//source/common/common:utility_lib", "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_features_lib", + "@com_github_google_quiche//:http2_adapter", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", diff --git a/source/common/http/http1/BUILD b/source/common/http/http1/BUILD index 84f8e2d7c3..0e31c65919 100644 --- a/source/common/http/http1/BUILD +++ b/source/common/http/http1/BUILD @@ -115,9 +115,9 @@ envoy_cc_library( name = "legacy_parser_lib", srcs = ["legacy_parser_impl.cc"], hdrs = ["legacy_parser_impl.h"], - external_deps = ["http_parser"], deps = [ ":parser_interface", + "//bazel/external/http_parser", "//source/common/common:assert_lib", ], ) diff --git a/source/common/http/http2/BUILD b/source/common/http/http2/BUILD index 26fa680b9f..281377db83 100644 --- a/source/common/http/http2/BUILD +++ b/source/common/http/http2/BUILD @@ -28,9 +28,6 @@ envoy_cc_library( name = "codec_lib", srcs = ["codec_impl.cc"], hdrs = ["codec_impl.h"], - external_deps = [ - "quiche_http2_adapter", - ], deps = [ ":codec_stats_lib", ":metadata_decoder_lib", @@ -61,6 +58,7 @@ envoy_cc_library( "//source/common/http:utility_lib", "//source/common/network:common_connection_filter_states_lib", "//source/common/runtime:runtime_features_lib", + "@com_github_google_quiche//:http2_adapter", "@com_google_absl//absl/algorithm", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:inlined_vector", @@ -97,14 +95,12 @@ envoy_cc_library( name = "metadata_encoder_lib", srcs = ["metadata_encoder.cc"], hdrs = ["metadata_encoder.h"], - external_deps = [ - "quiche_http2_adapter", - ], deps = [ "//envoy/http:codec_interface", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", + "@com_github_google_quiche//:http2_adapter", ], ) @@ -112,15 +108,13 @@ envoy_cc_library( name = "metadata_decoder_lib", srcs = ["metadata_decoder.cc"], hdrs = ["metadata_decoder.h"], - external_deps = [ - "quiche_http2_hpack_decoder", - ], deps = [ "//envoy/http:codec_interface", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", "//source/common/runtime:runtime_features_lib", + "@com_github_google_quiche//:http2_hpack_decoder_hpack_decoder_lib", ] + envoy_select_nghttp2([envoy_external_dep_path("nghttp2")]), ) diff --git a/source/common/io/BUILD b/source/common/io/BUILD index c3d8166152..2b6c2b029f 100644 --- a/source/common/io/BUILD +++ b/source/common/io/BUILD @@ -17,9 +17,9 @@ envoy_cc_library( hdrs = [ "io_uring_impl.h", ], - external_deps = ["uring"], tags = ["nocompdb"], deps = [ + "//bazel/foreign_cc:liburing_linux", "//envoy/common/io:io_uring_interface", "//envoy/thread_local:thread_local_interface", ], diff --git a/source/common/json/BUILD b/source/common/json/BUILD index 1ca1e30c9d..02ae19bfab 100644 --- a/source/common/json/BUILD +++ b/source/common/json/BUILD @@ -12,15 +12,13 @@ envoy_cc_library( name = "json_internal_lib", srcs = ["json_internal.cc"], hdrs = ["json_internal.h"], - external_deps = [ - "json", - ], deps = [ "//envoy/json:json_object_interface", "//source/common/common:assert_lib", "//source/common/common:hash_lib", "//source/common/common:utility_lib", "//source/common/protobuf:utility_lib", + "@com_github_nlohmann_json//:json", ], ) diff --git a/source/common/orca/BUILD b/source/common/orca/BUILD index c558cee04e..79aa433408 100644 --- a/source/common/orca/BUILD +++ b/source/common/orca/BUILD @@ -12,13 +12,11 @@ envoy_cc_library( name = "orca_parser", srcs = ["orca_parser.cc"], hdrs = ["orca_parser.h"], - external_deps = [ - "fmtlib", - ], deps = [ "//envoy/http:header_map_interface", "//source/common/common:base64_lib", "@com_github_cncf_xds//xds/data/orca/v3:pkg_cc_proto", + "@com_github_fmtlib_fmt//:fmtlib", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], @@ -28,15 +26,13 @@ envoy_cc_library( name = "orca_load_metrics_lib", srcs = ["orca_load_metrics.cc"], hdrs = ["orca_load_metrics.h"], - external_deps = [ - "fmtlib", - ], deps = [ ":orca_parser", "//envoy/http:header_map_interface", "//source/common/http:header_utility_lib", "//source/common/protobuf:utility_lib_header", "@com_github_cncf_xds//xds/data/orca/v3:pkg_cc_proto", + "@com_github_fmtlib_fmt//:fmtlib", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/source/common/protobuf/BUILD b/source/common/protobuf/BUILD index fa252699ff..8c60668ca2 100644 --- a/source/common/protobuf/BUILD +++ b/source/common/protobuf/BUILD @@ -34,7 +34,6 @@ envoy_cc_library( name = "message_validator_lib", srcs = ["message_validator_impl.cc"], hdrs = ["message_validator_impl.h"], - external_deps = ["protobuf"], deps = [ "//envoy/protobuf:message_validator_interface", "//envoy/stats:stats_interface", @@ -42,18 +41,17 @@ envoy_cc_library( "//source/common/common:hash_lib", "//source/common/common:logger_lib", "//source/common/common:macros", + "@com_google_protobuf//:protobuf", ], ) envoy_cc_library( name = "protobuf", hdrs = ["protobuf.h"], - external_deps = [ - "protobuf", - ], deps = [ ":cc_wkt_protos", "//envoy/common:base_includes", + "@com_google_protobuf//:protobuf", ], ) @@ -72,10 +70,6 @@ envoy_cc_library( envoy_cc_library( name = "yaml_utility_lib", srcs = envoy_select_enable_yaml(["yaml_utility.cc"]), - external_deps = [ - "protobuf", - "yaml_cpp", - ], deps = [ ":message_validator_lib", ":protobuf", @@ -88,6 +82,8 @@ envoy_cc_library( "//source/common/protobuf:visitor_lib", "//source/common/runtime:runtime_features_lib", "@com_github_cncf_xds//udpa/annotations:pkg_cc_proto", + "@com_github_jbeder_yaml_cpp//:yaml-cpp", + "@com_google_protobuf//:protobuf", "@envoy_api//envoy/annotations:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", "@utf8_range//:utf8_validity", @@ -99,11 +95,9 @@ envoy_cc_library( srcs = [ "create_reflectable_message.cc", ], - external_deps = [ - "protobuf", - ], deps = [ "utility_lib_header", + "@com_google_protobuf//:protobuf", ] + envoy_select_enable_lite_protos([ "@envoy_api//bazel/cc_proto_descriptor_library:create_dynamic_message", "@envoy_api//bazel/cc_proto_descriptor_library:text_format_transcoder", @@ -213,9 +207,6 @@ envoy_cc_library( envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], - external_deps = [ - "protobuf", - ], deps = [ ":deterministic_hash_lib", ":message_validator_lib", @@ -229,6 +220,7 @@ envoy_cc_library( "//source/common/protobuf:visitor_lib", "//source/common/runtime:runtime_features_lib", "@com_github_cncf_xds//udpa/annotations:pkg_cc_proto", + "@com_google_protobuf//:protobuf", "@envoy_api//envoy/annotations:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", ] + envoy_select_enable_yaml(["yaml_utility_lib"]), diff --git a/source/common/quic/BUILD b/source/common/quic/BUILD index 62a54d13e6..2186c5f600 100644 --- a/source/common/quic/BUILD +++ b/source/common/quic/BUILD @@ -21,13 +21,13 @@ envoy_cc_library( name = "envoy_quic_alarm_lib", srcs = ["envoy_quic_alarm.cc"], hdrs = ["envoy_quic_alarm.h"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ "//envoy/event:dispatcher_interface", "//envoy/event:timer_interface", "@com_github_google_quiche//:quic_core_alarm_lib", "@com_github_google_quiche//:quic_core_clock_lib", + "@com_github_google_quiche//:quic_platform", ], ) @@ -35,13 +35,13 @@ envoy_cc_library( name = "envoy_quic_alarm_factory_lib", srcs = ["envoy_quic_alarm_factory.cc"], hdrs = ["envoy_quic_alarm_factory.h"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ ":envoy_quic_alarm_lib", "@com_github_google_quiche//:quic_core_alarm_factory_lib", "@com_github_google_quiche//:quic_core_arena_scoped_ptr_lib", "@com_github_google_quiche//:quic_core_one_block_arena_lib", + "@com_github_google_quiche//:quic_platform", ], ) @@ -100,7 +100,6 @@ envoy_cc_library( name = "envoy_quic_proof_source_base_lib", srcs = ["envoy_quic_proof_source_base.cc"], hdrs = ["envoy_quic_proof_source_base.h"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ ":envoy_quic_utils_lib", @@ -109,6 +108,7 @@ envoy_cc_library( "@com_github_google_quiche//:quic_core_crypto_proof_source_lib", "@com_github_google_quiche//:quic_core_data_lib", "@com_github_google_quiche//:quic_core_versions_lib", + "@com_github_google_quiche//:quic_platform", ], ) @@ -116,9 +116,7 @@ envoy_cc_library( name = "envoy_quic_proof_source_lib", srcs = ["envoy_quic_proof_source.cc"], hdrs = ["envoy_quic_proof_source.h"], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], tags = ["nofips"], deps = [ ":envoy_quic_proof_source_base_lib", @@ -138,13 +136,13 @@ envoy_cc_library( name = "envoy_quic_proof_verifier_base_lib", srcs = ["envoy_quic_proof_verifier_base.cc"], hdrs = ["envoy_quic_proof_verifier_base.h"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ ":envoy_quic_utils_lib", "@com_github_google_quiche//:quic_core_crypto_certificate_view_lib", "@com_github_google_quiche//:quic_core_crypto_crypto_handshake_lib", "@com_github_google_quiche//:quic_core_versions_lib", + "@com_github_google_quiche//:quic_platform", ], ) @@ -152,13 +150,13 @@ envoy_cc_library( name = "envoy_quic_proof_verifier_lib", srcs = ["envoy_quic_proof_verifier.cc"], hdrs = ["envoy_quic_proof_verifier.h"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ ":envoy_quic_proof_verifier_base_lib", ":envoy_quic_utils_lib", ":quic_ssl_connection_info_lib", "//source/common/tls:context_lib", + "@com_github_google_quiche//:quic_platform", ], ) @@ -462,10 +460,7 @@ envoy_cc_library( name = "envoy_quic_utils_lib", srcs = ["envoy_quic_utils.cc"], hdrs = ["envoy_quic_utils.h"], - external_deps = [ - "quiche_quic_platform", - "ssl", - ], + external_deps = ["ssl"], tags = ["nofips"], deps = [ "//envoy/http:codec_interface", @@ -478,6 +473,7 @@ envoy_cc_library( "//source/common/quic:quic_io_handle_wrapper_lib", "@com_github_google_quiche//:quic_core_config_lib", "@com_github_google_quiche//:quic_core_http_header_list_lib", + "@com_github_google_quiche//:quic_platform", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", ], @@ -591,11 +587,11 @@ envoy_cc_library( name = "envoy_quic_packet_writer_lib", srcs = ["envoy_quic_packet_writer.cc"], hdrs = ["envoy_quic_packet_writer.h"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ ":envoy_quic_utils_lib", "@com_github_google_quiche//:quic_core_packet_writer_lib", + "@com_github_google_quiche//:quic_platform", ], ) @@ -606,7 +602,6 @@ envoy_cc_library( "//conditions:default": [], }), hdrs = ["udp_gso_batch_writer.h"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ ":envoy_quic_utils_lib", @@ -614,6 +609,7 @@ envoy_cc_library( "//source/common/network:io_socket_error_lib", "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_lib", + "@com_github_google_quiche//:quic_platform", ] + select({ "//bazel:linux": ["@com_github_google_quiche//:quic_core_batch_writer_gso_batch_writer_lib"], "//conditions:default": [], @@ -721,11 +717,9 @@ envoy_cc_library( name = "cert_compression_lib", srcs = ["cert_compression.cc"], hdrs = ["cert_compression.h"], - external_deps = [ - "ssl", - "zlib", - ], + external_deps = ["ssl"], deps = [ + "//bazel/foreign_cc:zlib", "//source/common/common:assert_lib", "//source/common/common:logger_lib", "//source/common/runtime:runtime_lib", diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD index b2852de43a..4e01a6e007 100644 --- a/source/common/stats/BUILD +++ b/source/common/stats/BUILD @@ -54,15 +54,13 @@ envoy_cc_library( name = "histogram_lib", srcs = ["histogram_impl.cc"], hdrs = ["histogram_impl.h"], - external_deps = [ - "libcircllhist", - ], deps = [ ":metric_impl_lib", "//source/common/common:assert_lib", "//source/common/common:hash_lib", "//source/common/common:matchers_lib", "//source/common/common:utility_lib", + "@com_github_openhistogram_libcircllhist//:libcircllhist", "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto", ], ) diff --git a/source/common/tls/BUILD b/source/common/tls/BUILD index 4a0b1b3b71..ea7576b8c2 100644 --- a/source/common/tls/BUILD +++ b/source/common/tls/BUILD @@ -63,9 +63,7 @@ envoy_cc_library( name = "ssl_socket_base", srcs = ["ssl_socket.cc"], hdrs = ["ssl_socket.h"], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], # TLS is core functionality. visibility = ["//visibility:public"], deps = [ @@ -98,9 +96,7 @@ envoy_cc_library( name = "client_ssl_socket_lib", srcs = ["client_ssl_socket.cc"], hdrs = ["client_ssl_socket.h"], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ ":ssl_socket_base", "@com_google_absl//absl/container:node_hash_set", @@ -114,9 +110,7 @@ envoy_cc_library( name = "server_ssl_socket_lib", srcs = ["server_ssl_socket.cc"], hdrs = ["server_ssl_socket.h"], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ ":ssl_socket_base", "@com_google_absl//absl/container:node_hash_set", @@ -138,9 +132,7 @@ envoy_cc_library( name = "context_config_lib", srcs = ["context_config_impl.cc"], hdrs = ["context_config_impl.h"], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], # TLS is core functionality. visibility = ["//visibility:public"], deps = [ @@ -185,9 +177,7 @@ envoy_cc_library( "context_impl.h", "context_manager_impl.h", ], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], # TLS is core functionality. visibility = ["//visibility:public"], deps = [ @@ -243,9 +233,7 @@ envoy_cc_library( name = "stats_lib", srcs = ["stats.cc"], hdrs = ["stats.h"], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ "//envoy/stats:stats_interface", "//envoy/stats:stats_macros", @@ -258,9 +246,7 @@ envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], hdrs = ["utility.h"], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ "//source/common/common:assert_lib", "//source/common/common:empty_string", diff --git a/source/common/tls/cert_validator/BUILD b/source/common/tls/cert_validator/BUILD index ed1619c6d5..63fbe3df35 100644 --- a/source/common/tls/cert_validator/BUILD +++ b/source/common/tls/cert_validator/BUILD @@ -23,9 +23,7 @@ envoy_cc_library( "san_matcher.h", "utility.h", ], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], visibility = ["//visibility:public"], deps = [ "//envoy/config:typed_config_interface", diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index e96b08a318..f974321a7b 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -227,9 +227,7 @@ envoy_cc_library( name = "health_checker_lib", srcs = ["health_checker_impl.cc"], hdrs = ["health_checker_impl.h"], - external_deps = [ - "grpc_health_proto", - ], + external_deps = ["grpc_health_proto"], deps = [ ":health_checker_event_logger_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/source/extensions/access_loggers/fluentd/BUILD b/source/extensions/access_loggers/fluentd/BUILD index 3a7151731b..73da28b991 100644 --- a/source/extensions/access_loggers/fluentd/BUILD +++ b/source/extensions/access_loggers/fluentd/BUILD @@ -23,14 +23,12 @@ envoy_cc_library( name = "fluentd_access_log_lib", srcs = ["fluentd_access_log_impl.cc"], hdrs = ["fluentd_access_log_impl.h"], - external_deps = [ - "msgpack", - ], deps = [ ":substitution_formatter_lib", "//envoy/access_log:access_log_interface", "//source/common/access_log:access_log_lib", "//source/extensions/access_loggers/common:access_log_base", + "@com_github_msgpack_cpp//:msgpack", "@envoy_api//envoy/extensions/access_loggers/fluentd/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/common/dubbo/BUILD b/source/extensions/common/dubbo/BUILD index ec22a68ea7..e3e114a4e1 100644 --- a/source/extensions/common/dubbo/BUILD +++ b/source/extensions/common/dubbo/BUILD @@ -12,13 +12,11 @@ envoy_cc_library( name = "hessian2_utils_lib", srcs = ["hessian2_utils.cc"], hdrs = ["hessian2_utils.h"], - external_deps = [ - "hessian2_codec_codec_impl", - "hessian2_codec_object_codec_lib", - ], deps = [ "//envoy/buffer:buffer_interface", "//source/common/singleton:const_singleton", + "@com_github_alibaba_hessian2_codec//hessian2:codec_impl_lib", + "@com_github_alibaba_hessian2_codec//hessian2/basic_codec:object_codec_lib", ], ) diff --git a/source/extensions/common/wasm/BUILD b/source/extensions/common/wasm/BUILD index c45b87e200..1931b851d5 100644 --- a/source/extensions/common/wasm/BUILD +++ b/source/extensions/common/wasm/BUILD @@ -97,8 +97,8 @@ envoy_cc_extension( deps = [ ":wasm_hdr", ":wasm_runtime_factory_interface", + "//bazel/foreign_cc:zlib", "//envoy/server:lifecycle_notifier_interface", - "//external:zlib", "//source/common/buffer:buffer_lib", "//source/common/common:enum_to_int", "//source/common/common:safe_memcpy_lib", diff --git a/source/extensions/common/wasm/ext/BUILD b/source/extensions/common/wasm/ext/BUILD index 3523551306..254f4b1276 100644 --- a/source/extensions/common/wasm/ext/BUILD +++ b/source/extensions/common/wasm/ext/BUILD @@ -97,7 +97,7 @@ cc_proto_library( name = "node_subset_cc_proto", deps = [ ":node_subset_proto", - # "//external:protobuf_clib", + # "@com_google_protobuf//:protoc_lib", ], ) diff --git a/source/extensions/compression/brotli/compressor/BUILD b/source/extensions/compression/brotli/compressor/BUILD index c8e6e3a8a4..6eb824ae6d 100644 --- a/source/extensions/compression/brotli/compressor/BUILD +++ b/source/extensions/compression/brotli/compressor/BUILD @@ -13,11 +13,11 @@ envoy_cc_library( name = "compressor_lib", srcs = ["brotli_compressor_impl.cc"], hdrs = ["brotli_compressor_impl.h"], - external_deps = ["brotlienc"], deps = [ "//envoy/compression/compressor:compressor_interface", "//source/common/buffer:buffer_lib", "//source/extensions/compression/brotli/common:brotli_base_lib", + "@org_brotli//:brotlienc", ], ) diff --git a/source/extensions/compression/brotli/decompressor/BUILD b/source/extensions/compression/brotli/decompressor/BUILD index 18155bfaae..c5239f0df1 100644 --- a/source/extensions/compression/brotli/decompressor/BUILD +++ b/source/extensions/compression/brotli/decompressor/BUILD @@ -13,7 +13,6 @@ envoy_cc_library( name = "decompressor_lib", srcs = ["brotli_decompressor_impl.cc"], hdrs = ["brotli_decompressor_impl.h"], - external_deps = ["brotlidec"], deps = [ "//envoy/compression/decompressor:decompressor_interface", "//envoy/stats:stats_interface", @@ -21,6 +20,7 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/runtime:runtime_features_lib", "//source/extensions/compression/brotli/common:brotli_base_lib", + "@org_brotli//:brotlidec", ], ) diff --git a/source/extensions/compression/gzip/common/BUILD b/source/extensions/compression/gzip/common/BUILD index 5c301a6a9a..c843301dd6 100644 --- a/source/extensions/compression/gzip/common/BUILD +++ b/source/extensions/compression/gzip/common/BUILD @@ -12,8 +12,8 @@ envoy_cc_library( name = "zlib_base_lib", srcs = ["base.cc"], hdrs = ["base.h"], - external_deps = ["zlib"], deps = [ + "//bazel/foreign_cc:zlib", "//source/common/buffer:buffer_lib", ], ) diff --git a/source/extensions/compression/gzip/compressor/BUILD b/source/extensions/compression/gzip/compressor/BUILD index c138afe6ac..a7d38764f3 100644 --- a/source/extensions/compression/gzip/compressor/BUILD +++ b/source/extensions/compression/gzip/compressor/BUILD @@ -13,8 +13,8 @@ envoy_cc_library( name = "compressor_lib", srcs = ["zlib_compressor_impl.cc"], hdrs = ["zlib_compressor_impl.h"], - external_deps = ["zlib"], deps = [ + "//bazel/foreign_cc:zlib", "//envoy/compression/compressor:compressor_interface", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", diff --git a/source/extensions/compression/gzip/decompressor/BUILD b/source/extensions/compression/gzip/decompressor/BUILD index 2090f49bcd..4904d33ac4 100644 --- a/source/extensions/compression/gzip/decompressor/BUILD +++ b/source/extensions/compression/gzip/decompressor/BUILD @@ -13,8 +13,8 @@ envoy_cc_library( name = "zlib_decompressor_impl_lib", srcs = ["zlib_decompressor_impl.cc"], hdrs = ["zlib_decompressor_impl.h"], - external_deps = ["zlib"], deps = [ + "//bazel/foreign_cc:zlib", "//envoy/compression/decompressor:decompressor_interface", "//envoy/stats:stats_interface", "//envoy/stats:stats_macros", diff --git a/source/extensions/compression/zstd/common/BUILD b/source/extensions/compression/zstd/common/BUILD index a0aa945564..eb8f90fb83 100644 --- a/source/extensions/compression/zstd/common/BUILD +++ b/source/extensions/compression/zstd/common/BUILD @@ -11,8 +11,8 @@ envoy_extension_package() envoy_cc_library( name = "zstd_dictionary_manager_lib", hdrs = ["dictionary_manager.h"], - external_deps = ["zstd"], deps = [ + "//bazel/foreign_cc:zstd", "//envoy/event:dispatcher_interface", "//envoy/thread_local:thread_local_interface", "//source/common/config:datasource_lib", diff --git a/source/extensions/config_subscription/rest/BUILD b/source/extensions/config_subscription/rest/BUILD index 360ed0cdac..e7f8051ddf 100644 --- a/source/extensions/config_subscription/rest/BUILD +++ b/source/extensions/config_subscription/rest/BUILD @@ -13,9 +13,7 @@ envoy_cc_extension( name = "http_subscription_lib", srcs = ["http_subscription_impl.cc"], hdrs = ["http_subscription_impl.h"], - external_deps = [ - "http_api_protos", - ], + external_deps = ["http_api_protos"], extra_visibility = [ # previously considered core code. "//test:__subpackages__", diff --git a/source/extensions/filters/common/lua/BUILD b/source/extensions/filters/common/lua/BUILD index 12d7d0554a..207abbac99 100644 --- a/source/extensions/filters/common/lua/BUILD +++ b/source/extensions/filters/common/lua/BUILD @@ -12,10 +12,8 @@ envoy_cc_library( name = "lua_lib", srcs = ["lua.cc"], hdrs = ["lua.h"], - external_deps = [ - "luajit", - ], deps = [ + "//bazel/foreign_cc:luajit", "//envoy/thread_local:thread_local_interface", "//source/common/common:assert_lib", "//source/common/common:c_smart_ptr_lib", diff --git a/source/extensions/filters/http/adaptive_concurrency/controller/BUILD b/source/extensions/filters/http/adaptive_concurrency/controller/BUILD index 60cb03dd31..abceffc527 100644 --- a/source/extensions/filters/http/adaptive_concurrency/controller/BUILD +++ b/source/extensions/filters/http/adaptive_concurrency/controller/BUILD @@ -19,9 +19,6 @@ envoy_cc_library( "controller.h", "gradient_controller.h", ], - external_deps = [ - "libcircllhist", - ], deps = [ "//envoy/common:time_interface", "//source/common/common:thread_synchronizer_lib", @@ -30,6 +27,7 @@ envoy_cc_library( "//source/common/runtime:runtime_lib", "//source/common/stats:isolated_store_lib", "//source/common/stats:stats_lib", + "@com_github_openhistogram_libcircllhist//:libcircllhist", "@envoy_api//envoy/extensions/filters/http/adaptive_concurrency/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/common/BUILD b/source/extensions/filters/http/common/BUILD index 7bc4eea7f4..73d4be20fb 100644 --- a/source/extensions/filters/http/common/BUILD +++ b/source/extensions/filters/http/common/BUILD @@ -31,12 +31,10 @@ envoy_cc_library( name = "jwks_fetcher_lib", srcs = ["jwks_fetcher.cc"], hdrs = ["jwks_fetcher.h"], - external_deps = [ - "jwt_verify_lib", - ], deps = [ "//envoy/upstream:cluster_manager_interface", "//source/common/http:utility_lib", + "@com_github_google_jwt_verify//:jwt_verify_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", ], diff --git a/source/extensions/filters/http/gcp_authn/BUILD b/source/extensions/filters/http/gcp_authn/BUILD index ecb021ee00..2dd2e7daa0 100644 --- a/source/extensions/filters/http/gcp_authn/BUILD +++ b/source/extensions/filters/http/gcp_authn/BUILD @@ -44,16 +44,14 @@ envoy_cc_library( envoy_cc_library( name = "token_cache", hdrs = ["token_cache.h"], - external_deps = [ - "jwt_verify_lib", - "simple_lru_cache_lib", - ], deps = [ "//source/common/http:headers_lib", "//source/common/http:message_lib", "//source/common/http:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", + "@com_github_google_jwt_verify//:jwt_verify_lib", + "@com_github_google_jwt_verify//:simple_lru_cache_lib", "@envoy_api//envoy/extensions/filters/http/gcp_authn/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/grpc_field_extraction/BUILD b/source/extensions/filters/http/grpc_field_extraction/BUILD index 929124b0a6..ae910dc834 100644 --- a/source/extensions/filters/http/grpc_field_extraction/BUILD +++ b/source/extensions/filters/http/grpc_field_extraction/BUILD @@ -24,9 +24,7 @@ envoy_cc_library( name = "extractor_impl", srcs = ["extractor_impl.cc"], hdrs = ["extractor_impl.h"], - external_deps = [ - "grpc_transcoding", - ], + external_deps = ["grpc_transcoding"], deps = [ "extractor", "//source/common/common:minimal_logger_lib", @@ -52,9 +50,7 @@ envoy_cc_library( name = "filter", srcs = ["filter.cc"], hdrs = ["filter.h"], - external_deps = [ - "grpc_transcoding", - ], + external_deps = ["grpc_transcoding"], deps = [ "extractor_impl", "filter_config", diff --git a/source/extensions/filters/http/grpc_json_transcoder/BUILD b/source/extensions/filters/http/grpc_json_transcoder/BUILD index 7a8455a241..fa88fe00db 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/BUILD +++ b/source/extensions/filters/http/grpc_json_transcoder/BUILD @@ -43,9 +43,7 @@ envoy_cc_library( name = "http_body_utils_lib", srcs = ["http_body_utils.cc"], hdrs = ["http_body_utils.h"], - external_deps = [ - "api_httpbody_protos", - ], + external_deps = ["api_httpbody_protos"], deps = [ "//source/common/grpc:codec_lib", "//source/common/protobuf", diff --git a/source/extensions/filters/http/jwt_authn/BUILD b/source/extensions/filters/http/jwt_authn/BUILD index ccf0af7f35..c01f977750 100644 --- a/source/extensions/filters/http/jwt_authn/BUILD +++ b/source/extensions/filters/http/jwt_authn/BUILD @@ -34,9 +34,6 @@ envoy_cc_library( name = "jwks_async_fetcher_lib", srcs = ["jwks_async_fetcher.cc"], hdrs = ["jwks_async_fetcher.h"], - external_deps = [ - "jwt_verify_lib", - ], deps = [ ":stats_lib", "//envoy/server:factory_context_interface", @@ -45,6 +42,7 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", "//source/common/tracing:http_tracer_lib", "//source/extensions/filters/http/common:jwks_fetcher_lib", + "@com_github_google_jwt_verify//:jwt_verify_lib", "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", ], ) @@ -53,13 +51,11 @@ envoy_cc_library( name = "jwks_cache_lib", srcs = ["jwks_cache.cc"], hdrs = ["jwks_cache.h"], - external_deps = [ - "jwt_verify_lib", - ], deps = [ "jwks_async_fetcher_lib", ":jwt_cache_lib", "//source/common/config:datasource_lib", + "@com_github_google_jwt_verify//:jwt_verify_lib", "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", ], ) @@ -83,14 +79,12 @@ envoy_cc_library( name = "filter_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], - external_deps = [ - "jwt_verify_lib", - ], deps = [ ":filter_config_lib", ":matchers_lib", "//envoy/http:filter_interface", "//source/common/http:headers_lib", + "@com_github_google_jwt_verify//:jwt_verify_lib", ], ) @@ -151,12 +145,10 @@ envoy_cc_library( name = "jwt_cache_lib", srcs = ["jwt_cache.cc"], hdrs = ["jwt_cache.h"], - external_deps = [ - "jwt_verify_lib", - "simple_lru_cache_lib", - ], deps = [ "//source/common/protobuf:utility_lib", + "@com_github_google_jwt_verify//:jwt_verify_lib", + "@com_github_google_jwt_verify//:simple_lru_cache_lib", "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/oauth2/BUILD b/source/extensions/filters/http/oauth2/BUILD index 5438db7cb9..5874d69036 100644 --- a/source/extensions/filters/http/oauth2/BUILD +++ b/source/extensions/filters/http/oauth2/BUILD @@ -44,9 +44,6 @@ envoy_cc_library( name = "oauth_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], - external_deps = [ - "jwt_verify_lib", - ], deps = [ ":oauth_client", "//envoy/server:filter_config_interface", @@ -59,6 +56,7 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", "//source/common/secret:secret_provider_impl_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", + "@com_github_google_jwt_verify//:jwt_verify_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/oauth2/v3:pkg_cc_proto", ], diff --git a/source/extensions/filters/listener/http_inspector/BUILD b/source/extensions/filters/listener/http_inspector/BUILD index a0fc9a0bfd..fc5dec01f9 100644 --- a/source/extensions/filters/listener/http_inspector/BUILD +++ b/source/extensions/filters/listener/http_inspector/BUILD @@ -15,8 +15,8 @@ envoy_cc_library( name = "http_inspector_lib", srcs = ["http_inspector.cc"], hdrs = ["http_inspector.h"], - external_deps = ["http_parser"], deps = [ + "//bazel/external/http_parser", "//envoy/event:dispatcher_interface", "//envoy/event:timer_interface", "//envoy/network:filter_interface", diff --git a/source/extensions/filters/network/dubbo_proxy/BUILD b/source/extensions/filters/network/dubbo_proxy/BUILD index 909bb29b7f..09edc7e358 100644 --- a/source/extensions/filters/network/dubbo_proxy/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/BUILD @@ -13,13 +13,11 @@ envoy_cc_library( name = "hessian_utils_lib", srcs = ["hessian_utils.cc"], hdrs = ["hessian_utils.h"], - external_deps = [ - "hessian2_codec_codec_impl", - "hessian2_codec_object_codec_lib", - ], deps = [ "//envoy/buffer:buffer_interface", "//source/common/singleton:const_singleton", + "@com_github_alibaba_hessian2_codec//hessian2:codec_impl_lib", + "@com_github_alibaba_hessian2_codec//hessian2/basic_codec:object_codec_lib", ], ) diff --git a/source/extensions/filters/udp/dns_filter/BUILD b/source/extensions/filters/udp/dns_filter/BUILD index ebb9bff329..e4315f1acc 100644 --- a/source/extensions/filters/udp/dns_filter/BUILD +++ b/source/extensions/filters/udp/dns_filter/BUILD @@ -24,8 +24,8 @@ envoy_cc_library( "dns_filter_utils.h", "dns_parser.h", ], - external_deps = ["ares"], deps = [ + "//bazel/foreign_cc:ares", "//envoy/buffer:buffer_interface", "//envoy/event:dispatcher_interface", "//envoy/network:address_interface", diff --git a/source/extensions/health_checkers/grpc/BUILD b/source/extensions/health_checkers/grpc/BUILD index 8507c82777..c06822d4e7 100644 --- a/source/extensions/health_checkers/grpc/BUILD +++ b/source/extensions/health_checkers/grpc/BUILD @@ -12,9 +12,7 @@ envoy_cc_extension( name = "health_checker_lib", srcs = ["health_checker_impl.cc"], hdrs = ["health_checker_impl.h"], - external_deps = [ - "grpc_health_proto", - ], + external_deps = ["grpc_health_proto"], extra_visibility = [ # previously considered core code. "//test:__subpackages__", diff --git a/source/extensions/network/dns_resolver/cares/BUILD b/source/extensions/network/dns_resolver/cares/BUILD index bb9c3a94f7..7a8a92fa61 100644 --- a/source/extensions/network/dns_resolver/cares/BUILD +++ b/source/extensions/network/dns_resolver/cares/BUILD @@ -12,9 +12,9 @@ envoy_cc_extension( name = "config", srcs = ["dns_impl.cc"], hdrs = ["dns_impl.h"], - external_deps = ["ares"], visibility = ["//visibility:public"], deps = [ + "//bazel/foreign_cc:ares", "//envoy/event:dispatcher_interface", "//envoy/event:file_event_interface", "//envoy/network:dns_interface", diff --git a/source/extensions/tracers/datadog/BUILD b/source/extensions/tracers/datadog/BUILD index cd7327a67d..85dfb3d1d0 100644 --- a/source/extensions/tracers/datadog/BUILD +++ b/source/extensions/tracers/datadog/BUILD @@ -37,9 +37,6 @@ envoy_cc_library( # equivalents of std::string_view and std::optional. "-DDD_USE_ABSEIL_FOR_ENVOY", ], - external_deps = [ - "dd_trace_cpp", - ], deps = [ "//source/common/config:utility_lib", "//source/common/http:async_client_utility_lib", @@ -51,6 +48,7 @@ envoy_cc_library( "//source/common/upstream:cluster_update_tracker_lib", "//source/common/version:version_lib", "//source/extensions/tracers/common:factory_base_lib", + "@com_github_datadog_dd_trace_cpp//:dd_trace_cpp", ], ) diff --git a/source/extensions/tracers/opencensus/BUILD b/source/extensions/tracers/opencensus/BUILD index f661bed6dd..51c454f51c 100644 --- a/source/extensions/tracers/opencensus/BUILD +++ b/source/extensions/tracers/opencensus/BUILD @@ -28,20 +28,18 @@ envoy_cc_library( srcs = ["opencensus_tracer_impl.cc"], hdrs = ["opencensus_tracer_impl.h"], copts = ["-Wno-unused-parameter"], - external_deps = [ - "opencensus_trace", - "opencensus_trace_b3", - "opencensus_trace_cloud_trace_context", - "opencensus_trace_grpc_trace_bin", - "opencensus_trace_trace_context", - "opencensus_exporter_ocagent", - "opencensus_exporter_stdout", - "opencensus_exporter_stackdriver", - "opencensus_exporter_zipkin", - ], deps = [ "//source/common/config:utility_lib", "//source/common/tracing:http_tracer_lib", "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + "@io_opencensus_cpp//opencensus/exporters/trace/ocagent:ocagent_exporter", + "@io_opencensus_cpp//opencensus/exporters/trace/stackdriver:stackdriver_exporter", + "@io_opencensus_cpp//opencensus/exporters/trace/stdout:stdout_exporter", + "@io_opencensus_cpp//opencensus/exporters/trace/zipkin:zipkin_exporter", + "@io_opencensus_cpp//opencensus/trace", + "@io_opencensus_cpp//opencensus/trace:b3", + "@io_opencensus_cpp//opencensus/trace:cloud_trace_context", + "@io_opencensus_cpp//opencensus/trace:grpc_trace_bin", + "@io_opencensus_cpp//opencensus/trace:trace_context", ] + envoy_select_google_grpc(["//source/common/grpc:google_async_client_lib"]), ) diff --git a/source/extensions/tracers/opentelemetry/BUILD b/source/extensions/tracers/opentelemetry/BUILD index bcc2e7526a..0b6e75abea 100644 --- a/source/extensions/tracers/opentelemetry/BUILD +++ b/source/extensions/tracers/opentelemetry/BUILD @@ -40,7 +40,6 @@ envoy_cc_library( # https://github.com/open-telemetry/opentelemetry-cpp/blob/v1.14.0/api/BUILD#L32 "-DHAVE_ABSEIL", ], - external_deps = ["opentelemetry_api"], deps = [ ":trace_exporter", "//envoy/thread_local:thread_local_interface", @@ -50,6 +49,7 @@ envoy_cc_library( "//source/extensions/tracers/opentelemetry/resource_detectors:resource_detector_lib", "//source/extensions/tracers/opentelemetry/samplers:sampler_lib", "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", + "@io_opentelemetry_cpp//api", "@opentelemetry_proto//:trace_cc_proto", ], ) @@ -67,7 +67,6 @@ envoy_cc_library( "otlp_utils.h", "trace_exporter.h", ], - external_deps = ["opentelemetry_api"], deps = [ "//envoy/grpc:async_client_manager_interface", "//envoy/upstream:cluster_manager_interface", @@ -80,6 +79,7 @@ envoy_cc_library( "//source/common/tracing:trace_context_lib", "//source/common/version:version_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@io_opentelemetry_cpp//api", "@opentelemetry_proto//:trace_cc_proto", ], ) diff --git a/source/extensions/tracers/skywalking/BUILD b/source/extensions/tracers/skywalking/BUILD index b4c33f8786..2c376371a7 100644 --- a/source/extensions/tracers/skywalking/BUILD +++ b/source/extensions/tracers/skywalking/BUILD @@ -15,14 +15,12 @@ envoy_cc_library( name = "trace_segment_reporter_lib", srcs = ["trace_segment_reporter.cc"], hdrs = ["trace_segment_reporter.h"], - external_deps = [ - "cpp2sky", - ], deps = [ ":skywalking_stats_lib", "//envoy/grpc:async_client_manager_interface", "//source/common/common:backoff_lib", "//source/common/grpc:async_client_lib", + "@com_github_skyapm_cpp2sky//source:cpp2sky_data_lib", "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", ], ) @@ -37,9 +35,6 @@ envoy_cc_library( "skywalking_tracer_impl.h", "tracer.h", ], - external_deps = [ - "cpp2sky", - ], deps = [ ":trace_segment_reporter_lib", "//envoy/common:time_interface", @@ -49,6 +44,7 @@ envoy_cc_library( "//source/common/http:header_map_lib", "//source/common/runtime:runtime_lib", "//source/common/tracing:http_tracer_lib", + "@com_github_skyapm_cpp2sky//source:cpp2sky_data_lib", "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/tracers/xray/BUILD b/source/extensions/tracers/xray/BUILD index e13753344d..78cfd6c785 100644 --- a/source/extensions/tracers/xray/BUILD +++ b/source/extensions/tracers/xray/BUILD @@ -36,7 +36,6 @@ envoy_cc_library( "xray_configuration.h", "xray_tracer_impl.h", ], - external_deps = [], deps = [ ":daemon_cc_proto", "//envoy/common:time_interface", diff --git a/source/extensions/transport_sockets/alts/BUILD b/source/extensions/transport_sockets/alts/BUILD index 7d66d8dd9b..cd32ac0ca8 100644 --- a/source/extensions/transport_sockets/alts/BUILD +++ b/source/extensions/transport_sockets/alts/BUILD @@ -18,9 +18,7 @@ envoy_cc_library( hdrs = [ "grpc_tsi.h", ], - external_deps = [ - "grpc", - ], + external_deps = ["grpc"], visibility = ["//visibility:private"], deps = [ "//source/common/common:c_smart_ptr_lib", @@ -115,9 +113,7 @@ envoy_cc_library( name = "alts_channel_pool", srcs = ["alts_channel_pool.cc"], hdrs = ["alts_channel_pool.h"], - external_deps = [ - "grpc", - ], + external_deps = ["grpc"], deps = [ "@com_google_absl//absl/random", ], @@ -132,9 +128,7 @@ envoy_cc_library( name = "alts_proxy", srcs = ["alts_proxy.cc"], hdrs = ["alts_proxy.h"], - external_deps = [ - "grpc", - ], + external_deps = ["grpc"], deps = [ ":handshaker_cc_grpc", "@com_google_absl//absl/memory", @@ -147,9 +141,7 @@ envoy_cc_library( name = "alts_tsi_handshaker", srcs = ["alts_tsi_handshaker.cc"], hdrs = ["alts_tsi_handshaker.h"], - external_deps = [ - "grpc", - ], + external_deps = ["grpc"], deps = [ ":alts_proxy", ":handshaker_cc_grpc", diff --git a/source/extensions/transport_sockets/starttls/BUILD b/source/extensions/transport_sockets/starttls/BUILD index 1117a147ab..ec4bbbe5d5 100644 --- a/source/extensions/transport_sockets/starttls/BUILD +++ b/source/extensions/transport_sockets/starttls/BUILD @@ -29,9 +29,7 @@ envoy_cc_library( name = "starttls_socket_lib", srcs = ["starttls_socket.cc"], hdrs = ["starttls_socket.h"], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ "//envoy/network:connection_interface", "//envoy/network:transport_socket_interface", diff --git a/source/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD b/source/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD index 150928e553..0de73d027e 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD +++ b/source/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD @@ -16,9 +16,7 @@ envoy_cc_extension( hdrs = [ "spiffe_validator.h", ], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ "//envoy/ssl:context_config_interface", "//envoy/ssl:ssl_socket_extended_info_interface", diff --git a/source/server/BUILD b/source/server/BUILD index 7420331d1f..300b859024 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -267,11 +267,10 @@ envoy_cc_library( envoy_cc_library( name = "options_lib", - # TCLAP command line parser needs this to support int64_t/uint64_t in several build environments. srcs = ["options_impl.cc"], hdrs = ["options_impl.h"], + # TCLAP command line parser needs this to support int64_t/uint64_t in several build environments. copts = ["-DHAVE_LONG_LONG"], - external_deps = ["tclap"], deps = [ ":options_base", "//envoy/network:address_interface", @@ -285,6 +284,7 @@ envoy_cc_library( "//source/common/stats:stats_lib", "//source/common/stats:tag_utility_lib", "//source/common/version:version_lib", + "@com_github_mirror_tclap//:tclap", "@envoy_api//envoy/admin/v3:pkg_cc_proto", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ], diff --git a/test/BUILD b/test/BUILD index d8ad1e080c..b4711e6551 100644 --- a/test/BUILD +++ b/test/BUILD @@ -56,10 +56,6 @@ envoy_cc_test_library( envoy_pch_library( name = "test_pch", testonly = True, - external_deps = [ - "googletest", - "spdlog", - ], includes = [ "envoy/config/bootstrap/v3/bootstrap.pb.h", "envoy/config/cluster/v3/cluster.pb.h", @@ -93,6 +89,8 @@ envoy_pch_library( "//test/mocks/server:factory_context_mocks", "//test/mocks/server:instance_mocks", "//test/mocks/stats:stats_mocks", + "@com_github_gabime_spdlog//:spdlog", + "@com_google_googletest//:gtest", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/test/benchmark/BUILD b/test/benchmark/BUILD index 7d14fd5f4c..73b358abb3 100644 --- a/test/benchmark/BUILD +++ b/test/benchmark/BUILD @@ -12,15 +12,13 @@ envoy_cc_test_library( name = "main", srcs = ["main.cc"], hdrs = ["main.h"], - external_deps = [ - "benchmark", - "tclap", - ], deps = [ "//source/common/common:minimal_logger_lib", "//source/common/common:thread_lib", "//test/test_common:environment_lib", "//test/test_common:printers_lib", "//test/test_common:test_runtime_lib", + "@com_github_google_benchmark//:benchmark", + "@com_github_mirror_tclap//:tclap", ], ) diff --git a/test/common/buffer/BUILD b/test/common/buffer/BUILD index 1a6841bb67..027e7aaf6d 100644 --- a/test/common/buffer/BUILD +++ b/test/common/buffer/BUILD @@ -117,12 +117,10 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "buffer_speed_test", srcs = ["buffer_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/buffer:buffer_lib", "//source/common/buffer:watermark_buffer_lib", + "@com_github_google_benchmark//:benchmark", "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", ], ) diff --git a/test/common/common/BUILD b/test/common/common/BUILD index cac07a646a..96ad095dab 100644 --- a/test/common/common/BUILD +++ b/test/common/common/BUILD @@ -197,8 +197,10 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "logger_speed_test", srcs = ["logger_speed_test.cc"], - external_deps = ["benchmark"], - deps = ["//source/common/common:minimal_logger_lib"], + deps = [ + "//source/common/common:minimal_logger_lib", + "@com_github_google_benchmark//:benchmark", + ], ) envoy_benchmark_test( @@ -371,10 +373,10 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "re_speed_test", srcs = ["re_speed_test.cc"], - external_deps = ["benchmark"], deps = [ "//source/common/common:assert_lib", "//source/common/common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@com_googlesource_code_re2//:re2", ], ) @@ -382,12 +384,10 @@ envoy_cc_benchmark_binary( envoy_cc_benchmark_binary( name = "utility_speed_test", srcs = ["utility_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/common:assert_lib", "//source/common/common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@com_google_absl//absl/strings", ], ) @@ -400,11 +400,9 @@ envoy_benchmark_test( envoy_cc_benchmark_binary( name = "trie_lookup_table_speed_test", srcs = ["trie_lookup_table_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/common:trie_lookup_table_lib", + "@com_github_google_benchmark//:benchmark", "@com_google_absl//absl/strings", ], ) @@ -529,8 +527,10 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "inline_map_speed_test", srcs = ["inline_map_speed_test.cc"], - external_deps = ["benchmark"], - deps = ["//source/common/common:inline_map"], + deps = [ + "//source/common/common:inline_map", + "@com_github_google_benchmark//:benchmark", + ], ) envoy_benchmark_test( diff --git a/test/common/crypto/BUILD b/test/common/crypto/BUILD index a3d8272da2..d85fe7c034 100644 --- a/test/common/crypto/BUILD +++ b/test/common/crypto/BUILD @@ -15,9 +15,7 @@ envoy_cc_test( srcs = [ "utility_test.cc", ], - external_deps = [ - "ssl", - ], + external_deps = ["ssl"], deps = [ "//source/common/buffer:buffer_lib", "//source/common/common:hex_lib", diff --git a/test/common/formatter/BUILD b/test/common/formatter/BUILD index 661e19a6af..544346222f 100644 --- a/test/common/formatter/BUILD +++ b/test/common/formatter/BUILD @@ -92,9 +92,6 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "substitution_formatter_speed_test", srcs = ["substitution_formatter_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/formatter:formatter_extension_lib", "//source/common/formatter:substitution_formatter_lib", @@ -104,6 +101,7 @@ envoy_cc_benchmark_binary( "//test/mocks/http:http_mocks", "//test/mocks/stream_info:stream_info_mocks", "//test/test_common:printers_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/common/grpc/BUILD b/test/common/grpc/BUILD index ac4ebe2684..40934f3e5e 100644 --- a/test/common/grpc/BUILD +++ b/test/common/grpc/BUILD @@ -225,9 +225,6 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "async_client_manager_benchmark", srcs = ["async_client_manager_benchmark.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/api:api_lib", "//source/common/grpc:async_client_manager_lib", @@ -238,6 +235,7 @@ envoy_cc_benchmark_binary( "//test/mocks/upstream:cluster_priority_set_mocks", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 98b1e1be8a..565dbfddf8 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -164,13 +164,11 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "codes_speed_test", srcs = ["codes_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/http:codes_lib", "//source/common/stats:isolated_store_lib", "//source/common/stats:stats_lib", + "@com_github_google_benchmark//:benchmark", ], ) @@ -333,11 +331,9 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "header_map_impl_speed_test", srcs = ["header_map_impl_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/http:header_map_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index f6a8835d42..04b9c8a8c2 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -15,9 +15,6 @@ envoy_cc_test( name = "codec_impl_test", size = "large", srcs = ["codec_impl_test.cc"], - external_deps = [ - "quiche_http2_adapter", - ], shard_count = 5, deps = [ ":codec_impl_test_util", @@ -46,19 +43,18 @@ envoy_cc_test( "//test/test_common:registry_lib", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", + "@com_github_google_quiche//:http2_adapter", ], ) envoy_cc_test_library( name = "codec_impl_test_util", hdrs = ["codec_impl_test_util.h"], - external_deps = [ - "quiche_http2_adapter", - ], deps = [ "//source/common/http/http2:codec_lib", "//test/mocks:common_lib", "//test/mocks/server:overload_manager_mocks", + "@com_github_google_quiche//:http2_adapter", "@com_google_absl//absl/types:optional", ], ) @@ -89,15 +85,13 @@ envoy_cc_test_library( name = "http2_frame", srcs = ["http2_frame.cc"], hdrs = ["http2_frame.h"], - external_deps = [ - "quiche_http2_hpack", - ], deps = [ "//envoy/http:metadata_interface_with_external_headers", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:hex_lib", "//source/common/common:macros", + "@com_github_google_quiche//:http2_hpack_hpack_lib", ], ) @@ -137,10 +131,6 @@ envoy_cc_test( envoy_cc_test( name = "metadata_encoder_test", srcs = ["metadata_encoder_test.cc"], - external_deps = [ - "quiche_http2_adapter", - "quiche_http2_test_tools", - ], deps = [ "//source/common/buffer:buffer_lib", "//source/common/common:random_generator_lib", @@ -151,6 +141,8 @@ envoy_cc_test( "//test/test_common:logging_lib", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", + "@com_github_google_quiche//:http2_adapter", + "@com_github_google_quiche//:http2_adapter_mock_http2_visitor", ], ) @@ -207,9 +199,7 @@ envoy_cc_fuzz_test( name = "hpack_fuzz_test", srcs = ["hpack_fuzz_test.cc"], corpus = "hpack_corpus", - external_deps = [ - "nghttp2", - ], + external_deps = ["nghttp2"], deps = [ ":hpack_fuzz_proto_cc_proto", "//test/test_common:utility_lib", @@ -226,9 +216,7 @@ envoy_cc_fuzz_test( name = "http2_connection_fuzz_test", srcs = ["http2_connection_fuzz_test.cc"], corpus = "http2_connection_corpus", - external_deps = [ - "nghttp2", - ], + external_deps = ["nghttp2"], deps = [ ":http2_connection_proto_cc_proto", "//source/common/http/http2:codec_lib", diff --git a/test/common/listener_manager/BUILD b/test/common/listener_manager/BUILD index 35780b1bfa..484a147a49 100644 --- a/test/common/listener_manager/BUILD +++ b/test/common/listener_manager/BUILD @@ -184,12 +184,7 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "filter_chain_benchmark_test", srcs = ["filter_chain_benchmark_test.cc"], - external_deps = [ - "benchmark", - "googletest", - ], deps = [ - "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", "//source/common/listener_manager:filter_chain_manager_lib", "//test/test_common:environment_lib", "//test/mocks/network:network_mocks", @@ -197,6 +192,9 @@ envoy_cc_benchmark_binary( "//test/mocks/stream_info:stream_info_mocks", # tranport socket config registration "//source/extensions/transport_sockets/tls:config", + "@com_github_google_benchmark//:benchmark", + "@com_google_googletest//:gtest", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", ], ) diff --git a/test/common/network/BUILD b/test/common/network/BUILD index 8c7e54a689..036c23c15d 100644 --- a/test/common/network/BUILD +++ b/test/common/network/BUILD @@ -50,11 +50,9 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "address_impl_speed_test", srcs = ["address_impl_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/network:address_lib", + "@com_github_google_benchmark//:benchmark", ], ) @@ -408,12 +406,10 @@ envoy_cc_fuzz_test( envoy_cc_benchmark_binary( name = "lc_trie_speed_test", srcs = ["lc_trie_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/network:lc_trie_lib", "//source/common/network:utility_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/common/orca/BUILD b/test/common/orca/BUILD index c63c06cd9e..35e25fc6b3 100644 --- a/test/common/orca/BUILD +++ b/test/common/orca/BUILD @@ -11,15 +11,13 @@ envoy_package() envoy_cc_test( name = "orca_load_metrics_test", srcs = ["orca_load_metrics_test.cc"], - external_deps = [ - "fmtlib", - ], deps = [ "//source/common/orca:orca_load_metrics_lib", "//source/common/upstream:upstream_lib", "//test/test_common:status_utility_lib", "//test/test_common:utility_lib", "@com_github_cncf_xds//xds/data/orca/v3:pkg_cc_proto", + "@com_github_fmtlib_fmt//:fmtlib", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", ], @@ -28,15 +26,13 @@ envoy_cc_test( envoy_cc_test( name = "orca_parser_test", srcs = ["orca_parser_test.cc"], - external_deps = [ - "fmtlib", - ], deps = [ "//source/common/common:base64_lib", "//source/common/orca:orca_parser", "//test/test_common:status_utility_lib", "//test/test_common:utility_lib", "@com_github_cncf_xds//xds/data/orca/v3:pkg_cc_proto", + "@com_github_fmtlib_fmt//:fmtlib", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", ], diff --git a/test/common/protobuf/BUILD b/test/common/protobuf/BUILD index befefa24b6..c1fe7afdeb 100644 --- a/test/common/protobuf/BUILD +++ b/test/common/protobuf/BUILD @@ -102,11 +102,11 @@ envoy_cc_fuzz_test( envoy_cc_benchmark_binary( name = "utility_speed_test", srcs = ["utility_speed_test.cc"], - external_deps = ["benchmark"], deps = [ ":deterministic_hash_test_proto_cc_proto", "//source/common/protobuf:utility_lib", "//test/test_common:test_runtime_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD index 938238aafd..71fa7de681 100644 --- a/test/common/quic/BUILD +++ b/test/common/quic/BUILD @@ -15,7 +15,6 @@ envoy_package() envoy_cc_test( name = "envoy_quic_alarm_test", srcs = ["envoy_quic_alarm_test.cc"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ "//source/common/quic:envoy_quic_alarm_factory_lib", @@ -23,6 +22,7 @@ envoy_cc_test( "//source/common/quic:envoy_quic_clock_lib", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", + "@com_github_google_quiche//:quic_platform", ], ) @@ -41,7 +41,6 @@ envoy_cc_test( envoy_cc_test( name = "envoy_quic_writer_test", srcs = ["envoy_quic_writer_test.cc"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ "//source/common/network:io_socket_error_lib", @@ -50,13 +49,13 @@ envoy_cc_test( "//test/mocks/api:api_mocks", "//test/mocks/network:network_mocks", "//test/test_common:threadsafe_singleton_injector_lib", + "@com_github_google_quiche//:quic_platform", ], ) envoy_cc_test( name = "envoy_quic_proof_source_test", srcs = ["envoy_quic_proof_source_test.cc"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ ":test_utils_lib", @@ -69,6 +68,7 @@ envoy_cc_test( "//test/mocks/ssl:ssl_mocks", "//test/test_common:test_runtime_lib", "@com_github_google_quiche//:quic_core_versions_lib", + "@com_github_google_quiche//:quic_platform", "@com_github_google_quiche//:quic_test_tools_test_certificates_lib", ], ) @@ -101,7 +101,6 @@ envoy_cc_test( envoy_cc_test( name = "envoy_quic_proof_verifier_test", srcs = ["envoy_quic_proof_verifier_test.cc"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ ":test_utils_lib", @@ -112,6 +111,7 @@ envoy_cc_test( "//test/mocks/event:event_mocks", "//test/mocks/server:server_factory_context_mocks", "//test/mocks/ssl:ssl_mocks", + "@com_github_google_quiche//:quic_platform", "@com_github_google_quiche//:quic_test_tools_test_certificates_lib", ], ) diff --git a/test/common/quic/platform/BUILD b/test/common/quic/platform/BUILD index ef8fc42c98..dca484cd50 100644 --- a/test/common/quic/platform/BUILD +++ b/test/common/quic/platform/BUILD @@ -23,7 +23,6 @@ envoy_cc_test( "//conditions:default": ["-Wno-unused-parameter"], }), data = ["//test/common/tls/test_data:certs"], - external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ "//source/common/memory:stats_lib", @@ -37,6 +36,7 @@ envoy_cc_test( "//test/test_common:utility_lib", "@com_github_google_quiche//:quic_core_error_codes_lib", "@com_github_google_quiche//:quic_core_types_lib", + "@com_github_google_quiche//:quic_platform", "@com_github_google_quiche//:quic_platform_expect_bug", "@com_github_google_quiche//:quic_platform_test", "@com_github_google_quiche//:quic_platform_test_output", diff --git a/test/common/router/BUILD b/test/common/router/BUILD index 414d769202..af12e529b7 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -65,14 +65,12 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "config_impl_headermap_benchmark_test", srcs = ["config_impl_headermap_benchmark_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/http:header_map_lib", "//source/common/router:config_lib", "//test/mocks/server:server_mocks", "//test/test_common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", ], ) @@ -519,15 +517,13 @@ envoy_proto_library( envoy_cc_benchmark_binary( name = "config_impl_speed_test", srcs = ["config_impl_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/common:assert_lib", "//source/common/router:config_lib", "//test/mocks/server:instance_mocks", "//test/mocks/stream_info:stream_info_mocks", "//test/test_common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", ], ) @@ -540,12 +536,10 @@ envoy_benchmark_test( envoy_cc_benchmark_binary( name = "header_formatter_speed_test", srcs = ["header_formatter_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/router:router_lib", "//test/common/stream_info:test_util", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index 95ccb0e0e0..7218abb28a 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -72,14 +72,12 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "recent_lookups_benchmark", srcs = ["recent_lookups_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/common:random_generator_lib", "//source/common/common:utility_lib", "//source/common/runtime:runtime_lib", "//source/common/stats:recent_lookups_lib", + "@com_github_google_benchmark//:benchmark", ], ) @@ -215,9 +213,6 @@ envoy_cc_benchmark_binary( "make_elements_helper.cc", "symbol_table_speed_test.cc", ], - external_deps = [ - "benchmark", - ], deps = [ ":make_elements_helper_lib", ":stat_test_utility_lib", @@ -229,6 +224,7 @@ envoy_cc_benchmark_binary( "//test/mocks/stats:stats_mocks", "//test/test_common:logging_lib", "//test/test_common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@com_google_absl//absl/strings", ], ) @@ -248,9 +244,6 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "deferred_creation_stats_benchmark", srcs = ["deferred_creation_stats_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ ":real_thread_test_base", "//source/common/common:random_generator_lib", @@ -260,6 +253,7 @@ envoy_cc_benchmark_binary( "//source/common/stats:isolated_store_lib", "//source/common/stats:symbol_table_lib", "//source/exe:process_wide_lib", + "@com_github_google_benchmark//:benchmark", ], ) @@ -279,15 +273,13 @@ envoy_cc_benchmark_binary( srcs = [ "stats_matcher_impl_speed_test.cc", ], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/memory:stats_lib", "//source/common/stats:stats_matcher_lib", "//test/mocks/server:server_factory_context_mocks", "//test/test_common:logging_lib", "//test/test_common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@com_google_absl//absl/strings", "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto", "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", @@ -325,11 +317,9 @@ envoy_cc_benchmark_binary( srcs = [ "tag_extractor_impl_speed_test.cc", ], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/stats:tag_producer_lib", + "@com_github_google_benchmark//:benchmark", "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto", ], ) @@ -366,9 +356,6 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "thread_local_store_speed_test", srcs = ["thread_local_store_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ ":stat_test_utility_lib", "//source/common/common:thread_lib", @@ -380,6 +367,7 @@ envoy_cc_benchmark_binary( "//test/test_common:simulated_time_system_lib", "//test/test_common:test_time_lib", "//test/test_common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@com_google_absl//absl/strings", "@envoy_api//envoy/config/metrics/v3:pkg_cc_proto", ], diff --git a/test/common/tls/BUILD b/test/common/tls/BUILD index 055c56ab68..6e2dc3207d 100644 --- a/test/common/tls/BUILD +++ b/test/common/tls/BUILD @@ -282,14 +282,12 @@ envoy_cc_benchmark_binary( data = [ "//test/common/tls/test_data:certs", ], - external_deps = [ - "benchmark", - "ssl", - ], + external_deps = ["ssl"], # Uses raw POSIX syscalls, does not build on Windows. tags = ["skip_on_windows"], deps = [ "//source/common/buffer:buffer_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 97fdd20407..216aca1bac 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -604,12 +604,10 @@ envoy_cc_fuzz_test( envoy_cc_benchmark_binary( name = "scheduler_benchmark", srcs = ["scheduler_benchmark.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/common:random_generator_lib", "//source/common/upstream:scheduler_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/dependencies/BUILD b/test/dependencies/BUILD index 1ef365b90f..688fedc331 100644 --- a/test/dependencies/BUILD +++ b/test/dependencies/BUILD @@ -11,7 +11,5 @@ envoy_package() envoy_cc_test( name = "curl_test", srcs = ["curl_test.cc"], - external_deps = [ - "curl", - ], + external_deps = ["curl"], ) diff --git a/test/extensions/access_loggers/fluentd/BUILD b/test/extensions/access_loggers/fluentd/BUILD index 0c6e832be6..a8a4d9acad 100644 --- a/test/extensions/access_loggers/fluentd/BUILD +++ b/test/extensions/access_loggers/fluentd/BUILD @@ -15,14 +15,12 @@ envoy_extension_cc_test( name = "fluentd_access_log_impl_test", srcs = ["fluentd_access_log_impl_test.cc"], extension_names = ["envoy.access_loggers.fluentd"], - external_deps = [ - "msgpack", - ], deps = [ "//source/extensions/access_loggers/fluentd:config", "//test/mocks/server:factory_context_mocks", "//test/test_common:environment_lib", "//test/test_common:utility_lib", + "@com_github_msgpack_cpp//:msgpack", "@envoy_api//envoy/config/accesslog/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/access_loggers/fluentd/v3:pkg_cc_proto", ], diff --git a/test/extensions/access_loggers/open_telemetry/BUILD b/test/extensions/access_loggers/open_telemetry/BUILD index f569984ae4..0e7ddef73e 100644 --- a/test/extensions/access_loggers/open_telemetry/BUILD +++ b/test/extensions/access_loggers/open_telemetry/BUILD @@ -130,9 +130,6 @@ envoy_extension_cc_test( envoy_cc_benchmark_binary( name = "substitution_formatter_speed_test", srcs = ["substitution_formatter_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/formatter:formatter_extension_lib", "//source/common/http:header_map_lib", @@ -140,6 +137,7 @@ envoy_cc_benchmark_binary( "//source/extensions/access_loggers/open_telemetry:substitution_formatter_lib", "//test/common/stream_info:test_util", "//test/mocks/stream_info:stream_info_mocks", + "@com_github_google_benchmark//:benchmark", "@opentelemetry_proto//:common_cc_proto", ], ) diff --git a/test/extensions/bootstrap/wasm/BUILD b/test/extensions/bootstrap/wasm/BUILD index fd54284031..5efd44307b 100644 --- a/test/extensions/bootstrap/wasm/BUILD +++ b/test/extensions/bootstrap/wasm/BUILD @@ -97,9 +97,6 @@ envoy_extension_cc_test_binary( "//test/extensions/bootstrap/wasm/test_data:speed_cpp.wasm", ]), extension_names = ["envoy.bootstrap.wasm"], - external_deps = [ - "benchmark", - ], tags = ["skip_on_windows"], deps = [ "//source/common/event:dispatcher_lib", @@ -113,6 +110,7 @@ envoy_extension_cc_test_binary( "//test/mocks/upstream:upstream_mocks", "//test/test_common:environment_lib", "//test/test_common:simulated_time_system_lib", + "@com_github_google_benchmark//:benchmark", "@com_google_absl//absl/types:optional", ], ) diff --git a/test/extensions/clusters/eds/BUILD b/test/extensions/clusters/eds/BUILD index 09e0f59093..90d2873959 100644 --- a/test/extensions/clusters/eds/BUILD +++ b/test/extensions/clusters/eds/BUILD @@ -45,9 +45,6 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "eds_speed_test", srcs = ["eds_speed_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//envoy/config:xds_resources_delegate_interface", "//source/common/config:protobuf_link_hacks", @@ -69,6 +66,7 @@ envoy_cc_benchmark_binary( "//test/mocks/upstream:cluster_manager_mocks", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", + "@com_github_google_benchmark//:benchmark", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", diff --git a/test/extensions/common/wasm/BUILD b/test/extensions/common/wasm/BUILD index f46b84d59e..7d0cf2c213 100644 --- a/test/extensions/common/wasm/BUILD +++ b/test/extensions/common/wasm/BUILD @@ -75,9 +75,6 @@ envoy_cc_test( envoy_cc_test_binary( name = "wasm_speed_test", srcs = ["wasm_speed_test.cc"], - external_deps = [ - "benchmark", - ], tags = ["skip_on_windows"], deps = [ "//source/common/event:dispatcher_lib", @@ -86,6 +83,7 @@ envoy_cc_test_binary( "//test/mocks/server:server_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:environment_lib", + "@com_github_google_benchmark//:benchmark", "@com_google_absl//absl/types:optional", ], ) diff --git a/test/extensions/filters/http/compressor/BUILD b/test/extensions/filters/http/compressor/BUILD index 5853091258..001aaa5d9c 100644 --- a/test/extensions/filters/http/compressor/BUILD +++ b/test/extensions/filters/http/compressor/BUILD @@ -73,10 +73,6 @@ envoy_extension_cc_test( envoy_cc_benchmark_binary( name = "compressor_filter_speed_test", srcs = ["compressor_filter_speed_test.cc"], - external_deps = [ - "benchmark", - "googletest", - ], deps = [ "//envoy/compression/compressor:compressor_factory_interface", "//source/common/protobuf:utility_lib", @@ -91,6 +87,8 @@ envoy_cc_benchmark_binary( "//test/mocks/runtime:runtime_mocks", "//test/test_common:printers_lib", "//test/test_common:utility_lib", + "@com_github_google_benchmark//:benchmark", + "@com_google_googletest//:gtest", "@envoy_api//envoy/extensions/filters/http/compressor/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/filters/http/ext_proc/BUILD b/test/extensions/filters/http/ext_proc/BUILD index 9d144ccd7c..b2dcff8c17 100644 --- a/test/extensions/filters/http/ext_proc/BUILD +++ b/test/extensions/filters/http/ext_proc/BUILD @@ -377,8 +377,6 @@ envoy_extension_cc_test_library( name = "tracer_test_filter_lib", srcs = ["tracer_test_filter.cc"], extension_names = ["envoy.filters.http.ext_proc"], - external_deps = [ - ], deps = [ ":tracer_test_filter_proto_cc_proto", "//source/common/config:utility_lib", diff --git a/test/extensions/filters/http/grpc_field_extraction/message_converter/BUILD b/test/extensions/filters/http/grpc_field_extraction/message_converter/BUILD index 67f4ecae2e..a26bf073a0 100644 --- a/test/extensions/filters/http/grpc_field_extraction/message_converter/BUILD +++ b/test/extensions/filters/http/grpc_field_extraction/message_converter/BUILD @@ -12,8 +12,6 @@ envoy_package() envoy_cc_test_library( name = "message_converter_test_lib", hdrs = ["message_converter_test_lib.h"], - external_deps = [ - ], deps = [ "//source/common/buffer:buffer_lib", "//source/common/grpc:codec_lib", diff --git a/test/extensions/filters/http/proto_message_extraction/extraction_util/BUILD b/test/extensions/filters/http/proto_message_extraction/extraction_util/BUILD index e6b9a20493..3d933ebb9d 100644 --- a/test/extensions/filters/http/proto_message_extraction/extraction_util/BUILD +++ b/test/extensions/filters/http/proto_message_extraction/extraction_util/BUILD @@ -23,7 +23,6 @@ envoy_cc_test( "@com_google_absl//absl/log:check", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@com_google_googletest//:gtest", "@com_google_protobuf//:protobuf", "@com_google_protoconverter//:all", "@com_google_protofieldextraction//:all_libs", diff --git a/test/extensions/filters/listener/tls_inspector/BUILD b/test/extensions/filters/listener/tls_inspector/BUILD index a843e2a574..410d6f9a58 100644 --- a/test/extensions/filters/listener/tls_inspector/BUILD +++ b/test/extensions/filters/listener/tls_inspector/BUILD @@ -58,9 +58,6 @@ envoy_extension_cc_benchmark_binary( name = "tls_inspector_benchmark", srcs = ["tls_inspector_benchmark.cc"], extension_names = ["envoy.filters.listener.tls_inspector"], - external_deps = [ - "benchmark", - ], deps = [ ":tls_utility_lib", "//source/common/http:utility_lib", @@ -71,6 +68,7 @@ envoy_extension_cc_benchmark_binary( "//test/mocks/network:network_mocks", "//test/mocks/stats:stats_mocks", "//test/test_common:threadsafe_singleton_injector_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/extensions/filters/network/redis_proxy/BUILD b/test/extensions/filters/network/redis_proxy/BUILD index 32c3bd39ed..4b8282b6a3 100644 --- a/test/extensions/filters/network/redis_proxy/BUILD +++ b/test/extensions/filters/network/redis_proxy/BUILD @@ -136,9 +136,6 @@ envoy_extension_cc_benchmark_binary( name = "command_lookup_speed_test", srcs = ["command_lookup_speed_test.cc"], extension_names = ["envoy.filters.network.redis_proxy"], - external_deps = [ - "benchmark", - ], deps = [ ":redis_mocks", "//source/common/stats:isolated_store_lib", @@ -148,6 +145,7 @@ envoy_extension_cc_benchmark_binary( "//test/mocks/network:network_mocks", "//test/test_common:printers_lib", "//test/test_common:simulated_time_system_lib", + "@com_github_google_benchmark//:benchmark", ], ) @@ -193,9 +191,6 @@ envoy_extension_cc_benchmark_binary( name = "command_split_speed_test", srcs = ["command_split_speed_test.cc"], extension_names = ["envoy.filters.network.redis_proxy"], - external_deps = [ - "benchmark", - ], deps = [ ":redis_mocks", "//source/common/stats:isolated_store_lib", @@ -204,6 +199,7 @@ envoy_extension_cc_benchmark_binary( "//source/extensions/filters/network/redis_proxy:router_lib", "//test/test_common:printers_lib", "//test/test_common:simulated_time_system_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/extensions/filters/udp/dns_filter/BUILD b/test/extensions/filters/udp/dns_filter/BUILD index 38b051cc6d..61b358d9b3 100644 --- a/test/extensions/filters/udp/dns_filter/BUILD +++ b/test/extensions/filters/udp/dns_filter/BUILD @@ -18,8 +18,8 @@ envoy_extension_cc_test_library( srcs = ["dns_filter_test_utils.cc"], hdrs = ["dns_filter_test_utils.h"], extension_names = ["envoy.filters.udp.dns_filter"], - external_deps = ["ares"], deps = [ + "//bazel/foreign_cc:ares", "//source/common/common:random_generator_lib", "//source/common/network:address_lib", "//source/common/network:utility_lib", diff --git a/test/extensions/load_balancing_policies/common/BUILD b/test/extensions/load_balancing_policies/common/BUILD index 48678bf4f7..2fa854c692 100644 --- a/test/extensions/load_balancing_policies/common/BUILD +++ b/test/extensions/load_balancing_policies/common/BUILD @@ -14,9 +14,6 @@ envoy_cc_test_library( name = "benchmark_base_tester_lib", srcs = ["benchmark_base_tester.cc"], hdrs = ["benchmark_base_tester.h"], - external_deps = [ - "benchmark", - ], deps = [ "//source/common/common:random_generator_lib", "//source/common/memory:stats_lib", @@ -25,6 +22,7 @@ envoy_cc_test_library( "//test/mocks/upstream:cluster_info_mocks", "//test/test_common:printers_lib", "//test/test_common:simulated_time_system_lib", + "@com_github_google_benchmark//:benchmark", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/load_balancing_policies/subset/BUILD b/test/extensions/load_balancing_policies/subset/BUILD index 689deaaf9d..b66dab8161 100644 --- a/test/extensions/load_balancing_policies/subset/BUILD +++ b/test/extensions/load_balancing_policies/subset/BUILD @@ -81,14 +81,12 @@ envoy_extension_cc_benchmark_binary( name = "subset_benchmark", srcs = ["subset_benchmark.cc"], extension_names = ["envoy.load_balancing_policies.subset"], - external_deps = [ - "benchmark", - ], deps = [ "//source/extensions/load_balancing_policies/random:config", "//source/extensions/load_balancing_policies/subset:config", "//test/extensions/load_balancing_policies/common:benchmark_base_tester_lib", "//test/mocks/upstream:load_balancer_mocks", + "@com_github_google_benchmark//:benchmark", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/random/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto", diff --git a/test/extensions/tracers/datadog/BUILD b/test/extensions/tracers/datadog/BUILD index 4eba5b3534..739bfd76ec 100644 --- a/test/extensions/tracers/datadog/BUILD +++ b/test/extensions/tracers/datadog/BUILD @@ -31,9 +31,6 @@ envoy_extension_cc_test( "-DDD_USE_ABSEIL_FOR_ENVOY", ], extension_names = ["envoy.tracers.datadog"], - external_deps = [ - "dd_trace_cpp", - ], # TODO(wrowe): envoy_extension_ rules don't currently exclude windows extensions tags = ["skip_on_windows"], deps = [ @@ -56,6 +53,7 @@ envoy_extension_cc_test( "//test/mocks/upstream:cluster_manager_mocks", "//test/mocks/upstream:thread_local_cluster_mocks", "//test/test_common:utility_lib", + "@com_github_datadog_dd_trace_cpp//:dd_trace_cpp", "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/tracers/opentelemetry/BUILD b/test/extensions/tracers/opentelemetry/BUILD index 8aa5bb003b..ac25ecbdb9 100644 --- a/test/extensions/tracers/opentelemetry/BUILD +++ b/test/extensions/tracers/opentelemetry/BUILD @@ -22,9 +22,6 @@ envoy_extension_cc_test( "-DHAVE_ABSEIL", ], extension_names = ["envoy.tracers.opentelemetry"], - external_deps = [ - "opentelemetry_api", - ], deps = [ "//envoy/common:time_interface", "//envoy/runtime:runtime_interface", @@ -49,6 +46,7 @@ envoy_extension_cc_test( "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", "@com_google_absl//absl/types:optional", + "@io_opentelemetry_cpp//api", ], ) diff --git a/test/extensions/tracers/skywalking/BUILD b/test/extensions/tracers/skywalking/BUILD index 36f31a7bce..e0e017783f 100644 --- a/test/extensions/tracers/skywalking/BUILD +++ b/test/extensions/tracers/skywalking/BUILD @@ -50,14 +50,12 @@ envoy_cc_library( envoy_extension_cc_test( name = "skywalking_test_helper", extension_names = ["envoy.tracers.skywalking"], - external_deps = [ - "cpp2sky", - ], deps = [ ":skywalking_test_helper_lib", "//source/common/common:base64_lib", "//source/common/common:hex_lib", "//test/test_common:utility_lib", + "@com_github_skyapm_cpp2sky//source:cpp2sky_data_lib", ], ) diff --git a/test/extensions/transport_sockets/alts/BUILD b/test/extensions/transport_sockets/alts/BUILD index d36c654994..87a9782044 100644 --- a/test/extensions/transport_sockets/alts/BUILD +++ b/test/extensions/transport_sockets/alts/BUILD @@ -117,9 +117,7 @@ envoy_extension_cc_test( name = "tsi_socket_test", srcs = ["tsi_socket_test.cc"], extension_names = ["envoy.transport_sockets.alts"], - external_deps = [ - "grpc", - ], + external_deps = ["grpc"], deps = [ "//envoy/event:dispatcher_interface", "//envoy/network:address_interface", @@ -149,9 +147,7 @@ envoy_extension_cc_test( size = "large", srcs = envoy_select_google_grpc(["alts_integration_test.cc"]), extension_names = ["envoy.transport_sockets.alts"], - external_deps = [ - "grpc", - ], + external_deps = ["grpc"], deps = [ "//source/common/common:utility_lib", "//source/common/event:dispatcher_includes", diff --git a/test/fuzz/BUILD b/test/fuzz/BUILD index 4de7a506c7..d9dd74d003 100644 --- a/test/fuzz/BUILD +++ b/test/fuzz/BUILD @@ -59,9 +59,6 @@ envoy_cc_test_library( name = "utility_lib", srcs = ["utility.cc"], hdrs = ["utility.h"], - external_deps = [ - "quiche_http2_adapter", - ], deps = [ ":common_proto_cc_proto", "//source/common/common:empty_string", @@ -72,6 +69,7 @@ envoy_cc_test_library( "//test/mocks/ssl:ssl_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:utility_lib", + "@com_github_google_quiche//:http2_adapter", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/test/server/BUILD b/test/server/BUILD index 3aea3d95c1..f984efcd48 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -406,15 +406,13 @@ envoy_cc_test( envoy_cc_benchmark_binary( name = "server_stats_flush_benchmark", srcs = ["server_stats_flush_benchmark_test.cc"], - external_deps = [ - "benchmark", - ], deps = [ "//envoy/stats:stats_interface", "//source/common/stats:thread_local_store_lib", "//source/server:server_lib", "//test/mocks/upstream:cluster_manager_mocks", "//test/test_common:simulated_time_system_lib", + "@com_github_google_benchmark//:benchmark", ], ) diff --git a/test/test_common/BUILD b/test/test_common/BUILD index f218ddd510..4cc616a430 100644 --- a/test/test_common/BUILD +++ b/test/test_common/BUILD @@ -17,9 +17,7 @@ envoy_cc_test_library( name = "environment_lib", srcs = ["environment.cc"], hdrs = ["environment.h"], - external_deps = [ - "bazel_runfiles", - ], + external_deps = ["bazel_runfiles"], deps = [ ":network_utility_lib", "//envoy/server:options_interface", diff --git a/test/tools/router_check/BUILD b/test/tools/router_check/BUILD index 01baa3acb9..b09a660e6d 100644 --- a/test/tools/router_check/BUILD +++ b/test/tools/router_check/BUILD @@ -36,7 +36,6 @@ envoy_cc_test_library( "router.h", ], copts = ["-DHAVE_LONG_LONG"], - external_deps = ["tclap"], deps = [ ":validation_proto_cc_proto", "//source/common/common:random_generator_lib", @@ -54,6 +53,7 @@ envoy_cc_test_library( "//test/test_common:printers_lib", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", + "@com_github_mirror_tclap//:tclap", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", diff --git a/test/tools/schema_validator/BUILD b/test/tools/schema_validator/BUILD index 4ea42313bd..ed852bdd0b 100644 --- a/test/tools/schema_validator/BUILD +++ b/test/tools/schema_validator/BUILD @@ -27,13 +27,13 @@ envoy_cc_test_library( hdrs = ["validator.h"], # TCLAP command line parser needs this to support int64_t/uint64_t in several build environments. copts = ["-DHAVE_LONG_LONG"], - external_deps = ["tclap"], deps = [ "//envoy/api:api_interface", "//source/common/protobuf:utility_lib", "//source/common/stats:isolated_store_lib", "//source/common/version:version_lib", "//test/test_common:utility_lib", + "@com_github_mirror_tclap//:tclap", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", diff --git a/test/tools/wee8_compile/BUILD b/test/tools/wee8_compile/BUILD index 1cad66e548..98a77602bf 100644 --- a/test/tools/wee8_compile/BUILD +++ b/test/tools/wee8_compile/BUILD @@ -23,5 +23,7 @@ envoy_cc_library( "-Wno-non-virtual-dtor", "-Wno-unused-parameter", ], - external_deps = ["wee8"], + deps = [ + "@v8//:wee8", + ], ) From 322cda13fc491aeac9852f693b15ebc204710b1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Sep 2024 10:39:22 +0100 Subject: [PATCH 19/28] build(deps): bump protobuf from 5.28.1 to 5.28.2 in /tools/base (#36207) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 7d8dd8884d..2acf5c379a 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -1056,18 +1056,18 @@ ply==3.11 \ --hash=sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3 \ --hash=sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce # via -r requirements.in -protobuf==5.28.1 \ - --hash=sha256:0dfd86d2b5edf03d91ec2a7c15b4e950258150f14f9af5f51c17fa224ee1931f \ - --hash=sha256:1b04bde117a10ff9d906841a89ec326686c48ececeb65690f15b8cabe7149495 \ - --hash=sha256:42597e938f83bb7f3e4b35f03aa45208d49ae8d5bcb4bc10b9fc825e0ab5e423 \ - --hash=sha256:4304e4fceb823d91699e924a1fdf95cde0e066f3b1c28edb665bda762ecde10f \ - --hash=sha256:4b4b9a0562a35773ff47a3df823177ab71a1f5eb1ff56d8f842b7432ecfd7fd2 \ - --hash=sha256:4c7f5cb38c640919791c9f74ea80c5b82314c69a8409ea36f2599617d03989af \ - --hash=sha256:51f09caab818707ab91cf09cc5c156026599cf05a4520779ccbf53c1b352fb25 \ - --hash=sha256:c529535e5c0effcf417682563719e5d8ac8d2b93de07a56108b4c2d436d7a29a \ - --hash=sha256:cabfe43044ee319ad6832b2fda332646f9ef1636b0130186a3ae0a52fc264bb4 \ - --hash=sha256:f24e5d70e6af8ee9672ff605d5503491635f63d5db2fffb6472be78ba62efd8f \ - --hash=sha256:fc063acaf7a3d9ca13146fefb5b42ac94ab943ec6e978f543cd5637da2d57957 +protobuf==5.28.2 \ + --hash=sha256:2c69461a7fcc8e24be697624c09a839976d82ae75062b11a0972e41fd2cd9132 \ + --hash=sha256:35cfcb15f213449af7ff6198d6eb5f739c37d7e4f1c09b5d0641babf2cc0c68f \ + --hash=sha256:52235802093bd8a2811abbe8bf0ab9c5f54cca0a751fdd3f6ac2a21438bffece \ + --hash=sha256:59379674ff119717404f7454647913787034f03fe7049cbef1d74a97bb4593f0 \ + --hash=sha256:5e8a95246d581eef20471b5d5ba010d55f66740942b95ba9b872d918c459452f \ + --hash=sha256:87317e9bcda04a32f2ee82089a204d3a2f0d3c8aeed16568c7daf4756e4f1fe0 \ + --hash=sha256:8ddc60bf374785fb7cb12510b267f59067fa10087325b8e1855b898a0d81d276 \ + --hash=sha256:a8b9403fc70764b08d2f593ce44f1d2920c5077bf7d311fefec999f8c40f78b7 \ + --hash=sha256:c0ea0123dac3399a2eeb1a1443d82b7afc9ff40241433296769f7da42d142ec3 \ + --hash=sha256:ca53faf29896c526863366a52a8f4d88e69cd04ec9571ed6082fa117fac3ab36 \ + --hash=sha256:eeea10f3dc0ac7e6b4933d32db20662902b4ab81bf28df12218aa389e9c2102d # via # -r requirements.in # envoy-base-utils From ec92cc38c6304d524fca818f732f3c435e7abab7 Mon Sep 17 00:00:00 2001 From: chenylh Date: Thu, 19 Sep 2024 21:16:22 +0800 Subject: [PATCH 20/28] dns: rename `Success` to `Completed` (#36143) Signed-off-by: chenyuliang5 --- envoy/network/dns.h | 6 +- .../network/connectivity_manager_test.cc | 8 +- .../logical_dns/logical_dns_cluster.cc | 2 +- .../clusters/redis/redis_cluster.cc | 4 +- .../clusters/strict_dns/strict_dns_cluster.cc | 2 +- .../dynamic_forward_proxy/dns_cache_impl.cc | 6 +- source/extensions/common/wasm/context.cc | 2 +- .../filters/udp/dns_filter/dns_filter.cc | 2 +- .../udp/dns_filter/dns_filter_resolver.cc | 4 +- .../filters/udp/dns_filter/dns_parser.h | 2 +- .../dns_resolver/apple/apple_dns_impl.cc | 2 +- .../dns_resolver/apple/apple_dns_impl.h | 2 +- .../network/dns_resolver/cares/dns_impl.cc | 4 +- .../network/dns_resolver/cares/dns_impl.h | 7 +- .../dns_resolver/getaddrinfo/getaddrinfo.cc | 4 +- .../upstream/cluster_manager_impl_test.cc | 20 +- test/common/upstream/upstream_impl_test.cc | 70 +++---- .../common/logical_host_integration_test.cc | 2 +- .../logical_dns/logical_dns_cluster_test.cc | 22 +- .../clusters/redis/redis_cluster_test.cc | 12 +- .../dns_cache_impl_test.cc | 140 ++++++------- test/extensions/common/wasm/wasm_test.cc | 8 +- .../aws_request_signing_integration_test.cc | 4 +- .../filters/udp/dns_filter/dns_filter_test.cc | 11 +- .../dns_resolver/apple/apple_dns_impl_test.cc | 192 +++++++++--------- .../dns_resolver/cares/dns_impl_test.cc | 124 +++++------ .../getaddrinfo/getaddrinfo_test.cc | 24 +-- 27 files changed, 347 insertions(+), 339 deletions(-) diff --git a/envoy/network/dns.h b/envoy/network/dns.h index 4aa5f655dd..c7847babcc 100644 --- a/envoy/network/dns.h +++ b/envoy/network/dns.h @@ -86,11 +86,11 @@ class DnsResolver { /** * Final status for a DNS resolution. - * TODO(abeyad): Rename `Success` to `Completed` or something similar. DNS resolution can return - * result statuses like NODATA and NONAME, which indicate successful completion of the query but + * DNS resolution can return result statuses like NODATA态SERVFAIL and NONAME, + * which indicate successful completion of the query but * no results, and `Completed` is a more accurate way of reflecting that. */ - enum class ResolutionStatus { Success, Failure }; + enum class ResolutionStatus { Completed, Failure }; /** * Called when a resolution attempt is complete. diff --git a/mobile/test/common/network/connectivity_manager_test.cc b/mobile/test/common/network/connectivity_manager_test.cc index 4230c3c9d4..61e36c7e83 100644 --- a/mobile/test/common/network/connectivity_manager_test.cc +++ b/mobile/test/common/network/connectivity_manager_test.cc @@ -81,15 +81,15 @@ TEST_F(ConnectivityManagerTest, WhenDrainPostDnsRefreshEnabledDrainsPostDnsRefre connectivity_manager_->onDnsResolutionComplete( "cached.example.com", std::make_shared(), - Network::DnsResolver::ResolutionStatus::Success); + Network::DnsResolver::ResolutionStatus::Completed); connectivity_manager_->onDnsResolutionComplete( "not-cached.example.com", std::make_shared(), - Network::DnsResolver::ResolutionStatus::Success); + Network::DnsResolver::ResolutionStatus::Completed); connectivity_manager_->onDnsResolutionComplete( "not-cached2.example.com", std::make_shared(), - Network::DnsResolver::ResolutionStatus::Success); + Network::DnsResolver::ResolutionStatus::Completed); } TEST_F(ConnectivityManagerTest, WhenDrainPostDnsNotEnabledDoesntDrainPostDnsRefresh) { @@ -102,7 +102,7 @@ TEST_F(ConnectivityManagerTest, WhenDrainPostDnsNotEnabledDoesntDrainPostDnsRefr EXPECT_CALL(cm_, drainConnections(_)).Times(0); connectivity_manager_->onDnsResolutionComplete( "example.com", std::make_shared(), - Network::DnsResolver::ResolutionStatus::Success); + Network::DnsResolver::ResolutionStatus::Completed); } TEST_F(ConnectivityManagerTest, diff --git a/source/extensions/clusters/logical_dns/logical_dns_cluster.cc b/source/extensions/clusters/logical_dns/logical_dns_cluster.cc index a29ac21949..ef3d443af7 100644 --- a/source/extensions/clusters/logical_dns/logical_dns_cluster.cc +++ b/source/extensions/clusters/logical_dns/logical_dns_cluster.cc @@ -107,7 +107,7 @@ void LogicalDnsCluster::startResolve() { // If the DNS resolver successfully resolved with an empty response list, the logical DNS // cluster does not update. This ensures that a potentially previously resolved address does // not stabilize back to 0 hosts. - if (status == Network::DnsResolver::ResolutionStatus::Success && !response.empty()) { + if (status == Network::DnsResolver::ResolutionStatus::Completed && !response.empty()) { info_->configUpdateStats().update_success_.inc(); const auto addrinfo = response.front().addrInfo(); // TODO(mattklein123): Move port handling into the DNS interface. diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index d667efea6d..ed96a0a49a 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -365,7 +365,7 @@ void RedisCluster::RedisDiscoverySession::resolveClusterHostnames( updateDnsStats(status, response.empty()); // If DNS resolution for a primary fails, we stop resolution for remaining, and reset // the timer. - if (status != Network::DnsResolver::ResolutionStatus::Success) { + if (status != Network::DnsResolver::ResolutionStatus::Completed) { ENVOY_LOG(error, "Unable to resolve cluster slot primary hostname {}", slot.primary_hostname_); resolve_timer_->enableTimer(parent_.cluster_refresh_rate_); @@ -419,7 +419,7 @@ void RedisCluster::RedisDiscoverySession::resolveReplicas( updateDnsStats(status, response.empty()); // If DNS resolution fails here, we move on to resolve other replicas in the list. // We log a warn message. - if (status != Network::DnsResolver::ResolutionStatus::Success) { + if (status != Network::DnsResolver::ResolutionStatus::Completed) { ENVOY_LOG(warn, "Unable to resolve cluster replica address {}", replica.first); } else { // Replica resolved diff --git a/source/extensions/clusters/strict_dns/strict_dns_cluster.cc b/source/extensions/clusters/strict_dns/strict_dns_cluster.cc index d2b1794ec5..b379ef1e89 100644 --- a/source/extensions/clusters/strict_dns/strict_dns_cluster.cc +++ b/source/extensions/clusters/strict_dns/strict_dns_cluster.cc @@ -130,7 +130,7 @@ void StrictDnsClusterImpl::ResolveTarget::startResolve() { std::chrono::milliseconds final_refresh_rate = parent_.dns_refresh_rate_ms_; - if (status == Network::DnsResolver::ResolutionStatus::Success) { + if (status == Network::DnsResolver::ResolutionStatus::Completed) { parent_.info_->configUpdateStats().update_success_.inc(); HostVector new_hosts; diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index b06e8ab4ae..d7841bad16 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -216,7 +216,7 @@ void DnsCacheImpl::startCacheLoad(const std::string& host, uint16_t default_port // If the DNS request was simply to create a host endpoint in a Dynamic Forward Proxy cluster, // fast fail the look-up as the address is not needed. if (is_proxy_lookup) { - finishResolve(host, Network::DnsResolver::ResolutionStatus::Success, "proxy_resolve", {}, {}, + finishResolve(host, Network::DnsResolver::ResolutionStatus::Completed, "proxy_resolve", {}, {}, true); } else { startResolve(host, *primary_host); @@ -503,7 +503,7 @@ void DnsCacheImpl::finishResolve(const std::string& host, runResolutionCompleteCallbacks(host, primary_host_info->host_info_, status); // Kick off the refresh timer. - if (status == Network::DnsResolver::ResolutionStatus::Success) { + if (status == Network::DnsResolver::ResolutionStatus::Completed) { primary_host_info->failure_backoff_strategy_->reset( std::chrono::duration_cast(dns_ttl).count()); primary_host_info->refresh_timer_->enableTimer(dns_ttl); @@ -682,7 +682,7 @@ void DnsCacheImpl::loadCacheEntries( key, accumulateToString(responses, [](const auto& dns_response) { return dns_response.addrInfo().address_->asString(); })); - finishResolve(key, Network::DnsResolver::ResolutionStatus::Success, "from_cache", + finishResolve(key, Network::DnsResolver::ResolutionStatus::Completed, "from_cache", std::move(responses), resolution_time); stats_.cache_load_.inc(); return KeyValueStore::Iterate::Continue; diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index d21983da96..8d717c38d2 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -202,7 +202,7 @@ void Context::onResolveDns(uint32_t token, Envoy::Network::DnsResolver::Resoluti if (wasm()->isFailed() || !wasm()->on_resolve_dns_) { return; } - if (status != Network::DnsResolver::ResolutionStatus::Success) { + if (status != Network::DnsResolver::ResolutionStatus::Completed) { buffer_.set(""); wasm()->on_resolve_dns_(this, id_, token, 0); return; diff --git a/source/extensions/filters/udp/dns_filter/dns_filter.cc b/source/extensions/filters/udp/dns_filter/dns_filter.cc index e971791fe4..d895ac8f5f 100644 --- a/source/extensions/filters/udp/dns_filter/dns_filter.cc +++ b/source/extensions/filters/udp/dns_filter/dns_filter.cc @@ -234,7 +234,7 @@ DnsFilter::DnsFilter(Network::UdpReadFilterCallbacks& callbacks, // We cannot retry the resolution if ares returns without a response. The ares context // is still dirty and will result in a segfault when it is freed during a subsequent resolve // call from here. We will retry resolutions for pending lookups only - if (context->resolution_status_ != Network::DnsResolver::ResolutionStatus::Success && + if (context->resolution_status_ != Network::DnsResolver::ResolutionStatus::Completed && !context->in_callback_ && context->retry_ > 0) { --context->retry_; ENVOY_LOG(debug, "resolving name [{}] via external resolvers [retry {}]", query->name_, diff --git a/source/extensions/filters/udp/dns_filter/dns_filter_resolver.cc b/source/extensions/filters/udp/dns_filter/dns_filter_resolver.cc index ae2a72dca5..9930ffcf45 100644 --- a/source/extensions/filters/udp/dns_filter/dns_filter_resolver.cc +++ b/source/extensions/filters/udp/dns_filter/dns_filter_resolver.cc @@ -30,7 +30,7 @@ void DnsFilterResolver::resolveExternalQuery(DnsQueryContextPtr context, // We don't support other lookups other than A and AAAA. Set success here so that we don't // retry for something that we are certain will fail. ENVOY_LOG(debug, "Unknown query type [{}] for upstream lookup", domain_query->type_); - ctx.query_context->resolution_status_ = Network::DnsResolver::ResolutionStatus::Success; + ctx.query_context->resolution_status_ = Network::DnsResolver::ResolutionStatus::Completed; ctx.resolver_status = DnsFilterResolverStatus::Complete; invokeCallback(ctx); return; @@ -87,7 +87,7 @@ void DnsFilterResolver::resolveExternalQuery(DnsQueryContextPtr context, ctx.resolver_status = DnsFilterResolverStatus::Complete; // C-ares doesn't expose the TTL in the data available here. - if (status == Network::DnsResolver::ResolutionStatus::Success) { + if (status == Network::DnsResolver::ResolutionStatus::Completed) { ctx.resolved_hosts.reserve(response.size()); for (const auto& resp : response) { const auto& addrinfo = resp.addrInfo(); diff --git a/source/extensions/filters/udp/dns_filter/dns_parser.h b/source/extensions/filters/udp/dns_filter/dns_parser.h index 3449c19d64..1b5b1a2eeb 100644 --- a/source/extensions/filters/udp/dns_filter/dns_parser.h +++ b/source/extensions/filters/udp/dns_filter/dns_parser.h @@ -200,7 +200,7 @@ class DnsQueryContext { uint64_t retry_; uint16_t id_; Network::DnsResolver::ResolutionStatus resolution_status_ = - Network::DnsResolver::ResolutionStatus::Success; + Network::DnsResolver::ResolutionStatus::Completed; DnsHeader header_; DnsHeader response_header_; DnsQueryPtrVec queries_; diff --git a/source/extensions/network/dns_resolver/apple/apple_dns_impl.cc b/source/extensions/network/dns_resolver/apple/apple_dns_impl.cc index cf2694c660..c842536933 100644 --- a/source/extensions/network/dns_resolver/apple/apple_dns_impl.cc +++ b/source/extensions/network/dns_resolver/apple/apple_dns_impl.cc @@ -72,7 +72,7 @@ AppleDnsResolverImpl::startResolution(const std::string& dns_name, ENVOY_LOG_EVENT(debug, "apple_dns_immediate_resolution", "DNS resolver resolved ({}) to ({}) without issuing call to Apple API", dns_name, address->asString()); - callback(DnsResolver::ResolutionStatus::Success, "apple_dns_success", + callback(DnsResolver::ResolutionStatus::Completed, "apple_dns_success", {DnsResponse(address, std::chrono::seconds(60))}); return {nullptr, true}; } diff --git a/source/extensions/network/dns_resolver/apple/apple_dns_impl.h b/source/extensions/network/dns_resolver/apple/apple_dns_impl.h index c89914e594..d8f01bd736 100644 --- a/source/extensions/network/dns_resolver/apple/apple_dns_impl.h +++ b/source/extensions/network/dns_resolver/apple/apple_dns_impl.h @@ -122,7 +122,7 @@ class AppleDnsResolverImpl : public DnsResolver, protected Logger::Loggablenodes != nullptr) { @@ -264,7 +264,7 @@ void DnsResolverImpl::AddrInfoPendingResolution::onAresGetAddrInfoCallback( } else if (isResponseWithNoRecords(status)) { // Treat `ARES_ENODATA` or `ARES_ENOTFOUND` here as success to populate back the // "empty records" response. - pending_response_.status_ = ResolutionStatus::Success; + pending_response_.status_ = ResolutionStatus::Completed; pending_response_.details_ = absl::StrCat("cares_norecords:", ares_strerror(status)); ASSERT(addrinfo == nullptr); } else { diff --git a/source/extensions/network/dns_resolver/cares/dns_impl.h b/source/extensions/network/dns_resolver/cares/dns_impl.h index af806315c0..a6ead67415 100644 --- a/source/extensions/network/dns_resolver/cares/dns_impl.h +++ b/source/extensions/network/dns_resolver/cares/dns_impl.h @@ -110,10 +110,9 @@ class DnsResolverImpl : public DnsResolver, protected Logger::LoggableasString(); })); - return std::make_pair(ResolutionStatus::Success, final_results); + return std::make_pair(ResolutionStatus::Completed, final_results); } // Background thread which wakes up and does resolutions. @@ -190,7 +190,7 @@ void GetAddrInfoDnsResolver::resolveThreadRoutine() { // https://github.com/envoyproxy/envoy/blob/099d85925b32ce8bf06e241ee433375a0a3d751b/source/extensions/network/dns_resolver/cares/dns_impl.h#L109-L111. ENVOY_LOG(debug, "getaddrinfo for host={} has no results rc={}", next_query->dns_name_, gai_strerror(rc.return_value_)); - response = std::make_pair(ResolutionStatus::Success, std::list()); + response = std::make_pair(ResolutionStatus::Completed, std::list()); } else { ENVOY_LOG(debug, "getaddrinfo failed for host={} with rc={} errno={}", next_query->dns_name_, gai_strerror(rc.return_value_), errorDetails(rc.errno_)); diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index ad2c80b5f4..8d9d2ca7b8 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -2824,7 +2824,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemove) { cluster_manager_->setInitializedCb([&]() -> void { initialized.ready(); }); EXPECT_CALL(initialized, ready()); - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); // After we are initialized, we should immediately get called back if someone asks for an @@ -2878,7 +2878,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemove) { // Remove the first host, this should lead to the first cp being drained. dns_timer_->invokeCallback(); - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2"})); cp1->idle_cb_(); cp1->idle_cb_ = nullptr; @@ -2912,7 +2912,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemove) { // Now add and remove a host that we never have a conn pool to. This should not lead to any // drain callbacks, etc. dns_timer_->invokeCallback(); - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.3"})); factory_.tls_.shutdownThread(); @@ -2980,7 +2980,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemoveWithTls) { cluster_manager_->setInitializedCb([&]() -> void { initialized.ready(); }); EXPECT_CALL(initialized, ready()); - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); // After we are initialized, we should immediately get called back if someone asks for an @@ -3076,7 +3076,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemoveWithTls) { // Remove the first host, this should lead to the first cp being drained. dns_timer_->invokeCallback(); - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2"})); cp1->idle_cb_(); cp1->idle_cb_ = nullptr; @@ -3134,7 +3134,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemoveWithTls) { // Now add and remove a host that we never have a conn pool to. This should not lead to any // drain callbacks, etc. dns_timer_->invokeCallback(); - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.3"})); factory_.tls_.shutdownThread(); @@ -3924,7 +3924,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemoveDefaultPriority) { EXPECT_FALSE(all_clusters.active_clusters_.at("cluster_1").get().info()->addedViaApi()); EXPECT_EQ(nullptr, cluster_manager_->getThreadLocalCluster("cluster_1")); - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2"})); EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) @@ -3957,7 +3957,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemoveDefaultPriority) { // crash. dns_timer_->invokeCallback(); // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({})); factory_.tls_.shutdownThread(); @@ -4018,7 +4018,7 @@ TEST_P(ClusterManagerLifecycleTest, ConnPoolDestroyWithDraining) { EXPECT_FALSE(all_clusters.active_clusters_.at("cluster_1").get().info()->addedViaApi()); EXPECT_EQ(nullptr, cluster_manager_->getThreadLocalCluster("cluster_1")); - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2"})); MockConnPoolWithDestroy* mock_cp = new MockConnPoolWithDestroy(); @@ -4043,7 +4043,7 @@ TEST_P(ClusterManagerLifecycleTest, ConnPoolDestroyWithDraining) { // Remove the first host, this should lead to the cp being drained. dns_timer_->invokeCallback(); // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({})); // The drained callback might get called when the CP is being destroyed. diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index be63bcf58d..034c6fdefa 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -235,7 +235,7 @@ TEST_P(StrictDnsParamTest, ImmediateResolve) { EXPECT_CALL(*dns_resolver, resolve("foo.bar.com", std::get<1>(GetParam()), _)) .WillOnce(Invoke([&](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { - cb(Network::DnsResolver::ResolutionStatus::Success, "", + cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse(std::get<2>(GetParam()))); return nullptr; })); @@ -480,7 +480,7 @@ TEST_F(StrictDnsClusterImplTest, ZeroHostsHealthChecker) { EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)); EXPECT_CALL(initialized, ready()); EXPECT_CALL(*resolver.timer_, enableTimer(_, _)); - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", {}); + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", {}); EXPECT_EQ(0UL, cluster->prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ(0UL, cluster->prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); } @@ -531,7 +531,7 @@ TEST_F(StrictDnsClusterImplTest, DontWaitForDNSOnInit) { EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); } @@ -642,7 +642,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { resolver1.expectResolve(*dns_resolver_); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -653,7 +653,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -662,7 +662,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -671,7 +671,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.3"})); EXPECT_THAT( std::list({"127.0.0.3:11001"}), @@ -680,7 +680,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { // Make sure we de-dup the same address. EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1", "10.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.3:11001", "10.0.0.1:11002"}), @@ -702,7 +702,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({})); EXPECT_THAT( std::list({"10.0.0.1:11002"}), @@ -774,7 +774,7 @@ TEST_F(StrictDnsClusterImplTest, HostRemovalActiveHealthSkipped) { EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)); EXPECT_CALL(*resolver.timer_, enableTimer(_, _)).Times(2); - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); // Verify that both endpoints are initially marked with FAILED_ACTIVE_HC, then @@ -792,7 +792,7 @@ TEST_F(StrictDnsClusterImplTest, HostRemovalActiveHealthSkipped) { } // Re-resolve the DNS name with only one record - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1"})); const auto& hosts = cluster->prioritySet().hostSetsPerPriority()[0]->hosts(); @@ -835,7 +835,7 @@ TEST_F(StrictDnsClusterImplTest, HostRemovalAfterHcFail) { EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)); EXPECT_CALL(*resolver.timer_, enableTimer(_, _)).Times(2); - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); // Verify that both endpoints are initially marked with FAILED_ACTIVE_HC, then @@ -857,7 +857,7 @@ TEST_F(StrictDnsClusterImplTest, HostRemovalAfterHcFail) { } // Re-resolve the DNS name with only one record, we should still have 2 hosts. - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1"})); { @@ -915,7 +915,7 @@ TEST_F(StrictDnsClusterImplTest, HostUpdateWithDisabledACEndpoint) { EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)); EXPECT_CALL(*resolver.timer_, enableTimer(_, _)).Times(2); - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); { @@ -930,7 +930,7 @@ TEST_F(StrictDnsClusterImplTest, HostUpdateWithDisabledACEndpoint) { } // Re-resolve the DNS name with only one record, we should have 1 host. - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1"})); { @@ -1058,7 +1058,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { resolver1.expectResolve(*dns_resolver_); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -1078,7 +1078,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -1092,7 +1092,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -1105,7 +1105,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1", "10.0.0.1"})); // We received a new set of hosts for localhost2. Should rebuild the cluster-> @@ -1114,7 +1114,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); // We again received the same set as before for localhost1. No rebuild this time. @@ -1123,7 +1123,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.3"})); EXPECT_THAT( std::list({"127.0.0.3:11001", "10.0.0.1:11002"}), @@ -1131,7 +1131,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { // Make sure we de-dup the same address. EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1", "10.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.3:11001", "10.0.0.1:11002"}), @@ -1146,7 +1146,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { // Make sure that we *don't* de-dup between resolve targets. EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver3.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver3.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); const auto hosts = cluster->prioritySet().hostSetsPerPriority()[0]->hosts(); @@ -1183,12 +1183,12 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({})); EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver3.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver3.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({})); // Ensure that we called the update membership callback. @@ -1275,7 +1275,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { resolver1.expectResolve(*dns_resolver_); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -1286,7 +1286,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -1295,7 +1295,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -1304,7 +1304,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver1.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.3"})); EXPECT_THAT( std::list({"127.0.0.3:11001"}), @@ -1313,7 +1313,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { // Make sure we de-dup the same address. EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver2.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1", "10.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.3:11001", "10.0.0.1:11002"}), @@ -1330,7 +1330,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); - resolver3.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver3.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"192.168.1.1", "192.168.1.2"})); // Make sure we have multiple priorities. @@ -1424,7 +1424,7 @@ TEST_F(StrictDnsClusterImplTest, FailureRefreshRateBackoffResetsWhenSuccessHappe // Successful call should reset the failure backoff strategy. EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({})); // Therefore, a subsequent failure should get a [0,base * 1] refresh. @@ -1475,13 +1475,13 @@ TEST_F(StrictDnsClusterImplTest, TtlAsDnsRefreshRateNoJitter) { EXPECT_CALL(membership_updated, ready()); EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(5000), _)); resolver.dns_callback_( - Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"192.168.1.1", "192.168.1.2"}, std::chrono::seconds(5))); // If the response is successful but empty, the cluster uses the cluster configured refresh rate. EXPECT_CALL(membership_updated, ready()); EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(4000), _)); - resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + resolver.dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({}, std::chrono::seconds(5))); // On failure, the cluster uses the cluster configured refresh rate. @@ -1529,7 +1529,7 @@ TEST_F(StrictDnsClusterImplTest, TtlAsDnsRefreshRateYesJitter) { enableTimer(std::chrono::milliseconds(ttl_s * 1000 + jitter_ms), _)); ON_CALL(random_, random()).WillByDefault(Return(random_return)); resolver.dns_callback_( - Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"192.168.1.1", "192.168.1.2"}, std::chrono::seconds(ttl_s))); } diff --git a/test/extensions/clusters/common/logical_host_integration_test.cc b/test/extensions/clusters/common/logical_host_integration_test.cc index 8a0763814d..66beadcdd2 100644 --- a/test/extensions/clusters/common/logical_host_integration_test.cc +++ b/test/extensions/clusters/common/logical_host_integration_test.cc @@ -54,7 +54,7 @@ TEST_P(LogicalHostIntegrationTest, LogicalDNSRaceCrashTest) { Network::DnsResolver::ResolveCb dns_callback) -> Network::ActiveDnsQuery* { const uint32_t address = address_ptr->address_; // Keep changing the returned addresses to force address update. - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({ // The only significant address is the first one; the other ones are // just used to populate a list whose maintenance is race-prone. diff --git a/test/extensions/clusters/logical_dns/logical_dns_cluster_test.cc b/test/extensions/clusters/logical_dns/logical_dns_cluster_test.cc index b781b28e26..8c4a1c68d4 100644 --- a/test/extensions/clusters/logical_dns/logical_dns_cluster_test.cc +++ b/test/extensions/clusters/logical_dns/logical_dns_cluster_test.cc @@ -99,7 +99,7 @@ class LogicalDnsClusterTest : public Event::TestUsingSimulatedTime, public testi EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); EXPECT_CALL(*resolve_timer_, enableTimer(std::chrono::milliseconds(4000), _)); - dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); @@ -129,7 +129,7 @@ class LogicalDnsClusterTest : public Event::TestUsingSimulatedTime, public testi // Should not cause any changes. EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); - dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2", "127.0.0.3"})); EXPECT_EQ("127.0.0.1:" + std::to_string(expected_hc_port), @@ -167,7 +167,7 @@ class LogicalDnsClusterTest : public Event::TestUsingSimulatedTime, public testi // Should cause a change. EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); - dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.3", "127.0.0.1", "127.0.0.2"})); EXPECT_EQ("127.0.0.3:" + std::to_string(expected_hc_port), @@ -196,10 +196,10 @@ class LogicalDnsClusterTest : public Event::TestUsingSimulatedTime, public testi .WillOnce(Return(new NiceMock())); logical_host->createConnection(server_context_.dispatcher_, nullptr, nullptr); - // Empty Success should not cause any change. + // Empty Completed should not cause any change. ON_CALL(random_, random()).WillByDefault(Return(6000)); EXPECT_CALL(*resolve_timer_, enableTimer(std::chrono::milliseconds(6000), _)); - dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", {}); + dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", {}); EXPECT_EQ(logical_host, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]); EXPECT_CALL(server_context_.dispatcher_, @@ -301,7 +301,7 @@ TEST_P(LogicalDnsParamTest, ImmediateResolve) { .WillOnce(Invoke([&](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); - cb(Network::DnsResolver::ResolutionStatus::Success, "", + cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse(std::get<2>(GetParam()))); return nullptr; })); @@ -350,7 +350,7 @@ TEST_F(LogicalDnsParamTest, FailureRefreshRateBackoffResetsWhenSuccessHappens) { // Successful call should reset the failure backoff strategy. EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(*resolve_timer_, enableTimer(std::chrono::milliseconds(4000), _)); - dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); @@ -390,12 +390,12 @@ TEST_F(LogicalDnsParamTest, TtlAsDnsRefreshRate) { EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); EXPECT_CALL(*resolve_timer_, enableTimer(std::chrono::milliseconds(5000), _)); - dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"}, std::chrono::seconds(5))); // If the response is successful but empty, the cluster uses the cluster configured refresh rate. EXPECT_CALL(*resolve_timer_, enableTimer(std::chrono::milliseconds(4000), _)); - dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({}, std::chrono::seconds(5))); // On failure, the cluster uses the cluster configured refresh rate. @@ -603,7 +603,7 @@ TEST_F(LogicalDnsClusterTest, DontWaitForDNSOnInit) { EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(*resolve_timer_, enableTimer(std::chrono::milliseconds(4000), _)); - dns_callback_(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback_(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); } @@ -643,7 +643,7 @@ TEST_F(LogicalDnsClusterTest, DNSRefreshHasJitter) { ON_CALL(random_, random()).WillByDefault(Return(random_return)); dns_callback_( - Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"}, std::chrono::seconds(3000))); } diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index d8236942e0..cf3e3570e3 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -173,7 +173,7 @@ class RedisClusterTest : public testing::Test, const std::string& expected_address, const std::list& resolved_addresses, Network::DnsResolver::ResolutionStatus status = - Network::DnsResolver::ResolutionStatus::Success) { + Network::DnsResolver::ResolutionStatus::Completed) { EXPECT_CALL(*dns_resolver_, resolve(expected_address, dns_lookup_family, _)) .WillOnce(Invoke([status, resolved_addresses]( const std::string&, Network::DnsLookupFamily, @@ -775,7 +775,7 @@ TEST_P(RedisDnsParamTest, ImmediateResolveDns) { .WillOnce(Invoke([&](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { std::list address_pair = std::get<2>(GetParam()); - cb(Network::DnsResolver::ResolutionStatus::Success, "", + cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse(address_pair)); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)); expectClusterSlotResponse( @@ -856,9 +856,9 @@ TEST_F(RedisClusterTest, AddressAsHostnameParallelResolution) { EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)); cluster_->initialize([&]() -> void { initialized_.ready(); }); expectClusterSlotResponse(twoSlotsPrimariesHostnames("primary1.com", "primary2.com", 22120)); - primary1_resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + primary1_resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse(std::list{"127.0.1.1"})); - primary2_resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + primary2_resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse(std::list{"127.0.1.2"})); expectHealthyHosts(std::list({ "127.0.1.1:22120", @@ -1241,7 +1241,7 @@ TEST_F(RedisClusterTest, MultipleDnsDiscovery) { EXPECT_CALL(*dns_resolver_, resolve("foo.bar.com", _, _)) .WillOnce(Invoke([&](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { - cb(Network::DnsResolver::ResolutionStatus::Success, "", + cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse(std::list({"127.0.0.1", "127.0.0.2"}))); return nullptr; })); @@ -1249,7 +1249,7 @@ TEST_F(RedisClusterTest, MultipleDnsDiscovery) { EXPECT_CALL(*dns_resolver_, resolve("foo1.bar.com", _, _)) .WillOnce(Invoke([&](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { - cb(Network::DnsResolver::ResolutionStatus::Success, "", + cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse(std::list({"127.0.0.3", "127.0.0.4"}))); return nullptr; })); diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index fd3a4819cf..2cba229e1f 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -148,11 +148,11 @@ TEST_F(DnsCacheImplTest, PreresolveSuccess) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete(authority, DnsHostInfoEquals("10.0.0.1:443", "bar.baz.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); initialize({{host, port}} /* preresolve_hostnames */); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -186,10 +186,10 @@ TEST_F(DnsCacheImplTest, DnsFirstResolveComplete) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("bar.baz.com:443", DnsHostInfoEquals("10.0.0.1:443", "bar.baz.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); initialize({{"bar.baz.com", 443}}); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -232,9 +232,9 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, @@ -254,9 +254,9 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, @@ -279,9 +279,9 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.2"})); checkStats(3 /* attempt */, 3 /* success */, 0 /* failure */, 2 /* address changed */, @@ -395,9 +395,9 @@ TEST_F(DnsCacheImplTest, Ipv4Address) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("127.0.0.1:80", DnsHostInfoEquals("127.0.0.1:80", "127.0.0.1", true), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1"})); } @@ -429,9 +429,9 @@ TEST_F(DnsCacheImplTest, Ipv4AddressWithPort) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("127.0.0.1:10000", DnsHostInfoEquals("127.0.0.1:10000", "127.0.0.1", true), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1"})); } @@ -460,9 +460,9 @@ TEST_F(DnsCacheImplTest, Ipv6Address) { EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("[::1]:80", "::1", true))); EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("[::1]:80", DnsHostInfoEquals("[::1]:80", "::1", true), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"::1"})); } @@ -491,9 +491,9 @@ TEST_F(DnsCacheImplTest, Ipv6AddressWithPort) { EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("[::1]:10000", "::1", true))); EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("[::1]:10000", DnsHostInfoEquals("[::1]:10000", "::1", true), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"::1"})); } @@ -528,9 +528,9 @@ TEST_F(DnsCacheImplTest, TTL) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(6000), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, @@ -549,9 +549,9 @@ TEST_F(DnsCacheImplTest, TTL) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(6000), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -607,9 +607,9 @@ TEST_F(DnsCacheImplTest, TTLWithMinRefreshRate) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(45000), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); } @@ -644,9 +644,9 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); // Re-resolve with ~30s passed. TTL should still be OK at 60s. @@ -659,9 +659,9 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); // Re-resolve with ~30s passed. TTL should expire. @@ -692,7 +692,7 @@ TEST_F(DnsCacheImplTest, InlineResolve) { EXPECT_CALL(*resolver_, resolve("localhost", _, _)) .WillOnce(Invoke([](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb callback) { - callback(Network::DnsResolver::ResolutionStatus::Success, "", + callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1"})); return nullptr; })); @@ -705,7 +705,7 @@ TEST_F(DnsCacheImplTest, InlineResolve) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("localhost:80", DnsHostInfoEquals("127.0.0.1:80", "localhost", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); post_cb(); } @@ -880,9 +880,10 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithEmptyResult) { EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoAddressIsNull())); EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoAddressIsNull(), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(configured_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", TestUtility::makeDnsResponse({})); + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", + TestUtility::makeDnsResponse({})); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 0 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -926,8 +927,8 @@ TEST_F(DnsCacheImplTest, CancelResolve) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed)); + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); } @@ -962,8 +963,8 @@ TEST_F(DnsCacheImplTest, MultipleResolveSameHost) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed)); + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); } @@ -1000,8 +1001,8 @@ TEST_F(DnsCacheImplTest, MultipleResolveDifferentHost) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("bar.com:443", DnsHostInfoEquals("10.0.0.1:443", "bar.com", false), - Network::DnsResolver::ResolutionStatus::Success)); - resolve_cb2(Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed)); + resolve_cb2(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); EXPECT_CALL( @@ -1012,8 +1013,8 @@ TEST_F(DnsCacheImplTest, MultipleResolveDifferentHost) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); - resolve_cb1(Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed)); + resolve_cb1(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.2"})); absl::flat_hash_map hosts; @@ -1054,8 +1055,8 @@ TEST_F(DnsCacheImplTest, CacheHit) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed)); + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); result = dns_cache_->loadDnsCacheEntry("foo.com", 80, false, callbacks); @@ -1087,8 +1088,8 @@ TEST_F(DnsCacheImplTest, CacheHitWithDifferentDefaultPort) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed)); + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); // Using the same DNS cache, resolve the same host but different default port 443 @@ -1136,8 +1137,9 @@ TEST_F(DnsCacheImplTest, InvalidPort) { EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoAddressIsNull())); EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:abc:80", DnsHostInfoAddressIsNull(), - Network::DnsResolver::ResolutionStatus::Success)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", TestUtility::makeDnsResponse({})); + Network::DnsResolver::ResolutionStatus::Completed)); + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", + TestUtility::makeDnsResponse({})); } // Max host overflow. @@ -1275,9 +1277,9 @@ TEST_F(DnsCacheImplTest, SetIpVersionToRemoveYieldsNonEmptyResponse) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("127.0.0.2:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "::2"})); // Verify that only the address is now set to an IPv4. result = dns_cache_->loadDnsCacheEntry("foo.com", 80, false, callbacks); @@ -1296,10 +1298,10 @@ TEST_F(DnsCacheImplTest, SetIpVersionToRemoveYieldsNonEmptyResponse) { onDnsHostAddOrUpdate("foo.com:80", DnsHostInfoEquals("[::2]:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("[::2]:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); dns_cache_->forceRefreshHosts(); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "::2"})); // Verify that only the address is now set to an IPv6. result = dns_cache_->loadDnsCacheEntry("foo.com", 80, false, callbacks); @@ -1320,10 +1322,10 @@ TEST_F(DnsCacheImplTest, SetIpVersionToRemoveYieldsNonEmptyResponse) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("127.0.0.2:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); dns_cache_->forceRefreshHosts(); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2", "::2"})); // Verify that only the address is now set to an IPv4 (the first entry in the DNS response). result = dns_cache_->loadDnsCacheEntry("foo.com", 80, false, callbacks); @@ -1361,9 +1363,9 @@ TEST_F(DnsCacheImplTest, SetIpVersionToRemoveYieldsEmptyResponse) { EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoAddressIsNull())); EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoAddressIsNull(), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(configured_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"::2"})); result = dns_cache_->loadDnsCacheEntry("foo.com", 80, false, callbacks); @@ -1406,9 +1408,9 @@ TEST_F(DnsCacheImplTest, SetIpVersionToRemoveIgnoreIPv4LoopbackAddress) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("127.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1"})); // Verify that only the address is now set to an IPv4. result = dns_cache_->loadDnsCacheEntry("foo.com", 80, false, callbacks); @@ -1447,9 +1449,9 @@ TEST_F(DnsCacheImplTest, SetIpVersionToRemoveIgnoreIPv6LoopbackAddress) { EXPECT_CALL(callbacks, onLoadDnsCacheComplete(DnsHostInfoEquals("[::1]:80", "foo.com", false))); EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("[::1]:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"::1"})); // Verify that only the address is now set to an IPv4. result = dns_cache_->loadDnsCacheEntry("foo.com", 80, false, callbacks); @@ -1472,16 +1474,16 @@ TEST_F(DnsCacheImplTest, SetIpVersionToRemoveWithDnsPreresolveHostnames) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("127.0.0.2:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)) + Network::DnsResolver::ResolutionStatus::Completed)) .Times(AtLeast(1)); initialize(/* preresolve_hostnames= */ {{"foo.com", 80}}); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2"})); dns_cache_->setIpVersionToRemove(Network::Address::IpVersion::v4); EXPECT_ENVOY_BUG( - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.2"})), "Unable to delete IP version addresses when DNS preresolve hostnames are not empty."); @@ -1763,9 +1765,9 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(6000), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, @@ -1786,9 +1788,9 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, @@ -1812,9 +1814,9 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(dns_ttl_), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.2", "10.0.0.1"})); checkStats(3 /* attempt */, 3 /* success */, 0 /* failure */, 2 /* address changed */, @@ -1834,9 +1836,9 @@ TEST_F(DnsCacheImplTest, ResolveSuccessWithCaching) { EXPECT_CALL(update_callbacks_, onDnsResolutionComplete("foo.com:80", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false), - Network::DnsResolver::ResolutionStatus::Success)); + Network::DnsResolver::ResolutionStatus::Completed)); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(40000), _)); - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"10.0.0.2", "10.0.0.1"}, std::chrono::seconds(40))); } diff --git a/test/extensions/common/wasm/wasm_test.cc b/test/extensions/common/wasm/wasm_test.cc index 7544b23493..e12e68eccf 100644 --- a/test/extensions/common/wasm/wasm_test.cc +++ b/test/extensions/common/wasm/wasm_test.cc @@ -301,7 +301,7 @@ TEST_P(WasmCommonTest, Segv) { // Subsequent calls should be NOOP(s). - root_context->onResolveDns(0, Envoy::Network::DnsResolver::ResolutionStatus::Success, {}); + root_context->onResolveDns(0, Envoy::Network::DnsResolver::ResolutionStatus::Completed, {}); Envoy::Stats::MockMetricSnapshot stats_snapshot; root_context->onStatsUpdate(stats_snapshot); } @@ -1353,7 +1353,7 @@ TEST_P(WasmCommonContextTest, OnDnsResolve) { EXPECT_CALL(rootContext(), log_(spdlog::level::warn, Eq("TestRootContext::onDone 1"))); dns_callback( - Network::DnsResolver::ResolutionStatus::Success, "", + Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"192.168.1.101", "192.168.1.102"}, std::chrono::seconds(1001))); rootContext().onResolveDns(1 /* token */, Envoy::Network::DnsResolver::ResolutionStatus::Failure, @@ -1366,7 +1366,7 @@ TEST_P(WasmCommonContextTest, OnDnsResolve) { } // Wait till the Wasm is destroyed and then the late callback should do nothing. deferred_runner_.setFunction([dns_callback] { - dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", + dns_callback(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"192.168.1.101", "192.168.1.102"}, std::chrono::seconds(1001))); }); @@ -1386,7 +1386,7 @@ TEST_P(WasmCommonContextTest, EmptyContext) { setup(code, "context", "empty"); setupContext(); - root_context_->onResolveDns(0, Envoy::Network::DnsResolver::ResolutionStatus::Success, {}); + root_context_->onResolveDns(0, Envoy::Network::DnsResolver::ResolutionStatus::Completed, {}); NiceMock stats_snapshot; root_context_->onStatsUpdate(stats_snapshot); root_context_->validateConfiguration("", plugin_); diff --git a/test/extensions/filters/http/aws_request_signing/aws_request_signing_integration_test.cc b/test/extensions/filters/http/aws_request_signing/aws_request_signing_integration_test.cc index 6ddd2e81ff..ab25ca80d0 100644 --- a/test/extensions/filters/http/aws_request_signing/aws_request_signing_integration_test.cc +++ b/test/extensions/filters/http/aws_request_signing/aws_request_signing_integration_test.cc @@ -248,7 +248,7 @@ class InitializeFilterTest : public ::testing::Test, public HttpIntegrationTest EXPECT_CALL(*dns_resolver_, resolve(expected_address, _, _)) .WillRepeatedly(Invoke([&](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { - cb(Network::DnsResolver::ResolutionStatus::Success, "", + cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); return nullptr; @@ -495,7 +495,7 @@ class CdsInteractionTest : public testing::Test, public HttpIntegrationTest { EXPECT_CALL(*dns_resolver_, resolve(expected_address, _, _)) .WillRepeatedly(Invoke([&](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { - cb(Network::DnsResolver::ResolutionStatus::Success, "", + cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); return nullptr; diff --git a/test/extensions/filters/udp/dns_filter/dns_filter_test.cc b/test/extensions/filters/udp/dns_filter/dns_filter_test.cc index aa698ba161..985090ba7e 100644 --- a/test/extensions/filters/udp/dns_filter/dns_filter_test.cc +++ b/test/extensions/filters/udp/dns_filter/dns_filter_test.cc @@ -759,7 +759,7 @@ TEST_F(DnsFilterTest, ExternalResolutionReturnSingleAddress) { EXPECT_CALL(*timeout_timer, disableTimer()).Times(AnyNumber()); // Execute resolve callback - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({expected_address})); // parse the result @@ -812,7 +812,7 @@ TEST_F(DnsFilterTest, ExternalResolutionIpv6SingleAddress) { EXPECT_CALL(*timeout_timer, disableTimer()); // Execute resolve callback - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({expected_address})); // parse the result @@ -865,7 +865,7 @@ TEST_F(DnsFilterTest, ExternalResolutionReturnMultipleAddresses) { EXPECT_CALL(*timeout_timer, disableTimer()); // Execute resolve callback - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({expected_address})); // parse the result @@ -917,7 +917,8 @@ TEST_F(DnsFilterTest, ExternalResolutionReturnNoAddresses) { EXPECT_CALL(*timeout_timer, disableTimer()); // Execute resolve callback - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", TestUtility::makeDnsResponse({})); + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", + TestUtility::makeDnsResponse({})); // parse the result response_ctx_ = ResponseValidator::createResponseContext(udp_response_, counters_); @@ -1003,7 +1004,7 @@ TEST_F(DnsFilterTest, ExternalResolutionTimeout2) { // Execute resolve callback. This should harmlessly return and not alter // the response received by the client. Even though we are returning a successful // response, the client does not get an answer - resolve_cb(Network::DnsResolver::ResolutionStatus::Success, "", + resolve_cb(Network::DnsResolver::ResolutionStatus::Completed, "", TestUtility::makeDnsResponse({"130.207.244.251"})); // parse the result diff --git a/test/extensions/network/dns_resolver/apple/apple_dns_impl_test.cc b/test/extensions/network/dns_resolver/apple/apple_dns_impl_test.cc index ae263ae91f..4318229b03 100644 --- a/test/extensions/network/dns_resolver/apple/apple_dns_impl_test.cc +++ b/test/extensions/network/dns_resolver/apple/apple_dns_impl_test.cc @@ -287,35 +287,35 @@ TEST_F(AppleDnsImplTest, DestructPending) { TEST_F(AppleDnsImplTest, LocalLookup) { EXPECT_NE(nullptr, resolveWithExpectations("localhost", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionAuto) { EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionV4Preferred) { EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::V4Preferred, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionV4Only) { EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionV6Only) { EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::All, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } @@ -326,114 +326,119 @@ TEST_F(AppleDnsImplTest, DnsIpAddressVersionV6Only) { TEST_F(AppleDnsImplTest, DnsIpAddressVersionAllSupportsV4Only) { EXPECT_NE(nullptr, resolveWithExpectations("ipv4.google.com", DnsLookupFamily::All, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionAllSupportsV6Only) { - auto* dns_query = resolver_->resolve("ipv6.google.com", DnsLookupFamily::All, - [=](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& results) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); - // On v4 only networks, there will be no results. - for (const auto& result : results) { - const auto& addrinfo = result.addrInfo(); - EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); - EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); - } - dispatcher_->exit(); - }); + auto* dns_query = + resolver_->resolve("ipv6.google.com", DnsLookupFamily::All, + [=](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& results) -> void { + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); + // On v4 only networks, there will be no results. + for (const auto& result : results) { + const auto& addrinfo = result.addrInfo(); + EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); + EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); + } + dispatcher_->exit(); + }); EXPECT_THAT(dns_query, NotNull()); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionV4OnlySupportsV4Only) { EXPECT_NE(nullptr, resolveWithExpectations("ipv4.google.com", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionV4OnlySupportsV6Only) { EXPECT_NE(nullptr, resolveWithExpectations("ipv6.google.com", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, false)); + DnsResolver::ResolutionStatus::Completed, false)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionV6OnlySupportsV4Only) { - auto* dns_query = resolver_->resolve("ipv4.google.com", DnsLookupFamily::V6Only, - [=](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& results) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); - for (const auto& result : results) { - const auto& addrinfo = result.addrInfo(); - EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); - EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); - } - dispatcher_->exit(); - }); + auto* dns_query = + resolver_->resolve("ipv4.google.com", DnsLookupFamily::V6Only, + [=](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& results) -> void { + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); + for (const auto& result : results) { + const auto& addrinfo = result.addrInfo(); + EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); + EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); + } + dispatcher_->exit(); + }); EXPECT_THAT(dns_query, NotNull()); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionV6OnlySupportsV6Only) { - auto* dns_query = resolver_->resolve("ipv6.google.com", DnsLookupFamily::V6Only, - [=](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& results) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); - EXPECT_FALSE(results.empty()); - for (const auto& result : results) { - const auto& addrinfo = result.addrInfo(); - EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); - EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); - } - dispatcher_->exit(); - }); + auto* dns_query = + resolver_->resolve("ipv6.google.com", DnsLookupFamily::V6Only, + [=](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& results) -> void { + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); + EXPECT_FALSE(results.empty()); + for (const auto& result : results) { + const auto& addrinfo = result.addrInfo(); + EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); + EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); + } + dispatcher_->exit(); + }); EXPECT_THAT(dns_query, NotNull()); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionAutoSupportsV4Only) { EXPECT_NE(nullptr, resolveWithExpectations("ipv4.google.com", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionAutoSupportsV6Only) { - auto* dns_query = resolver_->resolve("ipv6.google.com", DnsLookupFamily::Auto, - [=](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& results) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); - // On v4 only networks, there will be no results. - for (const auto& result : results) { - const auto& addrinfo = result.addrInfo(); - EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); - EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); - } - dispatcher_->exit(); - }); + auto* dns_query = + resolver_->resolve("ipv6.google.com", DnsLookupFamily::Auto, + [=](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& results) -> void { + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); + // On v4 only networks, there will be no results. + for (const auto& result : results) { + const auto& addrinfo = result.addrInfo(); + EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); + EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); + } + dispatcher_->exit(); + }); EXPECT_THAT(dns_query, NotNull()); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionV4PreferredSupportsV4Only) { EXPECT_NE(nullptr, resolveWithExpectations("ipv4.google.com", DnsLookupFamily::V4Preferred, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionV4PreferredSupportsV6Only) { - auto* dns_query = resolver_->resolve("ipv6.google.com", DnsLookupFamily::V4Preferred, - [=](DnsResolver::ResolutionStatus status, absl::string_view, - std::list&& results) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); - // On v4 only networks, there will be no results. - for (const auto& result : results) { - const auto& addrinfo = result.addrInfo(); - EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); - EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); - } - dispatcher_->exit(); - }); + auto* dns_query = + resolver_->resolve("ipv6.google.com", DnsLookupFamily::V4Preferred, + [=](DnsResolver::ResolutionStatus status, absl::string_view, + std::list&& results) -> void { + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); + // On v4 only networks, there will be no results. + for (const auto& result : results) { + const auto& addrinfo = result.addrInfo(); + EXPECT_THAT(addrinfo.address_->ip()->ipv6(), NotNull()); + EXPECT_THAT(addrinfo.address_->ip()->ipv4(), IsNull()); + } + dispatcher_->exit(); + }); EXPECT_THAT(dns_query, NotNull()); dispatcher_->run(Event::Dispatcher::RunType::Block); } @@ -444,43 +449,44 @@ TEST_F(AppleDnsImplTest, DnsIpAddressVersionV4PreferredSupportsV6Only) { // queries. TEST_F(AppleDnsImplTest, DoubleLookup) { EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DoubleLookupInOneLoop) { - EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, true, false)); + EXPECT_NE(nullptr, + resolveWithExpectations("google.com", DnsLookupFamily::V4Only, + DnsResolver::ResolutionStatus::Completed, true, false)); EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); dispatcher_->run(Event::Dispatcher::RunType::Block); } TEST_F(AppleDnsImplTest, DnsIpAddressVersionInvalid) { // The DNS queries are successful, but return no results. EXPECT_NE(nullptr, resolveWithExpectations("invalidDnsName", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, false)); + DnsResolver::ResolutionStatus::Completed, false)); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_NE(nullptr, resolveWithExpectations("invalidDnsName", DnsLookupFamily::V4Preferred, - DnsResolver::ResolutionStatus::Success, false)); + DnsResolver::ResolutionStatus::Completed, false)); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_NE(nullptr, resolveWithExpectations("invalidDnsName", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, false)); + DnsResolver::ResolutionStatus::Completed, false)); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_NE(nullptr, resolveWithExpectations("invalidDnsName", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, false)); + DnsResolver::ResolutionStatus::Completed, false)); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_NE(nullptr, resolveWithExpectations("invalidDnsName", DnsLookupFamily::All, - DnsResolver::ResolutionStatus::Success, false)); + DnsResolver::ResolutionStatus::Completed, false)); dispatcher_->run(Event::Dispatcher::RunType::Block); } @@ -501,7 +507,7 @@ TEST_F(AppleDnsImplTest, Cancel) { resolveWithUnreferencedParameters("some.domain", DnsLookupFamily::Auto, false); EXPECT_NE(nullptr, resolveWithExpectations("google.com", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, true)); + DnsResolver::ResolutionStatus::Completed, true)); ASSERT_NE(nullptr, query); query->cancel(Network::ActiveDnsQuery::CancelReason::QueryAbandoned); @@ -511,7 +517,7 @@ TEST_F(AppleDnsImplTest, Cancel) { TEST_F(AppleDnsImplTest, NonExistentDomain) { EXPECT_NE(nullptr, resolveWithExpectations("some.domain", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, false)); + DnsResolver::ResolutionStatus::Completed, false)); dispatcher_->run(Event::Dispatcher::RunType::Block); } @@ -520,7 +526,7 @@ TEST_F(AppleDnsImplTest, LocalResolution) { resolver_->resolve("0.0.0.0", DnsLookupFamily::Auto, [](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& results) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_EQ(1, results.size()); EXPECT_EQ("0.0.0.0:0", results.front().addrInfo().address_->asString()); EXPECT_EQ(std::chrono::seconds(60), results.front().addrInfo().ttl_); @@ -644,7 +650,7 @@ class AppleDnsImplFakeApiTest : public testing::Test { [&dns_callback_executed, dns_lookup_family, address_type, expected_address_size](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_EQ(expected_address_size, response.size()); if (dns_lookup_family == DnsLookupFamily::Auto) { @@ -757,7 +763,7 @@ TEST_F(AppleDnsImplFakeApiTest, ErrorInSocketAccess) { hostname, Network::DnsLookupFamily::Auto, [&dns_callback_executed](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) -> void { - // Status is success because it isn't possible to attach a file event + // Status is Completed because it isn't possible to attach a file event // error to a specific query. EXPECT_EQ(DnsResolver::ResolutionStatus::Failure, status); EXPECT_EQ(0, response.size()); @@ -898,7 +904,7 @@ TEST_F(AppleDnsImplFakeApiTest, QuerySynchronousCompletionUnroutableFamilies) { hostname, Network::DnsLookupFamily::Auto, [&dns_callback_executed](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_EQ(1, response.size()); EXPECT_EQ("1.2.3.4:0", response.front().addrInfo().address_->asString()); EXPECT_EQ(std::chrono::seconds(30), response.front().addrInfo().ttl_); @@ -937,7 +943,7 @@ TEST_F(AppleDnsImplFakeApiTest, QuerySynchronousCompletion) { hostname, Network::DnsLookupFamily::Auto, [&dns_callback_executed](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_EQ(1, response.size()); EXPECT_EQ("1.2.3.4:0", response.front().addrInfo().address_->asString()); EXPECT_EQ(std::chrono::seconds(30), response.front().addrInfo().ttl_); @@ -993,7 +999,7 @@ TEST_F(AppleDnsImplFakeApiTest, MultipleAddresses) { hostname, Network::DnsLookupFamily::Auto, [&dns_callback_executed](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_EQ(2, response.size()); dns_callback_executed.Notify(); }); @@ -1124,7 +1130,7 @@ TEST_F(AppleDnsImplFakeApiTest, MultipleQueries) { hostname, Network::DnsLookupFamily::Auto, [&dns_callback_executed](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_EQ(1, response.size()); EXPECT_EQ("1.2.3.4:0", response.front().addrInfo().address_->asString()); EXPECT_EQ(std::chrono::seconds(30), response.front().addrInfo().ttl_); @@ -1145,7 +1151,7 @@ TEST_F(AppleDnsImplFakeApiTest, MultipleQueries) { hostname2, Network::DnsLookupFamily::V4Only, [&dns_callback_executed2](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_EQ(1, response.size()); EXPECT_EQ("5.6.7.8:0", response.front().addrInfo().address_->asString()); EXPECT_EQ(std::chrono::seconds(30), response.front().addrInfo().ttl_); @@ -1199,7 +1205,7 @@ TEST_F(AppleDnsImplFakeApiTest, MultipleQueriesOneFails) { std::list&& response) -> void { // Even though the second query will fail, this one will flush with the // state it had. - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_EQ(1, response.size()); EXPECT_EQ("1.2.3.4:0", response.front().addrInfo().address_->asString()); EXPECT_EQ(std::chrono::seconds(30), response.front().addrInfo().ttl_); @@ -1261,7 +1267,7 @@ TEST_F(AppleDnsImplFakeApiTest, ResultWithOnlyNonAdditiveReplies) { hostname, Network::DnsLookupFamily::Auto, [&dns_callback_executed](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_TRUE(response.empty()); dns_callback_executed.Notify(); }); @@ -1338,7 +1344,7 @@ TEST_F(AppleDnsImplFakeApiTest, DeallocateOnDestruction) { hostname, Network::DnsLookupFamily::Auto, [&dns_callback_executed](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); EXPECT_EQ(1, response.size()); dns_callback_executed.Notify(); }); diff --git a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc index b57c5e4a80..44353d23af 100644 --- a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc +++ b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc @@ -841,7 +841,7 @@ class DnsImplTest : public testing::TestWithParam { return resolver_->resolve(address, lookup_family, [=, this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& results) -> void { - EXPECT_EQ(DnsResolver::ResolutionStatus::Success, status); + EXPECT_EQ(DnsResolver::ResolutionStatus::Completed, status); std::list address_as_string_list = getAddressAsStringList(results); EXPECT_EQ(0, address_as_string_list.size()); @@ -879,7 +879,7 @@ class DnsImplTest : public testing::TestWithParam { const DnsLookupFamily lookup_family, const std::list& expected_addresses, const DnsResolver::ResolutionStatus resolution_status = - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, const bool getifaddrs_supported = true, const bool getifaddrs_success = true) { server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); @@ -1036,7 +1036,7 @@ TEST_P(DnsImplTest, LocalLookup) { if (GetParam() == Address::IpVersion::v4) { EXPECT_EQ(nullptr, resolveWithExpectations("localhost", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"127.0.0.1"}, {"::1"}, absl::nullopt)); } @@ -1045,12 +1045,12 @@ TEST_P(DnsImplTest, LocalLookup) { "Synchronous DNS IPv6 localhost resolution failed. Please verify localhost resolves to ::1 " "in /etc/hosts, since this misconfiguration is a common cause of these failures."; EXPECT_EQ(nullptr, resolveWithExpectations("localhost", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, {"::1"}, + DnsResolver::ResolutionStatus::Completed, {"::1"}, {"127.0.0.1"}, absl::nullopt)) << error_msg; EXPECT_EQ(nullptr, resolveWithExpectations("localhost", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, {"::1"}, + DnsResolver::ResolutionStatus::Completed, {"::1"}, {"127.0.0.1"}, absl::nullopt)) << error_msg; } @@ -1059,14 +1059,14 @@ TEST_P(DnsImplTest, LocalLookup) { TEST_P(DnsImplTest, DnsIpAddressVersion) { server_->addHosts("some.good.domain", {"1.2.3.4"}, RecordType::A); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, {"1.2.3.4"}, + DnsResolver::ResolutionStatus::Completed, {"1.2.3.4"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(2 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, {"1.2.3.4"}, + DnsResolver::ResolutionStatus::Completed, {"1.2.3.4"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(3 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, @@ -1081,7 +1081,7 @@ TEST_P(DnsImplTest, DnsIpAddressVersion) { TEST_P(DnsImplTest, DnsIpAddressVersionV6) { server_->addHosts("some.good.domain", {"1::2"}, RecordType::AAAA); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, {"1::2"}, {}, + DnsResolver::ResolutionStatus::Completed, {"1::2"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1093,7 +1093,7 @@ TEST_P(DnsImplTest, DnsIpAddressVersionV6) { 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, {"1::2"}, {}, + DnsResolver::ResolutionStatus::Completed, {"1::2"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(3 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, @@ -1124,7 +1124,7 @@ TEST_P(DnsImplTest, DestroyChannelOnResetNetworking) { ASSERT_FALSE(peer_->isChannelDirty()); server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(2 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, @@ -1135,7 +1135,7 @@ TEST_P(DnsImplTest, DestroyChannelOnResetNetworking) { resetChannel(); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(4 /*resolve_total*/, 0 /*pending_resolutions*/, 2 /*not_found*/, @@ -1184,7 +1184,7 @@ TEST_P(DnsImplTest, DestroyChannelOnRefused) { 1 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_FALSE(peer_->isChannelDirty()); @@ -1192,7 +1192,7 @@ TEST_P(DnsImplTest, DestroyChannelOnRefused) { 1 /*get_addr_failure*/, 0 /*timeouts*/); } -// Validate success/fail lookup behavior via TestDnsServer. This exercises the +// Validate completed/fail lookup behavior via TestDnsServer. This exercises the // network event handling in DnsResolverImpl. TEST_P(DnsImplTest, RemoteAsyncLookup) { server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); @@ -1203,7 +1203,7 @@ TEST_P(DnsImplTest, RemoteAsyncLookup) { 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(4 /*resolve_total*/, 0 /*pending_resolutions*/, 3 /*not_found*/, @@ -1216,7 +1216,7 @@ TEST_P(DnsImplTest, MultiARecordLookup) { EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7", "123.4.5.6", "6.5.4.3"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(2 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, @@ -1230,7 +1230,7 @@ TEST_P(DnsImplTest, CNameARecordLookupV4) { server_->setCnameTtl(std::chrono::seconds(60)); EXPECT_NE(nullptr, resolveWithExpectations("root.cnam.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, std::chrono::seconds(60))); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1244,7 +1244,7 @@ TEST_P(DnsImplTest, CNameARecordLookupWithV6) { server_->setCnameTtl(std::chrono::seconds(60)); EXPECT_NE(nullptr, resolveWithExpectations("root.cnam.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, std::chrono::seconds(60))); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(2 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, @@ -1261,7 +1261,7 @@ TEST_P(DnsImplTest, CNameARecordLookupV4InvalidTTL) { server_->setCnameTtl(std::chrono::seconds(60)); EXPECT_NE(nullptr, resolveWithExpectations("root.cnam.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, std::chrono::seconds(0))); dispatcher_->run(Event::Dispatcher::RunType::Block); @@ -1270,7 +1270,7 @@ TEST_P(DnsImplTest, CNameARecordLookupV4InvalidTTL) { server_->setCnameTtl(std::chrono::seconds(2147483648)); EXPECT_NE(nullptr, resolveWithExpectations("root.cnam.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, std::chrono::seconds(0))); dispatcher_->run(Event::Dispatcher::RunType::Block); @@ -1280,7 +1280,7 @@ TEST_P(DnsImplTest, CNameARecordLookupV4InvalidTTL) { EXPECT_NE(nullptr, resolveWithExpectations("root.cnam.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, {"201.134.56.7"}, {}, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, std::chrono::seconds(2147483647))); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(3 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1293,21 +1293,21 @@ TEST_P(DnsImplTest, MultiARecordLookupWithV6) { EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7", "123.4.5.6", "6.5.4.3"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {{"1::2", "1::2:3", "1::2:3:4"}}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(2 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {{"1::2", "1::2:3", "1::2:3:4"}}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(3 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1319,8 +1319,8 @@ TEST_P(DnsImplTest, AutoOnlyV6IfBothV6andV4) { server_->addHosts("some.good.domain", {"1::2"}, RecordType::AAAA); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, {{"1::2"}}, {}, - absl::nullopt)); + DnsResolver::ResolutionStatus::Completed, {{"1::2"}}, + {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); @@ -1330,8 +1330,8 @@ TEST_P(DnsImplTest, AutoV6IfOnlyV6) { server_->addHosts("some.good.domain", {"1::2"}, RecordType::AAAA); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, {{"1::2"}}, {}, - absl::nullopt)); + DnsResolver::ResolutionStatus::Completed, {{"1::2"}}, + {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); @@ -1340,7 +1340,7 @@ TEST_P(DnsImplTest, AutoV6IfOnlyV6) { TEST_P(DnsImplTest, AutoV4IfOnlyV4) { server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {{"201.134.56.7"}}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(2 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, @@ -1352,7 +1352,7 @@ TEST_P(DnsImplTest, V4PreferredOnlyV4IfBothV6andV4) { server_->addHosts("some.good.domain", {"1::2"}, RecordType::AAAA); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Preferred, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {{"201.134.56.7"}}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1363,8 +1363,8 @@ TEST_P(DnsImplTest, V4PreferredV6IfOnlyV6) { server_->addHosts("some.good.domain", {"1::2"}, RecordType::AAAA); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Preferred, - DnsResolver::ResolutionStatus::Success, {{"1::2"}}, {}, - absl::nullopt)); + DnsResolver::ResolutionStatus::Completed, {{"1::2"}}, + {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(2 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); @@ -1373,7 +1373,7 @@ TEST_P(DnsImplTest, V4PreferredV6IfOnlyV6) { TEST_P(DnsImplTest, V4PreferredV4IfOnlyV4) { server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Preferred, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {{"201.134.56.7"}}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1385,7 +1385,7 @@ TEST_P(DnsImplTest, AllIfBothV6andV4) { server_->addHosts("some.good.domain", {"1::2"}, RecordType::AAAA); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::All, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {{"201.134.56.7"}, {"1::2"}}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1396,8 +1396,8 @@ TEST_P(DnsImplTest, AllV6IfOnlyV6) { server_->addHosts("some.good.domain", {"1::2"}, RecordType::AAAA); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::All, - DnsResolver::ResolutionStatus::Success, {{"1::2"}}, {}, - absl::nullopt)); + DnsResolver::ResolutionStatus::Completed, {{"1::2"}}, + {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); @@ -1406,7 +1406,7 @@ TEST_P(DnsImplTest, AllV6IfOnlyV6) { TEST_P(DnsImplTest, AllV4IfOnlyV4) { server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::All, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {{"201.134.56.7"}}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1421,7 +1421,7 @@ TEST_P(DnsImplTest, Cancel) { resolveWithUnreferencedParameters("some.domain", DnsLookupFamily::Auto, false); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); ASSERT_NE(nullptr, query); @@ -1442,7 +1442,7 @@ TEST_P(DnsImplTest, RecordTtlLookup) { uint64_t resolve_total = 0; if (GetParam() == Address::IpVersion::v4) { EXPECT_EQ(nullptr, resolveWithExpectations("localhost", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"127.0.0.1"}, {}, std::chrono::seconds(0))); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1452,16 +1452,16 @@ TEST_P(DnsImplTest, RecordTtlLookup) { if (GetParam() == Address::IpVersion::v6) { EXPECT_EQ(nullptr, resolveWithExpectations("localhost", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, {"::1"}, {}, - std::chrono::seconds(0))); + DnsResolver::ResolutionStatus::Completed, {"::1"}, + {}, std::chrono::seconds(0))); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(resolve_total + 1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); resolve_total++; EXPECT_EQ(nullptr, resolveWithExpectations("localhost", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, {"::1"}, {}, - std::chrono::seconds(0))); + DnsResolver::ResolutionStatus::Completed, {"::1"}, + {}, std::chrono::seconds(0))); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(resolve_total + 1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); @@ -1474,7 +1474,7 @@ TEST_P(DnsImplTest, RecordTtlLookup) { EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7", "123.4.5.6", "6.5.4.3"}, {"1::2", "1::2:3", "1::2:3:4"}, std::chrono::seconds(300))); dispatcher_->run(Event::Dispatcher::RunType::Block); @@ -1484,7 +1484,7 @@ TEST_P(DnsImplTest, RecordTtlLookup) { EXPECT_NE(nullptr, resolveWithExpectations( "some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, {"1::2", "1::2:3", "1::2:3:4"}, + DnsResolver::ResolutionStatus::Completed, {"1::2", "1::2:3", "1::2:3:4"}, {"201.134.56.7", "123.4.5.6", "6.5.4.3"}, std::chrono::seconds(300))); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(resolve_total + 1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1493,7 +1493,7 @@ TEST_P(DnsImplTest, RecordTtlLookup) { EXPECT_NE(nullptr, resolveWithExpectations( "some.good.domain", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, {"1::2", "1::2:3", "1::2:3:4"}, + DnsResolver::ResolutionStatus::Completed, {"1::2", "1::2:3", "1::2:3:4"}, {"201.134.56.7", "123.4.5.6", "6.5.4.3"}, std::chrono::seconds(300))); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(resolve_total + 1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1538,7 +1538,7 @@ TEST_P(DnsImplTest, PendingTimerEnable) { TEST_P(DnsImplTest, PendingResolutions) { server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {{"201.134.56.7"}}, {}, absl::nullopt)); checkStats(0 /*resolve_total*/, 1 /*pending_resolutions*/, 0 /*not_found*/, @@ -1579,7 +1579,7 @@ TEST_P(DnsImplTest, WithNoRecord) { TEST_P(DnsImplTest, WithARecord) { server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1591,21 +1591,21 @@ TEST_P(DnsImplTest, WithARecord) { 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(4 /*resolve_total*/, 0 /*pending_resolutions*/, 2 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Preferred, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(5 /*resolve_total*/, 0 /*pending_resolutions*/, 2 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::All, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(6 /*resolve_total*/, 0 /*pending_resolutions*/, 2 /*not_found*/, @@ -1620,28 +1620,28 @@ TEST_P(DnsImplTest, WithAAAARecord) { 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, {"1::2"}, {}, + DnsResolver::ResolutionStatus::Completed, {"1::2"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(2 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, {"1::2"}, {}, + DnsResolver::ResolutionStatus::Completed, {"1::2"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(3 /*resolve_total*/, 0 /*pending_resolutions*/, 1 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Preferred, - DnsResolver::ResolutionStatus::Success, {"1::2"}, {}, + DnsResolver::ResolutionStatus::Completed, {"1::2"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(5 /*resolve_total*/, 0 /*pending_resolutions*/, 2 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::All, - DnsResolver::ResolutionStatus::Success, {"1::2"}, {}, + DnsResolver::ResolutionStatus::Completed, {"1::2"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(6 /*resolve_total*/, 0 /*pending_resolutions*/, 2 /*not_found*/, @@ -1652,35 +1652,35 @@ TEST_P(DnsImplTest, WithBothAAndAAAARecord) { server_->addHosts("some.good.domain", {"201.134.56.7"}, RecordType::A); server_->addHosts("some.good.domain", {"1::2"}, RecordType::AAAA); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Only, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V6Only, - DnsResolver::ResolutionStatus::Success, {"1::2"}, {}, + DnsResolver::ResolutionStatus::Completed, {"1::2"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(2 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, {"1::2"}, {}, + DnsResolver::ResolutionStatus::Completed, {"1::2"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(3 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::V4Preferred, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(4 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::All, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7", "1::2"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); checkStats(5 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, @@ -1872,14 +1872,14 @@ TEST_P(DnsImplFilterUnroutableFamiliesTest, FilterAllV6) { TEST_P(DnsImplFilterUnroutableFamiliesTest, DontFilterIfGetifaddrsIsNotSupported) { testFilterAddresses({}, DnsLookupFamily::All, {"201.134.56.7", "1::2"}, - DnsResolver::ResolutionStatus::Success, false /* getifaddrs_supported */); + DnsResolver::ResolutionStatus::Completed, false /* getifaddrs_supported */); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); } TEST_P(DnsImplFilterUnroutableFamiliesTest, DontFilterIfThereIsAGetifaddrsFailure) { testFilterAddresses({}, DnsLookupFamily::All, {"201.134.56.7", "1::2"}, - DnsResolver::ResolutionStatus::Success, true /* getifaddrs_supported */, + DnsResolver::ResolutionStatus::Completed, true /* getifaddrs_supported */, false /* getifaddrs_success */); checkStats(1 /*resolve_total*/, 0 /*pending_resolutions*/, 0 /*not_found*/, 0 /*get_addr_failure*/, 0 /*timeouts*/); @@ -2099,7 +2099,7 @@ TEST_P(DnsImplCustomResolverTest, CustomResolverValidAfterChannelDestruction) { // server's address was passed as a custom resolver on construction, the new channel should still // point to the test dns server, and the query should succeed. EXPECT_NE(nullptr, resolveWithExpectations("some.good.domain", DnsLookupFamily::Auto, - DnsResolver::ResolutionStatus::Success, + DnsResolver::ResolutionStatus::Completed, {"201.134.56.7"}, {}, absl::nullopt)); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_FALSE(peer_->isChannelDirty()); diff --git a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc index 932fc270bd..b2a830cd21 100644 --- a/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc +++ b/test/extensions/network/dns_resolver/getaddrinfo/getaddrinfo_test.cc @@ -93,7 +93,7 @@ class GetAddrInfoDnsImplTest : public testing::Test { std::list&& response) { // Since we use AF_UNSPEC, depending on the CI environment we might get either 1 or 2 // addresses. - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_TRUE(response.size() == 1 || response.size() == 2); EXPECT_TRUE("127.0.0.1:0" == response.front().addrInfo().address_->asString() || "[::1]:0" == response.front().addrInfo().address_->asString()); @@ -166,7 +166,7 @@ TEST_F(GetAddrInfoDnsImplTest, NoData) { resolver_->resolve("localhost", DnsLookupFamily::All, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_TRUE(response.empty()); dispatcher_->exit(); @@ -183,7 +183,7 @@ TEST_F(GetAddrInfoDnsImplTest, NoName) { resolver_->resolve("localhost", DnsLookupFamily::All, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_TRUE(response.empty()); dispatcher_->exit(); @@ -203,7 +203,7 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainIndefinitelyAndSuccess) { resolver_->resolve("localhost", DnsLookupFamily::All, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_TRUE(response.empty()); dispatcher_->exit(); @@ -247,7 +247,7 @@ TEST_F(GetAddrInfoDnsImplTest, TryAgainWithNumRetriesAndSuccess) { resolver_->resolve("localhost", DnsLookupFamily::All, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_TRUE(response.empty()); dispatcher_->exit(); @@ -293,7 +293,7 @@ TEST_F(GetAddrInfoDnsImplTest, All) { "localhost", DnsLookupFamily::All, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_EQ(2, response.size()); EXPECT_EQ("[[::1]:0, 127.0.0.1:0]", accumulateToString(response, [](const auto& dns_response) { @@ -317,7 +317,7 @@ TEST_F(GetAddrInfoDnsImplTest, V4Only) { "localhost", DnsLookupFamily::V4Only, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_EQ(1, response.size()); EXPECT_EQ("[127.0.0.1:0]", accumulateToString(response, [](const auto& dns_response) { @@ -341,7 +341,7 @@ TEST_F(GetAddrInfoDnsImplTest, V6Only) { "localhost", DnsLookupFamily::V6Only, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_EQ(1, response.size()); EXPECT_EQ("[[::1]:0]", accumulateToString(response, [](const auto& dns_response) { @@ -365,7 +365,7 @@ TEST_F(GetAddrInfoDnsImplTest, V4Preferred) { "localhost", DnsLookupFamily::V4Preferred, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_EQ(1, response.size()); EXPECT_EQ("[127.0.0.1:0]", accumulateToString(response, [](const auto& dns_response) { @@ -389,7 +389,7 @@ TEST_F(GetAddrInfoDnsImplTest, V4PreferredNoV4) { "localhost", DnsLookupFamily::V4Preferred, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_EQ(1, response.size()); EXPECT_EQ("[[::1]:0]", accumulateToString(response, [](const auto& dns_response) { @@ -413,7 +413,7 @@ TEST_F(GetAddrInfoDnsImplTest, Auto) { "localhost", DnsLookupFamily::Auto, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_EQ(1, response.size()); EXPECT_EQ("[[::1]:0]", accumulateToString(response, [](const auto& dns_response) { @@ -437,7 +437,7 @@ TEST_F(GetAddrInfoDnsImplTest, AutoNoV6) { "localhost", DnsLookupFamily::Auto, [this](DnsResolver::ResolutionStatus status, absl::string_view, std::list&& response) { - EXPECT_EQ(status, DnsResolver::ResolutionStatus::Success); + EXPECT_EQ(status, DnsResolver::ResolutionStatus::Completed); EXPECT_EQ(1, response.size()); EXPECT_EQ("[127.0.0.1:0]", accumulateToString(response, [](const auto& dns_response) { From bf6012177c4b54d31f40f8edeaed72957c8dfd33 Mon Sep 17 00:00:00 2001 From: Raven Black Date: Thu, 19 Sep 2024 10:39:52 -0400 Subject: [PATCH 21/28] MockDispatcher to not accidentally run on other threads (#36134) Commit Message: MockDispatcher to not accidentally run on other threads Additional Description: Several issues have been found recently where the MockDispatcher's default behavior of just running the callback immediately causes tests to pass, but asan/tsan/release tests fail, because it ends up running the callback on a different thread, exactly the opposite of almost the whole point of posting things on a dispatcher which is to put them on a specific thread. This change makes MockDispatcher insist on being on whichever thread created the dispatcher, most likely the main thread in tests. This is still not great because the behavior of running the callback immediately is still quite different from the behavior of a dispatcher in production, e.g. ``` auto x = std::make_shared(0); this_thread_dispatcher.post([x]() { *x += 5; }); *x += 1; std::cerr << *x << std::endl; ``` would always output 6 with a default MockDispatcher, and would always output 1 with a production dispatcher, but at least cases like that can be debugged linearly and will be the same in all test environments, versus the running on a different (and incorrect) thread problem is very difficult to debug and sometimes only shows up (and sometimes flakily) in the harder to run test contexts. (But still pretty bad in that test coverage would say a thing works, then in production it could do a different thing than it did in tests.) Since it's a mock the behavior can also be manually overridden to be the old way. Arguably it should have just always required explicit manual interception because it's supposed to be a mock not a fake. Risk Level: Should be safe since it's test-only. Testing: This is test-only. Docs Changes: n/a Release Notes: n/a Platform Specific Features: n/a --------- Signed-off-by: Raven Black --- test/common/thread_local/thread_local_impl_test.cc | 4 +++- .../common/redis/cluster_refresh_manager_test.cc | 14 +++++++++++--- test/mocks/event/mocks.cc | 13 +++++++++++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/test/common/thread_local/thread_local_impl_test.cc b/test/common/thread_local/thread_local_impl_test.cc index 4ae9409fa4..fe95397605 100644 --- a/test/common/thread_local/thread_local_impl_test.cc +++ b/test/common/thread_local/thread_local_impl_test.cc @@ -378,7 +378,9 @@ TEST(ThreadLocalInstanceImplDispatcherTest, DestroySlotOnWorker) { EXPECT_CALL(main_dispatcher, isThreadSafe()).WillOnce(Return(false)); // Destroy the slot on worker thread and expect the post() of main dispatcher to be called. - EXPECT_CALL(main_dispatcher, post(_)); + // Override the behavior to do nothing, because the default mock behavior asserts that the + // callback must run on the same thread as the dispatcher. + EXPECT_CALL(main_dispatcher, post(_)).WillOnce([]() {}); slot.reset(); diff --git a/test/extensions/common/redis/cluster_refresh_manager_test.cc b/test/extensions/common/redis/cluster_refresh_manager_test.cc index e54c77037d..f834d6fc4a 100644 --- a/test/extensions/common/redis/cluster_refresh_manager_test.cc +++ b/test/extensions/common/redis/cluster_refresh_manager_test.cc @@ -25,11 +25,11 @@ namespace Common { namespace Redis { // TODO: rewrite the tests to fix the flaky test -class ClusterRefreshManagerTest : public testing::Test { +class ClusterRefreshManagerTest : public testing::Test, public Event::TestUsingSimulatedTime { public: ClusterRefreshManagerTest() : cluster_name_("fake_cluster"), refresh_manager_(std::make_shared( - dispatcher_, cm_, time_system_)) { + *dispatcher_, cm_, time_system_)) { time_system_.setMonotonicTime(std::chrono::seconds(1)); cluster_maps_.active_clusters_.emplace("fake_cluster", mock_cluster_); ON_CALL(cm_, clusters()).WillByDefault(Return(cluster_maps_)); @@ -100,7 +100,8 @@ class ClusterRefreshManagerTest : public testing::Test { } const std::string cluster_name_; - NiceMock dispatcher_; + Api::ApiPtr api_ = Api::createApiForTest(); + Event::DispatcherPtr dispatcher_ = api_->allocateDispatcher("test_thread"); NiceMock cm_; Upstream::ClusterManager::ClusterInfoMaps cluster_maps_; Upstream::MockClusterMockPrioritySet mock_cluster_; @@ -142,6 +143,7 @@ TEST_F(ClusterRefreshManagerTest, Basic) { advanceTime(MonotonicTime(std::chrono::seconds(3)), 2); thread_1->join(); thread_2->join(); + dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_GE(callback_count_, 2); EXPECT_EQ(cluster_info->redirects_count_, 0); @@ -187,6 +189,7 @@ TEST_F(ClusterRefreshManagerTest, BasicFailureEvents) { thread_1->join(); thread_2->join(); + dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_GE(callback_count_, 2); EXPECT_EQ(cluster_info->failures_count_, 0); EXPECT_EQ(cluster_info->last_callback_time_ms_.load(), 3000); @@ -197,6 +200,7 @@ TEST_F(ClusterRefreshManagerTest, BasicFailureEvents) { callback_count_ = 0; advanceTime(MonotonicTime(std::chrono::seconds(5))); + dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_FALSE(refresh_manager_->onFailure("unregistered_cluster_name")); EXPECT_EQ(callback_count_, 0); @@ -231,6 +235,7 @@ TEST_F(ClusterRefreshManagerTest, BasicDegradedEvents) { thread_1->join(); thread_2->join(); + dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_GE(callback_count_, 2); EXPECT_EQ(cluster_info->host_degraded_count_, 0); EXPECT_EQ(cluster_info->last_callback_time_ms_.load(), 3000); @@ -241,6 +246,7 @@ TEST_F(ClusterRefreshManagerTest, BasicDegradedEvents) { callback_count_ = 0; advanceTime(MonotonicTime(std::chrono::seconds(5))); + dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_FALSE(refresh_manager_->onHostDegraded("unregistered_cluster_name")); EXPECT_EQ(callback_count_, 0); @@ -290,6 +296,7 @@ TEST_F(ClusterRefreshManagerTest, HighVolume) { thread_1->join(); thread_2->join(); + dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_EQ(callback_count_, thread1_callback_count + thread2_callback_count); EXPECT_EQ(callback_count_, 30); } @@ -305,6 +312,7 @@ TEST_F(ClusterRefreshManagerTest, FeatureDisabled) { EXPECT_FALSE(refresh_manager_->onFailure(cluster_name_)); EXPECT_FALSE(refresh_manager_->onHostDegraded(cluster_name_)); + dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_GE(callback_count_, 0); EXPECT_EQ(cluster_info->redirects_count_, 0); EXPECT_EQ(cluster_info->failures_count_, 0); diff --git a/test/mocks/event/mocks.cc b/test/mocks/event/mocks.cc index e6a0b436f6..5a557332e3 100644 --- a/test/mocks/event/mocks.cc +++ b/test/mocks/event/mocks.cc @@ -20,6 +20,8 @@ MockDispatcher::MockDispatcher() : MockDispatcher("test_thread") {} MockDispatcher::MockDispatcher(const std::string& name) : name_(name) { time_system_ = std::make_unique(); + auto thread_factory = &Api::createApiForTest()->threadFactory(); + Envoy::Thread::ThreadId thread_id = thread_factory->currentThreadId(); ON_CALL(*this, initializeStats(_, _)).WillByDefault(Return()); ON_CALL(*this, clearDeferredDeleteList()).WillByDefault(Invoke([this]() -> void { to_delete_.clear(); @@ -28,14 +30,21 @@ MockDispatcher::MockDispatcher(const std::string& name) : name_(name) { ON_CALL(*this, createScaledTimer_(_, _)).WillByDefault(ReturnNew>()); ON_CALL(*this, createScaledTypedTimer_(_, _)) .WillByDefault(ReturnNew>()); - ON_CALL(*this, post(_)).WillByDefault(Invoke([](PostCb cb) -> void { cb(); })); + ON_CALL(*this, post(_)).WillByDefault([thread_factory, thread_id](PostCb cb) -> void { + ASSERT(thread_factory->currentThreadId() == thread_id, + "MockDispatcher tried to execute a callback on a different thread - a test with threads " + "should probably use a dispatcher from Api::createApiForTest() instead."); + cb(); + }); ON_CALL(buffer_factory_, createBuffer_(_, _, _)) .WillByDefault(Invoke([](std::function below_low, std::function above_high, std::function above_overflow) -> Buffer::Instance* { return new Buffer::WatermarkBuffer(below_low, above_high, above_overflow); })); - ON_CALL(*this, isThreadSafe()).WillByDefault(Return(true)); + ON_CALL(*this, isThreadSafe()).WillByDefault([thread_factory, thread_id]() { + return thread_factory->currentThreadId() == thread_id; + }); } MockDispatcher::~MockDispatcher() = default; From 4cbb5806a37e3ed5ea126faaa09d3636fdffc64d Mon Sep 17 00:00:00 2001 From: botengyao Date: Thu, 19 Sep 2024 10:44:16 -0400 Subject: [PATCH 22/28] sec-release: update the Q3 actual security release date (#35909) Signed-off-by: Boteng Yao --- RELEASES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index 258aa17c14..1435704556 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -138,6 +138,6 @@ Security releases are published on a 3-monthly cycle, around the mid point betwe | Quarter | Expected | Actual | Difference | |:-------:|:----------:|:----------:|:----------:| | 2024 Q2 | 2024/06/04 | 2024/06/04 | 0 days | -| 2024 Q3 | 2024/09/03 | +| 2024 Q3 | 2024/09/03 | 2024/09/19 | 16 days | NOTE: Zero-day vulnerabilities, and upstream vulnerabilities disclosed to us under embargo, may necessitate an emergency release with little or no warning. From f3ff3306f53fc0ebb6314ffad947896b56b23d29 Mon Sep 17 00:00:00 2001 From: Chao-Han Tsai Date: Thu, 19 Sep 2024 10:03:32 -0700 Subject: [PATCH 23/28] basic_auth: support authorization header override (#36046) Commit Message: basic_auth: support authorization header override Additional Description: provide a way to do basic authorization on headers other than `:Authorization`. Say we are doing two level of basic authorization. One at the proxy level with header name `Proxy-Authorization` and the other one at the application level with header name `Authorization`. Risk Level: low Testing: unit test Docs Changes: changelog Release Notes: Platform Specific Features: [Optional Runtime guard:] [Optional Fixes #Issue] [Optional Fixes commit #PR or SHA] [Optional Deprecated:] [Optional [API Considerations](https://github.com/envoyproxy/envoy/blob/main/api/review_checklist.md):] --------- Signed-off-by: Networking team Co-authored-by: Networking team --- .../http/basic_auth/v3/basic_auth.proto | 6 +++++ changelogs/current.yaml | 5 +++++ .../http/basic_auth/basic_auth_filter.cc | 11 +++++++++- .../http/basic_auth/basic_auth_filter.h | 5 ++++- .../filters/http/basic_auth/config.cc | 3 ++- .../filters/http/basic_auth/filter_test.cc | 22 ++++++++++++++++++- 6 files changed, 48 insertions(+), 4 deletions(-) diff --git a/api/envoy/extensions/filters/http/basic_auth/v3/basic_auth.proto b/api/envoy/extensions/filters/http/basic_auth/v3/basic_auth.proto index 995d2c3bca..af3c442c03 100644 --- a/api/envoy/extensions/filters/http/basic_auth/v3/basic_auth.proto +++ b/api/envoy/extensions/filters/http/basic_auth/v3/basic_auth.proto @@ -41,6 +41,12 @@ message BasicAuth { // If it is not specified, the username will not be forwarded. string forward_username_header = 2 [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME strict: false}]; + + // This field specifies the request header to load the basic credential from. + // + // If it is not specified, the filter loads the credential from the "Authorization" header. + string authentication_header = 3 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME strict: false}]; } // Extra settings that may be added to per-route configuration for diff --git a/changelogs/current.yaml b/changelogs/current.yaml index bfc2be2fc5..e17a1dd935 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -353,5 +353,10 @@ new_features: change: | Added two new methods ``oidsPeerCertificate()`` and ``oidsLocalCertificate()`` to SSL connection object API :ref:`SSL connection info object `. +- area: basic_auth + change: | + Added support to provide an override + :ref:`authentication_header ` + to load Basic Auth credential. deprecated: diff --git a/source/extensions/filters/http/basic_auth/basic_auth_filter.cc b/source/extensions/filters/http/basic_auth/basic_auth_filter.cc index fdb3465064..df7929f42b 100644 --- a/source/extensions/filters/http/basic_auth/basic_auth_filter.cc +++ b/source/extensions/filters/http/basic_auth/basic_auth_filter.cc @@ -2,6 +2,8 @@ #include +#include "envoy/http/header_map.h" + #include "source/common/common/base64.h" #include "source/common/http/header_utility.h" #include "source/common/http/headers.h" @@ -29,8 +31,10 @@ std::string computeSHA1(absl::string_view password) { } // namespace FilterConfig::FilterConfig(UserMap&& users, const std::string& forward_username_header, + const std::string& authentication_header, const std::string& stats_prefix, Stats::Scope& scope) : users_(std::move(users)), forward_username_header_(forward_username_header), + authentication_header_(Http::LowerCaseString(authentication_header)), stats_(generateStats(stats_prefix + "basic_auth.", scope)) {} BasicAuthFilter::BasicAuthFilter(FilterConfigConstSharedPtr config) : config_(std::move(config)) {} @@ -43,7 +47,12 @@ Http::FilterHeadersStatus BasicAuthFilter::decodeHeaders(Http::RequestHeaderMap& users = &route_specific_settings->users(); } - auto auth_header = headers.get(Http::CustomHeaders::get().Authorization); + Http::HeaderMap::GetResult auth_header; + if (!config_->authenticationHeader().get().empty()) { + auth_header = headers.get(config_->authenticationHeader()); + } else { + auth_header = headers.get(Http::CustomHeaders::get().Authorization); + } if (auth_header.empty()) { return onDenied("User authentication failed. Missing username and password.", diff --git a/source/extensions/filters/http/basic_auth/basic_auth_filter.h b/source/extensions/filters/http/basic_auth/basic_auth_filter.h index 4d1c9a8621..d257450416 100644 --- a/source/extensions/filters/http/basic_auth/basic_auth_filter.h +++ b/source/extensions/filters/http/basic_auth/basic_auth_filter.h @@ -45,10 +45,12 @@ using UserMap = absl::flat_hash_map; class FilterConfig { public: FilterConfig(UserMap&& users, const std::string& forward_username_header, - const std::string& stats_prefix, Stats::Scope& scope); + const std::string& authentication_header, const std::string& stats_prefix, + Stats::Scope& scope); const BasicAuthStats& stats() const { return stats_; } const std::string& forwardUsernameHeader() const { return forward_username_header_; } const UserMap& users() const { return users_; } + const Http::LowerCaseString& authenticationHeader() const { return authentication_header_; } private: static BasicAuthStats generateStats(const std::string& prefix, Stats::Scope& scope) { @@ -57,6 +59,7 @@ class FilterConfig { const UserMap users_; const std::string forward_username_header_; + const Http::LowerCaseString authentication_header_; BasicAuthStats stats_; }; using FilterConfigConstSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/http/basic_auth/config.cc b/source/extensions/filters/http/basic_auth/config.cc index 20e66b767a..828899b71c 100644 --- a/source/extensions/filters/http/basic_auth/config.cc +++ b/source/extensions/filters/http/basic_auth/config.cc @@ -68,7 +68,8 @@ Http::FilterFactoryCb BasicAuthFilterFactory::createFilterFactoryFromProtoTyped( Config::DataSource::read(proto_config.users(), false, context.serverFactoryContext().api()), std::string)); FilterConfigConstSharedPtr config = std::make_unique( - std::move(users), proto_config.forward_username_header(), stats_prefix, context.scope()); + std::move(users), proto_config.forward_username_header(), + proto_config.authentication_header(), stats_prefix, context.scope()); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(config)); }; diff --git a/test/extensions/filters/http/basic_auth/filter_test.cc b/test/extensions/filters/http/basic_auth/filter_test.cc index 87296f0eef..aeeb1a9909 100644 --- a/test/extensions/filters/http/basic_auth/filter_test.cc +++ b/test/extensions/filters/http/basic_auth/filter_test.cc @@ -18,7 +18,7 @@ class FilterTest : public testing::Test { UserMap users; users.insert({"user1", {"user1", "tESsBmE/yNY3lb6a0L6vVQEZNqw="}}); // user1:test1 users.insert({"user2", {"user2", "EJ9LPFDXsN9ynSmbxvjp75Bmlx8="}}); // user2:test2 - config_ = std::make_unique(std::move(users), "x-username", "stats", + config_ = std::make_unique(std::move(users), "x-username", "", "stats", *stats_.rootScope()); filter_ = std::make_shared(config_); filter_->setDecoderFilterCallbacks(decoder_filter_callbacks_); @@ -272,6 +272,26 @@ TEST_F(FilterTest, BasicAuthPerRouteEnabled) { filter_->decodeHeaders(invalid_credentials, true)); } +TEST_F(FilterTest, OverrideAuthorizationHeaderProvided) { + UserMap users; + users.insert({"user1", {"user1", "tESsBmE/yNY3lb6a0L6vVQEZNqw="}}); // user1:test1 + + FilterConfigConstSharedPtr config = std::make_unique( + std::move(users), "x-username", "x-authorization-override", "stats", *stats_.rootScope()); + std::shared_ptr filter = std::make_shared(config); + filter->setDecoderFilterCallbacks(decoder_filter_callbacks_); + + Http::TestRequestHeaderMapImpl request_headers_user1{ + {"x-authorization-override", "Basic dXNlcjE6dGVzdDE="}}; + request_headers_user1.setScheme("http"); + request_headers_user1.setHost("host"); + request_headers_user1.setPath("/"); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter->decodeHeaders(request_headers_user1, true)); + EXPECT_EQ("user1", request_headers_user1.get_("x-username")); +} + } // namespace BasicAuth } // namespace HttpFilters } // namespace Extensions From 5550967f21ffc621609a573a085ffb97311332b7 Mon Sep 17 00:00:00 2001 From: Kuat Yessenov Date: Tue, 9 Jul 2024 18:37:25 +0000 Subject: [PATCH 24/28] jwt: fix clear route cache with remote JWKs Change-Id: If54c7143ecb1bf936e88fbb3371d9c47f6d8f671 Signed-off-by: Kuat Yessenov Signed-off-by: Ryan Northey --- changelogs/current.yaml | 5 ++ .../filters/http/jwt_authn/authenticator.cc | 10 ++-- .../http/jwt_authn/filter_integration_test.cc | 51 +++++++++++++++++-- 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index e17a1dd935..3f59975758 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -143,6 +143,11 @@ bug_fixes: Fixed an inconsistency in how boolean values are loaded in RTDS, where they were previously converted to "1"/"0" instead of "true"/"false". The correct string representation ("true"/"false") will now be used. This change can be reverted by setting the runtime guard ``envoy.reloadable_features.boolean_to_string_fix`` to false. +- area: jwt + change: | + Fixed a bug where using ``clear_route_cache`` with remote JWKs works + incorrectly and may cause a crash when the modified request does not match + any route. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/source/extensions/filters/http/jwt_authn/authenticator.cc b/source/extensions/filters/http/jwt_authn/authenticator.cc index 62e175651c..cecbc2c0b0 100644 --- a/source/extensions/filters/http/jwt_authn/authenticator.cc +++ b/source/extensions/filters/http/jwt_authn/authenticator.cc @@ -480,6 +480,11 @@ void AuthenticatorImpl::doneWithStatus(const Status& status) { // Unless allowing failed or missing, all tokens must be verified successfully. if ((Status::Ok != status && !is_allow_failed_ && !is_allow_missing_) || tokens_.empty()) { tokens_.clear(); + if (clear_route_cache_ && clear_route_cb_) { + clear_route_cb_(); + } + clear_route_cb_ = nullptr; + if (is_allow_failed_) { callback_(Status::Ok); } else if (is_allow_missing_ && status == Status::JwtMissed) { @@ -489,11 +494,6 @@ void AuthenticatorImpl::doneWithStatus(const Status& status) { } callback_ = nullptr; - if (clear_route_cache_ && clear_route_cb_) { - clear_route_cb_(); - } - clear_route_cb_ = nullptr; - return; } diff --git a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc index 94f2dff75b..1203f4c008 100644 --- a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc @@ -20,7 +20,7 @@ namespace JwtAuthn { namespace { std::string getAuthFilterConfig(const std::string& config_str, bool use_local_jwks, - bool strip_failure_response) { + bool strip_failure_response, bool clear_route_cache = false) { JwtAuthentication proto_config; TestUtility::loadFromYaml(config_str, proto_config); proto_config.set_strip_failure_response(strip_failure_response); @@ -32,6 +32,11 @@ std::string getAuthFilterConfig(const std::string& config_str, bool use_local_jw local_jwks->set_inline_string(PublicKey); } + if (clear_route_cache) { + auto& provider0 = (*proto_config.mutable_providers())[std::string(ProviderName)]; + provider0.set_clear_route_cache(true); + } + HttpFilter filter; filter.set_name("envoy.filters.http.jwt_authn"); filter.mutable_typed_config()->PackFrom(proto_config); @@ -57,8 +62,10 @@ std::string getAsyncFetchFilterConfig(const std::string& config_str, bool fast_l return MessageUtil::getJsonStringFromMessageOrError(filter); } -std::string getFilterConfig(bool use_local_jwks, bool strip_failure_response) { - return getAuthFilterConfig(ExampleConfig, use_local_jwks, strip_failure_response); +std::string getFilterConfig(bool use_local_jwks, bool strip_failure_response, + bool clear_route_cache = false) { + return getAuthFilterConfig(ExampleConfig, use_local_jwks, strip_failure_response, + clear_route_cache); } class LocalJwksIntegrationTest : public HttpProtocolIntegrationTest {}; @@ -378,8 +385,8 @@ class RemoteJwksIntegrationTest : public HttpProtocolIntegrationTest { addFakeUpstream(GetParam().upstream_protocol); } - void initializeFilter(bool add_cluster) { - config_helper_.prependFilter(getFilterConfig(false, false)); + void initializeFilter(bool add_cluster, bool clear_route_cache = false) { + config_helper_.prependFilter(getFilterConfig(false, false, clear_route_cache)); if (add_cluster) { config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { @@ -390,6 +397,17 @@ class RemoteJwksIntegrationTest : public HttpProtocolIntegrationTest { } else { config_helper_.skipPortUsageValidation(); } + if (clear_route_cache) { + config_helper_.addConfigModifier( + [](envoy::extensions::filters::network::http_connection_manager::v3:: + HttpConnectionManager& hcm) { + auto* virtual_host = hcm.mutable_route_config()->mutable_virtual_hosts(0); + auto* route = virtual_host->mutable_routes(0); + auto* header = route->mutable_match()->add_headers(); + header->set_name("x-jwt-claim-sub"); + header->mutable_string_match()->set_exact("test"); + }); + } initialize(); } @@ -481,6 +499,29 @@ TEST_P(RemoteJwksIntegrationTest, WithGoodToken) { cleanup(); } +TEST_P(RemoteJwksIntegrationTest, WithGoodTokenClearRouteCache) { + initializeFilter(/*add_cluster=*/true, true); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest(Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-jwt-claim-sub", "test"}, + {"Authorization", "Bearer " + std::string(GoodToken)}, + }); + + waitForJwksResponse("200", PublicKey); + + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("404", response->headers().getStatusValue()); + + cleanup(); +} + // With remote Jwks, this test verifies a request is rejected even with a good Jwt token // when the remote jwks server replied with 500. TEST_P(RemoteJwksIntegrationTest, FetchFailedJwks) { From 0ee424bb06bbeb97549788946f392f9110ac1e33 Mon Sep 17 00:00:00 2001 From: Alyssa Wilk Date: Thu, 8 Aug 2024 20:00:11 +0000 Subject: [PATCH 25/28] internal_address_config: change the default to be more secure for service mesh environments Signed-off-by: Alyssa Wilk Signed-off-by: Boteng Yao Signed-off-by: Ryan Northey --- .../v3/http_connection_manager.proto | 28 +++++++++++++ changelogs/current.yaml | 8 ++++ source/common/http/conn_manager_config.h | 4 ++ source/common/runtime/runtime_features.cc | 2 + .../network/http_connection_manager/config.cc | 4 ++ .../http_connection_manager/config_test.cc | 41 +++++++++++++++++++ test/integration/http_integration.cc | 15 +++++++ test/mocks/http/mocks.h | 9 +++- 8 files changed, 110 insertions(+), 1 deletion(-) diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 712512ddb0..4cbbbc20d3 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -684,6 +684,34 @@ message HttpConnectionManager { // purposes. If unspecified, only RFC1918 IP addresses will be considered internal. // See the documentation for :ref:`config_http_conn_man_headers_x-envoy-internal` for more // information about internal/external addresses. + // + // .. warning:: + // In the next release, no IP addresses will be considered trusted. If you have tooling such as probes + // on your private network which need to be treated as trusted (e.g. changing arbitrary x-envoy headers) + // you will have to manually include those addresses or CIDR ranges like: + // + // .. validated-code-block:: yaml + // :type-name: envoy.extensions.filters.network.http_connection_manager.v3.InternalAddressConfig + // + // cidr_ranges: + // address_prefix: 10.0.0.0 + // prefix_len: 8 + // cidr_ranges: + // address_prefix: 192.168.0.0 + // prefix_len: 16 + // cidr_ranges: + // address_prefix: 172.16.0.0 + // prefix_len: 12 + // cidr_ranges: + // address_prefix: 127.0.0.1 + // prefix_len: 32 + // cidr_ranges: + // address_prefix: fd00:: + // prefix_len: 8 + // cidr_ranges: + // address_prefix: ::1 + // prefix_len: 128 + // InternalAddressConfig internal_address_config = 25; // If set, Envoy will not append the remote address to the diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 3f59975758..6d520b8b17 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -39,6 +39,14 @@ behavior_changes: change: | Added new tag extraction so that scoped rds stats have their :ref:'scope_route_config_name ' and stat prefix extracted. +- area: http + change: | + The default configuration of Envoy will continue to trust internal addresses while in the future it will not trust them by default. + If you have tooling such as probes on your private network which need to be treated as trusted (e.g. changing arbitrary ``x-envoy`` + headers) please explictily include those addresses or CIDR ranges into :ref:`internal_address_config + ` + See the config examples from the above ``internal_address_config`` link. This default no trust internal address can be turned on by + setting runtime guard ``envoy.reloadable_features.explicit_internal_address_config`` to ``true``. minor_behavior_changes: # *Changes that may cause incompatibilities for some users, but should not for most* diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 7a2b300b83..629eee958d 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -191,6 +191,10 @@ class InternalAddressConfig { class DefaultInternalAddressConfig : public Http::InternalAddressConfig { public: bool isInternalAddress(const Network::Address::Instance& address) const override { + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.explicit_internal_address_config")) { + return false; + } return Network::Utility::isInternalAddress(address); } }; diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index ac6d760161..0464f76248 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -152,6 +152,8 @@ FALSE_RUNTIME_GUARD(envoy_restart_features_xds_failover_support); FALSE_RUNTIME_GUARD(envoy_reloadable_features_dns_cache_set_ip_version_to_remove); // TODO(alyssawilk): evaluate and make this a config knob or remove. FALSE_RUNTIME_GUARD(envoy_reloadable_features_reset_brokenness_on_nework_change); +// TODO(botengyao): this will be default true in the next release after this warning release. +FALSE_RUNTIME_GUARD(envoy_reloadable_features_explicit_internal_address_config); // A flag to set the maximum TLS version for google_grpc client to TLS1.2, when needed for // compliance restrictions. diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index a8a036a70a..563ece44c9 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -77,6 +77,10 @@ std::unique_ptr createInternalAddressConfig( return std::make_unique(config.internal_address_config(), creation_status); } + ENVOY_LOG_ONCE_MISC(warn, + "internal_address_config is not configured. The existing default behaviour " + "will trust RFC1918 IP addresses, but this will be changed in next release. " + "Please explictily config internal address config as the migration step."); return std::make_unique(); } diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 23964105ce..5f39f9935a 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -708,6 +708,47 @@ TEST_F(HttpConnectionManagerConfigTest, UnixSocketInternalAddress) { EXPECT_FALSE(config.internalAddressConfig().isInternalAddress(externalIpAddress)); } +TEST_F(HttpConnectionManagerConfigTest, FutureDefaultInternalAddress) { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues( + {{"envoy.reloadable_features.explicit_internal_address_config", "true"}}); + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + http_filters: + - name: envoy.filters.http.router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + &scoped_routes_config_provider_manager_, tracer_manager_, + filter_config_provider_manager_, creation_status_); + ASSERT_TRUE(creation_status_.ok()); + // Envoy no longer considers RFC1918 IP addresses to be internal if runtime guard is enabled. + Network::Address::Ipv4Instance default_ip_address{"10.48.179.130", 0, nullptr}; + EXPECT_FALSE(config.internalAddressConfig().isInternalAddress(default_ip_address)); +} + +TEST_F(HttpConnectionManagerConfigTest, DefaultInternalAddress) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + http_filters: + - name: envoy.filters.http.router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromYaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + &scoped_routes_config_provider_manager_, tracer_manager_, + filter_config_provider_manager_, creation_status_); + ASSERT_TRUE(creation_status_.ok()); + // Previously, Envoy considered RFC1918 IP addresses to be internal, by default. + Network::Address::Ipv4Instance default_ip_address{"10.48.179.130", 0, nullptr}; + EXPECT_TRUE(config.internalAddressConfig().isInternalAddress(default_ip_address)); +} + TEST_F(HttpConnectionManagerConfigTest, CidrRangeBasedInternalAddress) { const std::string yaml_string = R"EOF( stat_prefix: ingress_http diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 47738157c3..d8bc506c18 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -341,6 +341,21 @@ HttpIntegrationTest::HttpIntegrationTest(Http::CodecType downstream_protocol, // Allow extension lookup by name in the integration tests. config_helper_.addRuntimeOverride("envoy.reloadable_features.no_extension_lookup_by_name", "false"); + + config_helper_.addConfigModifier( + [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { + auto* range = hcm.mutable_internal_address_config()->add_cidr_ranges(); + // Set loopback to be trusted so tests can set x-envoy headers. + range->set_address_prefix("127.0.0.1"); + range->mutable_prefix_len()->set_value(32); + // Legacy tests also set XFF: 10.0.0.1 + range->set_address_prefix("10.0.0.0"); + range->mutable_prefix_len()->set_value(8); + range = hcm.mutable_internal_address_config()->add_cidr_ranges(); + range->set_address_prefix("::1"); + range->mutable_prefix_len()->set_value(128); + }); } void HttpIntegrationTest::useAccessLog( diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 7dcf37ca99..db48b01f42 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -695,8 +695,15 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_METHOD(bool, appendXForwardedPort, (), (const)); MOCK_METHOD(bool, addProxyProtocolConnectionState, (), (const)); + class AllowInternalAddressConfig : public Http::InternalAddressConfig { + public: + bool isInternalAddress(const Network::Address::Instance& address) const override { + return Network::Utility::isInternalAddress(address); + } + }; + std::unique_ptr internal_address_config_ = - std::make_unique(); + std::make_unique(); std::vector early_header_mutation_extensions_; absl::optional scheme_; bool scheme_match_upstream_; From 34d8d36b60b9efe7cc907c3468636f46602e556c Mon Sep 17 00:00:00 2001 From: Boteng Yao Date: Wed, 24 Jul 2024 21:48:19 +0000 Subject: [PATCH 26/28] fix local reply in async client and destroy order Signed-off-by: Boteng Yao Signed-off-by: Ryan Northey --- changelogs/current.yaml | 3 + source/common/http/async_client_impl.cc | 45 +++++++++- source/common/http/async_client_impl.h | 28 ++---- .../ext_authz/ext_authz_integration_test.cc | 86 +++++++++++++++++++ 4 files changed, 138 insertions(+), 24 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 6d520b8b17..7f151c9113 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -156,6 +156,9 @@ bug_fixes: Fixed a bug where using ``clear_route_cache`` with remote JWKs works incorrectly and may cause a crash when the modified request does not match any route. +- area: http_async_client + change: | + Fixed the local reply and destroy order crashes when using the http async client for websocket handshake. removed_config_or_runtime: # *Normally occurs at the end of the* :ref:`deprecation period ` diff --git a/source/common/http/async_client_impl.cc b/source/common/http/async_client_impl.cc index a3b968c747..8fe0f75161 100644 --- a/source/common/http/async_client_impl.cc +++ b/source/common/http/async_client_impl.cc @@ -9,6 +9,7 @@ #include "source/common/grpc/common.h" #include "source/common/http/null_route_impl.h" #include "source/common/http/utility.h" +#include "source/common/local_reply/local_reply.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/stream_info/filter_state_impl.h" #include "source/common/tracing/http_tracer_impl.h" @@ -32,7 +33,8 @@ AsyncClientImpl::AsyncClientImpl(Upstream::ClusterInfoConstSharedPtr cluster, factory_context.api().randomGenerator(), std::move(shadow_writer), true, false, false, false, false, false, Protobuf::RepeatedPtrField{}, dispatcher.timeSource(), http_context, router_context)), - dispatcher_(dispatcher), runtime_(factory_context.runtime()) {} + dispatcher_(dispatcher), runtime_(factory_context.runtime()), + local_reply_(LocalReply::Factory::createDefault()) {} AsyncClientImpl::~AsyncClientImpl() { while (!active_streams_.empty()) { @@ -120,7 +122,7 @@ AsyncStreamImpl::AsyncStreamImpl(AsyncClientImpl& parent, AsyncClient::StreamCal ? options.filter_state : std::make_shared( StreamInfo::FilterState::LifeSpan::FilterChain)), - tracing_config_(Tracing::EgressConfig::get()), + tracing_config_(Tracing::EgressConfig::get()), local_reply_(*parent.local_reply_), retry_policy_(createRetryPolicy(parent, options, parent_.factory_context_, creation_status)), route_(std::make_shared( parent_.cluster_->name(), @@ -145,6 +147,35 @@ AsyncStreamImpl::AsyncStreamImpl(AsyncClientImpl& parent, AsyncClient::StreamCal // TODO(mattklein123): Correctly set protocol in stream info when we support access logging. } +void AsyncStreamImpl::sendLocalReply(Code code, absl::string_view body, + std::function modify_headers, + const absl::optional grpc_status, + absl::string_view details) { + if (encoded_response_headers_) { + resetStream(); + return; + } + Utility::sendLocalReply( + remote_closed_, + Utility::EncodeFunctions{ + [modify_headers](ResponseHeaderMap& headers) -> void { + if (modify_headers != nullptr) { + modify_headers(headers); + } + }, + [this](ResponseHeaderMap& response_headers, Code& code, std::string& body, + absl::string_view& content_type) -> void { + local_reply_.rewrite(request_headers_, response_headers, stream_info_, code, body, + content_type); + }, + [this, &details](ResponseHeaderMapPtr&& headers, bool end_stream) -> void { + encodeHeaders(std::move(headers), end_stream, details); + }, + [this](Buffer::Instance& data, bool end_stream) -> void { + encodeData(data, end_stream); + }}, + Utility::LocalReplyData{is_grpc_request_, code, body, grpc_status, is_head_request_}); +} void AsyncStreamImpl::encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, absl::string_view) { ENVOY_LOG(debug, "async http request response headers (end_stream={}):\n{}", end_stream, @@ -288,16 +319,24 @@ void AsyncStreamImpl::closeRemote(bool end_stream) { } void AsyncStreamImpl::reset() { - router_.onDestroy(); + routerDestroy(); resetStream(); } +void AsyncStreamImpl::routerDestroy() { + if (!router_destroyed_) { + router_destroyed_ = true; + router_.onDestroy(); + } +} + void AsyncStreamImpl::cleanup() { ASSERT(dispatcher().isThreadSafe()); local_closed_ = remote_closed_ = true; // This will destroy us, but only do so if we are actually in a list. This does not happen in // the immediate failure case. if (inserted()) { + routerDestroy(); dispatcher().deferredDelete(removeFromList(parent_.active_streams_)); } } diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index e0dc6562b3..2805cb6aeb 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -38,6 +38,7 @@ #include "source/common/common/linked_object.h" #include "source/common/http/message_impl.h" #include "source/common/http/null_route_impl.h" +#include "source/common/local_reply/local_reply.h" #include "source/common/router/config_impl.h" #include "source/common/router/router.h" #include "source/common/stream_info/stream_info_impl.h" @@ -84,6 +85,7 @@ class AsyncClientImpl final : public AsyncClient { Event::Dispatcher& dispatcher_; std::list> active_streams_; Runtime::Loader& runtime_; + const LocalReply::LocalReplyPtr local_reply_; friend class AsyncStreamImpl; friend class AsyncRequestSharedImpl; @@ -109,7 +111,7 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, } ~AsyncStreamImpl() override { - router_.onDestroy(); + routerDestroy(); // UpstreamRequest::cleanUp() is guaranteed to reset the high watermark calls. ENVOY_BUG(high_watermark_calls_ == 0, "Excess high watermark calls after async stream ended."); if (destructor_callback_.has_value()) { @@ -171,6 +173,7 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, void cleanup(); void closeRemote(bool end_stream); bool complete() { return local_closed_ && remote_closed_; } + void routerDestroy(); // Http::StreamDecoderFilterCallbacks OptRef connection() override { return {}; } @@ -201,26 +204,7 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, void sendLocalReply(Code code, absl::string_view body, std::function modify_headers, const absl::optional grpc_status, - absl::string_view details) override { - if (encoded_response_headers_) { - resetStream(); - return; - } - Utility::sendLocalReply( - remote_closed_, - Utility::EncodeFunctions{nullptr, nullptr, - [this, modify_headers, &details](ResponseHeaderMapPtr&& headers, - bool end_stream) -> void { - if (modify_headers != nullptr) { - modify_headers(*headers); - } - encodeHeaders(std::move(headers), end_stream, details); - }, - [this](Buffer::Instance& data, bool end_stream) -> void { - encodeData(data, end_stream); - }}, - Utility::LocalReplyData{is_grpc_request_, code, body, grpc_status, is_head_request_}); - } + absl::string_view details) override; // The async client won't pause if sending 1xx headers so simply swallow any. void encode1xxHeaders(ResponseHeaderMapPtr&&) override {} void encodeHeaders(ResponseHeaderMapPtr&& headers, bool end_stream, @@ -290,6 +274,7 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, StreamInfo::StreamInfoImpl stream_info_; Tracing::NullSpan active_span_; const Tracing::Config& tracing_config_; + const LocalReply::LocalReply& local_reply_; const std::unique_ptr retry_policy_; std::shared_ptr route_; uint32_t high_watermark_calls_{}; @@ -305,6 +290,7 @@ class AsyncStreamImpl : public virtual AsyncClient::Stream, bool is_head_request_{false}; bool send_xff_{true}; bool send_internal_{true}; + bool router_destroyed_{false}; friend class AsyncClientImpl; friend class AsyncClientImplUnitTest; diff --git a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc index 0586b80e7a..2cc89af51b 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_integration_test.cc @@ -1619,6 +1619,92 @@ TEST_P(ExtAuthzLocalReplyIntegrationTest, DeniedHeaderTest) { cleanup(); } +// This will trigger the http async client sendLocalReply since the websocket upgrade failed. +// Verify that there is no response code duplication and crash in the async stream destructor. +TEST_P(ExtAuthzLocalReplyIntegrationTest, AsyncClientSendLocalReply) { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* ext_authz_cluster = bootstrap.mutable_static_resources()->add_clusters(); + ext_authz_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + ext_authz_cluster->set_name("ext_authz"); + + envoy::extensions::filters::http::ext_authz::v3::ExtAuthz proto_config; + // Explicitly allow upgrade and connection header. + const std::string ext_authz_config = R"EOF( + http_service: + server_uri: + uri: "ext_authz:9000" + cluster: "ext_authz" + timeout: 300s + allowed_headers: + patterns: + - exact: upgrade + - exact: connection + )EOF"; + TestUtility::loadFromYaml(ext_authz_config, proto_config); + + envoy::config::listener::v3::Filter ext_authz_filter; + ext_authz_filter.set_name("envoy.filters.http.ext_authz"); + ext_authz_filter.mutable_typed_config()->PackFrom(proto_config); + config_helper_.prependFilter(MessageUtil::getJsonStringFromMessageOrError(ext_authz_filter)); + }); + + config_helper_.addConfigModifier( + [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { hcm.add_upgrade_configs()->set_upgrade_type("websocket"); }); + + const std::string local_reply_yaml = R"EOF( +body_format: + json_format: + code: "%RESPONSE_CODE%" + message: "%LOCAL_REPLY_BODY%" + )EOF"; + envoy::extensions::filters::network::http_connection_manager::v3::LocalReplyConfig + local_reply_config; + TestUtility::loadFromYaml(local_reply_yaml, local_reply_config); + config_helper_.setLocalReply(local_reply_config); + + HttpIntegrationTest::initialize(); + + auto conn = makeClientConnection(lookupPort("http")); + codec_client_ = makeHttpConnection(std::move(conn)); + auto response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"upgrade", "websocket"}, + {"connection", "Upgrade"}}); + + AssertionResult result = + fake_upstreams_.back()->waitForHttpConnection(*dispatcher_, fake_ext_authz_connection_); + RELEASE_ASSERT(result, result.message()); + FakeStreamPtr ext_authz_request; + result = fake_ext_authz_connection_->waitForNewStream(*dispatcher_, ext_authz_request); + RELEASE_ASSERT(result, result.message()); + + // This will fail the websocket upgrade. + Http::TestResponseHeaderMapImpl ext_authz_response_headers{ + {":status", "401"}, + {"content-type", "fake-type"}, + }; + ext_authz_request->encodeHeaders(ext_authz_response_headers, true); + + ASSERT_TRUE(response->waitForEndStream()); + EXPECT_TRUE(response->complete()); + + EXPECT_EQ("401", response->headers().Status()->value().getStringView()); + EXPECT_EQ("application/json", response->headers().ContentType()->value().getStringView()); + EXPECT_EQ("26", response->headers().ContentLength()->value().getStringView()); + + const std::string expected_body = R"({ + "code": 401, + "message": "" +})"; + EXPECT_TRUE(TestUtility::jsonStringEqual(response->body(), expected_body)); + + cleanup(); +} + TEST_P(ExtAuthzGrpcIntegrationTest, GoogleAsyncClientCreation) { initializeConfig(); setDownstreamProtocol(Http::CodecType::HTTP2); From 285a4084236bc6112079bfc1e23ee8186db9a545 Mon Sep 17 00:00:00 2001 From: Boteng Yao Date: Mon, 19 Aug 2024 18:27:48 +0000 Subject: [PATCH 27/28] flip oghttp flag Signed-off-by: Boteng Yao Signed-off-by: Ryan Northey --- changelogs/current.yaml | 4 ++++ source/common/runtime/runtime_features.cc | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 7f151c9113..1b7708944c 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -94,6 +94,10 @@ minor_behavior_changes: change: | Enhanced listener filter chain execution to include the case that listener filter has maxReadBytes() of 0, but may return StopIteration in onAccept to wait for asynchronous callback. +- area: http2 + change: | + Changes the default value of ``envoy.reloadable_features.http2_use_oghttp2`` to ``false``. This changes the codec used for HTTP/2 + requests and responses to address to address stability concerns. This behavior can be reverted by setting the feature to ``true``. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 0464f76248..28517bea8e 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -56,7 +56,6 @@ RUNTIME_GUARD(envoy_reloadable_features_http1_balsa_disallow_lone_cr_in_chunk_ex RUNTIME_GUARD(envoy_reloadable_features_http1_use_balsa_parser); RUNTIME_GUARD(envoy_reloadable_features_http2_discard_host_header); // Ignore the automated "remove this flag" issue: we should keep this for 1 year. -RUNTIME_GUARD(envoy_reloadable_features_http2_use_oghttp2); RUNTIME_GUARD(envoy_reloadable_features_http2_use_visitor_for_data); RUNTIME_GUARD(envoy_reloadable_features_http3_happy_eyeballs); RUNTIME_GUARD(envoy_reloadable_features_http3_remove_empty_trailers); @@ -123,6 +122,8 @@ FALSE_RUNTIME_GUARD(envoy_reloadable_features_test_feature_false); FALSE_RUNTIME_GUARD(envoy_reloadable_features_streaming_shadow); // TODO(adisuissa) reset to true to enable unified mux by default FALSE_RUNTIME_GUARD(envoy_reloadable_features_unified_mux); +// TODO(birenroy) flip after the security issue is addressed. +FALSE_RUNTIME_GUARD(envoy_reloadable_features_http2_use_oghttp2); // Used to track if runtime is initialized. FALSE_RUNTIME_GUARD(envoy_reloadable_features_runtime_initialized); // TODO(mattklein123): Flip this to true and/or remove completely once verified by Envoy Mobile. From 30c7d6cda5d4afafeeecb0ccb85b91c7c86b010c Mon Sep 17 00:00:00 2001 From: tyxia Date: Mon, 19 Aug 2024 00:52:55 +0000 Subject: [PATCH 28/28] escape invalid host name Signed-off-by: tyxia Signed-off-by: Boteng Yao Signed-off-by: Ryan Northey --- changelogs/current.yaml | 4 ++ source/common/common/utility.cc | 17 ++++++ source/common/common/utility.h | 9 +++ .../common/formatter/stream_info_formatter.cc | 10 ++- source/common/runtime/runtime_features.cc | 1 + .../formatter/substitution_formatter_test.cc | 61 +++++++++++++++++-- 6 files changed, 94 insertions(+), 8 deletions(-) diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 1b7708944c..5fd9945a3a 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -98,6 +98,10 @@ minor_behavior_changes: change: | Changes the default value of ``envoy.reloadable_features.http2_use_oghttp2`` to ``false``. This changes the codec used for HTTP/2 requests and responses to address to address stability concerns. This behavior can be reverted by setting the feature to ``true``. +- area: access_log + change: | + Sanitize SNI for potential log injection. The invalid character will be replaced by ``_`` with an ``invalid:`` marker. If runtime + flag ``envoy.reloadable_features.sanitize_sni_in_access_log`` is set to ``false``, the sanitize behavior is disabled. bug_fixes: # *Changes expected to improve the state of the world and are unlikely to have negative effects* diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc index 91dcd9d0f2..ab79b2cf56 100644 --- a/source/common/common/utility.cc +++ b/source/common/common/utility.cc @@ -586,6 +586,23 @@ void StringUtil::escapeToOstream(std::ostream& os, absl::string_view view) { } } +std::string StringUtil::sanitizeInvalidHostname(const absl::string_view source) { + std::string ret_str = std::string(source); + bool sanitized = false; + for (size_t i = 0; i < ret_str.size(); ++i) { + if (absl::ascii_isalnum(ret_str[i]) || ret_str[i] == '.' || ret_str[i] == '-') { + continue; + } + sanitized = true; + ret_str[i] = '_'; + } + + if (sanitized) { + ret_str = absl::StrCat("invalid:", ret_str); + } + return ret_str; +} + const std::string& getDefaultDateFormat(bool local_time) { if (local_time) { CONSTRUCT_ON_FIRST_USE(std::string, "%Y-%m-%dT%H:%M:%E3S%z"); diff --git a/source/common/common/utility.h b/source/common/common/utility.h index 47d1082da1..06a404fbd6 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -490,6 +490,15 @@ class StringUtil { */ static void escapeToOstream(std::ostream& os, absl::string_view view); + /** + * Sanitize host name strings for logging purposes. Replace invalid hostname characters (anything + * that's not alphanumeric, hyphen, or period) with underscore. The sanitized string is not a + * valid host name. + * @param source supplies the string to sanitize. + * @return sanitized string. + */ + static std::string sanitizeInvalidHostname(const absl::string_view source); + /** * Provide a default value for a string if empty. * @param s string. diff --git a/source/common/formatter/stream_info_formatter.cc b/source/common/formatter/stream_info_formatter.cc index 50681ffcda..c7f532db00 100644 --- a/source/common/formatter/stream_info_formatter.cc +++ b/source/common/formatter/stream_info_formatter.cc @@ -1353,8 +1353,14 @@ const StreamInfoFormatterProviderLookupTable& getKnownStreamInfoFormatterProvide [](const StreamInfo::StreamInfo& stream_info) { absl::optional result; if (!stream_info.downstreamAddressProvider().requestedServerName().empty()) { - result = std::string( - stream_info.downstreamAddressProvider().requestedServerName()); + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.sanitize_sni_in_access_log")) { + result = StringUtil::sanitizeInvalidHostname( + stream_info.downstreamAddressProvider().requestedServerName()); + } else { + result = std::string( + stream_info.downstreamAddressProvider().requestedServerName()); + } } return result; }); diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 28517bea8e..650205fcc8 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -86,6 +86,7 @@ RUNTIME_GUARD(envoy_reloadable_features_quic_upstream_socket_use_address_cache_f RUNTIME_GUARD(envoy_reloadable_features_reject_invalid_yaml); RUNTIME_GUARD(envoy_reloadable_features_report_stream_reset_error_code); RUNTIME_GUARD(envoy_reloadable_features_sanitize_http2_headers_without_nghttp2); +RUNTIME_GUARD(envoy_reloadable_features_sanitize_sni_in_access_log); RUNTIME_GUARD(envoy_reloadable_features_sanitize_te); RUNTIME_GUARD(envoy_reloadable_features_send_local_reply_when_no_buffer_and_upstream_request); RUNTIME_GUARD(envoy_reloadable_features_skip_dns_lookup_for_proxied_requests); diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index a87f2f447e..d7e1697ef5 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -973,20 +973,69 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("REQUESTED_SERVER_NAME"); - std::string requested_server_name = "stub_server"; + std::string requested_server_name; stream_info.downstream_connection_info_provider_->setRequestedServerName(requested_server_name); - EXPECT_EQ("stub_server", upstream_format.formatWithContext({}, stream_info)); + EXPECT_EQ(absl::nullopt, upstream_format.formatWithContext({}, stream_info)); EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), - ProtoEq(ValueUtil::stringValue("stub_server"))); + ProtoEq(ValueUtil::nullValue())); } { StreamInfoFormatter upstream_format("REQUESTED_SERVER_NAME"); - std::string requested_server_name; + std::string requested_server_name = "stub-server"; stream_info.downstream_connection_info_provider_->setRequestedServerName(requested_server_name); - EXPECT_EQ(absl::nullopt, upstream_format.formatWithContext({}, stream_info)); + EXPECT_EQ("stub-server", upstream_format.formatWithContext({}, stream_info)); EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), - ProtoEq(ValueUtil::nullValue())); + ProtoEq(ValueUtil::stringValue("stub-server"))); + } + + { + StreamInfoFormatter upstream_format("REQUESTED_SERVER_NAME"); + std::string requested_server_name = "stub_server\n"; + stream_info.downstream_connection_info_provider_->setRequestedServerName(requested_server_name); + EXPECT_EQ("invalid:stub_server_", upstream_format.formatWithContext({}, stream_info)); + EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), + ProtoEq(ValueUtil::stringValue("invalid:stub_server_"))); + } + + { + StreamInfoFormatter upstream_format("REQUESTED_SERVER_NAME"); + std::string requested_server_name = "\e[0;34m\n$(echo -e $blue)end"; + stream_info.downstream_connection_info_provider_->setRequestedServerName(requested_server_name); + EXPECT_EQ("invalid:__0_34m___echo_-e__blue_end_script_alert____script_", + upstream_format.formatWithContext({}, stream_info)); + EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), + ProtoEq(ValueUtil::stringValue( + "invalid:__0_34m___echo_-e__blue_end_script_alert____script_"))); + } + + { + StreamInfoFormatter upstream_format("REQUESTED_SERVER_NAME"); + std::string invalid_utf8_string("prefix"); + invalid_utf8_string.append(1, char(0xc3)); + invalid_utf8_string.append(1, char(0xc7)); + invalid_utf8_string.append("valid_middle"); + invalid_utf8_string.append(1, char(0xc4)); + invalid_utf8_string.append("valid_suffix"); + stream_info.downstream_connection_info_provider_->setRequestedServerName(invalid_utf8_string); + EXPECT_EQ("invalid:prefix__valid_middle_valid_suffix", + upstream_format.formatWithContext({}, stream_info)); + EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), + ProtoEq(ValueUtil::stringValue("invalid:prefix__valid_middle_valid_suffix"))); + } + + { + TestScopedRuntime scoped_runtime; + scoped_runtime.mergeValues({ + {"envoy.reloadable_features.sanitize_sni_in_access_log", "false"}, + }); + + StreamInfoFormatter upstream_format("REQUESTED_SERVER_NAME"); + std::string requested_server_name = "stub_server\n"; + stream_info.downstream_connection_info_provider_->setRequestedServerName(requested_server_name); + EXPECT_EQ("stub_server\n", upstream_format.formatWithContext({}, stream_info)); + EXPECT_THAT(upstream_format.formatValueWithContext({}, stream_info), + ProtoEq(ValueUtil::stringValue("stub_server\n"))); } {