diff --git a/api/envoy/extensions/geoip_providers/maxmind/v3/maxmind.proto b/api/envoy/extensions/geoip_providers/maxmind/v3/maxmind.proto index c1d7a2480ebd1..91e00c1dded74 100644 --- a/api/envoy/extensions/geoip_providers/maxmind/v3/maxmind.proto +++ b/api/envoy/extensions/geoip_providers/maxmind/v3/maxmind.proto @@ -25,9 +25,10 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // * :ref:`isp_db_path ` // * :ref:`asn_db_path ` // * :ref:`anon_db_path ` +// * :ref:`country_db_path ` // [#extension: envoy.geoip_providers.maxmind] -// [#next-free-field: 6] +// [#next-free-field: 7] message MaxMindConfig { // Full file path to the MaxMind city database, e.g., ``/etc/GeoLite2-City.mmdb``. // Database file is expected to have ``.mmdb`` extension. @@ -48,6 +49,13 @@ message MaxMindConfig { // ``isp_db`` instead. string isp_db_path = 5 [(validate.rules).string = {pattern: "^$|^.*\\.mmdb$"}]; + // Full file path to the MaxMind Country database, e.g., ``/etc/GeoLite2-Country.mmdb``. + // Database file is expected to have ``.mmdb`` extension. + // + // If ``country_db_path`` is not specified, country information will be fetched from + // ``city_db`` if ``city_db`` is configured. + string country_db_path = 6 [(validate.rules).string = {pattern: "^$|^.*\\.mmdb$"}]; + // Common provider configuration that specifies which geolocation headers will be populated with geolocation data. common.v3.CommonGeoipProviderConfig common_provider_config = 4 [(validate.rules).message = {required: true}]; diff --git a/changelogs/current.yaml b/changelogs/current.yaml index 743b26b2c235d..9826d0482ced4 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -413,6 +413,10 @@ new_features: change: | Propagated transport errors from tls_inspector to ``DownstreamTransportFailureReason`` in ``StreamInfo`` for access logging prior to the TLS handshake. +- area: geoip + change: | + Added support for MaxMind Country database via + :ref:`country_db_path `. - area: tls_inspector change: | Added configuration parameter to TLS inspector for maximum acceptable client hello size. diff --git a/source/extensions/geoip_providers/maxmind/geoip_provider.cc b/source/extensions/geoip_providers/maxmind/geoip_provider.cc index 1e5fea39bda30..3d7a6c870c2dd 100644 --- a/source/extensions/geoip_providers/maxmind/geoip_provider.cc +++ b/source/extensions/geoip_providers/maxmind/geoip_provider.cc @@ -23,6 +23,7 @@ static constexpr absl::string_view CITY_DB_TYPE = "city_db"; static constexpr absl::string_view ISP_DB_TYPE = "isp_db"; static constexpr absl::string_view ANON_DB_TYPE = "anon_db"; static constexpr absl::string_view ASN_DB_TYPE = "asn_db"; +static constexpr absl::string_view COUNTRY_DB_TYPE = "country_db"; } // namespace GeoipProviderConfig::GeoipProviderConfig( @@ -36,6 +37,9 @@ GeoipProviderConfig::GeoipProviderConfig( : absl::nullopt), asn_db_path_(!config.asn_db_path().empty() ? absl::make_optional(config.asn_db_path()) : absl::nullopt), + country_db_path_(!config.country_db_path().empty() + ? absl::make_optional(config.country_db_path()) + : absl::nullopt), stats_scope_(scope.createScope(absl::StrCat(stat_prefix, "maxmind."))), stat_name_set_(stats_scope_->symbolTable().makeSet("Maxmind")) { auto geo_headers_to_add = config.common_provider_config().geo_headers_to_add(); @@ -75,9 +79,9 @@ GeoipProviderConfig::GeoipProviderConfig( apple_private_relay_header_ = !geo_headers_to_add.apple_private_relay().empty() ? absl::make_optional(geo_headers_to_add.apple_private_relay()) : absl::nullopt; - if (!city_db_path_ && !anon_db_path_ && !asn_db_path_ && !isp_db_path_) { + if (!city_db_path_ && !anon_db_path_ && !asn_db_path_ && !isp_db_path_ && !country_db_path_) { throw EnvoyException("At least one geolocation database path needs to be configured: " - "city_db_path, isp_db_path, asn_db_path or anon_db_path"); + "city_db_path, isp_db_path, asn_db_path, anon_db_path or country_db_path"); } if (city_db_path_) { registerGeoDbStats(CITY_DB_TYPE); @@ -91,6 +95,9 @@ GeoipProviderConfig::GeoipProviderConfig( if (asn_db_path_) { registerGeoDbStats(ASN_DB_TYPE); } + if (country_db_path_) { + registerGeoDbStats(COUNTRY_DB_TYPE); + } }; void GeoipProviderConfig::registerGeoDbStats(const absl::string_view& db_type) { @@ -126,6 +133,9 @@ GeoipProvider::GeoipProvider(Event::Dispatcher& dispatcher, Api::Api& api, config_->anonDbPath() ? initMaxmindDb(config_->anonDbPath().value(), ANON_DB_TYPE) : nullptr; asn_db_ = config_->asnDbPath() ? initMaxmindDb(config_->asnDbPath().value(), ASN_DB_TYPE) : nullptr; + country_db_ = config_->countryDbPath() + ? initMaxmindDb(config_->countryDbPath().value(), COUNTRY_DB_TYPE) + : nullptr; mmdb_reload_dispatcher_ = api.allocateDispatcher("mmdb_reload_routine"); mmdb_watcher_ = dispatcher.createFilesystemWatcher(); mmdb_reload_thread_ = api.threadFactory().createThread( @@ -157,6 +167,13 @@ GeoipProvider::GeoipProvider(Event::Dispatcher& dispatcher, Api::Api& api, return onMaxmindDbUpdate(config_->asnDbPath().value(), ASN_DB_TYPE); })); } + if (config_->countryDbPath()) { + THROW_IF_NOT_OK(mmdb_watcher_->addWatch( + config_->countryDbPath().value(), Filesystem::Watcher::Events::MovedTo, + [this](uint32_t) { + return onMaxmindDbUpdate(config_->countryDbPath().value(), COUNTRY_DB_TYPE); + })); + } mmdb_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); }, Thread::Options{std::string("mmdb_reload_routine")}); @@ -177,6 +194,7 @@ void GeoipProvider::lookup(Geolocation::LookupRequest&& request, Geolocation::LookupGeoHeadersCallback&& cb) const { auto& remote_address = request.remoteAddress(); auto lookup_result = absl::flat_hash_map{}; + lookupInCountryDb(remote_address, lookup_result); lookupInCityDb(remote_address, lookup_result); lookupInAsnDb(remote_address, lookup_result); lookupInAnonDb(remote_address, lookup_result); @@ -187,9 +205,12 @@ void GeoipProvider::lookup(Geolocation::LookupRequest&& request, void GeoipProvider::lookupInCityDb( const Network::Address::InstanceConstSharedPtr& remote_address, absl::flat_hash_map& lookup_result) const { + // Country lookup falls back to City DB only if Country DB is not configured. + const bool should_lookup_country_from_city_db = + !config_->isCountryDbPathSet() && config_->isLookupEnabledForHeader(config_->countryHeader()); if (config_->isLookupEnabledForHeader(config_->cityHeader()) || config_->isLookupEnabledForHeader(config_->regionHeader()) || - config_->isLookupEnabledForHeader(config_->countryHeader())) { + should_lookup_country_from_city_db) { int mmdb_error; auto city_db_ptr = getCityDb(); // Used for testing. @@ -217,7 +238,8 @@ void GeoipProvider::lookupInCityDb( config_->regionHeader().value(), MMDB_REGION_LOOKUP_ARGS[0], MMDB_REGION_LOOKUP_ARGS[1], MMDB_REGION_LOOKUP_ARGS[2]); } - if (config_->isLookupEnabledForHeader(config_->countryHeader())) { + // Country lookup from City DB only when Country DB is not configured. + if (should_lookup_country_from_city_db) { populateGeoLookupResult(mmdb_lookup_result, lookup_result, config_->countryHeader().value(), MMDB_COUNTRY_LOOKUP_ARGS[0], MMDB_COUNTRY_LOOKUP_ARGS[1]); @@ -384,6 +406,51 @@ void GeoipProvider::lookupInIspDb( } } +void GeoipProvider::lookupInCountryDb( + const Network::Address::InstanceConstSharedPtr& remote_address, + absl::flat_hash_map& lookup_result) const { + if (config_->isLookupEnabledForHeader(config_->countryHeader())) { + // Country DB takes precedence if configured, otherwise fall back to City DB. + if (!config_->isCountryDbPathSet()) { + // Country lookup will be handled by lookupInCityDb. + return; + } + int mmdb_error; + auto country_db_ptr = getCountryDb(); + // Used for testing. + synchronizer_.syncPoint(std::string(COUNTRY_DB_TYPE).append("_lookup_pre_complete")); + if (!country_db_ptr) { + if (config_->isCityDbPathSet()) { + // Country information can be looked up from City database as well, so we don't need to + // throw an error if it is not set. + return; + } + IS_ENVOY_BUG("Maxmind country database must be initialised for performing lookups"); + return; + } + auto country_db = country_db_ptr.get(); + MMDB_lookup_result_s mmdb_lookup_result = MMDB_lookup_sockaddr( + country_db->mmdb(), reinterpret_cast(remote_address->sockAddr()), + &mmdb_error); + const uint32_t n_prev_hits = lookup_result.size(); + if (!mmdb_error && mmdb_lookup_result.found_entry) { + MMDB_entry_data_list_s* entry_data_list; + int status = MMDB_get_entry_data_list(&mmdb_lookup_result.entry, &entry_data_list); + if (status == MMDB_SUCCESS) { + populateGeoLookupResult(mmdb_lookup_result, lookup_result, config_->countryHeader().value(), + MMDB_COUNTRY_LOOKUP_ARGS[0], MMDB_COUNTRY_LOOKUP_ARGS[1]); + if (lookup_result.size() > n_prev_hits) { + config_->incHit(COUNTRY_DB_TYPE); + } + MMDB_free_entry_data_list(entry_data_list); + } else { + config_->incLookupError(COUNTRY_DB_TYPE); + } + } + config_->incTotal(COUNTRY_DB_TYPE); + } +} + MaxmindDbSharedPtr GeoipProvider::initMaxmindDb(const std::string& db_path, const absl::string_view& db_type, bool reload) { MMDB_s maxmind_db; @@ -398,7 +465,6 @@ MaxmindDbSharedPtr GeoipProvider::initMaxmindDb(const std::string& db_path, RELEASE_ASSERT(MMDB_SUCCESS == result_code, fmt::format("Unable to open Maxmind database file {}. Error {}", db_path, std::string(MMDB_strerror(result_code)))); - return nullptr; } config_->setDbBuildEpoch(db_type, maxmind_db.metadata.build_epoch); @@ -422,6 +488,9 @@ absl::Status GeoipProvider::mmdbReload(const MaxmindDbSharedPtr reloaded_db, } else if (db_type == ASN_DB_TYPE) { updateAsnDb(reloaded_db); config_->incDbReloadSuccess(db_type); + } else if (db_type == COUNTRY_DB_TYPE) { + updateCountryDb(reloaded_db); + config_->incDbReloadSuccess(db_type); } else { ENVOY_LOG(error, "Unsupported maxmind db type {}", db_type); return absl::InvalidArgumentError(fmt::format("Unsupported maxmind db type {}", db_type)); @@ -472,6 +541,17 @@ void GeoipProvider::updateAnonDb(MaxmindDbSharedPtr anon_db) ABSL_LOCKS_EXCLUDED anon_db_ = anon_db; } +MaxmindDbSharedPtr GeoipProvider::getCountryDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_) { + absl::ReaderMutexLock lock(&mmdb_mutex_); + return country_db_; +} + +void GeoipProvider::updateCountryDb(MaxmindDbSharedPtr country_db) + ABSL_LOCKS_EXCLUDED(mmdb_mutex_) { + absl::MutexLock lock(&mmdb_mutex_); + country_db_ = country_db; +} + absl::Status GeoipProvider::onMaxmindDbUpdate(const std::string& db_path, const absl::string_view& db_type) { MaxmindDbSharedPtr reloaded_db = initMaxmindDb(db_path, db_type, true /* reload */); diff --git a/source/extensions/geoip_providers/maxmind/geoip_provider.h b/source/extensions/geoip_providers/maxmind/geoip_provider.h index 91731fdf4407a..2483e1a768b64 100644 --- a/source/extensions/geoip_providers/maxmind/geoip_provider.h +++ b/source/extensions/geoip_providers/maxmind/geoip_provider.h @@ -23,10 +23,13 @@ class GeoipProviderConfig { const absl::optional& ispDbPath() const { return isp_db_path_; } const absl::optional& anonDbPath() const { return anon_db_path_; } const absl::optional& asnDbPath() const { return asn_db_path_; } + const absl::optional& countryDbPath() const { return country_db_path_; } bool isLookupEnabledForHeader(const absl::optional& header); bool isAsnDbPathSet() const { return asn_db_path_.has_value(); } bool isIspDbPathSet() const { return isp_db_path_.has_value(); } + bool isCountryDbPathSet() const { return country_db_path_.has_value(); } + bool isCityDbPathSet() const { return city_db_path_.has_value(); } const absl::optional& countryHeader() const { return country_header_; } const absl::optional& cityHeader() const { return city_header_; } @@ -82,6 +85,7 @@ class GeoipProviderConfig { absl::optional isp_db_path_; absl::optional anon_db_path_; absl::optional asn_db_path_; + absl::optional country_db_path_; absl::optional country_header_; absl::optional city_header_; @@ -140,6 +144,7 @@ class GeoipProvider : public Envoy::Geolocation::Driver, MaxmindDbSharedPtr isp_db_ ABSL_GUARDED_BY(mmdb_mutex_); MaxmindDbSharedPtr anon_db_ ABSL_GUARDED_BY(mmdb_mutex_); MaxmindDbSharedPtr asn_db_ ABSL_GUARDED_BY(mmdb_mutex_); + MaxmindDbSharedPtr country_db_ ABSL_GUARDED_BY(mmdb_mutex_); Thread::ThreadPtr mmdb_reload_thread_; Event::DispatcherPtr mmdb_reload_dispatcher_; Filesystem::WatcherPtr mmdb_watcher_; @@ -153,6 +158,8 @@ class GeoipProvider : public Envoy::Geolocation::Driver, absl::flat_hash_map& lookup_result) const; void lookupInIspDb(const Network::Address::InstanceConstSharedPtr& remote_address, absl::flat_hash_map& lookup_result) const; + void lookupInCountryDb(const Network::Address::InstanceConstSharedPtr& remote_address, + absl::flat_hash_map& lookup_result) const; absl::Status onMaxmindDbUpdate(const std::string& db_path, const absl::string_view& db_type); absl::Status mmdbReload(const MaxmindDbSharedPtr reloaded_db, const absl::string_view& db_type) ABSL_LOCKS_EXCLUDED(mmdb_mutex_); @@ -164,10 +171,12 @@ class GeoipProvider : public Envoy::Geolocation::Driver, MaxmindDbSharedPtr getIspDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_); MaxmindDbSharedPtr getAnonDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_); MaxmindDbSharedPtr getAsnDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_); + MaxmindDbSharedPtr getCountryDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_); void updateCityDb(MaxmindDbSharedPtr city_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_); void updateIspDb(MaxmindDbSharedPtr isp_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_); void updateAnonDb(MaxmindDbSharedPtr anon_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_); void updateAsnDb(MaxmindDbSharedPtr asn_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_); + void updateCountryDb(MaxmindDbSharedPtr country_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_); // A shared_ptr to keep the provider singleton alive as long as any of its providers are in use. const Singleton::InstanceSharedPtr owner_; // Used for testing only. diff --git a/test/extensions/filters/http/geoip/geoip_filter_integration_test.cc b/test/extensions/filters/http/geoip/geoip_filter_integration_test.cc index 9fa8eb429863c..4427f19921257 100644 --- a/test/extensions/filters/http/geoip/geoip_filter_integration_test.cc +++ b/test/extensions/filters/http/geoip/geoip_filter_integration_test.cc @@ -115,6 +115,40 @@ const std::string ConfigIsApplePrivateRelayOnly = R"EOF( isp_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-ISP-Test.mmdb" )EOF"; +const std::string ConfigWithCountryDb = R"EOF( +name: envoy.filters.http.geoip +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip + xff_config: + xff_num_trusted_hops: 1 + provider: + name: envoy.geoip_providers.maxmind + typed_config: + "@type": type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb" +)EOF"; + +const std::string ConfigWithCountryDbAndCityDb = R"EOF( +name: envoy.filters.http.geoip +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip + xff_config: + xff_num_trusted_hops: 1 + provider: + name: envoy.geoip_providers.maxmind + typed_config: + "@type": type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + city: "x-geo-city" + country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" +)EOF"; + class GeoipFilterIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: @@ -356,6 +390,47 @@ TEST_P(GeoipFilterIntegrationTest, MetricForDbBuildEpochIsEmitted) { test_server_->gauge("http.config_test.maxmind.city_db.db_build_epoch")->value()); } +TEST_P(GeoipFilterIntegrationTest, GeoDataPopulatedUseCountryDb) { + config_helper_.prependFilter(TestEnvironment::substitute(ConfigWithCountryDb)); + initialize(); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "216.160.83.56,9.10.11.12"}}; + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + EXPECT_EQ("US", headerValue("x-geo-country")); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + test_server_->waitForCounterEq("http.config_test.geoip.total", 1); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.hit")->value()); +} + +TEST_P(GeoipFilterIntegrationTest, GeoDataPopulatedUseCountryDbAndCityDb) { + config_helper_.prependFilter(TestEnvironment::substitute(ConfigWithCountryDbAndCityDb)); + initialize(); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "216.160.83.56,9.10.11.12"}}; + auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0); + EXPECT_EQ("US", headerValue("x-geo-country")); + EXPECT_EQ("Milton", headerValue("x-geo-city")); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + test_server_->waitForCounterEq("http.config_test.geoip.total", 1); + // Country should be looked up from Country DB. + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.hit")->value()); + // City should be looked up from City DB, but country should NOT be looked up from City DB. + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.total")->value()); + EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.hit")->value()); +} + } // namespace } // namespace Geoip } // namespace HttpFilters diff --git a/test/extensions/geoip_providers/maxmind/config_test.cc b/test/extensions/geoip_providers/maxmind/config_test.cc index c77c170ad32ad..db0a646b0885e 100644 --- a/test/extensions/geoip_providers/maxmind/config_test.cc +++ b/test/extensions/geoip_providers/maxmind/config_test.cc @@ -34,6 +34,9 @@ class GeoipProviderPeer { static const absl::optional& anonDbPath(const GeoipProvider& provider) { return provider.config_->anonDbPath(); } + static const absl::optional& countryDbPath(const GeoipProvider& provider) { + return provider.config_->countryDbPath(); + } static const absl::optional& countryHeader(const GeoipProvider& provider) { return provider.config_->countryHeader(); } @@ -61,6 +64,9 @@ class GeoipProviderPeer { static const absl::optional& ispHeader(const GeoipProvider& provider) { return provider.config_->ispHeader(); } + static bool isCityDbPathSet(const GeoipProvider& provider) { + return provider.config_->isCityDbPathSet(); + } }; MATCHER_P(HasCityDbPath, expected_db_path, "") { @@ -74,6 +80,16 @@ MATCHER_P(HasCityDbPath, expected_db_path, "") { return false; } +MATCHER_P(IsCityDbPathSet, expected, "") { + auto provider = std::static_pointer_cast(arg); + bool is_set = GeoipProviderPeer::isCityDbPathSet(*provider); + if (is_set == expected) { + return true; + } + *result_listener << "expected isCityDbPathSet()=" << expected << " but got " << is_set; + return false; +} + MATCHER_P(HasIspDbPath, expected_db_path, "") { auto provider = std::static_pointer_cast(arg); auto isp_db_path = GeoipProviderPeer::ispDbPath(*provider); @@ -96,6 +112,17 @@ MATCHER_P(HasAnonDbPath, expected_db_path, "") { return false; } +MATCHER_P(HasCountryDbPath, expected_db_path, "") { + auto provider = std::static_pointer_cast(arg); + auto country_db_path = GeoipProviderPeer::countryDbPath(*provider); + if (country_db_path && testing::Matches(expected_db_path)(country_db_path.value())) { + return true; + } + *result_listener << "expected country_db_path=" << expected_db_path + << " but country_db_path was not found in provider config"; + return false; +} + MATCHER_P(HasCountryHeader, expected_header, "") { auto provider = std::static_pointer_cast(arg); auto country_header = GeoipProviderPeer::countryHeader(*provider); @@ -276,10 +303,10 @@ TEST_F(MaxmindProviderConfigTest, ProviderConfigWithNoDbPaths) { TestUtility::loadFromYaml(provider_config_yaml, provider_config); NiceMock context; MaxmindProviderFactory factory; - EXPECT_THROW_WITH_MESSAGE(factory.createGeoipProviderDriver(provider_config, "maxmind", context), - Envoy::EnvoyException, - "At least one geolocation database path needs to be configured: " - "city_db_path, isp_db_path, asn_db_path or anon_db_path"); + EXPECT_THROW_WITH_MESSAGE( + factory.createGeoipProviderDriver(provider_config, "maxmind", context), Envoy::EnvoyException, + "At least one geolocation database path needs to be configured: " + "city_db_path, isp_db_path, asn_db_path, anon_db_path or country_db_path"); } TEST_F(MaxmindProviderConfigTest, ProviderConfigWithNoGeoHeaders) { @@ -394,6 +421,49 @@ TEST_F(MaxmindProviderConfigTest, DifferentProviderInstancesForDifferentProtoCon EXPECT_NE(driver1.get(), driver2.get()); } +TEST_F(MaxmindProviderConfigTest, ProviderConfigWithCountryDbPath) { + const auto provider_config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + country_db_path: %s + )EOF"; + MaxmindProviderConfig provider_config; + auto country_db_path = genGeoDbFilePath("GeoIP2-Country-Test.mmdb"); + auto processed_provider_config_yaml = absl::StrFormat(provider_config_yaml, country_db_path); + TestUtility::loadFromYaml(processed_provider_config_yaml, provider_config); + MaxmindProviderFactory factory; + Geolocation::DriverSharedPtr driver = + factory.createGeoipProviderDriver(provider_config, "maxmind", context_); + // City DB is not configured, so isCityDbPathSet() should return false. + EXPECT_THAT(driver, AllOf(HasCountryDbPath(country_db_path), HasCountryHeader("x-geo-country"), + IsCityDbPathSet(false))); +} + +TEST_F(MaxmindProviderConfigTest, ProviderConfigWithCountryDbAndCityDbPaths) { + const auto provider_config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + city: "x-geo-city" + country_db_path: %s + city_db_path: %s + )EOF"; + MaxmindProviderConfig provider_config; + auto country_db_path = genGeoDbFilePath("GeoIP2-Country-Test.mmdb"); + auto city_db_path = genGeoDbFilePath("GeoLite2-City-Test.mmdb"); + auto processed_provider_config_yaml = + absl::StrFormat(provider_config_yaml, country_db_path, city_db_path); + TestUtility::loadFromYaml(processed_provider_config_yaml, provider_config); + MaxmindProviderFactory factory; + Geolocation::DriverSharedPtr driver = + factory.createGeoipProviderDriver(provider_config, "maxmind", context_); + // Both Country DB and City DB are configured. + EXPECT_THAT(driver, AllOf(HasCountryDbPath(country_db_path), HasCityDbPath(city_db_path), + HasCountryHeader("x-geo-country"), HasCityHeader("x-geo-city"), + IsCityDbPathSet(true))); +} + } // namespace Maxmind } // namespace GeoipProviders } // namespace Extensions diff --git a/test/extensions/geoip_providers/maxmind/geoip_provider_test.cc b/test/extensions/geoip_providers/maxmind/geoip_provider_test.cc index f05691e0c269c..dd6b26aaed91f 100644 --- a/test/extensions/geoip_providers/maxmind/geoip_provider_test.cc +++ b/test/extensions/geoip_providers/maxmind/geoip_provider_test.cc @@ -36,6 +36,11 @@ class GeoipProviderPeer { auto provider = std::static_pointer_cast(driver); return provider->synchronizer_; } + static void setCountryDbToNull(const DriverSharedPtr& driver) { + auto provider = std::static_pointer_cast(driver); + absl::MutexLock lock(&provider->mmdb_mutex_); + provider->country_db_.reset(); + } }; namespace { @@ -100,6 +105,29 @@ const std::string default_anon_config_yaml = R"EOF( anon: "x-geo-anon" anon_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Anonymous-IP-Test.mmdb" )EOF"; + +// Country DB reload tests use City-Test databases since they contain country info and we have +// an updated version available. The `GeoIP2-Country-Test` DB is used for non-reload country tests. +const std::string default_country_db_path = + "{{ test_rundir " + "}}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb"; + +const std::string default_updated_country_db_path = + "{{ test_rundir " + "}}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test-Updated.mmdb"; + +const std::string default_country_config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + )EOF"; + +// Invalid DB path for reload error tests. +const std::string invalid_db_path = "{{ test_rundir " + "}}/test/extensions/geoip_providers/maxmind/test_data/" + "libmaxminddb-offset-integer-overflow.mmdb"; + } // namespace class GeoipProviderTestBase { @@ -663,60 +691,6 @@ TEST_F(GeoipProviderTest, DbEpochGaugeUpdatesWhenReloadedOnMmdbFileUpdate) { TestEnvironment::renameFile(city_db_path + "1", city_db_path); } -TEST_F(GeoipProviderTest, DbReloadError) { - constexpr absl::string_view config_yaml = R"EOF( - common_provider_config: - geo_headers_to_add: - country: "x-geo-country" - region: "x-geo-region" - city: "x-geo-city" - city_db_path: {} - )EOF"; - std::string city_db_path = TestEnvironment::substitute( - "{{ test_rundir " - "}}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb"); - std::string reloaded_invalid_city_db_path = - TestEnvironment::substitute("{{ test_rundir " - "}}/test/extensions/geoip_providers/maxmind/test_data/" - "libmaxminddb-offset-integer-overflow.mmdb"); - const std::string formatted_config = - fmt::format(config_yaml, TestEnvironment::substitute(city_db_path)); - auto cb_added_opt = absl::make_optional(); - initializeProvider(formatted_config, cb_added_opt); - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("81.2.69.144"); - Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; - testing::MockFunction lookup_cb; - auto lookup_cb_std = lookup_cb.AsStdFunction(); - EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); - provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); - EXPECT_EQ(3, captured_lookup_response_.size()); - const auto& city_it = captured_lookup_response_.find("x-geo-city"); - EXPECT_EQ("London", city_it->second); - TestEnvironment::renameFile(city_db_path, city_db_path + "1"); - TestEnvironment::renameFile(reloaded_invalid_city_db_path, city_db_path); - cb_added_opt.value().waitReady(); - { - absl::ReaderMutexLock guard(&mutex_); - EXPECT_TRUE(on_changed_cbs_[0](Filesystem::Watcher::Events::MovedTo).ok()); - } - // On mmdb reload error the old mmdb instance should be used for subsequent lookup requests. - expectReloadStats("city_db", 0, 1); - captured_lookup_response_.clear(); - EXPECT_EQ(0, captured_lookup_response_.size()); - remote_address = Network::Utility::parseInternetAddressNoThrow("81.2.69.144"); - Geolocation::LookupRequest lookup_rq2{std::move(remote_address)}; - testing::MockFunction lookup_cb2; - auto lookup_cb_std2 = lookup_cb2.AsStdFunction(); - EXPECT_CALL(lookup_cb2, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); - provider_->lookup(std::move(lookup_rq2), std::move(lookup_cb_std2)); - const auto& city1_it = captured_lookup_response_.find("x-geo-city"); - EXPECT_EQ("London", city1_it->second); - // Clean up modifications to mmdb file names. - TestEnvironment::renameFile(city_db_path, reloaded_invalid_city_db_path); - TestEnvironment::renameFile(city_db_path + "1", city_db_path); -} - using GeoipProviderDeathTest = GeoipProviderTest; TEST_F(GeoipProviderDeathTest, GeoDbPathDoesNotExist) { @@ -925,10 +899,278 @@ struct MmdbReloadTestCase mmdb_reload_test_cases[] = { "x-geo-asn", "237", "23742", "2806:2000::"}, {default_anon_config_yaml, "anon_db", default_anon_db_path, default_updated_anon_db_path, "x-geo-anon", "true", "false", "65.4.3.2"}, + // Country DB uses City-Test databases since they contain country info and we have an updated + // version. Both databases return GB for this IP. + {default_country_config_yaml, "country_db", default_country_db_path, + default_updated_country_db_path, "x-geo-country", "GB", "GB", "81.2.69.144"}, }; INSTANTIATE_TEST_SUITE_P(TestName, MmdbReloadImplTest, ::testing::ValuesIn(mmdb_reload_test_cases)); +// Parametrized test for reload errors. It verifies that when db reload fails, the old db is used. +struct MmdbReloadErrorTestCase { + MmdbReloadErrorTestCase() = default; + MmdbReloadErrorTestCase(const std::string& yaml_config, const std::string& db_type, + const std::string& source_db_file_path, + const std::string& expected_header_name, + const std::string& expected_header_value, const std::string& ip, + uint32_t expected_lookup_count) + : yaml_config_(yaml_config), db_type_(db_type), source_db_file_path_(source_db_file_path), + expected_header_name_(expected_header_name), expected_header_value_(expected_header_value), + ip_(ip), expected_lookup_count_(expected_lookup_count) {} + MmdbReloadErrorTestCase(const MmdbReloadErrorTestCase& rhs) = default; + + std::string yaml_config_; + std::string db_type_; + std::string source_db_file_path_; + std::string expected_header_name_; + std::string expected_header_value_; + std::string ip_; + uint32_t expected_lookup_count_; +}; + +class MmdbReloadErrorImplTest : public ::testing::TestWithParam, + public GeoipProviderTestBase {}; + +TEST_P(MmdbReloadErrorImplTest, MmdbReloadErrorUsesPreviousDb) { + MmdbReloadErrorTestCase test_case = GetParam(); + auto cb_added_opt = absl::make_optional(); + initializeProvider(test_case.yaml_config_, cb_added_opt); + std::string source_db_file_path = TestEnvironment::substitute(test_case.source_db_file_path_); + std::string invalid_db_file_path = TestEnvironment::substitute(invalid_db_path); + + // Initial lookup should succeed using the valid DB. + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow(test_case.ip_); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(test_case.expected_lookup_count_, captured_lookup_response_.size()); + const auto& header_it = captured_lookup_response_.find(test_case.expected_header_name_); + EXPECT_EQ(test_case.expected_header_value_, header_it->second); + + // Replace db with invalid file and trigger reload. + TestEnvironment::renameFile(source_db_file_path, source_db_file_path + "1"); + TestEnvironment::renameFile(invalid_db_file_path, source_db_file_path); + cb_added_opt.value().waitReady(); + { + absl::ReaderMutexLock guard(&mutex_); + EXPECT_TRUE(on_changed_cbs_[0](Filesystem::Watcher::Events::MovedTo).ok()); + } + // On reload error the old db instance should be used for subsequent lookup requests. + expectReloadStats(test_case.db_type_, 0, 1); + + // Lookup should still work using the previously loaded valid DB. + captured_lookup_response_.clear(); + EXPECT_EQ(0, captured_lookup_response_.size()); + remote_address = Network::Utility::parseInternetAddressNoThrow(test_case.ip_); + Geolocation::LookupRequest lookup_rq2{std::move(remote_address)}; + testing::MockFunction lookup_cb2; + auto lookup_cb_std2 = lookup_cb2.AsStdFunction(); + EXPECT_CALL(lookup_cb2, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq2), std::move(lookup_cb_std2)); + const auto& header2_it = captured_lookup_response_.find(test_case.expected_header_name_); + EXPECT_EQ(test_case.expected_header_value_, header2_it->second); + + // Clean up modifications to db file names. + TestEnvironment::renameFile(source_db_file_path, invalid_db_file_path); + TestEnvironment::renameFile(source_db_file_path + "1", source_db_file_path); +} + +// Config with full path for city_db reload error test. +const std::string city_db_reload_error_config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + region: "x-geo-region" + city: "x-geo-city" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + )EOF"; + +// Config with full path for country_db reload error test. +const std::string country_db_reload_error_config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb" + )EOF"; + +// Country DB reload error test uses actual Country-Test MMDB for testing. +const std::string country_db_reload_error_path = + "{{ test_rundir " + "}}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb"; + +struct MmdbReloadErrorTestCase mmdb_reload_error_test_cases[] = { + {city_db_reload_error_config_yaml, "city_db", default_city_db_path, "x-geo-city", "London", + "81.2.69.144", 3}, + {country_db_reload_error_config_yaml, "country_db", country_db_reload_error_path, + "x-geo-country", "SE", "89.160.20.112", 1}, +}; + +INSTANTIATE_TEST_SUITE_P(TestName, MmdbReloadErrorImplTest, + ::testing::ValuesIn(mmdb_reload_error_test_cases)); + +// Country DB specific tests. +TEST_F(GeoipProviderTest, ValidConfigCountryDbSuccessfulLookup) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb" + )EOF"; + initializeProvider(config_yaml, cb_added_nullopt); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("89.160.20.112"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(1, captured_lookup_response_.size()); + const auto& country_it = captured_lookup_response_.find("x-geo-country"); + EXPECT_EQ("SE", country_it->second); + expectStats("country_db"); +} + +TEST_F(GeoipProviderTest, CountryDbTakesPrecedenceOverCityDb) { + // When both Country DB and City DB are configured, Country DB should be used for country lookup. + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + city: "x-geo-city" + country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + )EOF"; + initializeProvider(config_yaml, cb_added_nullopt); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("89.160.20.112"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(2, captured_lookup_response_.size()); + const auto& country_it = captured_lookup_response_.find("x-geo-country"); + EXPECT_EQ("SE", country_it->second); + const auto& city_it = captured_lookup_response_.find("x-geo-city"); + EXPECT_EQ("Linköping", city_it->second); + // Country DB should be used for country lookup. + expectStats("country_db", 1, 1, 0); + // City DB should only be used for city lookup (not country). + expectStats("city_db", 1, 1, 0); +} + +TEST_F(GeoipProviderTest, CountryFallsBackToCityDbWhenCountryDbNotConfigured) { + // When only City DB is configured, country lookup should fall back to City DB. + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + city: "x-geo-city" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + )EOF"; + initializeProvider(config_yaml, cb_added_nullopt); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("89.160.20.112"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + EXPECT_EQ(2, captured_lookup_response_.size()); + const auto& country_it = captured_lookup_response_.find("x-geo-country"); + EXPECT_EQ("SE", country_it->second); + const auto& city_it = captured_lookup_response_.find("x-geo-city"); + EXPECT_EQ("Linköping", city_it->second); + // Country should be looked up from City DB. + expectStats("city_db", 1, 1, 0); +} + +// Test Country DB lookup error when MMDB_get_entry_data_list fails due to corrupted entry data. +TEST_F(GeoipProviderTest, CountryDbLookupErrorCorruptedEntryData) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/corrupted-entry-data.mmdb" + )EOF"; + initializeProvider(config_yaml, cb_added_nullopt); + // Use an IP that should be in the Country DB. + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("89.160.20.112"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + // With corrupted entry data, we expect lookup_error to be incremented. + expectStats("country_db", 1, 0, 1); +} + +// Test Country DB lookup when country_db is null but City DB is available. +TEST_F(GeoipProviderTest, CountryDbLookupWithNullDbAndCityFallback) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + city: "x-geo-city" + country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb" + city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb" + )EOF"; + initializeProvider(config_yaml, cb_added_nullopt); + + // Force country_db_ to be null using friend peer class. + GeoipProviderPeer::setCountryDbToNull(provider_); + + // Use an IP (London, UK) that works in City DB. + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("81.2.69.144"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + testing::MockFunction lookup_cb; + auto lookup_cb_std = lookup_cb.AsStdFunction(); + + EXPECT_CALL(lookup_cb, Call(_)).WillRepeatedly(SaveArg<0>(&captured_lookup_response_)); + provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); + + // Country header should be missing because the country DB is null and the city DB + // skipped the country lookup because isCountryDbPathSet() is true. + const auto& country_it = captured_lookup_response_.find("x-geo-country"); + EXPECT_EQ(captured_lookup_response_.end(), country_it); + + // City header should be present proving lookupInCityDb ran and provider didn't crash. + const auto& city_it = captured_lookup_response_.find("x-geo-city"); + EXPECT_NE(captured_lookup_response_.end(), city_it); + EXPECT_EQ("London", city_it->second); +} + +// Test Country DB lookup when country_db is null and NO City DB. +TEST_F(GeoipProviderTest, CountryDbLookupWithNullDbAndNoFallback) { + const std::string config_yaml = R"EOF( + common_provider_config: + geo_headers_to_add: + country: "x-geo-country" + country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb" + )EOF"; + initializeProvider(config_yaml, cb_added_nullopt); + + // Force country_db_ to be null. + GeoipProviderPeer::setCountryDbToNull(provider_); + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("81.2.69.144"); + Geolocation::LookupRequest lookup_rq{std::move(remote_address)}; + + auto lookup_cb_std = [&](Geolocation::LookupResult&&) {}; + + // Should trigger IS_ENVOY_BUG. + EXPECT_ENVOY_BUG( + { provider_->lookup(std::move(lookup_rq), std::move(lookup_cb_std)); }, + "Maxmind country database must be initialised for performing lookups"); +} + } // namespace Maxmind } // namespace GeoipProviders } // namespace Extensions diff --git a/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb b/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb new file mode 100644 index 0000000000000..d3de6f21b8224 Binary files /dev/null and b/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb differ diff --git a/test/extensions/geoip_providers/maxmind/test_data/corrupted-entry-data.mmdb b/test/extensions/geoip_providers/maxmind/test_data/corrupted-entry-data.mmdb new file mode 100644 index 0000000000000..75c281c4e0539 Binary files /dev/null and b/test/extensions/geoip_providers/maxmind/test_data/corrupted-entry-data.mmdb differ diff --git a/test/extensions/geoip_providers/maxmind/test_data/libmaxminddb-offset-integer-overflow.mmdb b/test/extensions/geoip_providers/maxmind/test_data/libmaxminddb-offset-integer-overflow.mmdb index 9908afc7e2d4c..b76f3546c7983 100644 Binary files a/test/extensions/geoip_providers/maxmind/test_data/libmaxminddb-offset-integer-overflow.mmdb and b/test/extensions/geoip_providers/maxmind/test_data/libmaxminddb-offset-integer-overflow.mmdb differ diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 603399abbdb55..c9d1391f48198 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -1601,6 +1601,7 @@ routable vhosts infos ElastiCache +Parametrized pinterest callouts streamable