Skip to content

geoip: add support for the MaxMind country db #42419

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
agrawroh wants to merge 1 commit into
base: main
Choose a base branch
from
+519 −10
Open

geoip: add support for the MaxMind country db #42419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion api/envoy/extensions/geoip_providers/maxmind/v3/maxmind.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
// * :ref:`isp_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.isp_db_path>`
// * :ref:`asn_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.asn_db_path>`
// * :ref:`anon_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.anon_db_path>`
// * :ref:`country_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.country_db_path>`
// [#extension: envoy.geoip_providers.maxmind]

// [#next-free-field: 6]
// [#next-free-field: 7]
message MaxMindConfig {
// Full file path to the MaxMind city database, e.g., ``/etc/GeoLite2-City.mmdb``.
// Database file is expected to have ``.mmdb`` extension.
Expand All @@ -48,6 +49,13 @@ message MaxMindConfig {
// ``isp_db`` instead.
string isp_db_path = 5 [(validate.rules).string = {pattern: "^$|^.*\\.mmdb$"}];

// Full file path to the MaxMind Country database, e.g., ``/etc/GeoLite2-Country.mmdb``.
// Database file is expected to have ``.mmdb`` extension.
//
// If ``country_db_path`` is not defined, country information will be fetched from
// ``city_db`` instead.
string country_db_path = 6 [(validate.rules).string = {pattern: "^$|^.*\\.mmdb$"}];

// Common provider configuration that specifies which geolocation headers will be populated with geolocation data.
common.v3.CommonGeoipProviderConfig common_provider_config = 4
[(validate.rules).message = {required: true}];
Expand Down
4 changes: 4 additions & 0 deletions changelogs/current.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,5 +463,9 @@ new_features:
- area: attributes
change: |
added :ref:`attributes <arch_overview_attributes>` for looking up request or response headers bytes.
- area: geoip
change: |
Added support for MaxMind Country database via
:ref:`country_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.country_db_path>`.

deprecated:
90 changes: 85 additions & 5 deletions source/extensions/geoip_providers/maxmind/geoip_provider.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ static constexpr absl::string_view CITY_DB_TYPE = "city_db";
static constexpr absl::string_view ISP_DB_TYPE = "isp_db";
static constexpr absl::string_view ANON_DB_TYPE = "anon_db";
static constexpr absl::string_view ASN_DB_TYPE = "asn_db";
static constexpr absl::string_view COUNTRY_DB_TYPE = "country_db";
} // namespace

GeoipProviderConfig::GeoipProviderConfig(
Expand All @@ -36,6 +37,9 @@ GeoipProviderConfig::GeoipProviderConfig(
: absl::nullopt),
asn_db_path_(!config.asn_db_path().empty() ? absl::make_optional(config.asn_db_path())
: absl::nullopt),
country_db_path_(!config.country_db_path().empty()
? absl::make_optional(config.country_db_path())
: absl::nullopt),
stats_scope_(scope.createScope(absl::StrCat(stat_prefix, "maxmind."))),
stat_name_set_(stats_scope_->symbolTable().makeSet("Maxmind")) {
auto geo_headers_to_add = config.common_provider_config().geo_headers_to_add();
Expand Down Expand Up @@ -75,9 +79,9 @@ GeoipProviderConfig::GeoipProviderConfig(
apple_private_relay_header_ = !geo_headers_to_add.apple_private_relay().empty()
? absl::make_optional(geo_headers_to_add.apple_private_relay())
: absl::nullopt;
if (!city_db_path_ && !anon_db_path_ && !asn_db_path_ && !isp_db_path_) {
if (!city_db_path_ && !anon_db_path_ && !asn_db_path_ && !isp_db_path_ && !country_db_path_) {
throw EnvoyException("At least one geolocation database path needs to be configured: "
"city_db_path, isp_db_path, asn_db_path or anon_db_path");
"city_db_path, isp_db_path, asn_db_path, anon_db_path or country_db_path");
}
if (city_db_path_) {
registerGeoDbStats(CITY_DB_TYPE);
Expand All @@ -91,6 +95,9 @@ GeoipProviderConfig::GeoipProviderConfig(
if (asn_db_path_) {
registerGeoDbStats(ASN_DB_TYPE);
}
if (country_db_path_) {
registerGeoDbStats(COUNTRY_DB_TYPE);
}
};

void GeoipProviderConfig::registerGeoDbStats(const absl::string_view& db_type) {
Expand Down Expand Up @@ -126,6 +133,9 @@ GeoipProvider::GeoipProvider(Event::Dispatcher& dispatcher, Api::Api& api,
config_->anonDbPath() ? initMaxmindDb(config_->anonDbPath().value(), ANON_DB_TYPE) : nullptr;
asn_db_ =
config_->asnDbPath() ? initMaxmindDb(config_->asnDbPath().value(), ASN_DB_TYPE) : nullptr;
country_db_ = config_->countryDbPath()
? initMaxmindDb(config_->countryDbPath().value(), COUNTRY_DB_TYPE)
: nullptr;
mmdb_reload_dispatcher_ = api.allocateDispatcher("mmdb_reload_routine");
mmdb_watcher_ = dispatcher.createFilesystemWatcher();
mmdb_reload_thread_ = api.threadFactory().createThread(
Expand Down Expand Up @@ -157,6 +167,13 @@ GeoipProvider::GeoipProvider(Event::Dispatcher& dispatcher, Api::Api& api,
return onMaxmindDbUpdate(config_->asnDbPath().value(), ASN_DB_TYPE);
}));
}
if (config_->countryDbPath()) {
THROW_IF_NOT_OK(mmdb_watcher_->addWatch(
config_->countryDbPath().value(), Filesystem::Watcher::Events::MovedTo,
[this](uint32_t) {
return onMaxmindDbUpdate(config_->countryDbPath().value(), COUNTRY_DB_TYPE);
}));
}
mmdb_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit);
},
Thread::Options{std::string("mmdb_reload_routine")});
Expand All @@ -177,6 +194,7 @@ void GeoipProvider::lookup(Geolocation::LookupRequest&& request,
Geolocation::LookupGeoHeadersCallback&& cb) const {
auto& remote_address = request.remoteAddress();
auto lookup_result = absl::flat_hash_map<std::string, std::string>{};
lookupInCountryDb(remote_address, lookup_result);
lookupInCityDb(remote_address, lookup_result);
lookupInAsnDb(remote_address, lookup_result);
lookupInAnonDb(remote_address, lookup_result);
Expand All @@ -187,9 +205,12 @@ void GeoipProvider::lookup(Geolocation::LookupRequest&& request,
void GeoipProvider::lookupInCityDb(
const Network::Address::InstanceConstSharedPtr& remote_address,
absl::flat_hash_map<std::string, std::string>& lookup_result) const {
// Country lookup falls back to City DB only if Country DB is not configured.
const bool should_lookup_country_from_city_db =
!config_->isCountryDbPathSet() && config_->isLookupEnabledForHeader(config_->countryHeader());
if (config_->isLookupEnabledForHeader(config_->cityHeader()) ||
config_->isLookupEnabledForHeader(config_->regionHeader()) ||
config_->isLookupEnabledForHeader(config_->countryHeader())) {
should_lookup_country_from_city_db) {
int mmdb_error;
auto city_db_ptr = getCityDb();
// Used for testing.
Expand Down Expand Up @@ -217,7 +238,8 @@ void GeoipProvider::lookupInCityDb(
config_->regionHeader().value(), MMDB_REGION_LOOKUP_ARGS[0],
MMDB_REGION_LOOKUP_ARGS[1], MMDB_REGION_LOOKUP_ARGS[2]);
}
if (config_->isLookupEnabledForHeader(config_->countryHeader())) {
// Country lookup from City DB only when Country DB is not configured.
if (should_lookup_country_from_city_db) {
populateGeoLookupResult(mmdb_lookup_result, lookup_result,
config_->countryHeader().value(), MMDB_COUNTRY_LOOKUP_ARGS[0],
MMDB_COUNTRY_LOOKUP_ARGS[1]);
Expand Down Expand Up @@ -384,6 +406,51 @@ void GeoipProvider::lookupInIspDb(
}
}

void GeoipProvider::lookupInCountryDb(
const Network::Address::InstanceConstSharedPtr& remote_address,
absl::flat_hash_map<std::string, std::string>& lookup_result) const {
if (config_->isLookupEnabledForHeader(config_->countryHeader())) {
// Country DB takes precedence if configured, otherwise fall back to City DB.
if (!config_->isCountryDbPathSet()) {
// Country lookup will be handled by lookupInCityDb.
return;
}
int mmdb_error;
auto country_db_ptr = getCountryDb();
// Used for testing.
synchronizer_.syncPoint(std::string(COUNTRY_DB_TYPE).append("_lookup_pre_complete"));
if (!country_db_ptr) {
if (config_->isCityDbPathSet()) {
// Country information can be looked up from City database as well, so we don't need to
// throw an error if it is not set.
return;
}
IS_ENVOY_BUG("Maxmind country database must be initialised for performing lookups");
return;
}
auto country_db = country_db_ptr.get();
MMDB_lookup_result_s mmdb_lookup_result = MMDB_lookup_sockaddr(
country_db->mmdb(), reinterpret_cast<const sockaddr*>(remote_address->sockAddr()),
&mmdb_error);
const uint32_t n_prev_hits = lookup_result.size();
if (!mmdb_error && mmdb_lookup_result.found_entry) {
MMDB_entry_data_list_s* entry_data_list;
int status = MMDB_get_entry_data_list(&mmdb_lookup_result.entry, &entry_data_list);
if (status == MMDB_SUCCESS) {
populateGeoLookupResult(mmdb_lookup_result, lookup_result, config_->countryHeader().value(),
MMDB_COUNTRY_LOOKUP_ARGS[0], MMDB_COUNTRY_LOOKUP_ARGS[1]);
if (lookup_result.size() > n_prev_hits) {
config_->incHit(COUNTRY_DB_TYPE);
}
MMDB_free_entry_data_list(entry_data_list);
} else {
config_->incLookupError(COUNTRY_DB_TYPE);
}
}
config_->incTotal(COUNTRY_DB_TYPE);
}
}

MaxmindDbSharedPtr GeoipProvider::initMaxmindDb(const std::string& db_path,
const absl::string_view& db_type, bool reload) {
MMDB_s maxmind_db;
Expand All @@ -398,7 +465,6 @@ MaxmindDbSharedPtr GeoipProvider::initMaxmindDb(const std::string& db_path,
RELEASE_ASSERT(MMDB_SUCCESS == result_code,
fmt::format("Unable to open Maxmind database file {}. Error {}", db_path,
std::string(MMDB_strerror(result_code))));
return nullptr;
}

config_->setDbBuildEpoch(db_type, maxmind_db.metadata.build_epoch);
Expand All @@ -422,6 +488,9 @@ absl::Status GeoipProvider::mmdbReload(const MaxmindDbSharedPtr reloaded_db,
} else if (db_type == ASN_DB_TYPE) {
updateAsnDb(reloaded_db);
config_->incDbReloadSuccess(db_type);
} else if (db_type == COUNTRY_DB_TYPE) {
updateCountryDb(reloaded_db);
config_->incDbReloadSuccess(db_type);
} else {
ENVOY_LOG(error, "Unsupported maxmind db type {}", db_type);
return absl::InvalidArgumentError(fmt::format("Unsupported maxmind db type {}", db_type));
Expand Down Expand Up @@ -472,6 +541,17 @@ void GeoipProvider::updateAnonDb(MaxmindDbSharedPtr anon_db) ABSL_LOCKS_EXCLUDED
anon_db_ = anon_db;
}

MaxmindDbSharedPtr GeoipProvider::getCountryDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
absl::ReaderMutexLock lock(&mmdb_mutex_);
return country_db_;
}

void GeoipProvider::updateCountryDb(MaxmindDbSharedPtr country_db)
ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
absl::MutexLock lock(&mmdb_mutex_);
country_db_ = country_db;
}

absl::Status GeoipProvider::onMaxmindDbUpdate(const std::string& db_path,
const absl::string_view& db_type) {
MaxmindDbSharedPtr reloaded_db = initMaxmindDb(db_path, db_type, true /* reload */);
Expand Down
9 changes: 9 additions & 0 deletions source/extensions/geoip_providers/maxmind/geoip_provider.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@ class GeoipProviderConfig {
const absl::optional<std::string>& ispDbPath() const { return isp_db_path_; }
const absl::optional<std::string>& anonDbPath() const { return anon_db_path_; }
const absl::optional<std::string>& asnDbPath() const { return asn_db_path_; }
const absl::optional<std::string>& countryDbPath() const { return country_db_path_; }

bool isLookupEnabledForHeader(const absl::optional<std::string>& header);
bool isAsnDbPathSet() const { return asn_db_path_.has_value(); }
bool isIspDbPathSet() const { return isp_db_path_.has_value(); }
bool isCountryDbPathSet() const { return country_db_path_.has_value(); }
bool isCityDbPathSet() const { return city_db_path_.has_value(); }

const absl::optional<std::string>& countryHeader() const { return country_header_; }
const absl::optional<std::string>& cityHeader() const { return city_header_; }
Expand Down Expand Up @@ -82,6 +85,7 @@ class GeoipProviderConfig {
absl::optional<std::string> isp_db_path_;
absl::optional<std::string> anon_db_path_;
absl::optional<std::string> asn_db_path_;
absl::optional<std::string> country_db_path_;

absl::optional<std::string> country_header_;
absl::optional<std::string> city_header_;
Expand Down Expand Up @@ -140,6 +144,7 @@ class GeoipProvider : public Envoy::Geolocation::Driver,
MaxmindDbSharedPtr isp_db_ ABSL_GUARDED_BY(mmdb_mutex_);
MaxmindDbSharedPtr anon_db_ ABSL_GUARDED_BY(mmdb_mutex_);
MaxmindDbSharedPtr asn_db_ ABSL_GUARDED_BY(mmdb_mutex_);
MaxmindDbSharedPtr country_db_ ABSL_GUARDED_BY(mmdb_mutex_);
Thread::ThreadPtr mmdb_reload_thread_;
Event::DispatcherPtr mmdb_reload_dispatcher_;
Filesystem::WatcherPtr mmdb_watcher_;
Expand All @@ -153,6 +158,8 @@ class GeoipProvider : public Envoy::Geolocation::Driver,
absl::flat_hash_map<std::string, std::string>& lookup_result) const;
void lookupInIspDb(const Network::Address::InstanceConstSharedPtr& remote_address,
absl::flat_hash_map<std::string, std::string>& lookup_result) const;
void lookupInCountryDb(const Network::Address::InstanceConstSharedPtr& remote_address,
absl::flat_hash_map<std::string, std::string>& lookup_result) const;
absl::Status onMaxmindDbUpdate(const std::string& db_path, const absl::string_view& db_type);
absl::Status mmdbReload(const MaxmindDbSharedPtr reloaded_db, const absl::string_view& db_type)
ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
Expand All @@ -164,10 +171,12 @@ class GeoipProvider : public Envoy::Geolocation::Driver,
MaxmindDbSharedPtr getIspDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
MaxmindDbSharedPtr getAnonDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
MaxmindDbSharedPtr getAsnDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
MaxmindDbSharedPtr getCountryDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
void updateCityDb(MaxmindDbSharedPtr city_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
void updateIspDb(MaxmindDbSharedPtr isp_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
void updateAnonDb(MaxmindDbSharedPtr anon_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
void updateAsnDb(MaxmindDbSharedPtr asn_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
void updateCountryDb(MaxmindDbSharedPtr country_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
// A shared_ptr to keep the provider singleton alive as long as any of its providers are in use.
const Singleton::InstanceSharedPtr owner_;
// Used for testing only.
Expand Down
75 changes: 75 additions & 0 deletions test/extensions/filters/http/geoip/geoip_filter_integration_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,40 @@ const std::string ConfigIsApplePrivateRelayOnly = R"EOF(
isp_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-ISP-Test.mmdb"
)EOF";

const std::string ConfigWithCountryDb = R"EOF(
name: envoy.filters.http.geoip
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip
xff_config:
xff_num_trusted_hops: 1
provider:
name: envoy.geoip_providers.maxmind
typed_config:
"@type": type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig
common_provider_config:
geo_headers_to_add:
country: "x-geo-country"
country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb"
)EOF";

const std::string ConfigWithCountryDbAndCityDb = R"EOF(
name: envoy.filters.http.geoip
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip
xff_config:
xff_num_trusted_hops: 1
provider:
name: envoy.geoip_providers.maxmind
typed_config:
"@type": type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig
common_provider_config:
geo_headers_to_add:
country: "x-geo-country"
city: "x-geo-city"
country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb"
city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb"
)EOF";

class GeoipFilterIntegrationTest : public testing::TestWithParam<Network::Address::IpVersion>,
public HttpIntegrationTest {
public:
Expand Down Expand Up @@ -356,6 +390,47 @@ TEST_P(GeoipFilterIntegrationTest, MetricForDbBuildEpochIsEmitted) {
test_server_->gauge("http.config_test.maxmind.city_db.db_build_epoch")->value());
}

TEST_P(GeoipFilterIntegrationTest, GeoDataPopulatedUseCountryDb) {
config_helper_.prependFilter(TestEnvironment::substitute(ConfigWithCountryDb));
initialize();
codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http")));
Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"},
{":path", "/"},
{":scheme", "http"},
{":authority", "host"},
{"x-forwarded-for", "216.160.83.56,9.10.11.12"}};
auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0);
EXPECT_EQ("US", headerValue("x-geo-country"));
ASSERT_TRUE(response->complete());
EXPECT_EQ("200", response->headers().getStatusValue());
test_server_->waitForCounterEq("http.config_test.geoip.total", 1);
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.total")->value());
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.hit")->value());
}

TEST_P(GeoipFilterIntegrationTest, GeoDataPopulatedUseCountryDbAndCityDb) {
config_helper_.prependFilter(TestEnvironment::substitute(ConfigWithCountryDbAndCityDb));
initialize();
codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http")));
Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"},
{":path", "/"},
{":scheme", "http"},
{":authority", "host"},
{"x-forwarded-for", "216.160.83.56,9.10.11.12"}};
auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0);
EXPECT_EQ("US", headerValue("x-geo-country"));
EXPECT_EQ("Milton", headerValue("x-geo-city"));
ASSERT_TRUE(response->complete());
EXPECT_EQ("200", response->headers().getStatusValue());
test_server_->waitForCounterEq("http.config_test.geoip.total", 1);
// Country should be looked up from Country DB.
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.total")->value());
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.hit")->value());
// City should be looked up from City DB, but country should NOT be looked up from City DB.
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.total")->value());
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.hit")->value());
}

} // namespace
} // namespace Geoip
} // namespace HttpFilters
Expand Down
Loading
Loading