Skip to content

Commit d870cef

Browse files
agrawrohagrawroh
committed
geoip: add support for the MaxMind country db
Signed-off-by: Rohit Agrawal <rohit.agrawal@databricks.com>
1 parent 3fb37a5 commit d870cef

File tree

9 files changed

+372
-9
lines changed

9 files changed

+372
-9
lines changed

‎api/envoy/extensions/geoip_providers/maxmind/v3/maxmind.proto‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,10 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
2525
// * :ref:`isp_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.isp_db_path>`
2626
// * :ref:`asn_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.asn_db_path>`
2727
// * :ref:`anon_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.anon_db_path>`
28+
// * :ref:`country_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.country_db_path>`
2829
// [#extension: envoy.geoip_providers.maxmind]
2930

30-
// [#next-free-field: 6]
31+
// [#next-free-field: 7]
3132
message MaxMindConfig {
3233
// Full file path to the MaxMind city database, e.g., ``/etc/GeoLite2-City.mmdb``.
3334
// Database file is expected to have ``.mmdb`` extension.
@@ -48,6 +49,13 @@ message MaxMindConfig {
4849
// ``isp_db`` instead.
4950
string isp_db_path = 5 [(validate.rules).string = {pattern: "^$|^.*\\.mmdb$"}];
5051

52+
// Full file path to the MaxMind Country database, e.g., ``/etc/GeoLite2-Country.mmdb``.
53+
// Database file is expected to have ``.mmdb`` extension.
54+
//
55+
// If ``country_db_path`` is not defined, country information will be fetched from
56+
// ``city_db`` instead.
57+
string country_db_path = 6 [(validate.rules).string = {pattern: "^$|^.*\\.mmdb$"}];
58+
5159
// Common provider configuration that specifies which geolocation headers will be populated with geolocation data.
5260
common.v3.CommonGeoipProviderConfig common_provider_config = 4
5361
[(validate.rules).message = {required: true}];

‎changelogs/current.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,5 +444,9 @@ new_features:
444444
- area: attributes
445445
change: |
446446
added :ref:`attributes <arch_overview_attributes>` for looking up request or response headers bytes.
447+
- area: geoip
448+
change: |
449+
Added support for MaxMind Country database via
450+
:ref:`country_db_path <envoy_v3_api_field_extensions.geoip_providers.maxmind.v3.MaxMindConfig.country_db_path>`.
447451
448452
deprecated:

‎source/extensions/geoip_providers/maxmind/geoip_provider.cc‎

Lines changed: 86 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ static constexpr absl::string_view CITY_DB_TYPE = "city_db";
2323
static constexpr absl::string_view ISP_DB_TYPE = "isp_db";
2424
static constexpr absl::string_view ANON_DB_TYPE = "anon_db";
2525
static constexpr absl::string_view ASN_DB_TYPE = "asn_db";
26+
static constexpr absl::string_view COUNTRY_DB_TYPE = "country_db";
2627
} // namespace
2728

2829
GeoipProviderConfig::GeoipProviderConfig(
@@ -36,6 +37,9 @@ GeoipProviderConfig::GeoipProviderConfig(
3637
: absl::nullopt),
3738
asn_db_path_(!config.asn_db_path().empty() ? absl::make_optional(config.asn_db_path())
3839
: absl::nullopt),
40+
country_db_path_(!config.country_db_path().empty()
41+
? absl::make_optional(config.country_db_path())
42+
: absl::nullopt),
3943
stats_scope_(scope.createScope(absl::StrCat(stat_prefix, "maxmind."))),
4044
stat_name_set_(stats_scope_->symbolTable().makeSet("Maxmind")) {
4145
auto geo_headers_to_add = config.common_provider_config().geo_headers_to_add();
@@ -75,9 +79,9 @@ GeoipProviderConfig::GeoipProviderConfig(
7579
apple_private_relay_header_ = !geo_headers_to_add.apple_private_relay().empty()
7680
? absl::make_optional(geo_headers_to_add.apple_private_relay())
7781
: absl::nullopt;
78-
if (!city_db_path_ && !anon_db_path_ && !asn_db_path_ && !isp_db_path_) {
82+
if (!city_db_path_ && !anon_db_path_ && !asn_db_path_ && !isp_db_path_ && !country_db_path_) {
7983
throw EnvoyException("At least one geolocation database path needs to be configured: "
80-
"city_db_path, isp_db_path, asn_db_path or anon_db_path");
84+
"city_db_path, isp_db_path, asn_db_path, anon_db_path or country_db_path");
8185
}
8286
if (city_db_path_) {
8387
registerGeoDbStats(CITY_DB_TYPE);
@@ -91,6 +95,9 @@ GeoipProviderConfig::GeoipProviderConfig(
9195
if (asn_db_path_) {
9296
registerGeoDbStats(ASN_DB_TYPE);
9397
}
98+
if (country_db_path_) {
99+
registerGeoDbStats(COUNTRY_DB_TYPE);
100+
}
94101
};
95102

96103
void GeoipProviderConfig::registerGeoDbStats(const absl::string_view& db_type) {
@@ -126,6 +133,9 @@ GeoipProvider::GeoipProvider(Event::Dispatcher& dispatcher, Api::Api& api,
126133
config_->anonDbPath() ? initMaxmindDb(config_->anonDbPath().value(), ANON_DB_TYPE) : nullptr;
127134
asn_db_ =
128135
config_->asnDbPath() ? initMaxmindDb(config_->asnDbPath().value(), ASN_DB_TYPE) : nullptr;
136+
country_db_ = config_->countryDbPath()
137+
? initMaxmindDb(config_->countryDbPath().value(), COUNTRY_DB_TYPE)
138+
: nullptr;
129139
mmdb_reload_dispatcher_ = api.allocateDispatcher("mmdb_reload_routine");
130140
mmdb_watcher_ = dispatcher.createFilesystemWatcher();
131141
mmdb_reload_thread_ = api.threadFactory().createThread(
@@ -157,6 +167,13 @@ GeoipProvider::GeoipProvider(Event::Dispatcher& dispatcher, Api::Api& api,
157167
return onMaxmindDbUpdate(config_->asnDbPath().value(), ASN_DB_TYPE);
158168
}));
159169
}
170+
if (config_->countryDbPath()) {
171+
THROW_IF_NOT_OK(mmdb_watcher_->addWatch(
172+
config_->countryDbPath().value(), Filesystem::Watcher::Events::MovedTo,
173+
[this](uint32_t) {
174+
return onMaxmindDbUpdate(config_->countryDbPath().value(), COUNTRY_DB_TYPE);
175+
}));
176+
}
160177
mmdb_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit);
161178
},
162179
Thread::Options{std::string("mmdb_reload_routine")});
@@ -177,6 +194,7 @@ void GeoipProvider::lookup(Geolocation::LookupRequest&& request,
177194
Geolocation::LookupGeoHeadersCallback&& cb) const {
178195
auto& remote_address = request.remoteAddress();
179196
auto lookup_result = absl::flat_hash_map<std::string, std::string>{};
197+
lookupInCountryDb(remote_address, lookup_result);
180198
lookupInCityDb(remote_address, lookup_result);
181199
lookupInAsnDb(remote_address, lookup_result);
182200
lookupInAnonDb(remote_address, lookup_result);
@@ -187,9 +205,12 @@ void GeoipProvider::lookup(Geolocation::LookupRequest&& request,
187205
void GeoipProvider::lookupInCityDb(
188206
const Network::Address::InstanceConstSharedPtr& remote_address,
189207
absl::flat_hash_map<std::string, std::string>& lookup_result) const {
208+
// Country lookup falls back to City DB only if Country DB is not configured.
209+
const bool should_lookup_country_from_city_db =
210+
!config_->isCountryDbPathSet() && config_->isLookupEnabledForHeader(config_->countryHeader());
190211
if (config_->isLookupEnabledForHeader(config_->cityHeader()) ||
191212
config_->isLookupEnabledForHeader(config_->regionHeader()) ||
192-
config_->isLookupEnabledForHeader(config_->countryHeader())) {
213+
should_lookup_country_from_city_db) {
193214
int mmdb_error;
194215
auto city_db_ptr = getCityDb();
195216
// Used for testing.
@@ -217,7 +238,8 @@ void GeoipProvider::lookupInCityDb(
217238
config_->regionHeader().value(), MMDB_REGION_LOOKUP_ARGS[0],
218239
MMDB_REGION_LOOKUP_ARGS[1], MMDB_REGION_LOOKUP_ARGS[2]);
219240
}
220-
if (config_->isLookupEnabledForHeader(config_->countryHeader())) {
241+
// Country lookup from City DB only when Country DB is not configured.
242+
if (should_lookup_country_from_city_db) {
221243
populateGeoLookupResult(mmdb_lookup_result, lookup_result,
222244
config_->countryHeader().value(), MMDB_COUNTRY_LOOKUP_ARGS[0],
223245
MMDB_COUNTRY_LOOKUP_ARGS[1]);
@@ -384,6 +406,52 @@ void GeoipProvider::lookupInIspDb(
384406
}
385407
}
386408

409+
void GeoipProvider::lookupInCountryDb(
410+
const Network::Address::InstanceConstSharedPtr& remote_address,
411+
absl::flat_hash_map<std::string, std::string>& lookup_result) const {
412+
if (config_->isLookupEnabledForHeader(config_->countryHeader())) {
413+
// Country DB takes precedence if configured, otherwise fall back to City DB.
414+
if (!config_->isCountryDbPathSet()) {
415+
// Country lookup will be handled by lookupInCityDb.
416+
return;
417+
}
418+
int mmdb_error;
419+
auto country_db_ptr = getCountryDb();
420+
// Used for testing.
421+
synchronizer_.syncPoint(std::string(COUNTRY_DB_TYPE).append("_lookup_pre_complete"));
422+
if (!country_db_ptr) {
423+
if (config_->isCityDbPathSet()) {
424+
// Country information can be looked up from City database as well, so we don't need to
425+
// throw an error if it is not set.
426+
return;
427+
}
428+
IS_ENVOY_BUG("Maxmind country database must be initialised for performing lookups");
429+
return;
430+
}
431+
auto country_db = country_db_ptr.get();
432+
MMDB_lookup_result_s mmdb_lookup_result = MMDB_lookup_sockaddr(
433+
country_db->mmdb(), reinterpret_cast<const sockaddr*>(remote_address->sockAddr()),
434+
&mmdb_error);
435+
const uint32_t n_prev_hits = lookup_result.size();
436+
if (!mmdb_error && mmdb_lookup_result.found_entry) {
437+
MMDB_entry_data_list_s* entry_data_list;
438+
int status = MMDB_get_entry_data_list(&mmdb_lookup_result.entry, &entry_data_list);
439+
if (status == MMDB_SUCCESS) {
440+
populateGeoLookupResult(mmdb_lookup_result, lookup_result,
441+
config_->countryHeader().value(), MMDB_COUNTRY_LOOKUP_ARGS[0],
442+
MMDB_COUNTRY_LOOKUP_ARGS[1]);
443+
if (lookup_result.size() > n_prev_hits) {
444+
config_->incHit(COUNTRY_DB_TYPE);
445+
}
446+
MMDB_free_entry_data_list(entry_data_list);
447+
} else {
448+
config_->incLookupError(COUNTRY_DB_TYPE);
449+
}
450+
}
451+
config_->incTotal(COUNTRY_DB_TYPE);
452+
}
453+
}
454+
387455
MaxmindDbSharedPtr GeoipProvider::initMaxmindDb(const std::string& db_path,
388456
const absl::string_view& db_type, bool reload) {
389457
MMDB_s maxmind_db;
@@ -422,6 +490,9 @@ absl::Status GeoipProvider::mmdbReload(const MaxmindDbSharedPtr reloaded_db,
422490
} else if (db_type == ASN_DB_TYPE) {
423491
updateAsnDb(reloaded_db);
424492
config_->incDbReloadSuccess(db_type);
493+
} else if (db_type == COUNTRY_DB_TYPE) {
494+
updateCountryDb(reloaded_db);
495+
config_->incDbReloadSuccess(db_type);
425496
} else {
426497
ENVOY_LOG(error, "Unsupported maxmind db type {}", db_type);
427498
return absl::InvalidArgumentError(fmt::format("Unsupported maxmind db type {}", db_type));
@@ -472,6 +543,17 @@ void GeoipProvider::updateAnonDb(MaxmindDbSharedPtr anon_db) ABSL_LOCKS_EXCLUDED
472543
anon_db_ = anon_db;
473544
}
474545

546+
MaxmindDbSharedPtr GeoipProvider::getCountryDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
547+
absl::ReaderMutexLock lock(&mmdb_mutex_);
548+
return country_db_;
549+
}
550+
551+
void GeoipProvider::updateCountryDb(MaxmindDbSharedPtr country_db)
552+
ABSL_LOCKS_EXCLUDED(mmdb_mutex_) {
553+
absl::MutexLock lock(&mmdb_mutex_);
554+
country_db_ = country_db;
555+
}
556+
475557
absl::Status GeoipProvider::onMaxmindDbUpdate(const std::string& db_path,
476558
const absl::string_view& db_type) {
477559
MaxmindDbSharedPtr reloaded_db = initMaxmindDb(db_path, db_type, true /* reload */);

‎source/extensions/geoip_providers/maxmind/geoip_provider.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,13 @@ class GeoipProviderConfig {
2323
const absl::optional<std::string>& ispDbPath() const { return isp_db_path_; }
2424
const absl::optional<std::string>& anonDbPath() const { return anon_db_path_; }
2525
const absl::optional<std::string>& asnDbPath() const { return asn_db_path_; }
26+
const absl::optional<std::string>& countryDbPath() const { return country_db_path_; }
2627

2728
bool isLookupEnabledForHeader(const absl::optional<std::string>& header);
2829
bool isAsnDbPathSet() const { return asn_db_path_.has_value(); }
2930
bool isIspDbPathSet() const { return isp_db_path_.has_value(); }
31+
bool isCountryDbPathSet() const { return country_db_path_.has_value(); }
32+
bool isCityDbPathSet() const { return city_db_path_.has_value(); }
3033

3134
const absl::optional<std::string>& countryHeader() const { return country_header_; }
3235
const absl::optional<std::string>& cityHeader() const { return city_header_; }
@@ -82,6 +85,7 @@ class GeoipProviderConfig {
8285
absl::optional<std::string> isp_db_path_;
8386
absl::optional<std::string> anon_db_path_;
8487
absl::optional<std::string> asn_db_path_;
88+
absl::optional<std::string> country_db_path_;
8589

8690
absl::optional<std::string> country_header_;
8791
absl::optional<std::string> city_header_;
@@ -140,6 +144,7 @@ class GeoipProvider : public Envoy::Geolocation::Driver,
140144
MaxmindDbSharedPtr isp_db_ ABSL_GUARDED_BY(mmdb_mutex_);
141145
MaxmindDbSharedPtr anon_db_ ABSL_GUARDED_BY(mmdb_mutex_);
142146
MaxmindDbSharedPtr asn_db_ ABSL_GUARDED_BY(mmdb_mutex_);
147+
MaxmindDbSharedPtr country_db_ ABSL_GUARDED_BY(mmdb_mutex_);
143148
Thread::ThreadPtr mmdb_reload_thread_;
144149
Event::DispatcherPtr mmdb_reload_dispatcher_;
145150
Filesystem::WatcherPtr mmdb_watcher_;
@@ -153,6 +158,8 @@ class GeoipProvider : public Envoy::Geolocation::Driver,
153158
absl::flat_hash_map<std::string, std::string>& lookup_result) const;
154159
void lookupInIspDb(const Network::Address::InstanceConstSharedPtr& remote_address,
155160
absl::flat_hash_map<std::string, std::string>& lookup_result) const;
161+
void lookupInCountryDb(const Network::Address::InstanceConstSharedPtr& remote_address,
162+
absl::flat_hash_map<std::string, std::string>& lookup_result) const;
156163
absl::Status onMaxmindDbUpdate(const std::string& db_path, const absl::string_view& db_type);
157164
absl::Status mmdbReload(const MaxmindDbSharedPtr reloaded_db, const absl::string_view& db_type)
158165
ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
@@ -164,10 +171,12 @@ class GeoipProvider : public Envoy::Geolocation::Driver,
164171
MaxmindDbSharedPtr getIspDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
165172
MaxmindDbSharedPtr getAnonDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
166173
MaxmindDbSharedPtr getAsnDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
174+
MaxmindDbSharedPtr getCountryDb() const ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
167175
void updateCityDb(MaxmindDbSharedPtr city_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
168176
void updateIspDb(MaxmindDbSharedPtr isp_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
169177
void updateAnonDb(MaxmindDbSharedPtr anon_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
170178
void updateAsnDb(MaxmindDbSharedPtr asn_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
179+
void updateCountryDb(MaxmindDbSharedPtr country_db) ABSL_LOCKS_EXCLUDED(mmdb_mutex_);
171180
// A shared_ptr to keep the provider singleton alive as long as any of its providers are in use.
172181
const Singleton::InstanceSharedPtr owner_;
173182
// Used for testing only.

‎test/extensions/filters/http/geoip/geoip_filter_integration_test.cc‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,40 @@ const std::string ConfigIsApplePrivateRelayOnly = R"EOF(
115115
isp_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-ISP-Test.mmdb"
116116
)EOF";
117117

118+
const std::string ConfigWithCountryDb = R"EOF(
119+
name: envoy.filters.http.geoip
120+
typed_config:
121+
"@type": type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip
122+
xff_config:
123+
xff_num_trusted_hops: 1
124+
provider:
125+
name: envoy.geoip_providers.maxmind
126+
typed_config:
127+
"@type": type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig
128+
common_provider_config:
129+
geo_headers_to_add:
130+
country: "x-geo-country"
131+
country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb"
132+
)EOF";
133+
134+
const std::string ConfigWithCountryDbAndCityDb = R"EOF(
135+
name: envoy.filters.http.geoip
136+
typed_config:
137+
"@type": type.googleapis.com/envoy.extensions.filters.http.geoip.v3.Geoip
138+
xff_config:
139+
xff_num_trusted_hops: 1
140+
provider:
141+
name: envoy.geoip_providers.maxmind
142+
typed_config:
143+
"@type": type.googleapis.com/envoy.extensions.geoip_providers.maxmind.v3.MaxMindConfig
144+
common_provider_config:
145+
geo_headers_to_add:
146+
country: "x-geo-country"
147+
city: "x-geo-city"
148+
country_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoIP2-Country-Test.mmdb"
149+
city_db_path: "{{ test_rundir }}/test/extensions/geoip_providers/maxmind/test_data/GeoLite2-City-Test.mmdb"
150+
)EOF";
151+
118152
class GeoipFilterIntegrationTest : public testing::TestWithParam<Network::Address::IpVersion>,
119153
public HttpIntegrationTest {
120154
public:
@@ -356,6 +390,47 @@ TEST_P(GeoipFilterIntegrationTest, MetricForDbBuildEpochIsEmitted) {
356390
test_server_->gauge("http.config_test.maxmind.city_db.db_build_epoch")->value());
357391
}
358392

393+
TEST_P(GeoipFilterIntegrationTest, GeoDataPopulatedUseCountryDb) {
394+
config_helper_.prependFilter(TestEnvironment::substitute(ConfigWithCountryDb));
395+
initialize();
396+
codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http")));
397+
Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"},
398+
{":path", "/"},
399+
{":scheme", "http"},
400+
{":authority", "host"},
401+
{"x-forwarded-for", "216.160.83.56,9.10.11.12"}};
402+
auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0);
403+
EXPECT_EQ("US", headerValue("x-geo-country"));
404+
ASSERT_TRUE(response->complete());
405+
EXPECT_EQ("200", response->headers().getStatusValue());
406+
test_server_->waitForCounterEq("http.config_test.geoip.total", 1);
407+
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.total")->value());
408+
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.hit")->value());
409+
}
410+
411+
TEST_P(GeoipFilterIntegrationTest, GeoDataPopulatedUseCountryDbAndCityDb) {
412+
config_helper_.prependFilter(TestEnvironment::substitute(ConfigWithCountryDbAndCityDb));
413+
initialize();
414+
codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http")));
415+
Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"},
416+
{":path", "/"},
417+
{":scheme", "http"},
418+
{":authority", "host"},
419+
{"x-forwarded-for", "216.160.83.56,9.10.11.12"}};
420+
auto response = sendRequestAndWaitForResponse(request_headers, 0, default_response_headers_, 0);
421+
EXPECT_EQ("US", headerValue("x-geo-country"));
422+
EXPECT_EQ("Milton", headerValue("x-geo-city"));
423+
ASSERT_TRUE(response->complete());
424+
EXPECT_EQ("200", response->headers().getStatusValue());
425+
test_server_->waitForCounterEq("http.config_test.geoip.total", 1);
426+
// Country should be looked up from Country DB.
427+
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.total")->value());
428+
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.country_db.hit")->value());
429+
// City should be looked up from City DB, but country should NOT be looked up from City DB.
430+
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.total")->value());
431+
EXPECT_EQ(1, test_server_->counter("http.config_test.maxmind.city_db.hit")->value());
432+
}
433+
359434
} // namespace
360435
} // namespace Geoip
361436
} // namespace HttpFilters

0 commit comments

Comments
 (0)