diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..bdac45f --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,8 @@ +image: Visual Studio 2017 +environment: + matrix: + - CMAKE_GENERATOR: "Visual Studio 15 2017 Win64" + - CMAKE_GENERATOR: "Visual Studio 15 2017" +build_script: + - cmd: git submodule update --init --recursive + - cmd: .\.ci\common\script\appveyor.bat "%CMAKE_GENERATOR%" diff --git a/.ci/common b/.ci/common new file mode 160000 index 0000000..30a13d9 --- /dev/null +++ b/.ci/common @@ -0,0 +1 @@ +Subproject commit 30a13d983baa92775306f99708762652e51dc500 diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..37e1393 --- /dev/null +++ b/.clang-format @@ -0,0 +1,4 @@ +BasedOnStyle: Google +ColumnLimit: 0 +AllowShortIfStatementsOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0593f08 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +/ALL_BUILD.vcxproj +/ALL_BUILD.vcxproj.filters +/CMakeCache.txt +/CMakeFiles/ +/CTestTestfile.cmake +/GeoLite2-ASN_20*/ +/GeoLite2-Country_20*/ +/INSTALL.vcxproj +/INSTALL.vcxproj.filters +/MK_DIST/ +/Makefile +/RUN_TESTS.vcxproj +/RUN_TESTS.vcxproj.filters +/Release/ +/Testing +/ZERO_CHECK.vcxproj +/ZERO_CHECK.vcxproj.filters +/argh.h +/asn.mmdb +/ca-bundle.pem +/catch.hpp +/cmake_install.cmake +/country.mmdb +/geolite2-asn.tar.gz +/geolite2-country.tar.gz +/install_manifest.txt +/libmkmmdb.a +/mkcurl.h +/mkdata.h +/mkmmdb-client +/mkmmdb-client.dir/ +/mkmmdb-client.vcxproj +/mkmmdb-client.vcxproj.filters +/mkmmdb.sln +/windows-curl-*.tar.gz +/windows-libmaxminddb-*.tar.gz +/x64/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..fed7891 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "cmake/Modules"] + path = cmake/Modules + url = https://github.com/measurement-kit/cmake-modules +[submodule ".ci/common"] + path = .ci/common + url = https://github.com/measurement-kit/ci-common diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c943e8a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: c++ +services: + - docker +sudo: required +matrix: + include: + - env: BUILD_TYPE="asan" + - env: BUILD_TYPE="clang" + - env: BUILD_TYPE="coverage" + - env: BUILD_TYPE="lsan" + - env: BUILD_TYPE="ubsan" + - env: BUILD_TYPE="vanilla" +script: + - ./.ci/common/script/travis $BUILD_TYPE diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..59989f4 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Simone Basso diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..9518e26 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,65 @@ +cmake_minimum_required(VERSION 3.1.0) +project(mkmmdb LANGUAGES CXX) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") +include(MkUtils) + +# Download dependencies +# --------------------- + +MkDownloadMMDBDatabases() + +if("${MSVC}") + MkDownloadMeasurementKitPrebuiltWindowsLibmaxminddb() + list(APPEND CMAKE_REQUIRED_INCLUDES "${MK_WINDOWS_LIBMAXMINDDB_INCLUDE_PATH}") + list(APPEND CMAKE_LIBRARY_PATH "${MK_WINDOWS_LIBMAXMINDDB_LIBRARY_PATH}") +endif() + +# Checks +# ------ + +include(CheckIncludeFiles) + +check_include_files(maxminddb.h MKMMDB_HAVE_MAXMINDDB_H) +find_library(LIBMAXMINDDB_LIBRARY maxminddb) +if (("${MKMMDB_HAVE_MAXMINDDB_H}" STREQUAL "") OR + ("${LIBMAXMINDDB_LIBRARY}" STREQUAL "LIBMAXMINDDB_LIBRARY-NOTFOUND")) + message(FATAL_ERROR "Cannot find libmaxminddb") +endif() +list(APPEND MKMMDB_LIBS "${LIBMAXMINDDB_LIBRARY}") + +find_package(CURL REQUIRED) + +# Compiler flags +# -------------- + +MkSetCompilerFlags() + +# Library and binary +# ------------------ + +set(MKMMDB_INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} + ${CURL_INCLUDE_DIRS} ${CMAKE_REQUIRED_INCLUDES}) + +add_library(mkmmdb mkmmdb.cpp) +add_executable(mkmmdb-client mkmmdb-client.cpp) +target_include_directories(mkmmdb-client PUBLIC ${MKMMDB_INCLUDES}) +if("${WIN32}" OR "${MINGW}") + list(APPEND MKMMDB_LIBS "ws2_32") + if ("${MINGW}") + list(APPEND MKMMDB_LIBS -static-libgcc -static-libstdc++) + endif() +endif() +list(APPEND MKMMDB_LIBS Threads::Threads) +target_link_libraries(mkmmdb-client "${MKMMDB_LIBS}" mkmmdb) + +# Testing +# ------- + +set(BUILD_TESTING "ON" CACHE BOOL "Whether to build tests") +if(${BUILD_TESTING}) + enable_testing() + add_test(NAME basic_test COMMAND mkmmdb-client 8.8.8.8) + add_test(NAME name_not_found_test COMMAND mkmmdb-client 127.0.0.1) + add_test(NAME invalid_ip_test COMMAND mkmmdb-client 8.8.8.x) +endif() diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4cb6d13 --- /dev/null +++ b/LICENSE @@ -0,0 +1,26 @@ +Copyright 2018 Open Observatory of Network Interference (OONI), The Tor Project + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..4de100a --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# Measurement Kit MMDB wrappers + +[![GitHub license](https://img.shields.io/github/license/measurement-kit/mkmmdb.svg)](https://raw.githubusercontent.com/measurement-kit/mkmmdb/master/LICENSE) [![Github Releases](https://img.shields.io/github/release/measurement-kit/mkmmdb.svg)](https://github.com/measurement-kit/mkmmdb/releases) [![Build Status](https://img.shields.io/travis/measurement-kit/mkmmdb/master.svg?label=travis)](https://travis-ci.org/measurement-kit/mkmmdb) [![codecov](https://codecov.io/gh/measurement-kit/mkmmdb/branch/master/graph/badge.svg)](https://codecov.io/gh/measurement-kit/mkmmdb) [![Build status](https://img.shields.io/appveyor/ci/bassosimone/mkmmdb/master.svg?label=appveyor)](https://ci.appveyor.com/project/bassosimone/mkmmdb/branch/master) + +Experimental library to use libmaxminddb from Measurement Kit. diff --git a/cmake/Modules b/cmake/Modules new file mode 160000 index 0000000..7041216 --- /dev/null +++ b/cmake/Modules @@ -0,0 +1 @@ +Subproject commit 70412163c491e575bb7d33b2b729d7cb15cd016a diff --git a/mkmmdb-client.cpp b/mkmmdb-client.cpp new file mode 100644 index 0000000..8679bdd --- /dev/null +++ b/mkmmdb-client.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + +#include "mkmmdb.h" + +int main(int argc, char **argv) { + if (argc != 2) { + std::clog << "Usage: mkmmdb-client " << std::endl; + exit(EXIT_FAILURE); + } + std::string ip = argv[1]; + { + mkmmdb_uptr db{mkmmdb_open("country.mmdb")}; + if (!mkmmdb_good(db.get())) { + std::clog << "Cannot open country.mmdb" << std::endl; + // FALLTHROUGH + } + std::string s = mkmmdb_lookup_cc(db.get(), ip.c_str()); + std::clog << "=== BEGIN LOGS === " << std::endl; + std::clog << mkmmdb_get_last_lookup_logs(db.get()); + std::clog << "=== END LOGS === " << std::endl; + std::clog << "CC: " << s << std::endl; + } + { + mkmmdb_uptr db{mkmmdb_open("asn.mmdb")}; + if (!mkmmdb_good(db.get())) { + std::clog << "Cannot open asn.mmdb" << std::endl; + // FALLTHROUGH + } + { + std::string s = mkmmdb_lookup_org(db.get(), ip.c_str()); + std::clog << "=== BEGIN LOGS === " << std::endl; + std::clog << mkmmdb_get_last_lookup_logs(db.get()); + std::clog << "=== END LOGS === " << std::endl; + std::clog << "ORG: " << s << std::endl; + } + { + int64_t v = mkmmdb_lookup_asn(db.get(), ip.c_str()); + std::clog << "=== BEGIN LOGS === " << std::endl; + std::clog << mkmmdb_get_last_lookup_logs(db.get()); + std::clog << "=== END LOGS === " << std::endl; + std::clog << "ASN: " << v << std::endl; + } + } +} diff --git a/mkmmdb.cpp b/mkmmdb.cpp new file mode 100644 index 0000000..5e0533f --- /dev/null +++ b/mkmmdb.cpp @@ -0,0 +1,2 @@ +#define MKMMDB_INLINE_IMPL +#include "mkmmdb.h" diff --git a/mkmmdb.h b/mkmmdb.h new file mode 100644 index 0000000..b9f0262 --- /dev/null +++ b/mkmmdb.h @@ -0,0 +1,286 @@ +// Part of Measurement Kit . +// Measurement Kit is free software under the BSD license. See AUTHORS +// and LICENSE for more information on the copying conditions. +#ifndef MEASUREMENT_KIT_MKMMDB_H +#define MEASUREMENT_KIT_MKMMDB_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// mkmmdb_t is a MMDB database. +typedef struct mkmmdb mkmmdb_t; + +/// mkmmdb_open opens a database pointing to @p path. It always returns a +/// valid pointer. Call mkmmdb_good to understand whether the database was +/// successfully openned or not. This function will call std::abort when +/// the @p path argument is a null pointer. +mkmmdb_t *mkmmdb_open(const char *path); + +/// mkmmdb_good returns true if the database has been successfully open +/// and false otherwise. This function aborts if @p mmdb is null. +int64_t mkmmdb_good(mkmmdb_t *mmdb); + +/// mkmmdb_lookup_cc returns the country code of @p ip using the +/// @p mmdb database, or the empty string on failure. The returned string +/// will be valid until @p mmdb is valid _and_ you don't call other +/// functions using the same @p mmdb instance. This function calls +/// std::abort if passed null parameters. +const char *mkmmdb_lookup_cc(mkmmdb_t *mmdb, const char *ip); + +/// mkmmdb_lookup_asn is like mkmmdb_lookup_cc but returns +/// the ASN on success and zero on failure. Also this function +/// calls std::abort if passed null parameters. +int64_t mkmmdb_lookup_asn(mkmmdb_t *mmdb, const char *ip); + +/// mkmmdb_lookup_org is like mkmmdb_lookup_cc but returns +/// the organization bound to @p ip on success, the empty string on +/// failure. The returned string will be valid until @p mmdb is +/// valid _and_ you don't call other lookup APIs using the +/// same @p mmdb instance. This function calls std::abort if +/// passed a null @p mmdb or a null @p ip. +const char *mkmmdb_lookup_org(mkmmdb_t *mmdb, const char *ip); + +/// mkmmdb_get_last_lookup_logs returns the logs of the last lookup +/// or an empty string. Calls std::abort if @p mmdb is null. +const char *mkmmdb_get_last_lookup_logs(mkmmdb_t *mmdb); + +/// mkmmdb_close closes @p mmdb. Note that @p mmdb MAY be null. +void mkmmdb_close(mkmmdb_t *mmdb); + +#ifdef __cplusplus +} // extern "C" + +#include +#include + +/// mkmmdb_deleter is a deleter for mkmmdb_t. +struct mkmmdb_deleter { + void operator()(mkmmdb_t *p) { mkmmdb_close(p); } +}; + +/// mkmmdb_uptr is a unique pointer to mkmmdb_t. +using mkmmdb_uptr = std::unique_ptr; + +// By default the implementation is not included. You can force it being +// included by providing the following definition to the compiler. +// +// If you're just into understanding the API, you can stop reading here. +#ifdef MKMMDB_INLINE_IMPL + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include +#include + +#include + +// mkMMDB_s_deleter is a deleter for a MMDB_s pointer. +struct mkMMDB_s_deleter { + void operator()(MMDB_s *p) { + MMDB_close(p); + delete p; + } +}; + +// mkMMDB_s_uptr is a unique pointer to MMDB_s. +using mkMMDB_s_uptr = std::unique_ptr; + +// mkmmdb is a MMDB database. +struct mkmmdb { + // mmdbs is a unique pointer to the real database instance. + mkMMDB_s_uptr mmdbs; + // last_lookup_result is the place where we save the latest lookup result. + std::string last_lookup_result; + // last_lookup_logs contains the logs of the last lookup. + std::string last_lookup_logs; +}; + +#ifndef MKMMDB_MMDB_OPEN +// MKMMDB_MMDB_OPEN allows to mock MMDB_open +#define MKMMDB_MMDB_OPEN MMDB_open +#endif + +#ifndef MKMMDB_ABORT +// MKMMDB_ABORT allows to mock std::abort +#define MKMMDB_ABORT std::abort +#endif + +mkmmdb_t *mkmmdb_open(const char *path) { + if (path == nullptr) { + MKMMDB_ABORT(); + } + mkmmdb_uptr mmdb{new mkmmdb_t}; + mmdb->mmdbs.reset(new MMDB_s); + if (MKMMDB_MMDB_OPEN(path, MMDB_MODE_MMAP, mmdb->mmdbs.get()) != 0) { + mmdb->mmdbs = nullptr; + } + return mmdb.release(); +} + +int64_t mkmmdb_good(mkmmdb_t *mmdb) { + if (mmdb == nullptr) { + MKMMDB_ABORT(); + } + return mmdb->mmdbs != nullptr; +} + +#ifndef MKMMDB_MMDB_GET_VALUE +// MKMMDB_MMDB_GET_VALUE allows to mock MMDB_get_value +#define MKMMDB_MMDB_GET_VALUE MMDB_get_value +#endif + +#ifndef MKMMDB_MMDB_LOOKUP_STRING +// MKMMDB_MMDB_LOOKUP_STRING allows to mock MMDB_lookup_string +#define MKMMDB_MMDB_LOOKUP_STRING MMDB_lookup_string +#endif + +static void mkmmdb_lookup_mmdb( + mkmmdb_t *mmdb, const std::string &ip, + std::function fun) { + if (mmdb == nullptr || fun == nullptr) { + MKMMDB_ABORT(); + } + { + mmdb->last_lookup_logs += "Start lookup for: "; + mmdb->last_lookup_logs += ip; + mmdb->last_lookup_logs += "\n"; + } + if (mmdb->mmdbs == nullptr) { + mmdb->last_lookup_logs += "The database is not open.\n"; + return; + } + auto gai_error = 0; + auto mmdb_error = 0; + auto record = MKMMDB_MMDB_LOOKUP_STRING(mmdb->mmdbs.get(), ip.c_str(), + &gai_error, &mmdb_error); + if (gai_error != 0) { + mmdb->last_lookup_logs += "mmdb_lookup_string: "; + mmdb->last_lookup_logs += gai_strerror(gai_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (mmdb_error != 0) { + mmdb->last_lookup_logs += "mmdb_lookup_string: "; + mmdb->last_lookup_logs += MMDB_strerror(mmdb_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (!record.found_entry) { + mmdb->last_lookup_logs += "mmdb_lookup_string: entry not found.\n"; + return; + } + fun(&record.entry); +} + +const char *mkmmdb_lookup_cc(mkmmdb_t *mmdb, const char *ip) { + if (mmdb == nullptr || ip == nullptr) { + MKMMDB_ABORT(); + } + mmdb->last_lookup_result = ""; + mmdb->last_lookup_logs = ""; + mkmmdb_lookup_mmdb( + mmdb, ip, [&](MMDB_entry_s *entry) { + MMDB_entry_data_s data{}; + auto mmdb_error = MKMMDB_MMDB_GET_VALUE( + entry, &data, "registered_country", "iso_code", nullptr); + if (mmdb_error != 0) { + mmdb->last_lookup_logs += "mmdb_get_value: "; + mmdb->last_lookup_logs += MMDB_strerror(mmdb_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (!data.has_data) { + mmdb->last_lookup_logs += "mmdb_get_value: no data for entry.\n"; + return; + } + if (data.type != MMDB_DATA_TYPE_UTF8_STRING) { + mmdb->last_lookup_logs += "mmdb_get_value: unexpected data type.\n"; + return; + } + mmdb->last_lookup_result = std::string{ + data.utf8_string, data.data_size}; + }); + return mmdb->last_lookup_result.c_str(); +} + +int64_t mkmmdb_lookup_asn(mkmmdb_t *mmdb, const char *ip) { + if (mmdb == nullptr || ip == nullptr) { + MKMMDB_ABORT(); + } + mmdb->last_lookup_logs = ""; + int64_t rv = {}; + mkmmdb_lookup_mmdb( + mmdb, ip, [&](MMDB_entry_s *entry) { + MMDB_entry_data_s data{}; + auto mmdb_error = MKMMDB_MMDB_GET_VALUE( + entry, &data, "autonomous_system_number", nullptr); + if (mmdb_error != 0) { + mmdb->last_lookup_logs += "mmdb_get_value: "; + mmdb->last_lookup_logs += MMDB_strerror(mmdb_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (!data.has_data) { + mmdb->last_lookup_logs += "mmdb_get_value: no data for entry.\n"; + return; + } + if (data.type != MMDB_DATA_TYPE_UINT32) { + mmdb->last_lookup_logs += "mmdb_get_value: unexpected data type.\n"; + return; + } + rv = data.uint32; + }); + return rv; +} + +const char *mkmmdb_lookup_org(mkmmdb_t *mmdb, const char *ip) { + if (mmdb == nullptr || ip == nullptr) { + MKMMDB_ABORT(); + } + mmdb->last_lookup_logs = ""; + mmdb->last_lookup_result = ""; + mkmmdb_lookup_mmdb( + mmdb, ip, [&](MMDB_entry_s *entry) { + MMDB_entry_data_s data{}; + auto mmdb_error = MKMMDB_MMDB_GET_VALUE( + entry, &data, "autonomous_system_organization", nullptr); + if (mmdb_error != 0) { + mmdb->last_lookup_logs += "mmdb_get_value: "; + mmdb->last_lookup_logs += MMDB_strerror(mmdb_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (!data.has_data) { + mmdb->last_lookup_logs += "mmdb_get_value: no data for entry.\n"; + return; + } + if (data.type != MMDB_DATA_TYPE_UTF8_STRING) { + mmdb->last_lookup_logs += "mmdb_get_value: unexpected data type.\n"; + return; + } + mmdb->last_lookup_result = std::string{ + data.utf8_string, data.data_size}; + }); + return mmdb->last_lookup_result.c_str(); +} + +const char *mkmmdb_get_last_lookup_logs(mkmmdb_t *mmdb) { + if (mmdb == nullptr) { + MKMMDB_ABORT(); + } + return mmdb->last_lookup_logs.c_str(); +} + +void mkmmdb_close(mkmmdb_t *mmdb) { delete mmdb; } + +#endif // MKMMDB_INLINE_IMPL +#endif // __cplusplus +#endif // MEASUREMENT_KIT_MKMMDB_H