Add support for GeoIP2 DB (#60)

- This commit will be tagged 1.50.0 Signed-off-by: Ondrej Vasko <ondrej.vaskoo@gmail.com>
Vnet-as · Mar 27, 2021 · 4bb1653 · 4bb1653
1 parent 8300d9d
commit 4bb1653
Show file tree

Hide file tree

Showing 17 changed files with 362 additions and 68 deletions.
diff --git a/.github/workflows/docker-compose-integration-test.yml b/.github/workflows/docker-compose-integration-test.yml
@@ -26,48 +26,74 @@ jobs:
         continue-on-error: true
         run: perlcritic postfwd-anti-spam.plugin
 
-  integration-test-postgresql:
-    name: "integration test postgresql (perl v${{matrix.perl-version}})"
+  integration-test-postgresql-geoip1:
+    name: "integration test postgresql + geoip v1 (perl v${{matrix.perl-version}})"
     strategy:
       matrix:
         perl-version: [ '5.32', '5.30', '5.28' ]
     runs-on: ubuntu-latest
     env:
       DATABASES: "postgresql"
+      POSTFWD_ANTISPAM_MAIN_CONFIG_PATH: "/etc/postfwd/02-dev-anti-spam-postgres-geoip1.conf"
     steps:
       - uses: actions/checkout@v2
       - name: docker-compose up
-        run: docker-compose -f tests/dev-compose-postgresql.yml
+        run: docker-compose -f tests/compose-dev-postgresql.yml
                             up
                             --build
                             -d
              && sleep 10
       - name: run integration tests
-        run: ./tests/integration-compose-test.sh
+        run: ./tests/integration-compose-test-geoip1.sh
       - name: read test logs
-        run: docker-compose -f tests/dev-compose-postgresql.yml logs postfwd-geoip-antispam
+        run: docker-compose -f tests/compose-dev-postgresql.yml logs postfwd-geoip-antispam
       - name: docker-compose down
-        run: docker-compose -f tests/dev-compose-postgresql.yml down
+        run: docker-compose -f tests/compose-dev-postgresql.yml down
 
-  integration-test-mysql:
-    name: "integration test mysql (perl v${{matrix.perl-version}})"
+  integration-test-mysql-geoip1:
+    name: "integration test mysql + geoip v1 (perl v${{matrix.perl-version}})"
     strategy:
       matrix:
         perl-version: [ '5.32', '5.30', '5.28' ]
     runs-on: ubuntu-latest
     env:
       DATABASES: "mysql"
+      POSTFWD_ANTISPAM_MAIN_CONFIG_PATH: "/etc/postfwd/01-dev-anti-spam-mysql-geoip1.conf"
     steps:
       - uses: actions/checkout@v2
       - name: docker-compose up
-        run: docker-compose -f tests/dev-compose-mysql.yml
+        run: docker-compose -f tests/compose-dev-mysql.yml
                             up
                             --build
                             -d
              && sleep 10
       - name: run integration tests
-        run: ./tests/integration-compose-test.sh
+        run: ./tests/integration-compose-test-geoip1.sh
       - name: read test logs
-        run: docker-compose -f tests/dev-compose-mysql.yml logs postfwd-geoip-antispam
+        run: docker-compose -f tests/compose-dev-mysql.yml logs postfwd-geoip-antispam
       - name: docker-compose down
-        run: docker-compose -f tests/dev-compose-mysql.yml down
+        run: docker-compose -f tests/compose-dev-mysql.yml down
+
+  integration-test-mysql-geoip2:
+    name: "integration test mysql + geoip v2 (perl v${{matrix.perl-version}})"
+    strategy:
+      matrix:
+        perl-version: [ '5.32', '5.30', '5.28' ]
+    runs-on: ubuntu-latest
+    env:
+      DATABASES: "mysql"
+      POSTFWD_ANTISPAM_MAIN_CONFIG_PATH: "/etc/postfwd/03-dev-anti-spam-mysql-geoip2.conf"
+    steps:
+      - uses: actions/checkout@v2
+      - name: docker-compose up
+        run: docker-compose -f tests/compose-dev-mysql.yml
+                            up
+                            --build
+                            -d
+             && sleep 10
+      - name: run integration tests
+        run: ./tests/integration-compose-test-geoip2.sh
+      - name: read test logs
+        run: docker-compose -f tests/compose-dev-mysql.yml logs postfwd-geoip-antispam
+      - name: docker-compose down
+        run: docker-compose -f tests/compose-dev-mysql.yml down
diff --git a/cpanfile b/cpanfile
@@ -1,7 +1,14 @@
 requires 'Geo::IP';
+requires 'GeoIP2::Database::Reader';
 requires 'Time::Piece';
 requires 'Config::Any';
 requires 'DBI';
 requires 'DBD::mysql';
 requires 'DBD::Pg';
 requires 'Net::Subnet';
+requires 'Net::SSLeay';
+requires 'IO::Socket::SSL';
+requires 'IO::Socket::SSL::Utils';
+requires 'LWP::Protocol::https';
+requires 'Class::XSAccessor';
+requires 'MaxMind::DB::Reader::XS';
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,6 +1,6 @@
 FROM postfwd/postfwd:v2.02
 
-LABEL maintainer="Postfwd GeoIp Spam Plugin Maintainer <ondrej.vaskoo@gmail.com>"
+LABEL maintainer="Postfwd GeoIP Spam Plugin Maintainer <ondrej.vaskoo@gmail.com>"
 
 ENV POSTFWD_ANTISPAM_MAIN_CONFIG_PATH=/etc/postfwd/anti-spam.conf
 ENV POSTFWD_ANTISPAM_SQL_STATEMENTS_CONFIG_PATH=/etc/postfwd/anti-spam-sql-st.conf
@@ -18,6 +18,8 @@ RUN    apk --no-cache update \
                           geoip-dev \
                           postgresql-dev \
                           mysql-dev \
+                          openssl \
+                          libmaxminddb-dev \
     && cpan App::cpanminus \
     && cpanm Geo::IP \
              IO::Handle \
@@ -30,6 +32,13 @@ RUN    apk --no-cache update \
              DBD::mysql \
              Net::Subnet \
              Sys::Mmap \
+             Net::SSLeay \
+             IO::Socket::SSL::Utils \
+             IO::Socket::SSL \
+             LWP::Protocol::https \
+             GeoIP2::Database::Reader \
+             Class::XSAccessor \
+             MaxMind::DB::Reader::XS --force \
     && apk del make \
                wget \
                gcc \

diff --git a/docker/README.md b/docker/README.md
@@ -1,6 +1,7 @@
 # Supported tags and respective `Dockerfile` links
 
 * [`latest` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/master/docker/Dockerfile)
+* [`v1.50.0` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/v1.50.0/docker/Dockerfile)
 * [`v1.40` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/v1.40/docker/Dockerfile)
 * [`v1.30` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/v1.30/docker/Dockerfile)
 * [`v1.21` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/v1.21/docker/Dockerfile)
@@ -19,7 +20,6 @@ Pull image with `docker pull lirt/postfwd-anti-geoip-spam-plugin:latest`.
 
 Prepare your configuration files and run this docker image with following command:
 
-
 ```bash
 docker run -d \
     -v </absolute/path/to/anti-spam.conf>:/etc/postfwd/anti-spam.conf \
@@ -32,7 +32,7 @@ docker run -d \
 
 If you don't have database by your hand, but **want to try the plugin**, you can try it with `docker-compose` template located in [official GitHub repository](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/tree/master/tests) in directory `./tests`.
 
-Run it with `docker-compose -f dev-compose.yml up` - this will bootstrap ready-to-work environment, where you can try the plugin.
+Run it with `docker-compose -f compose-dev-mysql.yml up` - this will bootstrap ready-to-work environment, where you can try the plugin.
 
 Then you can run from local shell `nc 127.0.0.1 10040 < <(./dev-request.sh <IP_ADDRESS>)`, to send artificial request into postfwd and watch logs what is happening.
 

diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
@@ -18,12 +18,7 @@ for arg; do
   esac
 done
 
-
 if [ "$1" = "postfwd3" ] && [ -z "$want_help" ]; then
-  if [ ! -f /etc/postfwd/anti-spam.conf ]; then
-    echo >&2 'ERROR: Anti-spam plugin configuration file /etc/postfwd/anti-spam.conf was not found. Perhaps you forgot to mount it using "-v </absolute/path/to/anti-spam.conf>:/etc/postfwd/anti-spam.conf".'
-    exit 1
-  fi
   chmod -R 644 /etc/postfwd/*
   chown -R postfw:postfw /etc/postfwd
 fi

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -2,7 +2,22 @@
 
 This changelog notes changes between versions of Postfwd GeoIP Anti-Spam plugin.
 
-## Version 1.40 [2. Mar 2020]
+## Version 1.50.0 [27. March 2021]
+
+GeoIP2 Feature.
+
+### Breaking changes
+
+- New dependencies for GeoIP2 module were added, please see changes in `cpanfile`.
+
+### Features / Enhancements
+
+- Added GeoIP2 support. Now you can use Maxmind GeoIP2 databases. Path to GeoIP file can be changed in config `app.geoip_db_path`.
+- Removed entrypoint check for configuration file. The config file path can be overriden using environment variable `POSTFWD_ANTISPAM_MAIN_CONFIG_PATH`.
+- Added integration tests for GeoIP2 (test structure changed overall).
+
+
+## Version 1.40 [2. March 2020]
 
 This stable release contains IP whitelisting feature (Reported as bug and requested by @csazku in https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/issues/50).
 
@@ -53,7 +68,7 @@ and does integration test with sample requests and verification through logs.
 Plugin item now exports `request{client_uniq_ip_login_count}`
 and `request{client_uniq_country_login_count}` instead of `result*`.
 
-## Version 1.2 [11. Mar 2019]
+## Version 1.2 [11. March 2019]
 
 This stable release has changes mainly in linting, readability and testability, but also
 contains several bugfixes.
@@ -80,7 +95,7 @@ Docker base image was updated from 1.37 to 1.39.
 - Script `lint.pl` was removed and replaced by more general/portable file
 `.perlcriticrc`.
 
-## Version 1.1 [5. Jan 2019]
+## Version 1.1 [5. January 2019]
 
 This stable version introduces docker image based on official postfwd docker image
 and other minor changes. All work done by @Lirt (ondrej.vaskoo@gmail.com),
@@ -123,7 +138,7 @@ postfwd logs and this postfwd plugin logs.
 
 ### Removed
 
-## Version 1.0 [12. Nov 2018]
+## Version 1.0 [12. November 2018]
 
 This is first official version and release.
 

diff --git a/docs/README.md b/docs/README.md
@@ -29,11 +29,13 @@ If you are interested in how your users got their mail accounts hacked, check ou
 
 | Plugin Version | Postfwd Version          |
 | :------------- | :----------------------- |
+| v1.50.0        | postfwd3 v2.xx           |
 | v1.40          | postfwd3 v2.xx           |
 | v1.30          | postfwd3 v2.xx           |
 | v1.21          | postfwd1, postfwd2 v1.xx |
 
-Supported database backends are **MySQL** and **PostgreSQL**.
+- Supported database backends are **MySQL** and **PostgreSQL**.
+- Supported GeoIP databases are both versions 1 and 2.
 
 To list changed between versions check release notes or look into the [Changelog](CHANGELOG.md).
 
@@ -43,7 +45,7 @@ Pre-built ready-to-use Docker image is located on DockerHub and can be simply pu
 
 ```bash
 # postfwd3 tags
-docker pull lirt/postfwd-anti-geoip-spam-plugin:v1.40
+docker pull lirt/postfwd-anti-geoip-spam-plugin:v1.50.0
 # postfwd1, postfwd2 tags
 docker pull lirt/postfwd-anti-geoip-spam-plugin:v1.21
 ```
@@ -54,7 +56,7 @@ To run postfwd with geoip-plugin, run docker with configuration files mounted as
 docker run \
     -v </absolute/path/to/anti-spam.conf>:/etc/postfwd/anti-spam.conf \
     -v </absolute/path/to/postfwd.cf>:/etc/postfwd/postfwd.cf \
-    lirt/postfwd-anti-geoip-spam-plugin:v1.40
+    lirt/postfwd-anti-geoip-spam-plugin:v1.50.0
 ```
 
 This will run `postfwd2` or `postfwd3` (based on docker tag) with default arguments, reading postfwd rules file from your mounted volume file `postfwd.cf` and using anti-spam configuration from your file `anti-spam.conf`.
@@ -87,7 +89,7 @@ CREATE INDEX postfwd_sasl_username ON postfwd_logins (sasl_username);
 - `Postfwd2` or `Postfwd3`.
 - Database (`MySQL` or `PostgreSQL`).
 - Perl modules - `Geo::IP`, `DBI`, `Time::Piece`, `Config::Any`, `Net::Subnet`, `DBD::mysql` or `DBD::Pg`.
-- GeoIP database.
+- GeoIP database (version 1 or 2).
 
 #### Cpanm
 
@@ -104,7 +106,13 @@ yum install -y 'perl(Geo::IP)' \
                'perl(DBI)' \
                'perl(DBD::mysql)' \
                'perl(DBD::Pg)' \
-               'perl(Net::Subnet)'
+               'perl(Net::Subnet)' \
+               'perl(GeoIP2::Database::Reader)' \
+               'perl(Net::SSLeay)' \
+               'perl(IO::Socket::SSL)' \
+               'perl(LWP::Protocol::https)' \
+               'perl(Class::XSAccessor)' \
+               'perl(MaxMind::DB::Reader::XS)'
 ```
 
 #### Dependencies on Debian based distributions
@@ -119,7 +127,13 @@ apt-get install -y libgeo-ip-perl \
                    libdbd-mysql-perl \
                    libdbd-pg-perl \
                    libnet-subnet-perl \
-                   geoip-database
+                   geoip-database \
+                   libnet-ssleay-perl \
+                   libio-socket-ssl-perl \
+                   liblwp-protocol-https-perl \
+                   libclass-xsaccessor-perl \
+                   libmaxmind-db-reader-xs-perl \
+                   libgeoip2-perl
 ```
 
 ## Configuration
@@ -218,14 +232,17 @@ Plugin stores interesting statistical information in the database. To query thos
 ### Prototyping with Docker
 
 Complete development environment with postfwd, anti-spam plugin and mysql/postgresql database configured together can be run with single command from directory `tests/`:
-- MySQL: `docker-compose -f dev-compose-mysql.yml up`
-- PostgreSQL: `docker-compose -f dev-compose-postgresql.yml up`
+- MySQL: `docker-compose -f compose-dev-mysql.yml up`
+- PostgreSQL: `docker-compose -f compose-dev-postgresql.yml up`
+- MySQL with GeoIP2: `export POSTFWD_ANTISPAM_MAIN_CONFIG_PATH=/etc/postfwd/03-dev-anti-spam-mysql-geoip2.conf docker-compose -f compose-dev-mysql.yml up`
 
 Note for overriding postfwd arguments:
 
 * Most important arguments to run `postfwd` in Docker are `--stdout` and `--nodaemon`. These arguments configure postfwd to log into standard output and stay in foreground.
 * For running postfwd plugin, you also need to set argument `--plugins <path-to-plugin>` to correct location of plugin.
 
+MaxMind test database `tests/GeoLite2-Country-Test.mmdb` was downloaded from [MaxMind-DB repository](https://github.com/maxmind/MaxMind-DB).
+
 ### Running tests
 
 Check for proper linting with `perlcritic postfwd-anti-spam.plugin`.
@@ -240,5 +257,5 @@ nc 127.0.0.1 10040 < <(envsubst < dev-request)
 
 # run testing script
 cd tests
-DATABASES="mysql postgresql" RUN_COMPOSE=1 ./integration-compose-test.sh
+DATABASES="mysql postgresql" RUN_COMPOSE=1 ./integration-compose-test-geoip1.sh
 ```