diff --git a/.github/workflows/docker-compose-integration-test.yml b/.github/workflows/docker-compose-integration-test.yml index 96a2e48..790e377 100644 --- a/.github/workflows/docker-compose-integration-test.yml +++ b/.github/workflows/docker-compose-integration-test.yml @@ -26,48 +26,74 @@ jobs: continue-on-error: true run: perlcritic postfwd-anti-spam.plugin - integration-test-postgresql: - name: "integration test postgresql (perl v${{matrix.perl-version}})" + integration-test-postgresql-geoip1: + name: "integration test postgresql + geoip v1 (perl v${{matrix.perl-version}})" strategy: matrix: perl-version: [ '5.32', '5.30', '5.28' ] runs-on: ubuntu-latest env: DATABASES: "postgresql" + POSTFWD_ANTISPAM_MAIN_CONFIG_PATH: "/etc/postfwd/02-dev-anti-spam-postgres-geoip1.conf" steps: - uses: actions/checkout@v2 - name: docker-compose up - run: docker-compose -f tests/dev-compose-postgresql.yml + run: docker-compose -f tests/compose-dev-postgresql.yml up --build -d && sleep 10 - name: run integration tests - run: ./tests/integration-compose-test.sh + run: ./tests/integration-compose-test-geoip1.sh - name: read test logs - run: docker-compose -f tests/dev-compose-postgresql.yml logs postfwd-geoip-antispam + run: docker-compose -f tests/compose-dev-postgresql.yml logs postfwd-geoip-antispam - name: docker-compose down - run: docker-compose -f tests/dev-compose-postgresql.yml down + run: docker-compose -f tests/compose-dev-postgresql.yml down - integration-test-mysql: - name: "integration test mysql (perl v${{matrix.perl-version}})" + integration-test-mysql-geoip1: + name: "integration test mysql + geoip v1 (perl v${{matrix.perl-version}})" strategy: matrix: perl-version: [ '5.32', '5.30', '5.28' ] runs-on: ubuntu-latest env: DATABASES: "mysql" + POSTFWD_ANTISPAM_MAIN_CONFIG_PATH: "/etc/postfwd/01-dev-anti-spam-mysql-geoip1.conf" steps: - uses: actions/checkout@v2 - name: docker-compose up - run: docker-compose -f tests/dev-compose-mysql.yml + run: docker-compose -f tests/compose-dev-mysql.yml up --build -d && sleep 10 - name: run integration tests - run: ./tests/integration-compose-test.sh + run: ./tests/integration-compose-test-geoip1.sh - name: read test logs - run: docker-compose -f tests/dev-compose-mysql.yml logs postfwd-geoip-antispam + run: docker-compose -f tests/compose-dev-mysql.yml logs postfwd-geoip-antispam - name: docker-compose down - run: docker-compose -f tests/dev-compose-mysql.yml down + run: docker-compose -f tests/compose-dev-mysql.yml down + + integration-test-mysql-geoip2: + name: "integration test mysql + geoip v2 (perl v${{matrix.perl-version}})" + strategy: + matrix: + perl-version: [ '5.32', '5.30', '5.28' ] + runs-on: ubuntu-latest + env: + DATABASES: "mysql" + POSTFWD_ANTISPAM_MAIN_CONFIG_PATH: "/etc/postfwd/03-dev-anti-spam-mysql-geoip2.conf" + steps: + - uses: actions/checkout@v2 + - name: docker-compose up + run: docker-compose -f tests/compose-dev-mysql.yml + up + --build + -d + && sleep 10 + - name: run integration tests + run: ./tests/integration-compose-test-geoip2.sh + - name: read test logs + run: docker-compose -f tests/compose-dev-mysql.yml logs postfwd-geoip-antispam + - name: docker-compose down + run: docker-compose -f tests/compose-dev-mysql.yml down diff --git a/cpanfile b/cpanfile index 2d9b2cc..203cbf6 100644 --- a/cpanfile +++ b/cpanfile @@ -1,7 +1,14 @@ requires 'Geo::IP'; +requires 'GeoIP2::Database::Reader'; requires 'Time::Piece'; requires 'Config::Any'; requires 'DBI'; requires 'DBD::mysql'; requires 'DBD::Pg'; requires 'Net::Subnet'; +requires 'Net::SSLeay'; +requires 'IO::Socket::SSL'; +requires 'IO::Socket::SSL::Utils'; +requires 'LWP::Protocol::https'; +requires 'Class::XSAccessor'; +requires 'MaxMind::DB::Reader::XS'; diff --git a/docker/Dockerfile b/docker/Dockerfile index 3424ab3..f03c364 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM postfwd/postfwd:v2.02 -LABEL maintainer="Postfwd GeoIp Spam Plugin Maintainer " +LABEL maintainer="Postfwd GeoIP Spam Plugin Maintainer " ENV POSTFWD_ANTISPAM_MAIN_CONFIG_PATH=/etc/postfwd/anti-spam.conf ENV POSTFWD_ANTISPAM_SQL_STATEMENTS_CONFIG_PATH=/etc/postfwd/anti-spam-sql-st.conf @@ -18,6 +18,8 @@ RUN apk --no-cache update \ geoip-dev \ postgresql-dev \ mysql-dev \ + openssl \ + libmaxminddb-dev \ && cpan App::cpanminus \ && cpanm Geo::IP \ IO::Handle \ @@ -30,6 +32,13 @@ RUN apk --no-cache update \ DBD::mysql \ Net::Subnet \ Sys::Mmap \ + Net::SSLeay \ + IO::Socket::SSL::Utils \ + IO::Socket::SSL \ + LWP::Protocol::https \ + GeoIP2::Database::Reader \ + Class::XSAccessor \ + MaxMind::DB::Reader::XS --force \ && apk del make \ wget \ gcc \ diff --git a/docker/README.md b/docker/README.md index 680594a..cd25b05 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,6 +1,7 @@ # Supported tags and respective `Dockerfile` links * [`latest` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/master/docker/Dockerfile) +* [`v1.50.0` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/v1.50.0/docker/Dockerfile) * [`v1.40` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/v1.40/docker/Dockerfile) * [`v1.30` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/v1.30/docker/Dockerfile) * [`v1.21` (Dockerfile)](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/blob/v1.21/docker/Dockerfile) @@ -19,7 +20,6 @@ Pull image with `docker pull lirt/postfwd-anti-geoip-spam-plugin:latest`. Prepare your configuration files and run this docker image with following command: - ```bash docker run -d \ -v :/etc/postfwd/anti-spam.conf \ @@ -32,7 +32,7 @@ docker run -d \ If you don't have database by your hand, but **want to try the plugin**, you can try it with `docker-compose` template located in [official GitHub repository](https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/tree/master/tests) in directory `./tests`. -Run it with `docker-compose -f dev-compose.yml up` - this will bootstrap ready-to-work environment, where you can try the plugin. +Run it with `docker-compose -f compose-dev-mysql.yml up` - this will bootstrap ready-to-work environment, where you can try the plugin. Then you can run from local shell `nc 127.0.0.1 10040 < <(./dev-request.sh )`, to send artificial request into postfwd and watch logs what is happening. diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 4b2e3a9..6e2eb20 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -18,12 +18,7 @@ for arg; do esac done - if [ "$1" = "postfwd3" ] && [ -z "$want_help" ]; then - if [ ! -f /etc/postfwd/anti-spam.conf ]; then - echo >&2 'ERROR: Anti-spam plugin configuration file /etc/postfwd/anti-spam.conf was not found. Perhaps you forgot to mount it using "-v :/etc/postfwd/anti-spam.conf".' - exit 1 - fi chmod -R 644 /etc/postfwd/* chown -R postfw:postfw /etc/postfwd fi diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index dfe8633..7d17898 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,7 +2,22 @@ This changelog notes changes between versions of Postfwd GeoIP Anti-Spam plugin. -## Version 1.40 [2. Mar 2020] +## Version 1.50.0 [27. March 2021] + +GeoIP2 Feature. + +### Breaking changes + +- New dependencies for GeoIP2 module were added, please see changes in `cpanfile`. + +### Features / Enhancements + +- Added GeoIP2 support. Now you can use Maxmind GeoIP2 databases. Path to GeoIP file can be changed in config `app.geoip_db_path`. +- Removed entrypoint check for configuration file. The config file path can be overriden using environment variable `POSTFWD_ANTISPAM_MAIN_CONFIG_PATH`. +- Added integration tests for GeoIP2 (test structure changed overall). + + +## Version 1.40 [2. March 2020] This stable release contains IP whitelisting feature (Reported as bug and requested by @csazku in https://github.com/Vnet-as/postfwd-anti-geoip-spam-plugin/issues/50). @@ -53,7 +68,7 @@ and does integration test with sample requests and verification through logs. Plugin item now exports `request{client_uniq_ip_login_count}` and `request{client_uniq_country_login_count}` instead of `result*`. -## Version 1.2 [11. Mar 2019] +## Version 1.2 [11. March 2019] This stable release has changes mainly in linting, readability and testability, but also contains several bugfixes. @@ -80,7 +95,7 @@ Docker base image was updated from 1.37 to 1.39. - Script `lint.pl` was removed and replaced by more general/portable file `.perlcriticrc`. -## Version 1.1 [5. Jan 2019] +## Version 1.1 [5. January 2019] This stable version introduces docker image based on official postfwd docker image and other minor changes. All work done by @Lirt (ondrej.vaskoo@gmail.com), @@ -123,7 +138,7 @@ postfwd logs and this postfwd plugin logs. ### Removed -## Version 1.0 [12. Nov 2018] +## Version 1.0 [12. November 2018] This is first official version and release. diff --git a/docs/README.md b/docs/README.md index cd62a06..3042719 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,11 +29,13 @@ If you are interested in how your users got their mail accounts hacked, check ou | Plugin Version | Postfwd Version | | :------------- | :----------------------- | +| v1.50.0 | postfwd3 v2.xx | | v1.40 | postfwd3 v2.xx | | v1.30 | postfwd3 v2.xx | | v1.21 | postfwd1, postfwd2 v1.xx | -Supported database backends are **MySQL** and **PostgreSQL**. +- Supported database backends are **MySQL** and **PostgreSQL**. +- Supported GeoIP databases are both versions 1 and 2. To list changed between versions check release notes or look into the [Changelog](CHANGELOG.md). @@ -43,7 +45,7 @@ Pre-built ready-to-use Docker image is located on DockerHub and can be simply pu ```bash # postfwd3 tags -docker pull lirt/postfwd-anti-geoip-spam-plugin:v1.40 +docker pull lirt/postfwd-anti-geoip-spam-plugin:v1.50.0 # postfwd1, postfwd2 tags docker pull lirt/postfwd-anti-geoip-spam-plugin:v1.21 ``` @@ -54,7 +56,7 @@ To run postfwd with geoip-plugin, run docker with configuration files mounted as docker run \ -v :/etc/postfwd/anti-spam.conf \ -v :/etc/postfwd/postfwd.cf \ - lirt/postfwd-anti-geoip-spam-plugin:v1.40 + lirt/postfwd-anti-geoip-spam-plugin:v1.50.0 ``` This will run `postfwd2` or `postfwd3` (based on docker tag) with default arguments, reading postfwd rules file from your mounted volume file `postfwd.cf` and using anti-spam configuration from your file `anti-spam.conf`. @@ -87,7 +89,7 @@ CREATE INDEX postfwd_sasl_username ON postfwd_logins (sasl_username); - `Postfwd2` or `Postfwd3`. - Database (`MySQL` or `PostgreSQL`). - Perl modules - `Geo::IP`, `DBI`, `Time::Piece`, `Config::Any`, `Net::Subnet`, `DBD::mysql` or `DBD::Pg`. -- GeoIP database. +- GeoIP database (version 1 or 2). #### Cpanm @@ -104,7 +106,13 @@ yum install -y 'perl(Geo::IP)' \ 'perl(DBI)' \ 'perl(DBD::mysql)' \ 'perl(DBD::Pg)' \ - 'perl(Net::Subnet)' + 'perl(Net::Subnet)' \ + 'perl(GeoIP2::Database::Reader)' \ + 'perl(Net::SSLeay)' \ + 'perl(IO::Socket::SSL)' \ + 'perl(LWP::Protocol::https)' \ + 'perl(Class::XSAccessor)' \ + 'perl(MaxMind::DB::Reader::XS)' ``` #### Dependencies on Debian based distributions @@ -119,7 +127,13 @@ apt-get install -y libgeo-ip-perl \ libdbd-mysql-perl \ libdbd-pg-perl \ libnet-subnet-perl \ - geoip-database + geoip-database \ + libnet-ssleay-perl \ + libio-socket-ssl-perl \ + liblwp-protocol-https-perl \ + libclass-xsaccessor-perl \ + libmaxmind-db-reader-xs-perl \ + libgeoip2-perl ``` ## Configuration @@ -218,14 +232,17 @@ Plugin stores interesting statistical information in the database. To query thos ### Prototyping with Docker Complete development environment with postfwd, anti-spam plugin and mysql/postgresql database configured together can be run with single command from directory `tests/`: -- MySQL: `docker-compose -f dev-compose-mysql.yml up` -- PostgreSQL: `docker-compose -f dev-compose-postgresql.yml up` +- MySQL: `docker-compose -f compose-dev-mysql.yml up` +- PostgreSQL: `docker-compose -f compose-dev-postgresql.yml up` +- MySQL with GeoIP2: `export POSTFWD_ANTISPAM_MAIN_CONFIG_PATH=/etc/postfwd/03-dev-anti-spam-mysql-geoip2.conf docker-compose -f compose-dev-mysql.yml up` Note for overriding postfwd arguments: * Most important arguments to run `postfwd` in Docker are `--stdout` and `--nodaemon`. These arguments configure postfwd to log into standard output and stay in foreground. * For running postfwd plugin, you also need to set argument `--plugins ` to correct location of plugin. +MaxMind test database `tests/GeoLite2-Country-Test.mmdb` was downloaded from [MaxMind-DB repository](https://github.com/maxmind/MaxMind-DB). + ### Running tests Check for proper linting with `perlcritic postfwd-anti-spam.plugin`. @@ -240,5 +257,5 @@ nc 127.0.0.1 10040 < <(envsubst < dev-request) # run testing script cd tests -DATABASES="mysql postgresql" RUN_COMPOSE=1 ./integration-compose-test.sh +DATABASES="mysql postgresql" RUN_COMPOSE=1 ./integration-compose-test-geoip1.sh ``` diff --git a/postfwd-anti-spam.plugin b/postfwd-anti-spam.plugin index c9a1b7b..cedd404 100644 --- a/postfwd-anti-spam.plugin +++ b/postfwd-anti-spam.plugin @@ -1,7 +1,7 @@ #!/usr/bin/env perl # Declare version -our $VERSION = 1.40; +our $VERSION = '1.50.0'; # English module for Perl::Critic compliance use English qw( -no_match_vars ); @@ -88,14 +88,15 @@ if ( !$config{logging}{logfile} } $log_file_fh = *STDOUT; mylog_info('Logging destination is STDOUT'); -} -else { +} else { open $log_file_fh, '>>', $config{logging}{logfile} or die "ERROR: Could not open file '$config{logging}{logfile}' $ERRNO\n"; $log_file_fh->autoflush; mylog_info("Logging destination is file '$config{logging}{logfile}'"); } +mylog_info("Configuration file $cfg_anti_spam_path was loaded successfully"); + # IP WHITELIST # Do not whitelist any IP addresses by default my $ip_whitelist = subnet_matcher qw( @@ -132,13 +133,97 @@ if ( $config{app}{ip_whitelist_path} || length $config{app}{ip_whitelist_path} ) } -# Geo IP +# GeoIP: +# Load GeoIP modules use Geo::IP; -my $gi = Geo::IP->open($config{app}{geoip_db_path}, GEOIP_MEMORY_CACHE | GEOIP_CHECK_CACHE); +use GeoIP2::Database::Reader; +my $gi; + +if (! -e $config{app}{geoip_db_path}) { + mylog_fatal("GeoIP Database file $config{app}{geoip_db_path} doesn't exist"); +} + +# Try to find out version of GeoIP DB +sub get_geoip_version { + mylog_info("Trying to get version of GeoIP Database from file $config{app}{geoip_db_path}"); + + # GeoIP Version 1 check + my $gi_v1 = Geo::IP->open($config{app}{geoip_db_path}, GEOIP_MEMORY_CACHE | GEOIP_CHECK_CACHE); + + if ( !length($gi_v1->database_info) || !($gi_v1->database_info)) { + mylog_info('Failed to get info about GeoIP database (v1 decoder), perhaps it is version 2') + } else { + $gi = $gi_v1; + mylog_info('GeoIP Database Informations: ', $gi->database_info); + mylog_info('GeoIP Database Edition: ', $gi->database_edition); + return 1 + } + + # GeoIP Version 2 check + my $gi_v2 = GeoIP2::Database::Reader->new( + file => $config{app}{geoip_db_path}, + locales => [ 'en' ] + ); + + my $gi_v2_metadata = eval { $gi_v2->metadata() }; + if ( $EVAL_ERROR ) { + mylog_info('Failed to get info about GeoIP database (v2 decoder)'); + } else { + $gi = $gi_v2; + mylog_info('[GeoIP2] Description: ', $gi_v2_metadata->description()->{en}); + mylog_info('[GeoIP2] Database Edition: ', $gi_v2_metadata->binary_format_major_version().$gi_v2_metadata->binary_format_minor_version()); + mylog_info('[GeoIP2] Database Type: ', $gi_v2_metadata->database_type()); + mylog_info('[GeoIP2] Build: ', $gi_v2_metadata->build_epoch()); + mylog_info('[GeoIP2] IP Version: ', $gi_v2_metadata->ip_version()); + return 2 + } + + return +} + +my $geoip_db_version = get_geoip_version(); +if ( !$geoip_db_version ) { + mylog_fatal("Cannot load or decode GeoIP database $config{app}{geoip_db_path}"); +} +mylog_info("GeoIP database $config{app}{geoip_db_path} with version $geoip_db_version was loaded successfully"); + +sub geoip_country_code { + my ($client_ip) = @_; + my $cc; + + if ($geoip_db_version == 1) { + $cc = $gi->country_code_by_addr($client_ip); + if ( !defined $cc ) { + mylog_info("Cannot find country code for IP address [$client_ip]"); + return; + } else { + if ( !length $cc || !($cc) ) { + mylog_info("Country code for IP address [$client_ip] is empty"); + return; + } + } + return $cc; + } + + if ($geoip_db_version == 2) { + my $country = eval { $gi->country( ip => $client_ip ) }; + if ( $EVAL_ERROR ) { + if ($EVAL_ERROR =~ m/No record found for IP addresds/ims ) { + mylog_info("Cannot find country code for IP address [$client_ip]"); + return; + } + } + my $country_rec = $country->country(); + + $cc = eval { $country_rec->iso_code() }; + if ( $EVAL_ERROR ) { + mylog_info("Country code for IP address [$client_ip] is empty"); + return; + } + return $cc; + } +} -# Print info about GeoIP database -mylog_info('GeoIP Database Informations: ', $gi->database_info); -mylog_info('GeoIP Database Edition: ', $gi->database_edition); # DB connection # Update values to your DB connection in config file /etc/postfix/anti-spam.conf @@ -162,8 +247,7 @@ for ( 1 .. DB_CONN_RETRIES ) { } if ( !defined $dbh ) { mylog_fatal 'Could not connect to configured database after 3 retries'; -} -else { +} else { mylog_info('Database connection successful'); } @@ -240,17 +324,10 @@ my $last_cache_flush = time; } # Get country code from GeoIP module - my $cc = $gi->country_code_by_addr($client_ip); + my $cc = geoip_country_code($client_ip); if ( !defined $cc ) { - mylog_info("Cannot find country code for IP address [$client_ip]"); return $result; } - else { - if ( !length $cc || !($cc) ) { - mylog_info("Country code for IP address [$client_ip] is empty"); - return $result; - } - } # Check if user with given IP already has record my $check_row_existence_sth = @@ -269,7 +346,6 @@ my $last_cache_flush = time; return $result; } if ( $row_count == 0 ) { - # Save new user mail into hash if it does not exists mylog_info("Inserting $user, $client_ip, $cc"); my $insert_sth = $dbh->prepare( $config_sql{insert_st} ) @@ -277,8 +353,7 @@ my $last_cache_flush = time; $insert_sth->execute( $user, $client_ip, $cc, localtime(time)->strftime('%F %T') ) or do { mylog_err( $insert_sth->errstr ); return $result; }; - } - else { + } else { # Increment or initialize login count for user and given IP/country mylog_info("Incrementing $user, $client_ip, $cc"); my $increment_sth = $dbh->prepare( $config_sql{increment_st} ) @@ -320,7 +395,6 @@ my $last_cache_flush = time; # Return number of logins from country last logged from return $result; - }, 'client_uniq_country_login_count' => sub { @@ -392,9 +466,8 @@ my $last_cache_flush = time; ); } - # Returns number of countries from which user logged in to an email via sasl + # Returns number of countries from which user logged in to an email via sasl return $result; - }, 'client_uniq_ip_login_count' => sub { @@ -466,9 +539,7 @@ my $last_cache_flush = time; # Returns number of IPs from which user logged in to an email via sasl return $result; - }, - ); 1; diff --git a/tests/dev-anti-spam-mysql.conf b/tests/01-dev-anti-spam-mysql-geoip1.conf similarity index 100% rename from tests/dev-anti-spam-mysql.conf rename to tests/01-dev-anti-spam-mysql-geoip1.conf diff --git a/tests/dev-anti-spam-postgres.conf b/tests/02-dev-anti-spam-postgres-geoip1.conf similarity index 100% rename from tests/dev-anti-spam-postgres.conf rename to tests/02-dev-anti-spam-postgres-geoip1.conf diff --git a/tests/03-dev-anti-spam-mysql-geoip2.conf b/tests/03-dev-anti-spam-mysql-geoip2.conf new file mode 100644 index 0000000..771736a --- /dev/null +++ b/tests/03-dev-anti-spam-mysql-geoip2.conf @@ -0,0 +1,28 @@ +[database] +driver = mysql +database = postfwd_antispam_test +host = 127.0.0.1 +port = 3306 +userid = testuser +password = testpasswordpostfwdantispam + +[logging] +# logfile = +autoflush = 1 + +[debugging] +# Enable(1) or disable(0) logging +debug = 1 +# Make log after exceeding unique country count limit +country_limit = 5 +# Make log after exceeding unique ip count limit +ip_limit = 20 + +[app] +# Flush database records with last login older than 1 day +db_flush_interval = 86400 +geoip_db_path = /usr/local/share/GeoIP/GeoLite2-Country-Test.mmdb +# IP whitelist must be valid comma separated strings in CIDR format without whitespaces. +# It specifies IP addresses which will NOT be counted into user logins database. +ip_whitelist = 198.51.100.0/24,203.0.113.123/32 +# ip_whitelist_path = /etc/postfwd/ip_whitelist.txt diff --git a/tests/04-dev-anti-spam-mysql-no-geoip.conf b/tests/04-dev-anti-spam-mysql-no-geoip.conf new file mode 100644 index 0000000..d8ce491 --- /dev/null +++ b/tests/04-dev-anti-spam-mysql-no-geoip.conf @@ -0,0 +1,28 @@ +[database] +driver = mysql +database = postfwd_antispam_test +host = 127.0.0.1 +port = 3306 +userid = testuser +password = testpasswordpostfwdantispam + +[logging] +# logfile = +autoflush = 1 + +[debugging] +# Enable(1) or disable(0) logging +debug = 1 +# Make log after exceeding unique country count limit +country_limit = 5 +# Make log after exceeding unique ip count limit +ip_limit = 20 + +[app] +# Flush database records with last login older than 1 day +db_flush_interval = 86400 +geoip_db_path = /usr/local/share/GeoIP/non-existing-file.mmdb +# IP whitelist must be valid comma separated strings in CIDR format without whitespaces. +# It specifies IP addresses which will NOT be counted into user logins database. +ip_whitelist = 198.51.100.0/24,203.0.113.123/32 +# ip_whitelist_path = /etc/postfwd/ip_whitelist.txt diff --git a/tests/GeoLite2-Country-Test.mmdb b/tests/GeoLite2-Country-Test.mmdb new file mode 100644 index 0000000..aa81cbe Binary files /dev/null and b/tests/GeoLite2-Country-Test.mmdb differ diff --git a/tests/dev-compose-mysql.yml b/tests/compose-dev-mysql.yml similarity index 61% rename from tests/dev-compose-mysql.yml rename to tests/compose-dev-mysql.yml index 4124ff4..8edc5c7 100644 --- a/tests/dev-compose-mysql.yml +++ b/tests/compose-dev-mysql.yml @@ -22,15 +22,25 @@ services: dockerfile: docker/Dockerfile environment: - PROG=postfwd3 + - POSTFWD_ANTISPAM_MAIN_CONFIG_PATH depends_on: - mysql-postfwd-db volumes: - type: bind - source: ./dev-anti-spam-mysql.conf - target: /etc/postfwd/anti-spam.conf + source: ./01-dev-anti-spam-mysql-geoip1.conf + target: /etc/postfwd/01-dev-anti-spam-mysql-geoip1.conf + - type: bind + source: ./03-dev-anti-spam-mysql-geoip2.conf + target: /etc/postfwd/03-dev-anti-spam-mysql-geoip2.conf + - type: bind + source: ./04-dev-anti-spam-mysql-no-geoip.conf + target: /etc/postfwd/04-dev-anti-spam-mysql-no-geoip.conf - type: bind source: ./dev-postfwd.cf target: /etc/postfwd/postfwd.cf - type: bind source: ./ip_whitelist.txt target: /etc/postfwd/ip_whitelist.txt + - type: bind + source: ./GeoLite2-Country-Test.mmdb + target: /usr/local/share/GeoIP/GeoLite2-Country-Test.mmdb diff --git a/tests/dev-compose-postgresql.yml b/tests/compose-dev-postgresql.yml similarity index 73% rename from tests/dev-compose-postgresql.yml rename to tests/compose-dev-postgresql.yml index dfb6736..187e3a9 100644 --- a/tests/dev-compose-postgresql.yml +++ b/tests/compose-dev-postgresql.yml @@ -20,15 +20,19 @@ services: dockerfile: docker/Dockerfile environment: - PROG=postfwd3 + - POSTFWD_ANTISPAM_MAIN_CONFIG_PATH depends_on: - postgres-postfwd-db volumes: - type: bind - source: ./dev-anti-spam-postgres.conf - target: /etc/postfwd/anti-spam.conf + source: ./02-dev-anti-spam-postgres-geoip1.conf + target: /etc/postfwd/02-dev-anti-spam-postgres-geoip1.conf - type: bind source: ./dev-postfwd.cf target: /etc/postfwd/postfwd.cf - type: bind source: ./ip_whitelist.txt target: /etc/postfwd/ip_whitelist.txt + - type: bind + source: ./GeoLite2-Country-Test.mmdb + target: /usr/local/share/GeoIP/GeoLite2-Country-Test.mmdb diff --git a/tests/integration-compose-test.sh b/tests/integration-compose-test-geoip1.sh similarity index 91% rename from tests/integration-compose-test.sh rename to tests/integration-compose-test-geoip1.sh index 9cd6e43..5bcc8e1 100755 --- a/tests/integration-compose-test.sh +++ b/tests/integration-compose-test-geoip1.sh @@ -17,7 +17,7 @@ sleep_duration=10 for DB in ${DATABASES}; do # Build and run compose if [ "${RUN_COMPOSE}" = "1" ]; then - docker-compose -f "${script_path}/dev-compose-${DB}.yml" up -d --build > /dev/null + docker-compose -f "${script_path}/compose-dev-${DB}.yml" up -d --build > /dev/null echo "Sleeping ${sleep_duration} seconds until compose initializes" sleep ${sleep_duration} echo "Sleep done" @@ -70,19 +70,19 @@ for DB in ${DATABASES}; do # 2. Check if spam-user exceeded country limit # 3. Check if spam-user exceeded IP address limit declare -a errors - if docker-compose -f "${script_path}/dev-compose-${DB}.yml" logs postfwd-geoip-antispam \ + if docker-compose -f "${script_path}/compose-dev-${DB}.yml" logs postfwd-geoip-antispam \ | grep -i "error\|fatal" \ | grep -E -v -e "ERROR.*: Retry [123]/3 - Can't connect to MySQL server on" \ -e "ERROR.*: Retry [123]/3 - could not connect to server: Connection refused"; then echo -e 'ERROR: Errors found in log.\nTEST FAILED!' errors+=("1") fi - if ! docker-compose -f "${script_path}/dev-compose-${DB}.yml" logs postfwd-geoip-antispam \ + if ! docker-compose -f "${script_path}/compose-dev-${DB}.yml" logs postfwd-geoip-antispam \ | grep "User spam-user1@example.com was logged from more than 5 countries([67])"; then echo -e 'ERROR: User did not exceed country login limit (5) but should!' errors+=("2") fi - if ! docker-compose -f "${script_path}/dev-compose-${DB}.yml" logs postfwd-geoip-antispam \ + if ! docker-compose -f "${script_path}/compose-dev-${DB}.yml" logs postfwd-geoip-antispam \ | grep "User spam-user2@example.com was logged from more than 20 IP addresses(2[45])"; then echo -e 'ERROR: User did not exceed IP address limit (20) but should!' errors+=("3") @@ -90,7 +90,7 @@ for DB in ${DATABASES}; do # Cleanup if [ "${RUN_COMPOSE}" = "1" ]; then - docker-compose -f "${script_path}/dev-compose-${DB}.yml" down > /dev/null + docker-compose -f "${script_path}/compose-dev-${DB}.yml" down > /dev/null fi done diff --git a/tests/integration-compose-test-geoip2.sh b/tests/integration-compose-test-geoip2.sh new file mode 100755 index 0000000..3891b70 --- /dev/null +++ b/tests/integration-compose-test-geoip2.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +script_path=$(dirname "${0}") + +function send_requests { + local sasl_username=${1} + local -n client_addresses=${2} + for client_address in "${client_addresses[@]}"; do + export SASL_USERNAME=${sasl_username} + export CLIENT_ADDRESS=${client_address} + nc 127.0.0.1 10040 -v -w 10 < <(envsubst < "${script_path}/dev-request") > /dev/null + done +} + + +# Valid user logging in with IP addresses from United Kingdom and Sweden +sasl_username="valid-user@example.com" +valid_user_addresses=( 2.125.160.216 + 81.2.69.142 + 81.2.69.144 + 81.2.69.192 + 89.160.20.112 + 89.160.20.128 +) +send_requests "$sasl_username" "valid_user_addresses" + + +# Spam user logging in with IP addresses from 7 different countries +# UK, US, BT, SE, CN, PH, GI +sasl_username="spam-user1@example.com" +spam_user1_addresses=( 2.125.160.216 + 50.114.0.12 216.160.83.56 + 67.43.156.0 + 89.160.20.112 + 111.235.160.1 + 202.196.224.0 + 217.65.48.0 +) +send_requests "$sasl_username" "spam_user1_addresses" + + +# Spam user logging in with 25 different IP addresses from SE +sasl_username="spam-user2@example.com" +spam_user2_addresses=( 89.160.20.128 89.160.20.129 89.160.20.130 + 89.160.20.131 89.160.20.132 89.160.20.133 + 89.160.20.134 89.160.20.135 89.160.20.136 + 89.160.20.137 89.160.20.138 89.160.20.139 + 89.160.20.140 89.160.20.141 89.160.20.142 + 89.160.20.143 89.160.20.144 89.160.20.145 + 89.160.20.146 89.160.20.147 89.160.20.148 + 89.160.20.149 89.160.20.150 89.160.20.151 + 89.160.20.152 +) +send_requests "$sasl_username" "spam_user2_addresses" + + +# Verify logs +# 1. Check for errors +# 2. Check if spam-user exceeded country limit +# 3. Check if spam-user exceeded IP address limit +declare -a errors +if docker-compose -f "${script_path}/compose-dev-mysql.yml" logs postfwd-geoip-antispam \ + | grep -i "error\|fatal" \ + | grep -E -v -e "ERROR.*: Retry [123]/3 - Can't connect to MySQL server on" \ + -e "ERROR.*: Retry [123]/3 - could not connect to server: Connection refused"; then + echo -e 'ERROR: Errors found in log.\nTEST FAILED!' + errors+=("1") +fi +if ! docker-compose -f "${script_path}/compose-dev-mysql.yml" logs postfwd-geoip-antispam \ + | grep "User spam-user1@example.com was logged from more than 5 countries([67])"; then + echo -e 'ERROR: User did not exceed country login limit (5) but should!' + errors+=("2") +fi +if ! docker-compose -f "${script_path}/compose-dev-mysql.yml" logs postfwd-geoip-antispam \ + | grep "User spam-user2@example.com was logged from more than 20 IP addresses(2[45])"; then + echo -e 'ERROR: User did not exceed IP address limit (20) but should!' + errors+=("3") +fi + +if [ "${#errors[@]}" -gt 0 ]; then + echo "Tests ended up with errors[${errors[*]}]." + exit 1 +fi +