Merge pull request #1171 from matsduf/merge-develop-into-master

Merge develop into master (Engine)

.gitignore

@@ -23,6 +23,7 @@ Zonemaster-*.tar.gz
 # Backup files from editor, by unknown and Emacs, respectively
 *.bak
 *~
+.*.swp
 
 # Unknown
 dev.pl

Changes

@@ -1,5 +1,49 @@
 Release history for Zonemaster component Zonemaster-Engine
 
+
+v4.6.0 2022-12-19 (public release version)
+
+ [Features]
+ - Updates implementation of test case Zone09 (#1109, #1163, #1103,
+   #1140, #1139)
+ - Updates implementation of test case Zone01 (#1035, #1161)
+ - Updates implementation of test case DNSSEC02 (#1158)
+ - Adds new implementation of test case Nameserver11 (#1034)
+ - Adds implementation of normalization specification replacing test
+   case Basic00 (#1040, #1157)
+ - Adds method for accessing translatable test case description (#1144)
+ - Rewrites implementation of test cases Connectivity01 and
+   Connectivity02 (#1143, #1136, #1137)
+ - Removes test case Basic04, replaced by updated test cases Connectivity01
+   and Connectivity02 (#1143)
+ - Makes root hints configurable (#1134, #850)
+ - Removes special treatment of SHA-1 in test case DNSSEC01 (#1116, #1115)
+ - Makes IPV4_DISABLED/IPV6_DISABLED tags being consistently outputted in
+   DEBUG level (#1102, #1117)
+
+ [Fixes]
+ - Fixes typo in message and removes zombie messages for Basic04 (#1168)
+ - Updates installation instructions (#1162, #1130)
+ - Cleanup system messages (#1142, #1164)
+ - Fixes Nameserver10 EDNS query (#1160)
+ - Fixes queries and response packets content for undelegated tests (#1150)
+ - Corrects the license statement in Engine.pm (#1152)
+ - Exposes the init methods (#1151)
+ - Updates logentry arguments (#1138, #1128, #1135, #1126)
+ - Removes need for double quotes for ASN Lookup (#1141)
+ - Limits old profile_example.json to only properties not used in
+   default profile and rename it (#1120)
+ - Fixes missing update of ns args in SYSTEM messages (#1097)
+ - Remove dependency on Net::IP, and use Net::IP::XS everywhere (#1119,
+   #1107, #1159)
+ - Use lowercase fragments to refer to internal headings in markdown
+   documents (#1127)
+ - Removes the use of a public resolver for test case Syntax06 (#1063)
+ - Adds check for undelegated test in DNSSEC11 (#1101, #1099)
+ - Refactors to avoid code duplication (#1098)
+ - Adds editorial update of msgid for test case DNSSEC01 (#1072)
+
+
 v4.5.1 2022-07-08 (public fix version)
 
  [Fixes]
@@ -11,7 +55,7 @@ v4.5.1 2022-07-08 (public fix version)
 
 v4.5.0 2022-06-09 (public release version)
 
-[Features]
+ [Features]
  - Use pre-built packages for ubuntu (#1079)
  - Updates implementation of test case Nameserver10 (#1061, #1060)
  - Updates implementation of test case DNSSEC02 (#1051, #1049, #1036)

Dockerfile

@@ -2,7 +2,10 @@ FROM zonemaster/ldns:local as build
 
 RUN apk add --no-cache \
     # Only needed for CPAN deps
+    gcc \
     make \
+    musl-dev \
+    perl-dev \
     # Transitive deps included to improve build speed
     perl-mailtools \
     perl-module-build-tiny \
@@ -17,7 +20,6 @@ RUN apk add --no-cache \
     perl-lwp-protocol-https \
     perl-module-install \
     perl-moose \
-    perl-net-ip \
     perl-pod-coverage \
     perl-test-differences \
     perl-test-exception \
@@ -28,7 +30,8 @@ RUN apk add --no-cache \
     Email::Valid \
     Locale::TextDomain \
     Module::Find \
-    MooseX::Singleton
+    MooseX::Singleton \
+    Net::IP::XS
 
 ARG version
 

MANIFEST

@@ -26,6 +26,8 @@ lib/Zonemaster/Engine/Logger.pm
 lib/Zonemaster/Engine/Logger/Entry.pm
 lib/Zonemaster/Engine/Nameserver.pm
 lib/Zonemaster/Engine/Nameserver/Cache.pm
+lib/Zonemaster/Engine/Normalization.pm
+lib/Zonemaster/Engine/Normalization/Error.pm
 lib/Zonemaster/Engine/Net/IP.pm
 lib/Zonemaster/Engine/NSArray.pm
 lib/Zonemaster/Engine/Overview.pod
@@ -51,19 +53,20 @@ Makefile.PL
 MANIFEST			This list of files
 META.yml
 README.md
+share/GNUmakefile
 share/iana-ipv4-special-registry.csv
 share/iana-ipv6-special-registry.csv
-share/modules.txt
-share/profile.json
-share/profile_example.json
 share/locale/da/LC_MESSAGES/Zonemaster-Engine.mo
 share/locale/es/LC_MESSAGES/Zonemaster-Engine.mo
 share/locale/fi/LC_MESSAGES/Zonemaster-Engine.mo
 share/locale/fr/LC_MESSAGES/Zonemaster-Engine.mo
 share/locale/nb/LC_MESSAGES/Zonemaster-Engine.mo
 share/locale/sv/LC_MESSAGES/Zonemaster-Engine.mo
 share/Makefile
-share/GNUmakefile
+share/modules.txt
+share/named.root
+share/profile.json
+share/profile_additional_properties.json
 t/00-load.t
 t/asn.data
 t/asn.t
@@ -74,6 +77,7 @@ t/nameserver-axfr.data
 t/nameserver-axfr.t
 t/nameserver.data
 t/nameserver.t
+t/normalization.t
 t/old-bugs.data
 t/old-bugs.t
 t/pod-coverage.t
@@ -274,6 +278,18 @@ t/Test-syntax06-L.data
 t/Test-syntax06-L.t
 t/Test-zone.data
 t/Test-zone.t
+t/Test-zone09-A.data
+t/Test-zone09-A.t
+t/Test-zone09-B.data
+t/Test-zone09-B.t
+t/Test-zone09-C.data
+t/Test-zone09-C.t
+t/Test-zone09-D.data
+t/Test-zone09-D.t
+t/Test-zone09-E.data
+t/Test-zone09-E.t
+t/Test-zone09-F.data
+t/Test-zone09-F.t
 t/translator.t
 t/undelegated.data
 t/undelegated.t

Makefile.PL

@@ -28,10 +28,11 @@ requires 'Locale::TextDomain' => 1.20;
 requires 'Module::Find'       => 0.10;
 requires 'Moose'              => 2.0401;
 requires 'MooseX::Singleton'  => 0.30;
-requires 'Net::IP'            => 1.26;
+requires 'Net::DNS'           => 0;
+requires 'Net::IP::XS'        => 0.21;
 requires 'Readonly'           => 0;
 requires 'Text::CSV'          => 0;
-requires 'Zonemaster::LDNS'   => 2.002002;
+requires 'Zonemaster::LDNS'   => 3.000000;
 
 test_requires 'Pod::Coverage'     => 0;
 test_requires 'Test::Differences' => 0;
@@ -46,8 +47,6 @@ if ($^O eq "freebsd") {
     requires_external_bin 'gmake';
 };
 
-recommends 'Net::IP::XS'  => 0;
-
 sub MY::postamble {
     my $pure_all;
     if ($^O eq "freebsd") {

README.md

@@ -47,6 +47,12 @@ For participation, contact and bug reporting, please see
 [Zonemaster/README.md](https://github.com/zonemaster/zonemaster/blob/master/README.md).
 
 
+## License
+
+This is free software under a 2-clause BSD license. The full text of the license can
+be found in the [LICENSE](LICENSE) file included in this respository.
+
+
 [CPAN site]:                                      https://metacpan.org/pod/Zonemaster::Engine
 [Docker Hub]:                                     https://hub.docker.com/u/zonemaster
 [Docker Image Creation]:                          https://github.com/zonemaster/zonemaster/blob/master/docs/internal-documentation/maintenance/ReleaseProcess-create-docker-image.md

docs/Installation.md

@@ -2,16 +2,16 @@
 
 ## Table of contents
 
-* [Overview](#Overview)
-* [Docker](#Docker)
-* [Prerequisites](#Prerequisites)
-* [Local installation](#Local-installation)
+* [Overview](#overview)
+* [Docker](#docker)
+* [Prerequisites](#prerequisites)
+* [Local installation](#local-installation)
   * [Installation on Rocky Linux]
   * [Installation on Debian and Ubuntu]
   * [Installation on FreeBSD]
   * [Installation on CentOS 7]
-* [Post-installation sanity check](#Post-installation-sanity-check)
-* [What to do next](#What-to-do-next)
+* [Post-installation sanity check](#post-installation-sanity-check)
+* [What to do next](#what-to-do-next)
 
 
 ## Overview
@@ -59,13 +59,13 @@ Zonemaster::Engine, see the [declaration of prerequisites].
 3) Install binary packages:
 
    ```sh
-   sudo dnf --assumeyes install cpanminus gcc libidn2-devel openssl-devel perl-Class-Accessor perl-Clone perl-core perl-Devel-CheckLib perl-Email-Valid perl-File-ShareDir perl-File-Slurp perl-libintl perl-IO-Socket-INET6 perl-List-MoreUtils perl-Module-Find perl-Module-Install perl-Moose perl-Net-IP perl-Pod-Coverage perl-Readonly perl-Test-Differences perl-Test-Exception perl-Test-Fatal perl-Test-NoWarnings perl-Test-Pod perl-Text-CSV perl-Test-Simple perl-YAML
+   sudo dnf --assumeyes install cpanminus gcc libidn2-devel openssl-devel perl-Class-Accessor perl-Clone perl-core perl-Devel-CheckLib perl-Email-Valid perl-File-ShareDir perl-File-Slurp perl-libintl perl-IO-Socket-INET6 perl-List-MoreUtils perl-Module-Find perl-Module-Install perl-Moose perl-Net-DNS perl-Pod-Coverage perl-Readonly perl-Test-Differences perl-Test-Exception perl-Test-Fatal perl-Test-NoWarnings perl-Test-Pod perl-Text-CSV perl-Test-Simple perl-YAML
    ```
 
 4) Install packages from CPAN:
 
    ```sh
-   sudo cpanm Module::Install::XSUtil MooseX::Singleton
+   sudo cpanm Module::Install::XSUtil MooseX::Singleton Net::IP::XS
    ```
 
 5) Install Zonemaster::LDNS and Zonemaster::Engine:
@@ -108,13 +108,13 @@ Using pre-built packages is the preferred method for Debian and Ubuntu.
 2) Install dependencies from binary packages:
 
    ```sh
-   sudo apt install autoconf automake build-essential cpanminus libclass-accessor-perl libclone-perl libdevel-checklib-perl libemail-valid-perl libfile-sharedir-perl libfile-slurp-perl libidn2-dev libintl-perl libio-socket-inet6-perl liblist-moreutils-perl libmodule-find-perl libmodule-install-perl libmodule-install-xsutil-perl libmoose-perl libmoosex-singleton-perl libnet-ip-perl libpod-coverage-perl libreadonly-perl libssl-dev libldns3 libldns-dev libtest-differences-perl libtest-exception-perl libtest-fatal-perl libtest-nowarnings-perl libtest-pod-perl libtext-csv-perl libtool m4
+   sudo apt install autoconf automake build-essential cpanminus libclass-accessor-perl libclone-perl libdevel-checklib-perl libemail-valid-perl libfile-sharedir-perl libfile-slurp-perl libidn2-dev libintl-perl libio-socket-inet6-perl liblist-moreutils-perl libmodule-find-perl libmodule-install-perl libmodule-install-xsutil-perl libmoose-perl libmoosex-singleton-perl libnet-dns-perl libnet-ip-xs-perl libpod-coverage-perl libreadonly-perl libssl-dev libldns3 libtest-differences-perl libtest-exception-perl libtest-fatal-perl libtest-nowarnings-perl libtest-pod-perl libtext-csv-perl libtool m4
    ```
 
 3) Install Zonemaster::LDNS and Zonemaster::Engine.
 
    ```sh
-   sudo cpanm --configure-args="--no-internal-ldns" Zonemaster::LDNS Zonemaster::Engine
+   sudo cpanm Zonemaster::LDNS Zonemaster::Engine
    ```
 
 ### Installation on FreeBSD
@@ -154,7 +154,7 @@ Using pre-built packages is the preferred method for Debian and Ubuntu.
 5) Install dependencies from binary packages:
 
    ```sh
-   pkg install devel/gmake libidn2 p5-App-cpanminus p5-Class-Accessor p5-Clone p5-Devel-CheckLib p5-Email-Valid p5-File-ShareDir p5-File-Slurp p5-IO-Socket-INET6 p5-List-MoreUtils p5-Locale-libintl p5-Module-Find p5-Module-Install p5-Module-Install-XSUtil p5-Moose p5-MooseX-Singleton p5-Net-IP-XS p5-Pod-Coverage p5-Readonly p5-Test-Differences p5-Test-Exception p5-Test-Fatal p5-Test-NoWarnings p5-Test-Pod p5-Text-CSV net-mgmt/p5-Net-IP dns/ldns
+   pkg install devel/gmake libidn2 p5-App-cpanminus p5-Class-Accessor p5-Clone p5-Devel-CheckLib p5-Email-Valid p5-File-ShareDir p5-File-Slurp p5-IO-Socket-INET6 p5-List-MoreUtils p5-Locale-libintl p5-Module-Find p5-Module-Install p5-Module-Install-XSUtil p5-Moose p5-MooseX-Singleton p5-Net-DNS p5-Net-IP-XS p5-Pod-Coverage p5-Readonly p5-Test-Differences p5-Test-Exception p5-Test-Fatal p5-Test-NoWarnings p5-Test-Pod p5-Text-CSV dns/ldns
    ```
 
 6) Install Zonemaster::LDNS:
@@ -187,13 +187,13 @@ Using pre-built packages is the preferred method for Debian and Ubuntu.
 2) Install binary packages:
 
    ```sh
-   sudo yum --assumeyes install cpanminus gcc libidn2-devel openssl-devel openssl11-devel perl-Class-Accessor perl-Clone perl-core perl-Devel-CheckLib perl-Email-Valid perl-File-ShareDir perl-File-Slurp perl-libintl perl-IO-Socket-INET6 perl-List-MoreUtils perl-Module-Find perl-Module-Install perl-Moose perl-Net-IP perl-Pod-Coverage perl-Readonly perl-Test-Differences perl-Test-Exception perl-Test-Fatal perl-Test-NoWarnings perl-Test-Pod perl-Text-CSV perl-Test-Simple perl-YAML
+   sudo yum --assumeyes install cpanminus gcc libidn2-devel openssl-devel openssl11-devel perl-Class-Accessor perl-Clone perl-core perl-Devel-CheckLib perl-Email-Valid perl-File-ShareDir perl-File-Slurp perl-libintl perl-IO-Socket-INET6 perl-List-MoreUtils perl-Module-Find perl-Module-Install perl-Moose perl-Net-DNS perl-Pod-Coverage perl-Readonly perl-Test-Differences perl-Test-Exception perl-Test-Fatal perl-Test-NoWarnings perl-Test-Pod perl-Text-CSV perl-Test-Simple perl-YAML
    ```
 
 3) Install packages from CPAN:
 
    ```sh
-   sudo cpanm Module::Install::XSUtil MooseX::Singleton
+   sudo cpanm Module::Install::XSUtil MooseX::Singleton Net::IP::XS
    ```
 
 4) Install Zonemaster::LDNS with support for DNSSEC algorithms 15 and 16:

docs/Profiles.md

@@ -11,10 +11,22 @@ always loaded by Zonemaster-Engine.
 ## Creating profiles
 
 Some properties are empty by default such as `logfilter` and
-`test_cases_vars`. These properties are not present in the default
-profile. For an example of their usage, refer to the example file,
-[profile_example.json].
+`test_cases_vars`. Those properties are not present in the default
+profile. For an example of their usage, refer to the additional file,
+[profile_additional_properties.json].
 
-[profile.json]:          ../share/profile.json
-[profile_example.json]:  ../share/profile_example.json
-[Profile properties]:    https://metacpan.org/pod/Zonemaster::Engine::Profile#PROFILE-PROPERTIES
+The content of the two files, as-is or modified, can be merged into a custom
+profile file that can be loaded by Zonemaster-Engine. Both Zonemaster-CLI and
+Zonemaster-Backend have direct options for loading a custom profile file.
+
+A custom profile file only has to contain those properties that are changed
+in comparison to the default profile file.
+
+You are adviced not to modify the default profile file in the default path on
+installed Zonemaster, since such modifications will be overwritten by updates
+of Zonemaster.
+
+
+[profile.json]:                        ../share/profile.json
+[profile_additional_properties.json]:  ../share/profile_additional_properties.json
+[Profile properties]:                  https://metacpan.org/pod/Zonemaster::Engine::Profile#PROFILE-PROPERTIES

docs/logentry_args.md

@@ -52,11 +52,12 @@ and updated messages (*msgids* and *msgstr*).
 | algo_descr     | Text                                     | The human readable description of a [DNSSEC algorithm].                 |
 | algo_mnemo     | Text                                     | The mnemonic of a [DNSSEC algorithm].                                   |
 | algo_num       | Non-negative integer                     | The numeric value for a [DNSSEC algorithm].                             |
-| domain         | Domain name                              | A domain name. If nsname is also applicable, use that one instead.      |
+| domain         | Domain name                              | A domain name. If "nsname" or "mailtarget" is also applicable, use that one instead.      |
 | ds_algo_descr  | Text                                     | The human readable description of a [DS Digest algorithm].              |
 | ds_algo_mnemo  | Text                                     | The mnemonic of a [DS Digest algorithm].                                |
 | ds_algo_num    | Non-negative integer                     | The numeric value for a [DS Digest algorithm].                          |
 | keytag         | Non-negative integer                     | A keytag for a DNSKEY record or a keytag used in a DS or RRSIG record.  |
+| label          | Domain name label                        | A single label, i.e. the string between the dots, from a domain name.   |
 | mailtarget     | Domain name                              | The domain name of the mailserver in an MX RDATA.                       |
 | mailtarget_list| List of domain names                     | A list of name servers, as specified by "mailtarget", separated by ";". |
 | module         | A Zonemaster test module, or `all`       | The name of a Zonemaster test module.                                   |
@@ -68,8 +69,11 @@ and updated messages (*msgids* and *msgstr*).
 | nsname         | Domain name                              | The domain name of a name server.                                       |
 | nsname_list    | List of domain names                     | A list of name servers, as specified by "nsname", separated by ";".     |
 | rcode          | An RCODE Name                            | An RCODE Name (not numeric code) from [DNS RCODEs].                     |
+| rrtype         | A Resource Record TYPE Name              | A Resource Record TYPE Name (not numeric code) from [DNS RR TYPEs].     |
+| soaserial      | Non-negative integer                     | The numeric value for the SERIAL field in an SOA record. Integer in range 0-4,294,967,295 |
+| soaserial_list | List of non-negative integers            | A list of non-negative integers, as specified by "soaserial", separated by ";". |
 | testcase       | A Zonemaster test case, or `all`         | A test case identifier.                                                 |
-
+| unicode_name   | Unicode name of a code point             | The name is a string in ASCII only and in upper case, e.g. "LATIN SMALL LETTER A"|
 
 ## Preliminary or proposed arguments
 
@@ -90,7 +94,6 @@ defined *arguments*.
 || DNS packet size| The size in octets of a DNS packets.|
 || DNSKEY key length| The key length for a DNSKEY. The interpretation of this value various quite a bit with the algorithm. Be careful when using it for algorithms that aren't RSA-based.|
 || DNSSEC delegation verification failure reason| A somewhat human-readable reason why the delegation step between the tested zone and its parent is not secure.|
-| dlabel (?) | Domain name label| A single label from a domain name.|
 | dlength (?) | Domain name label length| The length of a domain name label.|
 || Duration in seconds| An integer number of seconds.|
 | fqdn (?) | FQDN| A fully qualified domain name (with terminating dot).|
@@ -141,6 +144,7 @@ Message names maked with a question mark should not be considered stable.
 
 
 [Basic.pm]:                                  ../lib/Zonemaster/Engine/Test/Basic.pm
+[DNS RR TYPEs]:                              https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4
 [DNS RCODEs]:                                https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-6
 [DNSSEC algorithm]:                          https://www.iana.org/assignments/dns-sec-alg-numbers/dns-sec-alg-numbers.xhtml
 [DS Digest algorithm]:                       https://www.iana.org/assignments/ds-rr-types/ds-rr-types.xhtml

lib/Zonemaster/Engine.pm

@@ -1,6 +1,6 @@
 package Zonemaster::Engine;
 
-use version; our $VERSION = version->declare("v4.5.1");
+use version; our $VERSION = version->declare("v4.6.0");
 
 use 5.014002;
 
@@ -24,9 +24,20 @@ use Zonemaster::Engine::Test;
 use Zonemaster::Engine::Recursor;
 use Zonemaster::Engine::ASNLookup;
 
+INIT {
+    init_engine();
+}
+
 our $logger;
 our $recursor = Zonemaster::Engine::Recursor->new;
 
+my $init_done = 0;
+
+sub init_engine {
+    return if $init_done++;
+    Zonemaster::Engine::Recursor::init_recursor;
+}
+
 sub logger {
     return $logger //= Zonemaster::Engine::Logger->new;
 }
@@ -237,6 +248,10 @@ This manual describes the main L<Zonemaster::Engine> module. If what you're afte
 
 =over
 
+=item init_engine()
+
+Run the inititalization tasks if they have not been run already. This method is called automatically in INIT block.
+
 =item test_zone($name)
 
 Runs all available tests and returns a list of L<Zonemaster::Engine::Logger::Entry> objects.
@@ -418,12 +433,8 @@ Calle Dybedahl <calle at init.se>
 
 =head1 LICENSE
 
-This is free software, licensed under:
-
-The (three-clause) BSD License
-
-The full text of the license can be found in the
-F<LICENSE> file included with this distribution.
+This is free software under a 2-clause BSD license. The full text of the license can
+be found in the F<LICENSE> file included with this distribution.
 
 =cut