Ensure OWASP CRS v3.1 uses new upstream Modsecurity Docker image (#1438)

* Updating v3.1 docker build for travis testing

* Add the commit branch to travis

* Backporting 2.9.3 regression test fixes to OWASP CRS 3.1

.travis.yml

@@ -7,12 +7,12 @@ python:
 before_install:
   - |
     if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
-      docker build --build-arg REPO=$TRAVIS_PULL_REQUEST_SLUG --build-arg COMMIT=$TRAVIS_PULL_REQUEST_SHA -t modsecurity-crs ./util/docker/
+      docker build --build-arg REPO=$TRAVIS_PULL_REQUEST_SLUG --build-arg BRANCH=$TRAVIS_PULL_REQUEST_BRANCH --build-arg COMMIT=$TRAVIS_PULL_REQUEST_SHA -t modsecurity-crs ./util/docker/
     else
       docker build -t modsecurity-crs ./util/docker/
     fi
   - docker run -ti -e PARANOIA=5 -d -p 80:80 -v /var/log/apache2:/var/log/apache2/ --name "$TRAVIS_BUILD_ID" modsecurity-crs
-install: 
+install:
   - pip install -r ./util/integration/requirements.txt
   - pip install -r ./util/regression-tests/requirements.txt
 script:

util/docker/Dockerfile

@@ -1,27 +1,26 @@
-FROM owasp/modsecurity:v2-ubuntu-apache
-MAINTAINER Chaim Sanders chaim.sanders@gmail.com
+FROM owasp/modsecurity:2.9-apache
+LABEL maintainer="Chaim Sanders <chaim.sanders@gmail.com>"
 
 ARG COMMIT=v3.1/dev
+ARG BRANCH=v3.1/dev
 ARG REPO=SpiderLabs/owasp-modsecurity-crs
+ENV WEBSERVER=Apache
 ENV PARANOIA=1
 
 RUN apt-get update && \
-    apt-get -y install python git ca-certificates iproute2
+  apt-get -y install python git ca-certificates iproute2 && \
+  mkdir /opt/owasp-modsecurity-crs-3.1 && \
+  cd /opt/owasp-modsecurity-crs-3.1 && \
+  git init && \
+  git remote add origin https://github.com/${REPO} && \
+  git fetch --depth 1 origin ${BRANCH} && \
+  git checkout ${COMMIT} && \
+  mv crs-setup.conf.example crs-setup.conf && \
+  ln -sv /opt/owasp-modsecurity-crs-3.1 /etc/modsecurity.d/owasp-crs && \
+  printf "include /etc/modsecurity.d/owasp-crs/crs-setup.conf\ninclude /etc/modsecurity.d/owasp-crs/rules/*.conf" >> /etc/modsecurity.d/include.conf && \
+  sed -i -e 's/SecRuleEngine DetectionOnly/SecRuleEngine On/g' /etc/modsecurity.d/modsecurity.conf
 
-RUN cd /opt && \
-  git clone https://github.com/${REPO}.git owasp-modsecurity-crs-3.1 && \
-  cd owasp-modsecurity-crs-3.1 && \
-  git checkout -qf ${COMMIT} 
-
-RUN cd /opt && \
-  cp -R /opt/owasp-modsecurity-crs-3.1/ /etc/apache2/modsecurity.d/owasp-crs/ && \
-  mv /etc/apache2/modsecurity.d/owasp-crs/crs-setup.conf.example /etc/apache2/modsecurity.d/owasp-crs/crs-setup.conf && \
-  cd /etc/apache2/modsecurity.d && \
-  printf "include modsecurity.d/owasp-crs/crs-setup.conf\ninclude modsecurity.d/owasp-crs/rules/*.conf" > include.conf && \
-  sed -i -e 's/SecRuleEngine DetectionOnly/SecRuleEngine On/g' /etc/apache2/modsecurity.d/modsecurity.conf && \
-  a2enmod proxy proxy_http
-
-COPY proxy.conf           /etc/apache2/modsecurity.d/proxy.conf
+COPY proxy.conf           /etc/modsecurity.d/proxy.conf
 COPY docker-entrypoint.sh /
 
 EXPOSE 80

util/docker/docker-entrypoint.sh

@@ -1,15 +1,20 @@
 #!/bin/bash
-python -c "import re;import os;out=re.sub('(#SecAction[\S\s]{7}id:900000[\s\S]*tx\.paranoia_level=1\")','SecAction \\\\\n  \"id:900000, \\\\\n   phase:1, \\\\\n   nolog, \\\\\n   pass, \\\\\n   t:none, \\\\\n   setvar:tx.paranoia_level='+os.environ['PARANOIA']+'\"',open('/etc/apache2/modsecurity.d/owasp-crs/crs-setup.conf','r').read());open('/etc/apache2/modsecurity.d/owasp-crs/crs-setup.conf','w').write(out)" && \
-python -c "import re;import os;out=re.sub('(#SecAction[\S\s]{6}id:900330[\s\S]*total_arg_length=64000\")','SecAction \\\\\n \"id:900330, \\\\\n phase:1, \\\\\n nolog, \\\\\n pass, \\\\\n t:none, \\\\\n setvar:tx.total_arg_length=64000\"',open('/etc/apache2/modsecurity.d/owasp-crs/crs-setup.conf','r').read());open('/etc/apache2/modsecurity.d/owasp-crs/crs-setup.conf','w').write(out)" && \
-
-if [ ! -z $PROXY ]; then
-  if [ $PROXY -eq 1 ]; then
-    APACHE_ARGUMENTS='-D crs_proxy'
-    if [ -z "$UPSTREAM" ]; then
-      export UPSTREAM=$(/sbin/ip route | grep ^default | perl -pe 's/^.*?via ([\d.]+).*/$1/g'):81
+
+python -c "import re;import os;out=re.sub('(#SecAction[\S\s]{7}id:900000[\s\S]*tx\.paranoia_level=1\")','SecAction \\\\\n  \"id:900000, \\\\\n   phase:1, \\\\\n   nolog, \\\\\n   pass, \\\\\n   t:none, \\\\\n   setvar:tx.paranoia_level='+os.environ['PARANOIA']+'\"',open('/etc/modsecurity.d/owasp-crs/crs-setup.conf','r').read());open('/etc/modsecurity.d/owasp-crs/crs-setup.conf','w').write(out)" && \
+python -c "import re;import os;out=re.sub('(#SecAction[\S\s]{6}id:900330[\s\S]*total_arg_length=64000\")','SecAction \\\\\n \"id:900330, \\\\\n phase:1, \\\\\n nolog, \\\\\n pass, \\\\\n t:none, \\\\\n setvar:tx.total_arg_length=64000\"',open('/etc/modsecurity.d/owasp-crs/crs-setup.conf','r').read());open('/etc/modsecurity.d/owasp-crs/crs-setup.conf','w').write(out)" && \
+
+if [ $WEBSERVER = "Apache" ]; then
+  if [ ! -z $PROXY ]; then
+    if [ $PROXY -eq 1 ]; then
+      WEBSERVER_ARGUMENTS='-D crs_proxy'
+      if [ -z "$UPSTREAM" ]; then
+        export UPSTREAM=$(/sbin/ip route | grep ^default | perl -pe 's/^.*?via ([\d.]+).*/$1/g'):81
+      fi
     fi
   fi
+elif [ $WEBSERVER = "Nginx" ]; then
+  WEBSERVER_ARGUMENTS=''
 fi
 
 
-exec "$@" $APACHE_ARGUMENTS
+exec "$@" $WEBSERVER_ARGUMENTS

util/regression-tests/tests/REQUEST-920-PROTOCOL-ENFORCEMENT/920120.yaml

@@ -1,21 +1,21 @@
 ---
-  meta: 
+  meta:
     author: "csanders-git"
     enabled: true
     name: "920120.yaml"
     description: "Tests to trigger rule 920120"
-  tests: 
-    - 
+  tests:
+    -
       test_title: 920120-1
-      stages: 
-        - 
-          stage: 
+      stages:
+        -
+          stage:
             input:
               dest_addr: "127.0.0.1"
               method: "POST"
               port: 80
               headers:
-                  User-Agent: "ModSecurity CRS 3 Tests"              
+                  User-Agent: "ModSecurity CRS 3 Tests"
                   Host: "localhost"
                   Accept: "*/*"
                   Accept-Language: "en"
@@ -30,13 +30,13 @@
                 - "555-555-0199@example.com"
                 - "----------397236876--"
               protocol: "http"
-            output: 
+            output:
               log_contains: "id \"920120\""
-    - 
+    -
       test_title: 920120-2
       desc: Attempted multipart/form-data bypass (920120) from old modsec regressions
       stages:
-      - 
+      -
         stage:
           input:
             dest_addr: 127.0.0.1
@@ -57,7 +57,7 @@
             version: HTTP/1.1
             data:
             - '-----------------------------627652292512397580456702590'
-            - 'Content-Disposition: form-data; name=x'';filename="'';name=contact.txt;"'
+            - 'Content-Disposition: form-data; name="fi=le"; filename="test"'
             - 'Content-Type: text/plain'
             - ''
             - 'email: security@modsecurity.org'
@@ -69,11 +69,11 @@
             - '-----------------------------627652292512397580456702590--'
           output:
             log_contains: id "920120"
-    - 
+    -
       test_title: 920120-3
       desc: Invalid Request Body (920120)  from old modsec regressions
       stages:
-      - 
+      -
         stage:
           input:
             dest_addr: 127.0.0.1
@@ -109,4 +109,3 @@
             - '-----------------------------265001916915724--'
           output:
             log_contains: id "920120"
-

util/regression-tests/tests/REQUEST-920-PROTOCOL-ENFORCEMENT/920130.yaml

@@ -1,7 +1,7 @@
 ---
   meta: 
     author: "csanders-git"
-    enabled: true
+    enabled: false
     name: "920130.yaml"
     description: "Tests to trigger rule 920130"
   tests: 
@@ -82,4 +82,4 @@
             - </SOAP-ENV:Envelope>
           output:
             log_contains: id "920130"
-
+