Add support to regressions tests for matching multiple values in erro…

…r_log.

- Update `Variable offset - FILES_NAMES` regression test to specify
  expected error log output with two strings, because the order in which
  they're processed can vary in different systems.
  - The test currently expects a specific order, which is not guaranteed
    as the underlying structure in `AnchoredSetVariable::resolve` is a
    `std::unordered_multimap`.
  - The test is now enabled on Windows builds too.
- Simplified `Test match variable (*/n)` tests not to depend on order
  and enabled them on Windows builds too.

.github/workflows/ci.yml

@@ -169,14 +169,6 @@ jobs:
           md \bin
           copy "C:\Program Files\Git\usr\bin\echo.exe" \bin
           copy "C:\Program Files\Git\usr\bin\echo.exe" \bin\echo
-      - name: Disable tests that don't work on Windows
-        working-directory: test\test-cases\regression
-        shell: cmd
-        run: |
-          jq "map(if .title == \"Test match variable (1/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
-          jq "map(if .title == \"Test match variable (2/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
-          jq "map(if .title == \"Test match variable (3/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
-          jq "map(if .title == \"Variable offset - FILES_NAMES\" then .enabled = 0 else . end)" offset-variable.json > tmp.json && move /Y tmp.json offset-variable.json
       - name: Run tests
         working-directory: build\win32\build
         run: |

build/win32/docker/Dockerfile

@@ -86,21 +86,6 @@ RUN cmd.exe /C md \bin
 RUN cmd.exe /C copy "C:\Program Files\GIT\usr\bin" \bin > NUL
 RUN cmd.exe /C copy "C:\Program Files\GIT\usr\bin\echo.exe" \bin\echo > NUL
 
-# disable tests that don't work on windows
-ARG JQ_VERSION=1.7.1
-ARG JQ_BINARY=jq-windows-amd64.exe
-ARG JQ_URL=https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/${JQ_BINARY}
-
-ARG JQ_BIN=C:\TEMP\jq.exe
-ADD ${JQ_URL} ${JQ_BIN}
-
-WORKDIR ${MOD_SECURITY_DIR}\test\test-cases\regression
-
-RUN %JQ_BIN% "map(if .title == \"Test match variable (1/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
-RUN %JQ_BIN% "map(if .title == \"Test match variable (2/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
-RUN %JQ_BIN% "map(if .title == \"Test match variable (3/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json
-RUN %JQ_BIN% "map(if .title == \"Variable offset - FILES_NAMES\" then .enabled = 0 else . end)" offset-variable.json > tmp.json && move /Y tmp.json offset-variable.json
-
 # run tests
 WORKDIR ${MOD_SECURITY_DIR}\build\win32\build
 

test/regression/regression.cc

@@ -341,7 +341,9 @@ void perform_unit_test(const ModSecurityTest<RegressionTest> &test,
                 std::to_string(t->http_code) +
                 " got: " + std::to_string(r.status) + "\n";
             testRes->passed = false;
-        } else if (!contains(context.m_server_log.str(), t->error_log)) {
+        } else if (auto errit = std::find_if(t->error_log.begin(), t->error_log.end(), 
+            [&context](const auto &x) { return !contains(context.m_server_log.str(), x); });
+            errit != t->error_log.end()) {
             if (test.m_automake_output) {
                 std::cout << ":test-result: FAIL " << filename \
                     << ":" << t->name << std::endl;
@@ -351,7 +353,7 @@ void perform_unit_test(const ModSecurityTest<RegressionTest> &test,
             testRes->reason << "Error log was not matching the " \
                 << "expected results." << std::endl;
             testRes->reason << KWHT << "Expecting: " << RESET \
-                << t->error_log + "";
+                << *errit + "";
             testRes->passed = false;
         } else if (!t->audit_log.empty() && !contains(getAuditLogContent(modsec_transaction.m_rules->m_auditLog->m_path1), t->audit_log)) {
             if (test.m_automake_output) {

test/regression/regression_test.cc

@@ -189,7 +189,14 @@ RegressionTest *RegressionTest::from_yajl_node(const yajl_val &node) {
                     u->debug_log = YAJL_GET_STRING(val2);
                 }
                 if (strcmp(key2, "error_log") == 0) {
-                    u->error_log = YAJL_GET_STRING(val2);
+                    if (val2->u.array.len == 0)
+                        u->error_log.insert(YAJL_GET_STRING(val2));
+                    else {
+                        for (int k = 0; k < val2->u.array.len; k++) {
+                            yajl_val vale = val2->u.array.values[k];
+                            u->error_log.insert(YAJL_GET_STRING(vale));
+                        }
+                    }
                 }
                 if (strcmp(key2, "http_code") == 0) {
                     u->http_code = YAJL_GET_INTEGER(val2);

test/regression/regression_test.h

@@ -20,6 +20,7 @@
 #include <unordered_map>
 #include <map>
 #include <vector>
+#include <set>
 #include <string>
 #include <utility>
 
@@ -54,7 +55,7 @@ class RegressionTest {
 
     std::string audit_log;
     std::string debug_log;
-    std::string error_log;
+    std::set<std::string> error_log;
     std::string parser_error;
 
     std::string clientIp;

test/test-cases/regression/issue-2423-msg-in-chain.json

@@ -14,7 +14,6 @@
     },
     "request":{
       "headers":{
-        "Host":"localhost",
         "Transfer-Encoding": "deflate"
       },
       "uri":"/match-this",
@@ -45,7 +44,6 @@
     },
     "request":{
       "headers":{
-        "Host":"localhost",
         "Transfer-Encoding": "deflate"
       },
       "uri":"/match-this",
@@ -76,7 +74,6 @@
     },
     "request":{
       "headers":{
-        "Host":"localhost",
         "Transfer-Encoding": "deflate"
       },
       "uri":"/match-this",

test/test-cases/regression/offset-variable.json

@@ -1509,7 +1509,10 @@
       ]
     },
     "expected":{
-      "error_log":"o0,8o0,8v491,8t:trimo0,16o0,16v709,16t:trim"
+      "error_log":[
+        "o0,8o0,8v491,8t:trim",
+        "o0,16o0,16v709,16t:trim"
+      ]
     },
     "rules":[
       "SecRequestBodyAccess On",