From a081f20177e5ce21d146069e3882a43f2786a34d Mon Sep 17 00:00:00 2001
From: Max Leske <maxleske@gmail.com>
Date: Fri, 10 May 2024 22:50:11 +0200
Subject: [PATCH] feat: update to newest ftw-tests-schema (#209)

* feat: add run_once option for tests

- add new `retry_once` option
- add test case for `retry_once`
- make test infrastructure more modular

* feat: update code and tests to new schema version
* feat: implement platform overrides

* chore: fix lint issues
* chore: use PR branch of schema until ready
* feat: add --fail-fast flag

- add flag to stop test run on first test failure
- fix statistics to record at test, not stage level

* feat: add --log-file flag

The log file was the last remaining setting that could not be configured
from the command line.

* fix: increase default read timeout from 1 to 10 seconds

The default read timeout is too short for some of the more expensive
tests.

* feat: improve console output

- always print test ID on failure; especially useful with
  --show-failures-only
- use the same indentation for all output messages

* chore: update schema, drop legacy koanf
---
 check/base.go                                 |  49 ++--
 check/base_test.go                            |  51 +---
 check/error.go                                |  17 +-
 check/error_test.go                           |   9 +-
 check/logs.go                                 |  78 +++++-
 check/logs_test.go                            |  80 +++++-
 check/response.go                             |  20 +-
 check/response_test.go                        |   8 -
 check/status.go                               |  43 ++-
 check/status_test.go                          | 138 +++++++--
 cmd/check_test.go                             |  35 ++-
 cmd/root.go                                   |  17 +-
 cmd/run.go                                    |  17 +-
 cmd/run_test.go                               |  35 ++-
 config/config.go                              |  73 ++++-
 config/config_test.go                         |  67 ++++-
 config/types.go                               |  39 ++-
 ftwhttp/client.go                             |   3 +-
 ftwhttp/client_test.go                        |  30 +-
 ftwhttp/connection.go                         |   2 +-
 ftwhttp/request.go                            |   3 -
 go.mod                                        |   4 +-
 go.sum                                        |  14 +-
 output/output.go                              |  10 +-
 runner/run.go                                 | 122 ++++----
 runner/run_input_override_test.go             |  32 ++-
 runner/run_test.go                            | 144 ++++++++--
 runner/stats.go                               |  15 +-
 runner/testdata/TestBrokenOverrideRun.yaml    |  19 +-
 .../testdata/TestBrokenPortOverrideRun.yaml   |  19 +-
 runner/testdata/TestCloudRun.yaml             | 165 ++++++-----
 runner/testdata/TestDisabledRun.yaml          |  19 +-
 runner/testdata/TestFailFast.yaml             |  45 +++
 runner/testdata/TestFailedTestsRun.yaml       |  22 +-
 runner/testdata/TestIgnoredTestsRun.yaml      | 113 ++++----
 runner/testdata/TestLogsRun.yaml              |  44 ++-
 runner/testdata/TestOverrideRun.yaml          |  22 +-
 runner/testdata/TestRetryOnce.yaml            |  21 ++
 runner/testdata/TestRunMultipleMatches.yaml   |  36 ++-
 runner/testdata/TestRunTests_Run.yaml         | 113 ++++----
 runner/types.go                               |   3 +
 test/defaults.go                              |   7 +-
 test/errors.go                                |  40 ---
 test/errors_test.go                           |  66 -----
 test/files.go                                 |  10 +-
 test/files_test.go                            |  51 ++--
 .../TestCheckBenchmarkCheckFiles.yaml         |   1 +
 test/types.go                                 | 113 ++++++--
 test/types_test.go                            | 261 +++++++++---------
 test/yaml.go                                  |   6 +-
 waflog/read.go                                |   4 +-
 51 files changed, 1387 insertions(+), 968 deletions(-)
 create mode 100644 runner/testdata/TestFailFast.yaml
 create mode 100644 runner/testdata/TestRetryOnce.yaml
 delete mode 100644 test/errors.go
 delete mode 100644 test/errors_test.go

diff --git a/check/base.go b/check/base.go
index 91f208c3..5b48ae1d 100644
--- a/check/base.go
+++ b/check/base.go
@@ -6,6 +6,7 @@ package check
 import (
 	"bytes"
 
+	schema "github.com/coreruleset/ftw-tests-schema/types"
 	"github.com/coreruleset/go-ftw/config"
 	"github.com/coreruleset/go-ftw/test"
 	"github.com/coreruleset/go-ftw/waflog"
@@ -39,8 +40,8 @@ func (c *FTWCheck) SetExpectTestOutput(t *test.Output) {
 }
 
 // SetExpectStatus sets to expect the HTTP status from the test to be in the integer range passed
-func (c *FTWCheck) SetExpectStatus(s []int) {
-	c.expected.Status = s
+func (c *FTWCheck) SetExpectStatus(status int) {
+	c.expected.Status = status
 }
 
 // SetExpectResponse sets the response we expect in the text from the server
@@ -54,39 +55,43 @@ func (c *FTWCheck) SetExpectError(expect bool) {
 }
 
 // SetLogContains sets the string to look for in logs
-func (c *FTWCheck) SetLogContains(contains string) {
-	c.expected.LogContains = contains
+func (c *FTWCheck) SetLogContains(regex string) {
+	//nolint:staticcheck
+	c.expected.LogContains = regex
+	c.expected.Log.MatchRegex = regex
 }
 
 // SetNoLogContains sets the string to look that should not present in logs
-func (c *FTWCheck) SetNoLogContains(contains string) {
-	c.expected.NoLogContains = contains
+func (c *FTWCheck) SetNoLogContains(regex string) {
+	//nolint:staticcheck
+	c.expected.NoLogContains = regex
+	c.expected.Log.NoMatchRegex = regex
 }
 
-// ForcedIgnore check if this id need to be ignored from results
-func (c *FTWCheck) ForcedIgnore(id string) bool {
+// ForcedIgnore check if this ID need to be ignored from results
+func (c *FTWCheck) ForcedIgnore(testCase *schema.Test) bool {
 	for re := range c.cfg.TestOverride.Ignore {
-		if re.MatchString(id) {
+		if re.MatchString(testCase.IdString()) {
 			return true
 		}
 	}
 	return false
 }
 
-// ForcedPass check if this id need to be ignored from results
-func (c *FTWCheck) ForcedPass(id string) bool {
+// ForcedPass check if this ID need to be ignored from results
+func (c *FTWCheck) ForcedPass(testCase *schema.Test) bool {
 	for re := range c.cfg.TestOverride.ForcePass {
-		if re.MatchString(id) {
+		if re.MatchString(testCase.IdString()) {
 			return true
 		}
 	}
 	return false
 }
 
-// ForcedFail check if this id need to be ignored from results
-func (c *FTWCheck) ForcedFail(id string) bool {
+// ForcedFail check if this ID need to be ignored from results
+func (c *FTWCheck) ForcedFail(testCase *schema.Test) bool {
 	for re := range c.cfg.TestOverride.ForceFail {
-		if re.MatchString(id) {
+		if re.MatchString(testCase.IdString()) {
 			return true
 		}
 	}
@@ -98,20 +103,6 @@ func (c *FTWCheck) CloudMode() bool {
 	return c.cfg.RunMode == config.CloudRunMode
 }
 
-// SetCloudMode alters the values for expected logs and status code
-func (c *FTWCheck) SetCloudMode() {
-	var status = c.expected.Status
-
-	if c.expected.LogContains != "" {
-		status = append(status, 403)
-		c.expected.LogContains = ""
-	} else if c.expected.NoLogContains != "" {
-		status = append(status, 200, 404, 405)
-		c.expected.NoLogContains = ""
-	}
-	c.expected.Status = status
-}
-
 // SetStartMarker sets the log line that marks the start of the logs to analyze
 func (c *FTWCheck) SetStartMarker(marker []byte) {
 	c.log.StartMarker = bytes.ToLower(marker)
diff --git a/check/base_test.go b/check/base_test.go
index 5d09fca2..8d58a275 100644
--- a/check/base_test.go
+++ b/check/base_test.go
@@ -4,11 +4,11 @@
 package check
 
 import (
-	"sort"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
 
+	schema "github.com/coreruleset/ftw-tests-schema/types"
 	"github.com/coreruleset/go-ftw/config"
 	"github.com/coreruleset/go-ftw/test"
 	"github.com/coreruleset/go-ftw/utils"
@@ -60,7 +60,7 @@ func (s *checkBaseTestSuite) TestNewCheck() {
 	}
 
 	to := test.Output{
-		Status:           []int{200},
+		Status:           200,
 		ResponseContains: "",
 		LogContains:      "nothing",
 		NoLogContains:    "",
@@ -72,6 +72,7 @@ func (s *checkBaseTestSuite) TestNewCheck() {
 
 	c.SetNoLogContains("nologcontains")
 
+	//nolint:staticcheck
 	s.Equal(c.expected.NoLogContains, "nologcontains", "Problem setting nologcontains")
 }
 
@@ -79,52 +80,18 @@ func (s *checkBaseTestSuite) TestForced() {
 	c, err := NewCheck(s.cfg)
 	s.Require().NoError(err)
 
-	s.True(c.ForcedIgnore("942200-1"), "Can't find ignored value")
+	s.True(c.ForcedIgnore(&schema.Test{RuleId: 942200, TestId: 1}), "Can't find ignored value")
 
-	s.False(c.ForcedFail("1245"), "Value should not be found")
+	s.False(c.ForcedFail(&schema.Test{RuleId: 12345, TestId: 1}), "Value should not be found")
 
-	s.False(c.ForcedPass("1234"), "Value should not be found")
+	s.False(c.ForcedPass(&schema.Test{RuleId: 12345, TestId: 1}), "Value should not be found")
 
-	s.True(c.ForcedPass("1245"), "Value should be found")
+	s.True(c.ForcedPass(&schema.Test{RuleId: 1245, TestId: 1}), "Value should be found")
 
-	s.True(c.ForcedFail("6789"), "Value should be found")
+	s.True(c.ForcedFail(&schema.Test{RuleId: 6789, TestId: 1}), "Value should be found")
 
 	s.cfg.TestOverride.Ignore = make(map[*config.FTWRegexp]string)
-	s.Falsef(c.ForcedIgnore("anything"), "Should not find ignored value in empty map")
-
-}
-
-func (s *checkBaseTestSuite) TestCloudMode() {
-	c, err := NewCheck(s.cfg)
-	s.Require().NoError(err)
-
-	s.True(c.CloudMode(), "couldn't detect cloud mode")
-
-	status := []int{200, 301}
-	c.SetExpectStatus(status)
-	c.SetLogContains("this text")
-	// this should override logcontains
-	c.SetCloudMode()
-
-	cloudStatus := c.expected.Status
-	sort.Ints(cloudStatus)
-	res := sort.SearchInts(cloudStatus, 403)
-	s.Equalf(2, res, "couldn't find expected 403 status in %#v -> %d", cloudStatus, res)
-
-	c.SetLogContains("")
-	c.SetNoLogContains("no log contains")
-	// this should override logcontains
-	c.SetCloudMode()
-
-	cloudStatus = c.expected.Status
-	sort.Ints(cloudStatus)
-	found := false
-	for _, n := range cloudStatus {
-		if n == 200 {
-			found = true
-		}
-	}
-	s.True(found, "couldn't find expected 200 status")
+	s.Falsef(c.ForcedIgnore(&schema.Test{RuleId: 1234, TestId: 1}), "Should not find ignored value in empty map")
 
 }
 
diff --git a/check/error.go b/check/error.go
index a85dbb77..57e042b3 100644
--- a/check/error.go
+++ b/check/error.go
@@ -6,14 +6,15 @@ package check
 import "github.com/rs/zerolog/log"
 
 // AssertExpectError helper to check if this error was expected or not
-func (c *FTWCheck) AssertExpectError(err error) bool {
-	if err != nil {
-		log.Debug().Msgf("ftw/check: expected error? -> %t, and error is %s", *c.expected.ExpectError, err.Error())
+func (c *FTWCheck) AssertExpectError(err error) (bool, bool) {
+	errorExpected := c.expected.ExpectError != nil && *c.expected.ExpectError
+	var errorString string
+	if err == nil {
+		errorString = "-"
 	} else {
-		log.Debug().Msgf("ftw/check: expected error? -> %t, and error is nil", *c.expected.ExpectError)
+		errorString = err.Error()
 	}
-	if *c.expected.ExpectError && err != nil {
-		return true
-	}
-	return false
+	log.Debug().Caller().Msgf("Error expected: %t. Found: %s", errorExpected, errorString)
+
+	return errorExpected, (errorExpected && err != nil) || (!errorExpected && err == nil)
 }
diff --git a/check/error_test.go b/check/error_test.go
index 889455e9..256c9caf 100644
--- a/check/error_test.go
+++ b/check/error_test.go
@@ -46,12 +46,15 @@ func (s *checkErrorTestSuite) SetupTest() {
 	s.Require().NoError(err)
 	s.cfg.WithLogfile(logName)
 }
+
 func (s *checkErrorTestSuite) TestAssertResponseErrorOK() {
 	c, err := NewCheck(s.cfg)
 	s.Require().NoError(err)
 	for _, e := range expectedOKTests {
 		c.SetExpectError(e.expected)
-		s.Equal(e.expected, c.AssertExpectError(e.err))
+		expected, succeeded := c.AssertExpectError(e.err)
+		s.Equal(e.expected, expected)
+		s.True(succeeded)
 	}
 }
 
@@ -61,6 +64,8 @@ func (s *checkErrorTestSuite) TestAssertResponseFail() {
 
 	for _, e := range expectedFailTests {
 		c.SetExpectError(e.expected)
-		s.False(c.AssertExpectError(e.err))
+		expected, succeeded := c.AssertExpectError(e.err)
+		s.Equal(e.expected, expected)
+		s.False(succeeded)
 	}
 }
diff --git a/check/logs.go b/check/logs.go
index ab1d4ea8..d13dba88 100644
--- a/check/logs.go
+++ b/check/logs.go
@@ -3,28 +3,76 @@
 
 package check
 
-// AssertNoLogContains returns true is the string is not found in the logs
-func (c *FTWCheck) AssertNoLogContains() bool {
-	if c.expected.NoLogContains != "" {
-		return !c.log.Contains(c.expected.NoLogContains)
+import (
+	"fmt"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+)
+
+func (c *FTWCheck) AssertLogs() bool {
+	if c.CloudMode() {
+		// No logs to check in cloud mode
+		return true
 	}
-	return false
+
+	return c.assertLogContains() && c.assertNoLogContains()
 }
 
-// NoLogContainsRequired checks that the test requires no_log_contains
-func (c *FTWCheck) NoLogContainsRequired() bool {
-	return c.expected.NoLogContains != ""
+// AssertNoLogContains returns true is the string is not found in the logs
+func (c *FTWCheck) assertNoLogContains() bool {
+	logExpectations := c.expected.Log
+	result := true
+	if logExpectations.NoMatchRegex != "" {
+		result = !c.log.Contains(logExpectations.NoMatchRegex)
+		if !result {
+			log.Debug().Msgf("Unexpectedly found match for '%s'", logExpectations.NoMatchRegex)
+		}
+	}
+	if result && len(logExpectations.NoExpectIds) > 0 {
+		result = !c.log.Contains(generateIdRegex(logExpectations.NoExpectIds))
+		if !result {
+			log.Debug().Msg("Unexpectedly found IDs")
+		}
+	}
+	return result
 }
 
 // AssertLogContains returns true when the logs contain the string
-func (c *FTWCheck) AssertLogContains() bool {
-	if c.expected.LogContains != "" {
-		return c.log.Contains(c.expected.LogContains)
+func (c *FTWCheck) assertLogContains() bool {
+	logExpectations := c.expected.Log
+	result := true
+	if logExpectations.MatchRegex != "" {
+		result = c.log.Contains(logExpectations.MatchRegex)
+		if !result {
+			log.Debug().Msgf("Failed to find match for match_regex. Expected to find '%s'", logExpectations.MatchRegex)
+		}
+	}
+	if result && len(logExpectations.ExpectIds) > 0 {
+		result = c.log.Contains(generateIdRegex(logExpectations.ExpectIds))
+		if !result {
+			log.Debug().Msg("Failed to find expected IDs")
+		}
 	}
-	return false
+	return result
 }
 
-// LogContainsRequired checks that the test requires log_contains
-func (c *FTWCheck) LogContainsRequired() bool {
-	return c.expected.LogContains != ""
+// Search for both standard ModSecurity, and JSON output
+func generateIdRegex(ids []uint) string {
+	modSecLogSyntax := strings.Builder{}
+	jsonLogSyntax := strings.Builder{}
+	modSecLogSyntax.WriteString(`\[id "(?:`)
+	jsonLogSyntax.WriteString(`"id":\s*"?(?:`)
+	for index, id := range ids {
+		if index > 0 {
+			modSecLogSyntax.WriteRune('|')
+			jsonLogSyntax.WriteRune('|')
+		}
+		modSecLogSyntax.WriteString(fmt.Sprint(id))
+		jsonLogSyntax.WriteString(fmt.Sprint(id))
+	}
+	modSecLogSyntax.WriteString(`)"\]`)
+	jsonLogSyntax.WriteString(`)"?`)
+
+	return modSecLogSyntax.String() + "|" + jsonLogSyntax.String()
 }
diff --git a/check/logs_test.go b/check/logs_test.go
index 38e87b26..eb2cd432 100644
--- a/check/logs_test.go
+++ b/check/logs_test.go
@@ -42,27 +42,87 @@ func (s *checkLogsTestSuite) TearDownTest() {
 	err := os.Remove(s.logName)
 	s.Require().NoError(err)
 }
-func (s *checkLogsTestSuite) TestAssertLogContainsOK() {
+
+func (s *checkLogsTestSuite) TestLogContains() {
 	c, err := NewCheck(s.cfg)
 	s.Require().NoError(err)
 
 	c.SetLogContains(`id "920300"`)
-	s.True(c.AssertLogContains(), "did not find expected content 'id \"920300\"'")
+	s.True(c.AssertLogs(), "did not find expected content 'id \"920300\"'")
 
 	c.SetLogContains(`SOMETHING`)
-	s.False(c.AssertLogContains(), "found something that is not there")
-	s.True(c.LogContainsRequired(), "if LogContains is not empty it should return true")
+	s.False(c.AssertLogs(), "found something that is not there")
 
 	c.SetLogContains("")
-	s.False(c.AssertLogContains(), "empty LogContains should return false")
+	s.True(c.AssertLogs(), "empty LogContains should return true")
+}
 
-	c.SetNoLogContains("SOMETHING")
-	s.True(c.AssertNoLogContains(), "found something that is not there")
+func (s *checkLogsTestSuite) TestNoLogContains() {
+	c, err := NewCheck(s.cfg)
+	s.Require().NoError(err)
 
 	c.SetNoLogContains(`id "920300"`)
-	s.False(c.AssertNoLogContains(), "did not find expected content")
+	s.False(c.AssertLogs(), "did not find expected content")
+
+	c.SetNoLogContains("SOMETHING")
+	s.True(c.AssertLogs(), "found something that is not there")
 
 	c.SetNoLogContains("")
-	s.False(c.AssertNoLogContains(), "should return false when empty string is passed")
-	s.False(c.NoLogContainsRequired(), "if NoLogContains is an empty string is passed should return false")
+	s.True(c.AssertLogs(), "should return true when empty string is passed")
+}
+
+func (s *checkLogsTestSuite) TestAssertLogMatchRegex() {
+	c, err := NewCheck(s.cfg)
+	s.Require().NoError(err)
+
+	c.expected.Log.MatchRegex = `id\s"920300"`
+	s.True(c.AssertLogs(), `did not find expected content 'id\s"920300"'`)
+
+	c.expected.Log.MatchRegex = `SOMETHING`
+	s.False(c.AssertLogs(), "found something that is not there")
+
+	c.expected.Log.MatchRegex = ""
+	s.True(c.AssertLogs(), "empty LogContains should return true")
+}
+
+func (s *checkLogsTestSuite) TestAssertLogNoMatchRegex() {
+	c, err := NewCheck(s.cfg)
+	s.Require().NoError(err)
+
+	c.expected.Log.NoMatchRegex = `id\s"920300"`
+	s.False(c.AssertLogs(), `expected to find 'id\s"920300"'`)
+
+	c.expected.Log.NoMatchRegex = `SOMETHING`
+	s.True(c.AssertLogs(), "expected to _not_ find SOMETHING")
+
+	c.expected.Log.NoMatchRegex = ""
+	s.True(c.AssertLogs(), "empty LogContains should return true")
+}
+
+func (s *checkLogsTestSuite) TestAssertLogExpectIds() {
+	c, err := NewCheck(s.cfg)
+	s.Require().NoError(err)
+
+	c.expected.Log.ExpectIds = []uint{920300}
+	s.True(c.AssertLogs(), `did not find expected content 'id\s"920300"'`)
+
+	c.expected.Log.ExpectIds = []uint{123456}
+	s.False(c.AssertLogs(), "found something that is not there")
+
+	c.expected.Log.ExpectIds = []uint{}
+	s.True(c.AssertLogs(), "empty LogContains should return true")
+}
+
+func (s *checkLogsTestSuite) TestAssertLogNoExpectId() {
+	c, err := NewCheck(s.cfg)
+	s.Require().NoError(err)
+
+	c.expected.Log.NoExpectIds = []uint{920300}
+	s.False(c.AssertLogs(), `expected to find 'id\s"920300"'`)
+
+	c.expected.Log.NoExpectIds = []uint{123456}
+	s.True(c.AssertLogs(), "expected to _not_ find SOMETHING")
+
+	c.expected.Log.NoExpectIds = []uint{}
+	s.True(c.AssertLogs(), "empty LogContains should return true")
 }
diff --git a/check/response.go b/check/response.go
index 6f38bbcb..ce06457c 100644
--- a/check/response.go
+++ b/check/response.go
@@ -4,18 +4,22 @@
 package check
 
 import (
-	"strings"
+	"regexp"
+
+	"github.com/rs/zerolog/log"
 )
 
 // AssertResponseContains checks that the http response contains the needle
 func (c *FTWCheck) AssertResponseContains(response string) bool {
 	if c.expected.ResponseContains != "" {
-		return strings.Contains(response, c.expected.ResponseContains)
+		found, err := regexp.MatchString(c.expected.ResponseContains, response)
+		if err != nil {
+			log.Fatal().Msgf("Invalid regular expression for matching response contents: '%s'", c.expected.ResponseContains)
+		}
+		if !found {
+			log.Debug().Msgf("Failed to match response contents. Expected to find '%s'", c.expected.ResponseContains)
+		}
+		return found
 	}
-	return false
-}
-
-// ResponseContainsRequired checks that the test requires to check the response
-func (c *FTWCheck) ResponseContainsRequired() bool {
-	return c.expected.ResponseContains != ""
+	return true
 }
diff --git a/check/response_test.go b/check/response_test.go
index cb26a8cf..99365bc5 100644
--- a/check/response_test.go
+++ b/check/response_test.go
@@ -70,11 +70,3 @@ func (s *checkResponseTestSuite) TestAssertResponseTextChecksFullResponseOK() {
 		s.Truef(c.AssertResponseContains(e.response), "unexpected response: %v", e.response)
 	}
 }
-
-func (s *checkResponseTestSuite) TestAssertResponseContainsRequired() {
-	c, err := NewCheck(s.cfg)
-	s.Require().NoError(err)
-	c.SetExpectResponse("")
-	s.False(c.AssertResponseContains(""), "response shouldn't contain text")
-	s.False(c.ResponseContainsRequired(), "response shouldn't contain text")
-}
diff --git a/check/status.go b/check/status.go
index 8196f4b2..8682a3e6 100644
--- a/check/status.go
+++ b/check/status.go
@@ -3,17 +3,44 @@
 
 package check
 
+import (
+	"slices"
+
+	"github.com/rs/zerolog/log"
+)
+
+var negativeExpectedStatuses = []int{200, 404, 405}
+
 // AssertStatus will match the expected status list with the one received in the response
 func (c *FTWCheck) AssertStatus(status int) bool {
-	for _, i := range c.expected.Status {
-		if i == status {
-			return true
-		}
+	// No status code expectation defined
+	if c.expected.Status == 0 {
+		return true
 	}
-	return false
+
+	if c.CloudMode() {
+		return c.assertCloudStatus(status)
+	}
+
+	found := c.expected.Status == status
+	if !found {
+		log.Debug().Msgf("Failed to match response status. Expected: %d, found: %d", c.expected.Status, status)
+	}
+	return found
+
 }
 
-// StatusCodeRequired checks that the test requires to check the returned status code
-func (c *FTWCheck) StatusCodeRequired() bool {
-	return c.expected.Status != nil
+func (c *FTWCheck) assertCloudStatus(status int) bool {
+	logExpectations := c.expected.Log
+	if (logExpectations.MatchRegex != "" || len(logExpectations.ExpectIds) > 0) && status == 403 {
+		return true
+	}
+	if (logExpectations.NoMatchRegex != "" || len(logExpectations.NoExpectIds) > 0) && slices.Contains(negativeExpectedStatuses, status) {
+		return true
+	}
+	found := c.expected.Status == status
+	if !found {
+		log.Debug().Msgf("Failed to match response status (cloud mode). Expected: %d, found: %d", c.expected.Status, status)
+	}
+	return found
 }
diff --git a/check/status_test.go b/check/status_test.go
index e450a990..06beb921 100644
--- a/check/status_test.go
+++ b/check/status_test.go
@@ -4,6 +4,7 @@
 package check
 
 import (
+	"slices"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
@@ -12,23 +13,6 @@ import (
 	"github.com/coreruleset/go-ftw/utils"
 )
 
-var statusOKTests = []struct {
-	status         int
-	expectedStatus []int
-}{
-	{400, []int{0, 100, 200, 400}},
-	{400, []int{400}},
-}
-
-var statusFailTests = []struct {
-	status         int
-	expectedStatus []int
-}{
-	{400, []int{0, 100, 200}},
-	{200, []int{400}},
-	{200, []int{0}},
-}
-
 type checkStatusTestSuite struct {
 	suite.Suite
 	cfg *config.FTWConfiguration
@@ -50,26 +34,122 @@ func (s *checkStatusTestSuite) TestStatusOK() {
 	c, err := NewCheck(s.cfg)
 	s.Require().NoError(err)
 
-	for _, expected := range statusOKTests {
-		c.SetExpectStatus(expected.expectedStatus)
-		s.True(c.AssertStatus(expected.status))
-	}
+	c.SetExpectStatus(0)
+	s.checkStatus(c, []int{})
+
+	c.SetExpectStatus(200)
+	s.checkStatus(c, []int{200})
+
+	c.SetExpectStatus(303)
+	s.checkStatus(c, []int{303})
+
+	c.SetExpectStatus(400)
+	s.checkStatus(c, []int{400})
+
+	c.SetExpectStatus(403)
+	s.checkStatus(c, []int{403})
+
+	c.SetExpectStatus(500)
+	s.checkStatus(c, []int{500})
 }
 
-func (s *checkStatusTestSuite) TestStatusFail() {
+// always match since no status expectation set
+func (s *checkStatusTestSuite) TestCloudModePositiveMatch_AlwaysMatch() {
 	c, err := NewCheck(s.cfg)
 	s.Require().NoError(err)
+	c.cfg.RunMode = config.CloudRunMode
+	s.True(c.CloudMode(), "couldn't detect cloud mode")
 
-	for _, expected := range statusFailTests {
-		c.SetExpectStatus(expected.expectedStatus)
-		s.False(c.AssertStatus(expected.status))
-	}
+	// negative regex match set
+	c.SetLogContains("log contains")
+	s.checkStatus(c, []int{})
+
+	// negative regex match and ID negative ID match set
+	c.expected.Log.ExpectIds = []uint{123456}
+	s.checkStatus(c, []int{})
+
+	// negative ID match set
+	c.SetLogContains("")
+	s.checkStatus(c, []int{})
 }
 
-func (s *checkStatusTestSuite) TestStatusCodeRequired() {
+func (s *checkStatusTestSuite) TestCloudModePositiveMatch() {
 	c, err := NewCheck(s.cfg)
 	s.Require().NoError(err)
+	c.cfg.RunMode = config.CloudRunMode
+	s.True(c.CloudMode(), "couldn't detect cloud mode")
+
+	// nothing set
+	c.SetExpectStatus(418) // I'm a teapot
+	s.checkStatus(c, []int{418})
+
+	// regex match set
+	c.SetLogContains("this text")
+	s.checkStatus(c, []int{403, 418})
+
+	// regex match and ID match set
+	c.expected.Log.ExpectIds = []uint{123456}
+	s.checkStatus(c, []int{403, 418})
+
+	// ID match set
+	c.SetLogContains("")
+	s.checkStatus(c, []int{403, 418})
+}
 
-	c.SetExpectStatus([]int{200})
-	s.True(c.StatusCodeRequired(), "status code should be required")
+// always match since no status expectation set
+func (s *checkStatusTestSuite) TestCloudModeNegativeMatch_AlwaysMatch() {
+	c, err := NewCheck(s.cfg)
+	s.Require().NoError(err)
+	c.cfg.RunMode = config.CloudRunMode
+	s.True(c.CloudMode(), "couldn't detect cloud mode")
+
+	// negative regex match set
+	c.SetNoLogContains("no log contains")
+	s.checkStatus(c, []int{})
+
+	// negative regex match and ID negative ID match set
+	c.expected.Log.NoExpectIds = []uint{123456}
+	s.checkStatus(c, []int{})
+
+	// negative ID match set
+	c.SetNoLogContains("")
+	s.checkStatus(c, []int{})
+}
+
+// status expectation set, only match specific statuses
+func (s *checkStatusTestSuite) TestCloudModeNegativeMatch_SpecificMatch() {
+	c, err := NewCheck(s.cfg)
+	s.Require().NoError(err)
+	c.cfg.RunMode = config.CloudRunMode
+	s.True(c.CloudMode(), "couldn't detect cloud mode")
+
+	// nothing set
+	c.SetExpectStatus(418) // I'm a teapot
+	s.checkStatus(c, []int{418})
+
+	// negative regex match set
+	c.SetNoLogContains("no log contains")
+	s.checkStatus(c, []int{200, 404, 418, 405})
+
+	// negative regex match and ID negative ID match set
+	c.expected.Log.NoExpectIds = []uint{123456}
+	s.checkStatus(c, []int{200, 404, 418, 405})
+
+	// negative ID match set
+	c.SetNoLogContains("")
+	s.checkStatus(c, []int{200, 404, 418, 405})
+}
+
+func (s *checkStatusTestSuite) checkStatus(c *FTWCheck, expectedSuccesses []int) {
+	if len(expectedSuccesses) == 0 {
+		s.True(c.AssertStatus(-1), "Expected successful check because no expectation set")
+		return
+	}
+	for status := 100; status < 600; status++ {
+		if slices.Contains(expectedSuccesses, status) {
+			s.True(c.AssertStatus(status), "Unexpected result for status %d", status)
+		} else {
+			s.False(c.AssertStatus(status), "Unexpected result for status %d", status)
+		}
+	}
 }
diff --git a/cmd/check_test.go b/cmd/check_test.go
index d84ab3ce..0ecdc9a9 100644
--- a/cmd/check_test.go
+++ b/cmd/check_test.go
@@ -21,22 +21,21 @@ meta:
   description: "Test file for go-ftw"
 tests:
   - # Standard GET request
-    test_title: 1234
+    test_id: 1234
     stages:
-      - stage:
-          input:
-            dest_addr: "127.0.0.1"
-            method: "GET"
-            port: 1234
-            headers:
-              User-Agent: "OWASP CRS test agent"
-              Host: "localhost"
-              Accept: "*/*"
-            protocol: "http"
-            uri: "/"
-            version: "HTTP/1.1"
-          output:
-            status: [200]
+      - input:
+          dest_addr: "127.0.0.1"
+          method: "GET"
+          port: 1234
+          headers:
+            User-Agent: "OWASP CRS test agent"
+            Host: "localhost"
+            Accept: "*/*"
+          protocol: "http"
+          uri: "/"
+          version: "HTTP/1.1"
+        output:
+          status: 200
 `
 
 type checkCmdTestSuite struct {
@@ -46,11 +45,9 @@ type checkCmdTestSuite struct {
 }
 
 func (s *checkCmdTestSuite) SetupTest() {
-	tempDir, err := os.MkdirTemp("", "go-ftw-tests")
-	s.Require().NoError(err)
-	s.tempDir = tempDir
+	s.tempDir = s.T().TempDir()
 
-	err = os.MkdirAll(s.tempDir, fs.ModePerm)
+	err := os.MkdirAll(s.tempDir, fs.ModePerm)
 	s.Require().NoError(err)
 	testFileContents, err := os.CreateTemp(s.tempDir, "mock-test-*.yaml")
 	s.Require().NoError(err)
diff --git a/cmd/root.go b/cmd/root.go
index 779b2bd9..6ad5d11c 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -14,10 +14,11 @@ import (
 )
 
 var (
-	cfgFile string
-	debug   bool
-	trace   bool
-	cloud   bool
+	cfgFile       string
+	overridesFile string
+	debug         bool
+	trace         bool
+	cloud         bool
 )
 
 var cfg = config.NewDefaultConfig()
@@ -28,7 +29,8 @@ func NewRootCommand() *cobra.Command {
 		Use:   "ftw run",
 		Short: "Framework for Testing WAFs - Go Version",
 	}
-	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "override config file (default is $PWD/.ftw.yaml)")
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "specify config file (default is $PWD/.ftw.yaml)")
+	rootCmd.PersistentFlags().StringVar(&overridesFile, "overrides", "", "specify file with platform specific overrides")
 	rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "", false, "debug output")
 	rootCmd.PersistentFlags().BoolVarP(&trace, "trace", "", false, "trace output: really, really verbose")
 	rootCmd.PersistentFlags().BoolVarP(&cloud, "cloud", "", false, "cloud mode: rely only on HTTP status codes for determining test success or failure (will not process any logs)")
@@ -71,5 +73,8 @@ func initConfig() {
 	if cloud {
 		cfg.RunMode = config.CloudRunMode
 	}
-
+	err = cfg.LoadPlatformOverrides(overridesFile)
+	if err != nil {
+		log.Fatal("failed to load platform overrides")
+	}
 }
diff --git a/cmd/run.go b/cmd/run.go
index 590b5ae0..0c4551ed 100644
--- a/cmd/run.go
+++ b/cmd/run.go
@@ -33,14 +33,14 @@ func NewRunCommand() *cobra.Command {
 
 	runCmd.Flags().StringP("exclude", "e", "", "exclude tests matching this Go regular expression (e.g. to exclude all tests beginning with \"91\", use \"^91.*\"). \nIf you want more permanent exclusion, check the 'exclude' option in the config file.")
 	runCmd.Flags().StringP("include", "i", "", "include only tests matching this Go regular expression (e.g. to include only tests beginning with \"91\", use \"^91.*\"). \\nIf you want more permanent inclusion, check the 'include' option in the config file.\"")
-	_ = runCmd.Flags().MarkDeprecated("id", "This flag will be removed in v2.0. Use --include matching your test only.")
 	runCmd.Flags().StringP("dir", "d", ".", "recursively find yaml tests in this directory")
 	runCmd.Flags().StringP("output", "o", "normal", "output type for ftw tests. \"normal\" is the default.")
 	runCmd.Flags().StringP("file", "f", "", "output file path for ftw tests. Prints to standard output by default.")
+	runCmd.Flags().StringP("log-file", "l", "", "path to log file to watch for WAF events")
 	runCmd.Flags().BoolP("time", "t", false, "show time spent per test")
 	runCmd.Flags().BoolP("show-failures-only", "", false, "shows only the results of failed tests")
 	runCmd.Flags().Duration("connect-timeout", 3*time.Second, "timeout for connecting to endpoints during test execution")
-	runCmd.Flags().Duration("read-timeout", 1*time.Second, "timeout for receiving responses during test execution")
+	runCmd.Flags().Duration("read-timeout", 10*time.Second, "timeout for receiving responses during test execution")
 	runCmd.Flags().Int("max-marker-retries", 20, "maximum number of times the search for log markers will be repeated.\nEach time an additional request is sent to the web server, eventually forcing the log to be flushed")
 	runCmd.Flags().Int("max-marker-log-lines", 500, "maximum number of lines to search for a marker before aborting")
 	runCmd.Flags().String("wait-for-host", "", "Wait for host to be available before running tests.")
@@ -55,23 +55,26 @@ func NewRunCommand() *cobra.Command {
 	runCmd.Flags().Bool("wait-for-insecure-skip-tls-verify", http.DefaultInsecureSkipTLSVerify, "Skips tls certificate checks for the HTTPS request.")
 	runCmd.Flags().Bool("wait-for-no-redirect", http.DefaultNoRedirect, "Do not follow HTTP 3xx redirects.")
 	runCmd.Flags().DurationP("rate-limit", "r", 0, "Limit the request rate to the server to 1 request per specified duration. 0 is the default, and disables rate limiting.")
+	runCmd.Flags().Bool("fail-fast", false, "Fail on first failed test")
 
 	return runCmd
 }
 
+//gocyclo:ignore
 func runE(cmd *cobra.Command, _ []string) error {
 	cmd.SilenceUsage = true
 	exclude, _ := cmd.Flags().GetString("exclude")
 	include, _ := cmd.Flags().GetString("include")
 	dir, _ := cmd.Flags().GetString("dir")
 	outputFilename, _ := cmd.Flags().GetString("file")
+	logFilePath, _ := cmd.Flags().GetString("log-file")
 	showTime, _ := cmd.Flags().GetBool("time")
 	showOnlyFailed, _ := cmd.Flags().GetBool("show-failures-only")
 	wantedOutput, _ := cmd.Flags().GetString("output")
 	connectTimeout, _ := cmd.Flags().GetDuration("connect-timeout")
 	readTimeout, _ := cmd.Flags().GetDuration("read-timeout")
-	maxMarkerRetries, _ := cmd.Flags().GetInt("max-marker-retries")
-	maxMarkerLogLines, _ := cmd.Flags().GetInt("max-marker-log-lines")
+	maxMarkerRetries, _ := cmd.Flags().GetUint("max-marker-retries")
+	maxMarkerLogLines, _ := cmd.Flags().GetUint("max-marker-log-lines")
 	// wait4x flags
 	waitForHost, _ := cmd.Flags().GetString("wait-for-host")
 	timeout, _ := cmd.Flags().GetDuration("wait-for-timeout")
@@ -85,6 +88,7 @@ func runE(cmd *cobra.Command, _ []string) error {
 	insecureSkipTLSVerify, _ := cmd.Flags().GetBool("wait-for-insecure-skip-tls-verify")
 	noRedirect, _ := cmd.Flags().GetBool("wait-for-no-redirect")
 	rateLimit, _ := cmd.Flags().GetDuration("rate-limit")
+	failFast, _ := cmd.Flags().GetBool("fail-fast")
 
 	if exclude != "" && include != "" {
 		cmd.SilenceUsage = false
@@ -96,6 +100,10 @@ func runE(cmd *cobra.Command, _ []string) error {
 	if maxMarkerLogLines != 0 {
 		cfg.WithMaxMarkerLogLines(maxMarkerLogLines)
 	}
+	if logFilePath != "" {
+		cfg.LogFile = logFilePath
+	}
+
 	files := fmt.Sprintf("%s/**/*.yaml", dir)
 	tests, err := test.GetTestsFromFiles(files)
 
@@ -164,6 +172,7 @@ func runE(cmd *cobra.Command, _ []string) error {
 		ConnectTimeout: connectTimeout,
 		ReadTimeout:    readTimeout,
 		RateLimit:      rateLimit,
+		FailFast:       failFast,
 	}, out)
 
 	if err != nil {
diff --git a/cmd/run_test.go b/cmd/run_test.go
index e188358b..cae1dcd6 100644
--- a/cmd/run_test.go
+++ b/cmd/run_test.go
@@ -28,22 +28,21 @@ meta:
   description: "Test file for go-ftw"
 tests:
   - # Standard GET request
-    test_title: 1234
+    test_id: 1234
     stages:
-      - stage:
-          input:
-            dest_addr: "127.0.0.1"
-            method: "GET"
-            port: {{ .Port }}
-            headers:
-              User-Agent: "OWASP CRS test agent"
-              Host: "localhost"
-              Accept: "*/*"
-            protocol: "http"
-            uri: "/"
-            version: "HTTP/1.1"
-          output:
-            status: [200]
+      - input:
+          dest_addr: "127.0.0.1"
+          method: "GET"
+          port: {{ .Port }}
+          headers:
+            User-Agent: "OWASP CRS test agent"
+            Host: "localhost"
+            Accept: "*/*"
+          protocol: "http"
+          uri: "/"
+          version: "HTTP/1.1"
+        output:
+          status: 200
 `
 
 type runCmdTestSuite struct {
@@ -69,12 +68,10 @@ func (s *runCmdTestSuite) setupMockHTTPServer() *httptest.Server {
 }
 
 func (s *runCmdTestSuite) SetupTest() {
-	tempDir, err := os.MkdirTemp("", "go-ftw-tests")
-	s.Require().NoError(err)
-	s.tempDir = tempDir
+	s.tempDir = s.T().TempDir()
 
 	s.testHTTPServer = s.setupMockHTTPServer()
-	err = os.MkdirAll(s.tempDir, fs.ModePerm)
+	err := os.MkdirAll(s.tempDir, fs.ModePerm)
 	s.Require().NoError(err)
 	testUrl, err := url.Parse(s.testHTTPServer.URL)
 	s.Require().NoError(err)
diff --git a/config/config.go b/config/config.go
index 9d277453..cda1ae73 100644
--- a/config/config.go
+++ b/config/config.go
@@ -4,21 +4,24 @@
 package config
 
 import (
+	"fmt"
 	"os"
 	"strings"
 
+	schema "github.com/coreruleset/ftw-tests-schema/types/overrides"
 	"github.com/knadh/koanf/parsers/yaml"
 	"github.com/knadh/koanf/providers/env"
 	"github.com/knadh/koanf/providers/file"
 	"github.com/knadh/koanf/providers/rawbytes"
 	koanfv2 "github.com/knadh/koanf/v2"
+	"github.com/rs/zerolog/log"
 )
 
 // NewDefaultConfig initializes the configuration with default values
 func NewDefaultConfig() *FTWConfiguration {
 	cfg := &FTWConfiguration{
 		LogFile:             "",
-		TestOverride:        FTWTestOverride{},
+		PlatformOverrides:   PlatformOverrides{},
 		LogMarkerHeaderName: DefaultLogMarkerHeaderName,
 		RunMode:             DefaultRunMode,
 		MaxMarkerRetries:    DefaultMaxMarkerRetries,
@@ -70,7 +73,7 @@ func NewConfigFromFile(cfgFile string) (*FTWConfiguration, error) {
 		return nil, err
 	}
 
-	return unmarshal(k)
+	return unmarshalConfig(k)
 }
 
 // NewConfigFromEnv reads configuration information from environment variables that start with `FTW_`
@@ -85,7 +88,7 @@ func NewConfigFromEnv() (*FTWConfiguration, error) {
 		return nil, err
 	}
 
-	return unmarshal(k)
+	return unmarshalConfig(k)
 }
 
 // NewConfigFromString initializes the configuration from a yaml formatted string. Useful for testing.
@@ -96,7 +99,52 @@ func NewConfigFromString(conf string) (*FTWConfiguration, error) {
 		return nil, err
 	}
 
-	return unmarshal(k)
+	return unmarshalConfig(k)
+}
+
+// LoadPlatformOverrides loads platform overrides from the specified file path
+func (c *FTWConfiguration) LoadPlatformOverrides(overridesFile string) error {
+	if overridesFile == "" {
+		log.Trace().Msg("No overrides file specified, skipping.")
+		return nil
+	}
+	if _, err := os.Stat(overridesFile); err != nil {
+		return fmt.Errorf("could not find overrides file '%s'", overridesFile)
+	}
+
+	log.Debug().Msgf("Loading platform overrides from '%s'", overridesFile)
+
+	k := getKoanfInstance()
+	err := k.Load(file.Provider(overridesFile), yaml.Parser())
+	if err != nil {
+		return err
+	}
+
+	overrides, err := unmarshalPlatformOverrides(k)
+	if err != nil {
+		return err
+	}
+
+	c.PlatformOverrides.FTWOverrides = *overrides
+	c.populatePlatformOverridesMap()
+
+	return nil
+}
+
+func (c *FTWConfiguration) populatePlatformOverridesMap() {
+	rulesMap := map[uint][]*schema.TestOverride{}
+	for i := 0; i < len(c.PlatformOverrides.TestOverrides); i++ {
+		testOverride := &c.PlatformOverrides.TestOverrides[i]
+		var list []*schema.TestOverride
+		list, ok := rulesMap[testOverride.RuleId]
+		if !ok {
+			list = []*schema.TestOverride{}
+		}
+		list = append(list, testOverride)
+		rulesMap[testOverride.RuleId] = list
+
+	}
+	c.PlatformOverrides.OverridesMap = rulesMap
 }
 
 // WithLogfile changes the logfile in the configuration.
@@ -120,17 +168,17 @@ func (c *FTWConfiguration) WithLogMarkerHeaderName(name string) {
 }
 
 // WithMaxMarkerRetries sets the new amount of retries we are doing to find markers in the logfile.
-func (c *FTWConfiguration) WithMaxMarkerRetries(retries int) {
+func (c *FTWConfiguration) WithMaxMarkerRetries(retries uint) {
 	c.MaxMarkerRetries = retries
 }
 
 // WithMaxMarkerLogLines sets the new amount of lines we go back in the logfile attempting to find markers.
-func (c *FTWConfiguration) WithMaxMarkerLogLines(amount int) {
+func (c *FTWConfiguration) WithMaxMarkerLogLines(amount uint) {
 	c.MaxMarkerLogLines = amount
 }
 
 // Unmarshal the loaded koanf instance into a configuration object
-func unmarshal(k *koanfv2.Koanf) (*FTWConfiguration, error) {
+func unmarshalConfig(k *koanfv2.Koanf) (*FTWConfiguration, error) {
 	config := NewDefaultConfig()
 	err := k.UnmarshalWithConf("", config, koanfv2.UnmarshalConf{Tag: "koanf"})
 	if err != nil {
@@ -140,6 +188,17 @@ func unmarshal(k *koanfv2.Koanf) (*FTWConfiguration, error) {
 	return config, nil
 }
 
+// Unmarshal the loaded koanf instance into an FTWOverrides object
+func unmarshalPlatformOverrides(k *koanfv2.Koanf) (*schema.FTWOverrides, error) {
+	overrides := &schema.FTWOverrides{}
+	err := k.UnmarshalWithConf("", overrides, koanfv2.UnmarshalConf{Tag: "yaml"})
+	if err != nil {
+		return nil, err
+	}
+
+	return overrides, nil
+}
+
 // Get the global koanf instance
 func getKoanfInstance() *koanfv2.Koanf {
 	// Use "." as the key path delimiter. This can be "/" or any character.
diff --git a/config/config_test.go b/config/config_test.go
index d8d52c4b..30900a0e 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -10,7 +10,6 @@ import (
 
 	"github.com/stretchr/testify/suite"
 
-	"github.com/coreruleset/go-ftw/test"
 	"github.com/coreruleset/go-ftw/utils"
 )
 
@@ -219,7 +218,7 @@ func (s *fileTestSuite) TestNewDefaultConfigWithParams() {
 	cfg.WithLogfile("mylogfile.log")
 	s.Equal("mylogfile.log", cfg.LogFile)
 	overrides := FTWTestOverride{
-		Overrides: test.Overrides{},
+		Overrides: Overrides{},
 		Ignore:    nil,
 		ForcePass: nil,
 		ForceFail: nil,
@@ -235,8 +234,68 @@ func (s *fileTestSuite) TestNewDefaultConfigWithParams() {
 func (s *baseTestSuite) TestWithMaxMarker() {
 	cfg := NewDefaultConfig()
 	cfg.WithMaxMarkerRetries(19)
-	s.Equal(19, cfg.MaxMarkerRetries)
+	s.Equal(uint(19), cfg.MaxMarkerRetries)
 	cfg.WithMaxMarkerLogLines(111)
-	s.Equal(111, cfg.MaxMarkerLogLines)
+	s.Equal(uint(111), cfg.MaxMarkerLogLines)
 
 }
+
+func (s *baseTestSuite) TestPlatformOverridesDefaults() {
+	overrides := NewDefaultConfig().PlatformOverrides
+	meta := overrides.Meta
+	s.Empty(meta.Annotations)
+	s.Empty(meta.Engine)
+	s.Empty(meta.Platform)
+	s.Empty(overrides.Version)
+	s.Empty(overrides.TestOverrides)
+}
+
+func (s *baseTestSuite) TestLoadPlatformOverrides() {
+	tempDir := s.T().TempDir()
+	overridesFile, err := os.CreateTemp(tempDir, "overrides.yaml")
+	s.Require().NoError(err)
+	_, err = overridesFile.WriteString(`---
+version: "v0.0.0"
+meta:
+  engine: "coraza"
+  platform: "go"
+  annotations:
+    - purpose: "Test loading overrides"
+test_overrides:
+  - rule_id: 920100
+    test_ids: [4, 8]
+    reason: 'Invalid uri, Coraza not reached - 404 page not found'
+    output:
+      status: 404
+      log:
+        match_regex: 'match.*me'
+        no_expect_ids: [1234]
+      response_contains: '404'`)
+
+	s.Require().NoError(err)
+
+	cfg := NewDefaultConfig()
+	err = cfg.LoadPlatformOverrides(overridesFile.Name())
+	s.Require().NoError(err)
+
+	overrides := cfg.PlatformOverrides
+	meta := overrides.Meta
+	s.Equal("v0.0.0", overrides.Version)
+	s.Equal("coraza", meta.Engine)
+	s.Equal("go", meta.Platform)
+	s.Len(meta.Annotations, 1)
+	value, ok := meta.Annotations["purpose"]
+	s.True(ok)
+	s.Equal("Test loading overrides", value)
+
+	s.Len(overrides.TestOverrides, 1)
+	entry := overrides.TestOverrides[0]
+	s.Equal(uint(920100), entry.RuleId)
+	s.ElementsMatch([]uint{4, 8}, entry.TestIds)
+	s.Equal("Invalid uri, Coraza not reached - 404 page not found", entry.Reason)
+	s.Equal(404, entry.Output.Status)
+	s.Equal("match.*me", entry.Output.Log.MatchRegex)
+	s.Len(entry.Output.Log.NoExpectIds, 1)
+	s.Equal(uint(1234), entry.Output.Log.NoExpectIds[0])
+	s.Equal("404", entry.Output.ResponseContains)
+}
diff --git a/config/types.go b/config/types.go
index be000c62..d5ebeca8 100644
--- a/config/types.go
+++ b/config/types.go
@@ -7,7 +7,8 @@ import (
 	"fmt"
 	"regexp"
 
-	"github.com/coreruleset/go-ftw/test"
+	schema "github.com/coreruleset/ftw-tests-schema/types/overrides"
+	"github.com/coreruleset/go-ftw/ftwhttp"
 )
 
 // RunMode represents the mode of the test run
@@ -21,15 +22,17 @@ const (
 	// DefaultLogMarkerHeaderName is the default log marker header name
 	DefaultLogMarkerHeaderName string = "X-CRS-Test"
 	// DefaultMaxMarkerRetries is the default amount of retries that will be attempted to find the log markers
-	DefaultMaxMarkerRetries int = 20
+	DefaultMaxMarkerRetries uint = 20
 	// DefaultMaxMarkerLogLines is the default lines we are going read back in a logfile to find the markers
-	DefaultMaxMarkerLogLines int = 500
+	DefaultMaxMarkerLogLines uint = 500
 )
 
 // FTWConfiguration FTW global Configuration
 type FTWConfiguration struct {
 	// Logfile is the path to the file that contains the WAF logs to check. The path may be absolute or relative, in which case it will be interpreted as relative to the current working directory.
 	LogFile string `koanf:"logfile"`
+	// PlatformOverrides holds platform specific overrides for tests in the test suite
+	PlatformOverrides PlatformOverrides `koanf:"platformoverrides"`
 	// TestOverride holds the test overrides that will apply globally
 	TestOverride FTWTestOverride `koanf:"testoverride"`
 	// LogMarkerHeaderName is the name of the header that will be used by the test framework to mark positions in the log file
@@ -37,15 +40,20 @@ type FTWConfiguration struct {
 	// RunMode stores the mode used to interpret test results. See https://github.com/coreruleset/go-ftw#%EF%B8%8F-cloud-mode.
 	RunMode RunMode `koanf:"mode"`
 	// MaxMarkerRetries is the maximum number of times the search for log markers will be repeated; each time an additional request is sent to the web server, eventually forcing the log to be flushed
-	MaxMarkerRetries int `koanf:"maxmarkerretries"`
+	MaxMarkerRetries uint `koanf:"maxmarkerretries"`
 	// MaxMarkerLogLines is the maximum number of lines to search for a marker before aborting
-	MaxMarkerLogLines int `koanf:"maxmarkerloglines"`
+	MaxMarkerLogLines uint `koanf:"maxmarkerloglines"`
 	// IncludeTests is a list of tests to include (same as --include)
 	IncludeTests map[*FTWRegexp]string `koanf:"include"`
 	// ExcludeTests is a list of tests to exclude (same as --exclude)
 	ExcludeTests map[*FTWRegexp]string `koanf:"exclude"`
 }
 
+type PlatformOverrides struct {
+	schema.FTWOverrides
+	OverridesMap map[uint][]*schema.TestOverride
+}
+
 // FTWTestOverride holds four lists:
 //
 //	Overrides allows you to override input parameters in tests. An example usage is if you want to change the `dest_addr` of all tests to point to an external IP or host.
@@ -53,12 +61,31 @@ type FTWConfiguration struct {
 //	ForcePass is for tests you want to pass unconditionally. You should add a comment on why you force to pass the test
 //	ForceFail is for tests you want to fail unconditionally. You should add a comment on why you force to fail the test
 type FTWTestOverride struct {
-	Overrides test.Overrides        `koanf:"input"`
+	Overrides Overrides             `koanf:"input"`
 	Ignore    map[*FTWRegexp]string `koanf:"ignore"`
 	ForcePass map[*FTWRegexp]string `koanf:"forcepass"`
 	ForceFail map[*FTWRegexp]string `koanf:"forcefail"`
 }
 
+// Overrides represents the overridden inputs that have to be applied to tests
+type Overrides struct {
+	DestAddr   *string        `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
+	Port       *int           `yaml:"port,omitempty" koanf:"port,omitempty"`
+	Protocol   *string        `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
+	URI        *string        `yaml:"uri,omitempty" koanf:"uri,omitempty"`
+	Version    *string        `yaml:"version,omitempty" koanf:"version,omitempty"`
+	Headers    ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"`
+	Method     *string        `yaml:"method,omitempty" koanf:"method,omitempty"`
+	Data       *string        `yaml:"data,omitempty" koanf:"data,omitempty"`
+	SaveCookie *bool          `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
+	// Deprecated: replaced with AutocompleteHeaders
+	StopMagic               *bool   `yaml:"stop_magic" koanf:"stop_magic,omitempty"`
+	AutocompleteHeaders     *bool   `yaml:"autocomplete_headers" koanf:"autocomplete_headers,omitempty"`
+	EncodedRequest          *string `yaml:"encoded_request,omitempty" koanf:"encoded_request,omitempty"`
+	RAWRequest              *string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"`
+	OverrideEmptyHostHeader *bool   `yaml:"override_empty_host_header,omitempty" koanf:"override_empty_host_header,omitempty"`
+}
+
 // FTWRegexp is a wrapper around regexp.Regexp that implements the Unmarshaler interface
 type FTWRegexp regexp.Regexp
 
diff --git a/ftwhttp/client.go b/ftwhttp/client.go
index 8558ff2f..e4b7832a 100644
--- a/ftwhttp/client.go
+++ b/ftwhttp/client.go
@@ -8,12 +8,13 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"golang.org/x/time/rate"
 	"net"
 	"net/http/cookiejar"
 	"strings"
 	"time"
 
+	"golang.org/x/time/rate"
+
 	"github.com/rs/zerolog/log"
 	"golang.org/x/net/publicsuffix"
 )
diff --git a/ftwhttp/client_test.go b/ftwhttp/client_test.go
index 61f548c8..bab928aa 100644
--- a/ftwhttp/client_test.go
+++ b/ftwhttp/client_test.go
@@ -6,12 +6,13 @@ package ftwhttp
 import (
 	"bytes"
 	"fmt"
-	"golang.org/x/time/rate"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 	"time"
 
+	"golang.org/x/time/rate"
+
 	"github.com/stretchr/testify/suite"
 )
 
@@ -51,6 +52,13 @@ func (s *clientTestSuite) httpHandler() http.HandlerFunc {
 		} else {
 			w.WriteHeader(http.StatusOK)
 		}
+
+		if r.URL.Path == "/sleep" {
+			duration, err := time.ParseDuration(r.URL.Query().Get("milliseconds") + "ms")
+			s.Require().NoError(err)
+			time.Sleep(duration)
+		}
+
 		resp := new(bytes.Buffer)
 		for key, value := range r.Header {
 			_, err := fmt.Fprintf(resp, "%s=%s,", key, value)
@@ -114,15 +122,13 @@ func (s *clientTestSuite) TestDoRequest() {
 }
 
 func (s *clientTestSuite) TestGetTrackedTime() {
-	d := &Destination{
-		DestAddr: "httpbingo.org",
-		Port:     443,
-		Protocol: "https",
-	}
+	s.httpTestServer(insecureServer)
+	d, err := DestinationFromString(s.ts.URL)
+	s.Require().NoError(err, "This should not error")
 
 	rl := &RequestLine{
 		Method:  "POST",
-		URI:     "/post",
+		URI:     "/sleep?milliseconds=50",
 		Version: "HTTP/1.1",
 	}
 
@@ -131,7 +137,7 @@ func (s *clientTestSuite) TestGetTrackedTime() {
 	data := []byte(`test=me&one=two&one=twice`)
 	req := NewRequest(rl, h, data, true)
 
-	err := s.client.NewConnection(*d)
+	err = s.client.NewConnection(*d)
 	s.Require().NoError(err, "This should not error")
 
 	s.client.StartTrackingTime()
@@ -144,7 +150,7 @@ func (s *clientTestSuite) TestGetTrackedTime() {
 	s.Equal(http.StatusOK, resp.Parsed.StatusCode, "Error in calling website")
 
 	rtt := s.client.GetRoundTripTime()
-	s.GreaterOrEqual(int(rtt.RoundTripDuration()), 0, "Error getting RTT")
+	s.GreaterOrEqual(rtt.RoundTripDuration().Milliseconds(), int64(50), "Error getting RTT")
 }
 
 func (s *clientTestSuite) TestClientMultipartFormDataRequest() {
@@ -250,8 +256,10 @@ func (s *clientTestSuite) TestClientRateLimits() {
 
 	// We need to do at least 2 calls so there is a wait between both.
 	before := time.Now()
-	_, err = s.client.Do(*req)
-	_, err = s.client.Do(*req)
+	//nolint:errcheck
+	s.client.Do(*req)
+	//nolint:errcheck
+	s.client.Do(*req)
 	after := time.Now()
 
 	s.GreaterOrEqual(after.Sub(before), waitTime, "Rate limiter did not work as expected")
diff --git a/ftwhttp/connection.go b/ftwhttp/connection.go
index ac9bd5e7..557cbe81 100644
--- a/ftwhttp/connection.go
+++ b/ftwhttp/connection.go
@@ -119,7 +119,7 @@ func (c *Connection) Response() (*Response, error) {
 	}
 
 	data := buf.Bytes()
-	log.Trace().Msgf("ftw/http: received data - %q", data)
+	log.Debug().Msgf("ftw/http: received data - %q", data)
 
 	response := Response{
 		RAW:    data,
diff --git a/ftwhttp/request.go b/ftwhttp/request.go
index 4d8b4979..4ba1dde5 100644
--- a/ftwhttp/request.go
+++ b/ftwhttp/request.go
@@ -91,9 +91,6 @@ func (r Request) RawData() []byte {
 
 // Headers return request headers
 func (r Request) Headers() Header {
-	if r.headers == nil {
-		return nil
-	}
 	return r.headers
 }
 
diff --git a/go.mod b/go.mod
index 7cb47c33..4d5d1cfd 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
-	github.com/coreruleset/ftw-tests-schema v1.1.0
+	github.com/coreruleset/ftw-tests-schema v1.1.1-0.20240509130026-48cda85f1d5f
 	github.com/go-logr/zerologr v1.2.3
 	github.com/goccy/go-yaml v1.9.2
 	github.com/google/uuid v1.6.0
@@ -56,5 +56,3 @@ require (
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
-
-replace github.com/knadh/koanf/maps => github.com/knadh/koanf/maps v0.1.1
diff --git a/go.sum b/go.sum
index 7a2e381d..40b31e7a 100644
--- a/go.sum
+++ b/go.sum
@@ -9,8 +9,8 @@ github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6e
 github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes=
 github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreruleset/ftw-tests-schema v1.1.0 h1:3+NYrdLE3HVmOc3nGrisRBBvY9lGjePUrV+YkT5Ay3s=
-github.com/coreruleset/ftw-tests-schema v1.1.0/go.mod h1:gRd9wBxjUI85HypWRDxJzbk1JqHC4KTxl0l/Y2p9QK4=
+github.com/coreruleset/ftw-tests-schema v1.1.1-0.20240509130026-48cda85f1d5f h1:lncLdzrrw40U6m5C26lQ7T/EYzzWIlQonnb3JYSaKPE=
+github.com/coreruleset/ftw-tests-schema v1.1.1-0.20240509130026-48cda85f1d5f/go.mod h1:stf2FA6YhkPEMq30FzKvs97Qn5P9F7LHqWQbNGvGhNk=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -42,8 +42,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
 github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
-github.com/icza/backscanner v0.0.0-20240221180818-f23e3ba0e79f h1:EKPpaKkARuHjoV/ZKzk3vqbSJXULRSivDCQhL+tF77Y=
-github.com/icza/backscanner v0.0.0-20240221180818-f23e3ba0e79f/go.mod h1:GYeBD1CF7AqnKZK+UCytLcY3G+UKo0ByXX/3xfdNyqQ=
 github.com/icza/backscanner v0.0.0-20240328210400-b40c3a86dec5 h1:FcxwOojw6pUiPpsf7Q6Fw/pI+7cR6FlapLBEGV/902A=
 github.com/icza/backscanner v0.0.0-20240328210400-b40c3a86dec5/go.mod h1:GYeBD1CF7AqnKZK+UCytLcY3G+UKo0ByXX/3xfdNyqQ=
 github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k=
@@ -62,8 +60,6 @@ github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf
 github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA=
 github.com/knadh/koanf/providers/rawbytes v0.1.0 h1:dpzgu2KO6uf6oCb4aP05KDmKmAmI51k5pe8RYKQ0qME=
 github.com/knadh/koanf/providers/rawbytes v0.1.0/go.mod h1:mMTB1/IcJ/yE++A2iEZbY1MLygX7vttU+C+S/YmPu9c=
-github.com/knadh/koanf/v2 v2.1.0 h1:eh4QmHHBuU8BybfIJ8mB8K8gsGCD/AUQTdwGq/GzId8=
-github.com/knadh/koanf/v2 v2.1.0/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es=
 github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM=
 github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es=
 github.com/kyokomi/emoji/v2 v2.2.12 h1:sSVA5nH9ebR3Zji1o31wu3yOwD1zKXQA2z0zUyeit60=
@@ -117,8 +113,6 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
 golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -127,8 +121,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
 golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -146,8 +138,6 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
 golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
diff --git a/output/output.go b/output/output.go
index fef96040..bbfc0b78 100644
--- a/output/output.go
+++ b/output/output.go
@@ -30,19 +30,19 @@ type catalog map[string]string
 // this catalog is used to translate text from basic terminals to enhanced ones that support emoji, just
 // because we are fancy. If we are not using a normal output, then just use the key from this map.
 var normalCatalog = catalog{
-	"** Starting tests!":                    ":hammer_and_wrench: Starting tests!",
+	"** Starting tests!":                    ":hammer_and_wrench:Starting tests!",
 	"** Running go-ftw!":                    ":rocket:Running go-ftw!",
 	"=> executing tests in file %s":         ":point_right:executing tests in file %s",
 	"+ passed in %s (RTT %s)":               ":check_mark:passed in %s (RTT %s)",
-	"- failed in %s (RTT %s)":               ":collision:failed in %s (RTT %s)",
+	"- %s failed in %s (RTT %s)":            ":collision:%s failed in %s (RTT %s)",
 	"= test ignored":                        ":information:test ignored",
 	"= test forced to fail":                 ":information:test forced to fail",
 	"= test forced to pass":                 ":information:test forced to pass",
 	"¯\\_(ツ)_/¯ No tests were run":          ":person_shrugging:No tests were run",
 	"+ run %d total tests in %s":            ":plus:run %d total tests in %s",
-	">> skipped %d tests":                   ":next_track_button: skipped %d tests",
-	"^ ignored %d tests":                    ":index_pointing_up: ignored %d tests",
-	"^ forced to pass %d tests":             ":index_pointing_up: forced to pass %d tests",
+	">> skipped %d tests":                   ":next_track_button:skipped %d tests",
+	"^ ignored %d tests":                    ":index_pointing_up:ignored %d tests",
+	"^ forced to pass %d tests":             ":index_pointing_up:forced to pass %d tests",
 	"\\o/ All tests successful!":            ":tada:All tests successful!",
 	"- %d test(s) failed to run: %+q":       ":thumbs_down:%d test(s) failed to run: %+q",
 	"- %d test(s) were forced to fail: %+q": ":index_pointing_up:%d test(s) were forced to fail: %+q",
diff --git a/runner/run.go b/runner/run.go
index d8937578..6d6836fe 100644
--- a/runner/run.go
+++ b/runner/run.go
@@ -6,14 +6,15 @@ package runner
 import (
 	"errors"
 	"fmt"
-	"golang.org/x/time/rate"
 	"regexp"
 	"time"
 
+	"golang.org/x/time/rate"
+
 	"github.com/google/uuid"
 	"github.com/rs/zerolog/log"
 
-	"github.com/coreruleset/ftw-tests-schema/types"
+	schema "github.com/coreruleset/ftw-tests-schema/types"
 	"github.com/coreruleset/go-ftw/check"
 	"github.com/coreruleset/go-ftw/config"
 	"github.com/coreruleset/go-ftw/ftwhttp"
@@ -51,6 +52,7 @@ func Run(cfg *config.FTWConfiguration, tests []*test.FTWTest, c RunnerConfig, ou
 
 	runContext := &TestRunContext{
 		Config:         cfg,
+		RunnerConfig:   &c,
 		Include:        c.Include,
 		Exclude:        c.Exclude,
 		ShowTime:       c.ShowTime,
@@ -65,6 +67,9 @@ func Run(cfg *config.FTWConfiguration, tests []*test.FTWTest, c RunnerConfig, ou
 		if err := RunTest(runContext, tc); err != nil {
 			return &TestRunContext{}, err
 		}
+		if c.FailFast && runContext.Stats.TotalFailed() > 0 {
+			break
+		}
 	}
 
 	runContext.Stats.printSummary(out)
@@ -81,14 +86,13 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 	changed := true
 
 	for _, testCase := range ftwTest.Tests {
-		// if we received a particular testid, skip until we find it
-		if needToSkipTest(runContext.Include, runContext.Exclude, testCase.TestTitle, *ftwTest.Meta.Enabled) {
-			runContext.Stats.addResultToStats(Skipped, testCase.TestTitle, 0)
-			if !*ftwTest.Meta.Enabled && !runContext.ShowOnlyFailed {
-				runContext.Output.Println("\tskipping %s - (enabled: false) in file.", testCase.TestTitle)
-			}
+		// if we received a particular test ID, skip until we find it
+		if needToSkipTest(runContext.Include, runContext.Exclude, &testCase) {
+			runContext.Stats.addResultToStats(Skipped, &testCase)
+			log.Trace().Msgf("\tskipping %s", testCase.IdString())
 			continue
 		}
+		test.ApplyPlatformOverrides(runContext.Config, &testCase)
 		// this is just for printing once the next test
 		if changed && !runContext.ShowOnlyFailed {
 			runContext.Output.Println(runContext.Output.Message("=> executing tests in file %s"), ftwTest.Meta.Name)
@@ -96,7 +100,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 		}
 
 		if !runContext.ShowOnlyFailed {
-			runContext.Output.Printf("\trunning %s: ", testCase.TestTitle)
+			runContext.Output.Printf("\trunning %s: ", testCase.IdString())
 		}
 		// Iterate over stages
 		for _, stage := range testCase.Stages {
@@ -104,10 +108,21 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 			if err != nil {
 				return err
 			}
-			if err := RunStage(runContext, ftwCheck, testCase, stage.SD); err != nil {
-				return err
+			if err := RunStage(runContext, ftwCheck, testCase, stage); err != nil {
+				if err.Error() == "retry-once" {
+					log.Info().Msgf("Retrying test once: %s", testCase.IdString())
+					if err = RunStage(runContext, ftwCheck, testCase, stage); err != nil {
+						return err
+					}
+				} else {
+					return err
+				}
 			}
 		}
+		runContext.Stats.addResultToStats(runContext.Result, &testCase)
+		if runContext.RunnerConfig.FailFast && runContext.Stats.TotalFailed() > 0 {
+			break
+		}
 	}
 
 	return nil
@@ -118,12 +133,14 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 // ftwCheck is the current check utility
 // testCase is the test case the stage belongs to
 // stage is the stage you want to run
-func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase types.Test, stage types.StageData) error {
+//
+//gocyclo:ignore
+func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase schema.Test, stage schema.Stage) error {
 	stageStartTime := time.Now()
 	stageID := uuid.NewString()
 	// Apply global overrides initially
 	testInput := (test.Input)(stage.Input)
-	test.ApplyInputOverrides(&runContext.Config.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(runContext.Config, &testInput)
 	expectedOutput := stage.Output
 	expectErr := false
 	if expectedOutput.ExpectError != nil {
@@ -136,9 +153,9 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase typ
 	}
 
 	// Do not even run test if result is overridden. Just use the override and display the overridden result.
-	if overridden := overriddenTestResult(ftwCheck, testCase.TestTitle); overridden != Failed {
-		runContext.Stats.addResultToStats(overridden, testCase.TestTitle, 0)
-		displayResult(runContext, overridden, time.Duration(0), time.Duration(0))
+	if overridden := overriddenTestResult(ftwCheck, &testCase); overridden != Failed {
+		runContext.Stats.addResultToStats(overridden, &testCase)
+		displayResult(&testCase, runContext, overridden, time.Duration(0), time.Duration(0))
 		return nil
 	}
 
@@ -189,19 +206,19 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase typ
 
 	// now get the test result based on output
 	testResult := checkResult(ftwCheck, response, responseErr)
+	if testResult == Failed && expectedOutput.RetryOnce != nil && *expectedOutput.RetryOnce {
+		return errors.New("retry-once")
+	}
 
 	roundTripTime := runContext.Client.GetRoundTripTime().RoundTripDuration()
 	stageTime := time.Since(stageStartTime)
 
-	runContext.Stats.addResultToStats(testResult, testCase.TestTitle, stageTime)
-
 	runContext.Result = testResult
 
 	// show the result unless quiet was passed in the command line
-	displayResult(runContext, testResult, roundTripTime, stageTime)
+	displayResult(&testCase, runContext, testResult, roundTripTime, stageTime)
 
-	runContext.Stats.Run++
-	runContext.Stats.TotalTime += stageTime
+	runContext.Stats.addStageResultToStats(&testCase, stageTime)
 
 	return nil
 }
@@ -244,15 +261,10 @@ func markAndFlush(runContext *TestRunContext, dest *ftwhttp.Destination, stageID
 	return nil, fmt.Errorf("can't find log marker. Am I reading the correct log? Log file: %s", runContext.Config.LogFile)
 }
 
-func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, title string, enabled bool) bool {
-	// skip disabled tests
-	if !enabled {
-		return true
-	}
-
+func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, testCase *schema.Test) bool {
 	// never skip enabled explicit inclusions
 	if include != nil {
-		if include.MatchString(title) {
+		if include.MatchString(testCase.IdString()) {
 			// inclusion always wins over exclusion
 			return false
 		}
@@ -262,7 +274,7 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, title string
 	// if we need to exclude tests, and the title matches,
 	// it needs to be skipped
 	if exclude != nil {
-		if exclude.MatchString(title) {
+		if exclude.MatchString(testCase.IdString()) {
 			result = true
 		}
 	}
@@ -270,7 +282,7 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, title string
 	// if we need to include tests, but the title does not match
 	// it needs to be skipped
 	if include != nil {
-		if !include.MatchString(title) {
+		if !include.MatchString(testCase.IdString()) {
 			result = true
 		}
 	}
@@ -280,18 +292,20 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, title string
 
 func checkTestSanity(testInput test.Input) bool {
 	return (utils.IsNotEmpty(testInput.Data) && testInput.EncodedRequest != "") ||
+		//nolint:staticcheck
 		(utils.IsNotEmpty(testInput.Data) && testInput.RAWRequest != "") ||
+		//nolint:staticcheck
 		(testInput.EncodedRequest != "" && testInput.RAWRequest != "")
 }
 
-func displayResult(rc *TestRunContext, result TestResult, roundTripTime time.Duration, stageTime time.Duration) {
+func displayResult(testCase *schema.Test, rc *TestRunContext, result TestResult, roundTripTime time.Duration, stageTime time.Duration) {
 	switch result {
 	case Success:
 		if !rc.ShowOnlyFailed {
 			rc.Output.Println(rc.Output.Message("+ passed in %s (RTT %s)"), stageTime, roundTripTime)
 		}
 	case Failed:
-		rc.Output.Println(rc.Output.Message("- failed in %s (RTT %s)"), stageTime, roundTripTime)
+		rc.Output.Println(rc.Output.Message("- %s failed in %s (RTT %s)"), testCase.IdString(), stageTime, roundTripTime)
 	case Ignored:
 		if !rc.ShowOnlyFailed {
 			rc.Output.Println(rc.Output.Message(":information:test ignored"))
@@ -307,16 +321,16 @@ func displayResult(rc *TestRunContext, result TestResult, roundTripTime time.Dur
 	}
 }
 
-func overriddenTestResult(c *check.FTWCheck, id string) TestResult {
-	if c.ForcedIgnore(id) {
+func overriddenTestResult(c *check.FTWCheck, testCase *schema.Test) TestResult {
+	if c.ForcedIgnore(testCase) {
 		return Ignored
 	}
 
-	if c.ForcedFail(id) {
+	if c.ForcedFail(testCase) {
 		return ForceFail
 	}
 
-	if c.ForcedPass(id) {
+	if c.ForcedPass(testCase) {
 		return ForcePass
 	}
 
@@ -326,35 +340,33 @@ func overriddenTestResult(c *check.FTWCheck, id string) TestResult {
 // checkResult has the logic for verifying the result for the test sent
 func checkResult(c *check.FTWCheck, response *ftwhttp.Response, responseError error) TestResult {
 	// Request might return an error, but it could be expected, we check that first
-	if responseError != nil && c.AssertExpectError(responseError) {
-		return Success
+	if expected, succeeded := c.AssertExpectError(responseError); expected {
+		if succeeded {
+			return Success
+		}
+		return Failed
 	}
 
-	// If there was no error, perform the remaining checks
+	// In case of an unexpected error skip other checks
 	if responseError != nil {
+		log.Debug().Msgf("Encountered unexpected error: %v", responseError)
 		return Failed
 	}
-	if c.CloudMode() {
-		// Cloud mode assumes that we cannot read logs. So we rely entirely on status code and response
-		c.SetCloudMode()
+
+	// We should have a response here
+	if response == nil {
+		log.Error().Msg("No response to check")
+		return Failed
 	}
 
-	// If we didn't expect an error, check the actual response from the waf
-	if response != nil {
-		if c.StatusCodeRequired() && !c.AssertStatus(response.Parsed.StatusCode) {
-			return Failed
-		}
-		// Check if text is contained in the full raw response
-		if c.ResponseContainsRequired() && !c.AssertResponseContains(response.GetFullResponse()) {
-			return Failed
-		}
+	if !c.AssertStatus(response.Parsed.StatusCode) {
+		return Failed
 	}
-	// Lastly, check logs
-	if c.LogContainsRequired() && !c.AssertLogContains() {
+	if !c.AssertResponseContains(response.GetFullResponse()) {
 		return Failed
 	}
-	// We assume that they were already setup, for comparing
-	if c.NoLogContainsRequired() && !c.AssertNoLogContains() {
+	// Lastly, check logs
+	if !c.AssertLogs() {
 		return Failed
 	}
 
diff --git a/runner/run_input_override_test.go b/runner/run_input_override_test.go
index 07a87b60..1710b2f7 100644
--- a/runner/run_input_override_test.go
+++ b/runner/run_input_override_test.go
@@ -162,14 +162,14 @@ func (s *inputOverrideTestSuite) TestSetHostFromDestAddr() {
 	}
 	cfg := &config.FTWConfiguration{
 		TestOverride: config.FTWTestOverride{
-			Overrides: test.Overrides{
+			Overrides: config.Overrides{
 				DestAddr:                &overrideHost,
 				OverrideEmptyHostHeader: func() *bool { b := true; return &b }(),
 			},
 		},
 	}
 
-	test.ApplyInputOverrides(&cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(cfg, &testInput)
 
 	s.Equal(overrideHost, *testInput.DestAddr, "`dest_addr` should have been overridden")
 
@@ -189,7 +189,7 @@ func (s *inputOverrideTestSuite) TestSetHostFromHostHeaderOverride() {
 		DestAddr: &originalDestAddr,
 	}
 
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	hostHeader := testInput.GetHeaders().Get("Host")
 	s.NotEqual("", hostHeader, "Host header must be set after overriding the `Host` header")
@@ -211,7 +211,7 @@ func (s *inputOverrideTestSuite) TestSetHeaderOverridingExistingOne() {
 
 	s.NotNil(testInput.Headers, "Header map must exist before overriding any header")
 
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	overriddenHeader := testInput.GetHeaders().Get("unique_id")
 	s.NotEqual("", overriddenHeader, "unique_id header must be set after overriding it")
@@ -229,7 +229,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrides() {
 
 	s.NotNil(testInput.Headers, "Header map must exist before overriding any header")
 
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	overriddenHeader := testInput.GetHeaders().Get("unique_id")
 	s.NotEqual("", overriddenHeader, "unique_id header must be set after overriding it")
@@ -245,7 +245,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideURI() {
 		URI: &originalURI,
 	}
 
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	s.Equal(overrideURI, *testInput.URI, "`URI` should have been overridden")
 }
@@ -258,7 +258,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideVersion() {
 	testInput := test.Input{
 		Version: &originalVersion,
 	}
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	s.Equal(overrideVersion, *testInput.Version, "`Version` should have been overridden")
 }
@@ -271,7 +271,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideMethod() {
 	testInput := test.Input{
 		Method: &originalMethod,
 	}
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	s.Equal(overrideMethod, *testInput.Method, "`Method` should have been overridden")
 }
@@ -284,7 +284,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideData() {
 	testInput := test.Input{
 		Data: &originalData,
 	}
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	s.Equal(overrideData, *testInput.Data, "`Data` should have been overridden")
 }
@@ -297,7 +297,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideStopMagic() {
 	testInput := test.Input{
 		StopMagic: func() *bool { b := false; return &b }(),
 	}
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	// nolint
 	s.Equal(overrideStopMagic, *testInput.StopMagic, "`StopMagic` should have been overridden")
@@ -311,7 +311,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideAutocompleteHeaders() {
 	testInput := test.Input{
 		AutocompleteHeaders: func() *bool { b := false; return &b }(),
 	}
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	// nolint
 	s.Equal(overrideAutocompleteHeaders, *testInput.AutocompleteHeaders, "`AutocompleteHeaders` should have been overridden")
@@ -324,7 +324,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideNoAutocompleteHeaders() {
 	s.Nil(s.cfg.TestOverride.Overrides.AutocompleteHeaders)
 	//nolint:staticcheck
 	s.Nil(s.cfg.TestOverride.Overrides.StopMagic)
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
 	s.False(*testInput.AutocompleteHeaders, "`AutocompleteHeaders` should not have been overridden")
 }
@@ -336,8 +336,9 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideNoStopMagic() {
 	s.Nil(s.cfg.TestOverride.Overrides.AutocompleteHeaders)
 	//nolint:staticcheck
 	s.Nil(s.cfg.TestOverride.Overrides.StopMagic)
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
+	//nolint:staticcheck
 	s.True(*testInput.StopMagic, "`AutocompleteHeaders` should not have been overridden")
 }
 
@@ -348,7 +349,7 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideEncodedRequest() {
 	testInput := test.Input{
 		EncodedRequest: originalEncodedRequest,
 	}
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 	s.NoError(err, "Failed to apply input overrides")
 	s.Equal(overrideEncodedRequest, testInput.EncodedRequest, "`EncodedRequest` should have been overridden")
 }
@@ -362,7 +363,8 @@ func (s *inputOverrideTestSuite) TestApplyInputOverrideRAWRequest() {
 		RAWRequest: originalRAWRequest,
 	}
 
-	test.ApplyInputOverrides(&s.cfg.TestOverride.Overrides, &testInput)
+	test.ApplyInputOverrides(s.cfg, &testInput)
 
+	//nolint:staticcheck
 	s.Equal(overrideRAWRequest, testInput.RAWRequest, "`RAWRequest` should have been overridden")
 }
diff --git a/runner/run_test.go b/runner/run_test.go
index 062b3ab8..999738dd 100644
--- a/runner/run_test.go
+++ b/runner/run_test.go
@@ -52,11 +52,11 @@ testoverride:
 	"TestIgnoredTestsRun": `---
 testoverride:
   ignore:
-    "001": "This test result must be ignored"
+    ".*-1": "This test result must be ignored"
   forcefail:
-    "008": "This test should pass, but it is going to fail"
+    ".*-8": "This test should pass, but it is going to fail"
   forcepass:
-    "099": "This test failed, but it shall pass!"
+    ".*-99": "This test failed, but it shall pass!"
 `,
 	"TestOverrideRun": `---
 testoverride:
@@ -108,22 +108,36 @@ type runTestSuite struct {
 	tempFileName string
 }
 
-// Error checking omitted for brevity
 func (s *runTestSuite) newTestServer(logLines string) {
+	s.newTestServerWithHandlerGenerator(nil, logLines)
+}
+
+// Error checking omitted for brevity
+func (s *runTestSuite) newTestServerWithHandlerGenerator(serverHandler func(logLines string) http.HandlerFunc, logLines string) {
 	var err error
+	var handler http.HandlerFunc
 	s.setUpLogFileForTestServer()
 
-	s.ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		_, _ = w.Write([]byte("Hello, client"))
-
-		s.writeTestServerLog(logLines, r)
-	}))
+	if serverHandler == nil {
+		handler = s.getDefaultTestServerHandler(logLines)
+	} else {
+		handler = serverHandler(logLines)
+	}
+	s.ts = httptest.NewServer(handler)
 
 	s.dest, err = ftwhttp.DestinationFromString((s.ts).URL)
 	s.Require().NoError(err, "cannot get destination from string")
 }
 
+func (s *runTestSuite) getDefaultTestServerHandler(logLines string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("Hello, client"))
+
+		s.writeMarkerOrMessageToTestServerLog(logLines, r)
+	}
+}
+
 func (s *runTestSuite) setUpLogFileForTestServer() {
 	// log to the configured file
 	if s.cfg != nil && s.cfg.RunMode == config.DefaultRunMode {
@@ -137,7 +151,23 @@ func (s *runTestSuite) setUpLogFileForTestServer() {
 	}
 }
 
-func (s *runTestSuite) writeTestServerLog(logLines string, r *http.Request) {
+func (s *runTestSuite) writeTestServerLog(logLines string) {
+	file, err := os.OpenFile(s.logFilePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
+	s.Require().NoError(err, "cannot open file")
+
+	defer file.Close()
+
+	n, err := file.WriteString(logLines)
+	s.Len(logLines, n, "cannot write log message to file")
+	s.Require().NoError(err, "cannot write log message to file")
+
+	if logLines[len(logLines)-1] != '\n' {
+		_, err = file.WriteString("\n")
+		s.Require().NoError(err)
+	}
+}
+
+func (s *runTestSuite) writeMarkerOrMessageToTestServerLog(logLines string, r *http.Request) {
 	// write supplied log lines, emulating the output of the rule engine
 	logMessage := logLines
 	// if the request has the special test header, log the request instead
@@ -145,17 +175,14 @@ func (s *runTestSuite) writeTestServerLog(logLines string, r *http.Request) {
 	if r.Header.Get(s.cfg.LogMarkerHeaderName) != "" {
 		logMessage = fmt.Sprintf("request line: %s %s %s, headers: %s\n", r.Method, r.RequestURI, r.Proto, r.Header)
 	}
-	file, err := os.OpenFile(s.logFilePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
-	s.Require().NoError(err, "cannot open file")
-
-	defer file.Close()
+	s.writeTestServerLog(logMessage)
+}
 
-	n, err := file.WriteString(logMessage)
-	s.Len(logMessage, n, "cannot write log message to file")
-	s.Require().NoError(err, "cannot write log message to file")
+func (s *runTestSuite) TearDownTest() {
+	s.ts.Close()
 }
 
-func (s *runTestSuite) SetupTest() {
+func (s *runTestSuite) BeforeTest(_ string, name string) {
 	s.cfg = config.NewDefaultConfig()
 	// setup test webserver (not a waf)
 	s.newTestServer(logText)
@@ -163,18 +190,12 @@ func (s *runTestSuite) SetupTest() {
 		s.cfg.WithLogfile(s.logFilePath)
 	}
 
-	s.out = output.NewOutput("normal", os.Stdout)
-}
-
-func (s *runTestSuite) TearDownTest() {
-	s.ts.Close()
-}
-
-func (s *runTestSuite) BeforeTest(_ string, name string) {
 	var err error
 	var cfg string
 	var ok bool
 
+	s.out = output.NewOutput("normal", os.Stdout)
+
 	// if we have a configuration for this test, use it
 	// else use the default configuration
 	if cfg, ok = testConfigMap[name]; !ok {
@@ -215,7 +236,8 @@ func (s *runTestSuite) BeforeTest(_ string, name string) {
 	}
 
 	// create a temporary file to hold the test
-	testFileContents, err := os.CreateTemp("testdata", "mock-test-*.yaml")
+	tempDir := s.T().TempDir()
+	testFileContents, err := os.CreateTemp(tempDir, "mock-test-*.yaml")
 	s.Require().NoError(err, "cannot create temporary file")
 	err = tmpl.Execute(testFileContents, vars)
 	s.Require().NoError(err, "cannot execute template")
@@ -442,3 +464,69 @@ func (s *runTestSuite) TestGetRequestFromTestWithoutAutocompleteHeaders() {
 	s.Equal("", request.Headers().Get("Content-Length"), "Autocompletion is disabled")
 	s.Equal("", request.Headers().Get("Connection"), "Autocompletion is disabled")
 }
+
+// This test case verifies that the `retry_once` option works around a race condition in phase 5,
+// where the log entry for a phase 5 rule may appear after the end marker of the last test.
+// The race condition doesn't occur often, so retrying once should usually fix the issue.
+func (s *runTestSuite) TestRetryOnce() {
+	stageId := ""
+	serverHitCount := 0
+
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("Hello, client"))
+
+		// write logs on each start marker request
+		nextStageId := r.Header.Get(s.cfg.LogMarkerHeaderName)
+		if nextStageId != "" && nextStageId != stageId {
+			stageId = nextStageId
+			logMessage := ""
+			switch serverHitCount {
+			case 0:
+				// start marker
+				logMessage = fmt.Sprintf(
+					`[Sat Mar 18 16:47:21.474075 2023] [security2:error] [pid 193:tid 140523746522880] [client 172.18.0.1:39150] [client 172.18.0.1] ModSecurity: Warning. Pattern match "^.*$" at REQUEST_HEADERS:X-CRS-Test. [file "/etc/modsecurity.d/owasp-crs/crs-setup.conf"] [line "737"] [id "999999"] [msg "%s"] [tag "modsecurity"] [hostname "localhost"] [uri "/status/200"] [unique_id "ZBXrGVXqtKqlnATxdUEg7QAAANg"]`,
+					stageId)
+			case 1:
+				// hit without match + end marker
+				logMessage = fmt.Sprintf(
+					`[Sat Mar 18 16:47:21.476378 2023] [security2:error] [pid 193:tid 140524082149120] [client 172.18.0.1:39164] [client 172.18.0.1] ModSecurity: Warning. Pattern match "(?:^([\\\\d.]+|\\\\[[\\\\da-f:]+\\\\]|[\\\\da-f:]+)(:[\\\\d]+)?$)" at REQUEST_HEADERS:Host. [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf"] [line "761"] [id "920350"] [msg "Host header is a numeric IP address"] [data "127.0.0.1"] [severity "WARNING"] [ver "OWASP_CRS/4.0.0-rc1"] [tag "modsecurity"] [tag "application-multi"] [tag "language-multi"] [tag "platform-multi"] [tag "attack-protocol"] [tag "paranoia-level/1"] [tag "OWASP_CRS"] [tag "capec/1000/210/272"] [tag "PCI/6.5.10"] [hostname "127.0.0.1"] [uri "/"] [unique_id "ZBXrGVXqtKqlnATxdUEg7gAAAMQ"]
+[Sat Mar 18 16:47:21.480771 2023] [security2:error] [pid 193:tid 140524098930432] [client 172.18.0.1:39172] [client 172.18.0.1] ModSecurity: Warning. Pattern match "^.*$" at REQUEST_HEADERS:X-CRS-Test. [file "/etc/modsecurity.d/owasp-crs/crs-setup.conf"] [line "737"] [id "999999"] [msg "%s"] [tag "modsecurity"] [hostname "localhost"] [uri "/status/200"] [unique_id "ZBXrGVXqtKqlnATxdUEg7wAAAMM"]`,
+					stageId)
+			case 2:
+				// late flushed phase 5 hit + start marker
+				logMessage = fmt.Sprintf(
+					`[Sat Mar 18 16:47:21.483333 2023] [security2:error] [pid 193:tid 140524082149120] [client 172.18.0.1:39164] [client 172.18.0.1] ModSecurity: Warning. Unconditional match in SecAction. [file "/etc/modsecurity.d/owasp-crs/rules/RESPONSE-980-CORRELATION.conf"] [line "96"] [id "980170"] [msg "Anomaly Scores: (Inbound Scores: blocking=3, detection=3, per_pl=3-0-0-0, threshold=5) - (Outbound Scores: blocking=0, detection=0, per_pl=0-0-0-0, threshold=4) - (SQLI=0, XSS=0, RFI=0, LFI=0, RCE=0, PHPI=0, HTTP=0, SESS=0, COMBINED_SCORE=3)"] [ver "OWASP_CRS/4.0.0-rc1"] [tag "modsecurity"] [tag "reporting"] [hostname "127.0.0.1"] [uri "/"] [unique_id "ZBXrGVXqtKqlnATxdUEg7gAAAMQ"]
+[Sat Mar 18 16:47:21.474075 2023] [security2:error] [pid 193:tid 140523746522880] [client 172.18.0.1:39150] [client 172.18.0.1] ModSecurity: Warning. Pattern match "^.*$" at REQUEST_HEADERS:X-CRS-Test. [file "/etc/modsecurity.d/owasp-crs/crs-setup.conf"] [line "737"] [id "999999"] [msg "%s"] [tag "modsecurity"] [hostname "localhost"] [uri "/status/200"] [unique_id "ZBXrGVXqtKqlnATxdUEg7QAAANg"]`,
+					stageId)
+			default:
+				// hit with match + end marker
+				logMessage = fmt.Sprintf(
+					`[Sat Mar 18 16:47:21.476378 2023] [security2:error] [pid 193:tid 140524082149120] [client 172.18.0.1:39164] [client 172.18.0.1] ModSecurity: Warning. Pattern match "(?:^([\\\\d.]+|\\\\[[\\\\da-f:]+\\\\]|[\\\\da-f:]+)(:[\\\\d]+)?$)" at REQUEST_HEADERS:Host. [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf"] [line "761"] [id "920350"] [msg "Host header is a numeric IP address"] [data "127.0.0.1"] [severity "WARNING"] [ver "OWASP_CRS/4.0.0-rc1"] [tag "modsecurity"] [tag "application-multi"] [tag "language-multi"] [tag "platform-multi"] [tag "attack-protocol"] [tag "paranoia-level/1"] [tag "OWASP_CRS"] [tag "capec/1000/210/272"] [tag "PCI/6.5.10"] [hostname "127.0.0.1"] [uri "/"] [unique_id "ZBXrGVXqtKqlnATxdUEg7gAAAMQ"]
+[Sat Mar 18 16:47:21.483333 2023] [security2:error] [pid 193:tid 140524082149120] [client 172.18.0.1:39164] [client 172.18.0.1] ModSecurity: Warning. Unconditional match in SecAction. [file "/etc/modsecurity.d/owasp-crs/rules/RESPONSE-980-CORRELATION.conf"] [line "96"] [id "980170"] [msg "Anomaly Scores: (Inbound Scores: blocking=3, detection=3, per_pl=3-0-0-0, threshold=5) - (Outbound Scores: blocking=0, detection=0, per_pl=0-0-0-0, threshold=4) - (SQLI=0, XSS=0, RFI=0, LFI=0, RCE=0, PHPI=0, HTTP=0, SESS=0, COMBINED_SCORE=3)"] [ver "OWASP_CRS/4.0.0-rc1"] [tag "modsecurity"] [tag "reporting"] [hostname "127.0.0.1"] [uri "/"] [unique_id "ZBXrGVXqtKqlnATxdUEg7gAAAMQ"]
+[Sat Mar 18 16:47:21.480771 2023] [security2:error] [pid 193:tid 140524098930432] [client 172.18.0.1:39172] [client 172.18.0.1] ModSecurity: Warning. Pattern match "^.*$" at REQUEST_HEADERS:X-CRS-Test. [file "/etc/modsecurity.d/owasp-crs/crs-setup.conf"] [line "737"] [id "999999"] [msg "%s"] [tag "modsecurity"] [hostname "localhost"] [uri "/status/200"] [unique_id "ZBXrGVXqtKqlnATxdUEg7wAAAMM"]`,
+					stageId)
+			}
+
+			s.writeTestServerLog(logMessage)
+		}
+
+		serverHitCount++
+	}
+
+	s.ts.Config.Handler = http.HandlerFunc(handler)
+	res, err := Run(s.cfg, s.ftwTests, RunnerConfig{
+		Output: output.Quiet,
+	}, s.out)
+	s.Require().NoError(err)
+	s.Equalf(res.Stats.TotalFailed(), 0, "Oops, %d tests failed to run!", res.Stats.TotalFailed())
+}
+
+func (s *runTestSuite) TestFailFast() {
+	s.Equal(3, len(s.ftwTests[0].Tests))
+
+	res, err := Run(s.cfg, s.ftwTests, RunnerConfig{FailFast: true}, s.out)
+	s.Require().NoError(err)
+	s.Equal(1, res.Stats.TotalFailed(), "Oops, test run failed!")
+	s.Equal(2, res.Stats.Run)
+}
diff --git a/runner/stats.go b/runner/stats.go
index b37aab75..1ba34f3a 100644
--- a/runner/stats.go
+++ b/runner/stats.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/rs/zerolog/log"
 
+	schema "github.com/coreruleset/ftw-tests-schema/types"
 	"github.com/coreruleset/go-ftw/output"
 )
 
@@ -63,29 +64,33 @@ func (stats *RunStats) TotalFailed() int {
 	return len(stats.Failed) + len(stats.ForcedFail)
 }
 
-func (stats *RunStats) addResultToStats(result TestResult, title string, testTime time.Duration) {
+func (stats *RunStats) addResultToStats(result TestResult, testCase *schema.Test) {
+	title := testCase.IdString()
+	stats.Run++
+
 	switch result {
 	case Success:
 		stats.Success = append(stats.Success, title)
-		stats.RunTime[title] = testTime
 	case Failed:
 		stats.Failed = append(stats.Failed, title)
-		stats.RunTime[title] = testTime
 	case Skipped:
 		stats.Skipped = append(stats.Skipped, title)
 	case Ignored:
 		stats.Ignored = append(stats.Ignored, title)
 	case ForceFail:
 		stats.ForcedFail = append(stats.ForcedFail, title)
-		stats.RunTime[title] = testTime
 	case ForcePass:
 		stats.ForcedPass = append(stats.ForcedPass, title)
-		stats.RunTime[title] = testTime
 	default:
 		log.Info().Msgf("runner/stats: don't know how to handle TestResult %d", result)
 	}
 }
 
+func (stats *RunStats) addStageResultToStats(testCase *schema.Test, stageTime time.Duration) {
+	stats.RunTime[testCase.IdString()] += stageTime
+	stats.TotalTime += stageTime
+}
+
 func (stats *RunStats) printSummary(out *output.Output) {
 	if stats.Run > 0 {
 		if out.IsJson() {
diff --git a/runner/testdata/TestBrokenOverrideRun.yaml b/runner/testdata/TestBrokenOverrideRun.yaml
index ab6b84bf..e7da3d7a 100644
--- a/runner/testdata/TestBrokenOverrideRun.yaml
+++ b/runner/testdata/TestBrokenOverrideRun.yaml
@@ -5,15 +5,14 @@ meta:
   name: "TestBrokenOverrideRun.yaml"
   description: "Example Override Test"
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "access real external site"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Host: "{{ .TestAddr }}"
-          output:
-            expect_error: False
-            status: [200]
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Host: "{{ .TestAddr }}"
+        output:
+          expect_error: False
+          status: 200
diff --git a/runner/testdata/TestBrokenPortOverrideRun.yaml b/runner/testdata/TestBrokenPortOverrideRun.yaml
index b5dd26a3..484a8f1c 100644
--- a/runner/testdata/TestBrokenPortOverrideRun.yaml
+++ b/runner/testdata/TestBrokenPortOverrideRun.yaml
@@ -5,15 +5,14 @@ meta:
   name: "TestBrokenPortOverrideRun.yaml"
   description: "Example Override Test"
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "access real external site"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Host: "{{ .TestAddr }}"
-          output:
-            expect_error: False
-            status: [200]
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Host: "{{ .TestAddr }}"
+        output:
+          expect_error: False
+          status: 200
diff --git a/runner/testdata/TestCloudRun.yaml b/runner/testdata/TestCloudRun.yaml
index 7d7dedc3..1a6c3fed 100644
--- a/runner/testdata/TestCloudRun.yaml
+++ b/runner/testdata/TestCloudRun.yaml
@@ -5,102 +5,95 @@ meta:
   name: "TestCloudRun.yaml"
   description: "Example Test"
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "access real external site"
     stages:
-      - stage:
-          input:
-            uri: "/200"
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "{{ .TestAddr }}"
-          output:
-            expect_error: False
-            status: [200]
-  - test_title: "403"
+      - input:
+          uri: "/200"
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          expect_error: False
+          status: 200
+  - test_id: 403
     description: "using log_contains should return a 403 response."
     stages:
-      - stage:
-          input:
-            uri: "/403"
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "{{ .TestAddr }}"
-          output:
-            log_contains: "ModSecurity: Access denied with code 403"
-  - test_title: "405"
+      - input:
+          uri: "/403"
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          log_contains: "ModSecurity: Access denied with code 403"
+  - test_id: 405
     description: "using no_log_contains should return a 405 response."
     stages:
-      - stage:
-          input:
-            uri: "/405"
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "{{ .TestAddr }}"
-          output:
-            no_log_contains: "ModSecurity: Access denied with code 403"
-  - test_title: "008"
+      - input:
+          uri: "/405"
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          no_log_contains: "ModSecurity: Access denied with code 403"
+  - test_id: 8
     description: "this test is number 8"
     stages:
-      - stage:
-          input:
-            uri: "/200"
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            status: [200]
-  - test_title: "010"
+      - input:
+          uri: "/200"
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          status: 200
+  - test_id: 10
     stages:
-      - stage:
-          input:
-            uri: "/200"
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            version: "HTTP/1.1"
-            method: "OTHER"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            response_contains: "Hello, client"
-  - test_title: "101"
+      - input:
+          uri: "/200"
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          version: "HTTP/1.1"
+          method: "OTHER"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          response_contains: "Hello, client"
+  - test_id: 101
     description: "this tests exceptions (connection timeout)"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: 8090
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "none.host"
-          output:
-            expect_error: True
-  - test_title: "102"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: 8090
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "none.host"
+        output:
+          expect_error: True
+  - test_id: 102
     description: "this tests exceptions (connection timeout)"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: 8090
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Host: "none.host"
-              Accept: "*/*"
-            encoded_request: "UE9TVCAvaW5kZXguaHRtbCBIVFRQLzEuMQ0KSG9zdDogMTkyLjE2OC4xLjIzDQpVc2VyLUFnZW50OiBjdXJsLzcuNDMuMA0KQWNjZXB0OiAqLyoNCkNvbnRlbnQtTGVuZ3RoOiA2NA0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi94LXd3dy1mb3JtLXVybGVuY29kZWQNCkNvbm5lY3Rpb246IGNsb3NlDQoNCmQ9MTsyOzM7NDs1XG4xO0BTVU0oMSsxKSpjbWR8JyBwb3dlcnNoZWxsIElFWCh3Z2V0IDByLnBlL3ApJ1whQTA7Mw=="
-          output:
-            expect_error: True
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: 8090
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Host: "none.host"
+            Accept: "*/*"
+          encoded_request: "UE9TVCAvaW5kZXguaHRtbCBIVFRQLzEuMQ0KSG9zdDogMTkyLjE2OC4xLjIzDQpVc2VyLUFnZW50OiBjdXJsLzcuNDMuMA0KQWNjZXB0OiAqLyoNCkNvbnRlbnQtTGVuZ3RoOiA2NA0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi94LXd3dy1mb3JtLXVybGVuY29kZWQNCkNvbm5lY3Rpb246IGNsb3NlDQoNCmQ9MTsyOzM7NDs1XG4xO0BTVU0oMSsxKSpjbWR8JyBwb3dlcnNoZWxsIElFWCh3Z2V0IDByLnBlL3ApJ1whQTA7Mw=="
+        output:
+          expect_error: True
diff --git a/runner/testdata/TestDisabledRun.yaml b/runner/testdata/TestDisabledRun.yaml
index 0433bb97..bf1a03d2 100644
--- a/runner/testdata/TestDisabledRun.yaml
+++ b/runner/testdata/TestDisabledRun.yaml
@@ -5,15 +5,14 @@ meta:
   name: "TestDisabledRun.yaml"
   description: "we do not care, this test is disabled"
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "access real external site"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port:  {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Host: "{{ .TestAddr }}"
-          output:
-            status: [1234]
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port:  {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Host: "{{ .TestAddr }}"
+        output:
+          status: 1234
diff --git a/runner/testdata/TestFailFast.yaml b/runner/testdata/TestFailFast.yaml
new file mode 100644
index 00000000..e6594a83
--- /dev/null
+++ b/runner/testdata/TestFailFast.yaml
@@ -0,0 +1,45 @@
+---
+meta:
+  author: "tester"
+  name: "TestFailFast.yaml"
+  description: "Example Test"
+tests:
+  - test_id: 1
+    description: "succeed"
+    stages:
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          status: 200
+  - test_id: 2
+    description: "fail"
+    stages:
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          log:
+            match_regex: '--nothing--'
+  - test_id: 3
+    description: "not executed with fail-fast = true"
+    stages:
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          version: "HTTP/1.1"
+          method: "OTHER"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          response_contains: "Hello, client"
diff --git a/runner/testdata/TestFailedTestsRun.yaml b/runner/testdata/TestFailedTestsRun.yaml
index 54140dfc..d1f44f44 100644
--- a/runner/testdata/TestFailedTestsRun.yaml
+++ b/runner/testdata/TestFailedTestsRun.yaml
@@ -5,17 +5,15 @@ meta:
   name: "TestFailedTestsRun.yaml"
   description: "Example Test"
 tests:
-  - test_title: "990"
+  - test_id: 990
     description: test that fails
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            # -1 designates port value must be replaced by test setup
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "none.host"
-          output:
-            status: [413]
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "none.host"
+        output:
+          status: 413
diff --git a/runner/testdata/TestIgnoredTestsRun.yaml b/runner/testdata/TestIgnoredTestsRun.yaml
index 8b5f5f92..f8248275 100644
--- a/runner/testdata/TestIgnoredTestsRun.yaml
+++ b/runner/testdata/TestIgnoredTestsRun.yaml
@@ -5,71 +5,66 @@ meta:
   name: "TestIgnoredTestsRun.yaml"
   description: "Example Test"
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "access real external site"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "{{ .TestAddr }}"
-          output:
-            expect_error: False
-            status: [200]
-  - test_title: "008"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          expect_error: False
+          status: 200
+  - test_id: 8
     description: "this test is number 8"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            status: [200]
-  - test_title: "010"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          status: 200
+  - test_id: 10
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            version: "HTTP/1.1"
-            method: "OTHER"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            response_contains: "Hello, client"
-  - test_title: "101"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          version: "HTTP/1.1"
+          method: "OTHER"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          response_contains: "Hello, client"
+  - test_id: 101
     description: "this tests exceptions (connection timeout)"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: 8090
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "none.host"
-          output:
-            expect_error: True
-  - test_title: "102"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: 8090
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "none.host"
+        output:
+          expect_error: True
+  - test_id: 102
     description: "this tests exceptions (connection timeout)"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: 8090
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Host: "none.host"
-              Accept: "*/*"
-            encoded_request: "UE9TVCAvaW5kZXguaHRtbCBIVFRQLzEuMQ0KSG9zdDogMTkyLjE2OC4xLjIzDQpVc2VyLUFnZW50OiBjdXJsLzcuNDMuMA0KQWNjZXB0OiAqLyoNCkNvbnRlbnQtTGVuZ3RoOiA2NA0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi94LXd3dy1mb3JtLXVybGVuY29kZWQNCkNvbm5lY3Rpb246IGNsb3NlDQoNCmQ9MTsyOzM7NDs1XG4xO0BTVU0oMSsxKSpjbWR8JyBwb3dlcnNoZWxsIElFWCh3Z2V0IDByLnBlL3ApJ1whQTA7Mw=="
-          output:
-            expect_error: True
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: 8090
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Host: "none.host"
+            Accept: "*/*"
+          encoded_request: "UE9TVCAvaW5kZXguaHRtbCBIVFRQLzEuMQ0KSG9zdDogMTkyLjE2OC4xLjIzDQpVc2VyLUFnZW50OiBjdXJsLzcuNDMuMA0KQWNjZXB0OiAqLyoNCkNvbnRlbnQtTGVuZ3RoOiA2NA0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi94LXd3dy1mb3JtLXVybGVuY29kZWQNCkNvbm5lY3Rpb246IGNsb3NlDQoNCmQ9MTsyOzM7NDs1XG4xO0BTVU0oMSsxKSpjbWR8JyBwb3dlcnNoZWxsIElFWCh3Z2V0IDByLnBlL3ApJ1whQTA7Mw=="
+        output:
+          expect_error: True
diff --git a/runner/testdata/TestLogsRun.yaml b/runner/testdata/TestLogsRun.yaml
index 5c62b860..94e077d5 100644
--- a/runner/testdata/TestLogsRun.yaml
+++ b/runner/testdata/TestLogsRun.yaml
@@ -5,29 +5,25 @@ meta:
   name: "TestLogsRun.yaml"
   description: "Example Test"
 tests:
-  - test_title: "200"
+  - test_id: 200
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            # -1 designates port value must be replaced by test setup
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            log_contains: id \"949110\"
-  - test_title: "201"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          log_contains: id \"949110\"
+  - test_id: 201
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            # -1 designates port value must be replaced by test setup
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            no_log_contains: ABCDE
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          no_log_contains: ABCDE
diff --git a/runner/testdata/TestOverrideRun.yaml b/runner/testdata/TestOverrideRun.yaml
index 08c448ee..88ecf01a 100644
--- a/runner/testdata/TestOverrideRun.yaml
+++ b/runner/testdata/TestOverrideRun.yaml
@@ -5,17 +5,15 @@ meta:
   name: "TestOverrideRun.yaml"
   description: "Example Override Test"
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "access real external site"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            # -1 designates port value must be replaced by test setup
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Host: "{{ .TestAddr }}
-          output:
-            expect_error: False
-            status: [200]
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Host: "{{ .TestAddr }}"
+        output:
+          expect_error: False
+          status: 200
diff --git a/runner/testdata/TestRetryOnce.yaml b/runner/testdata/TestRetryOnce.yaml
new file mode 100644
index 00000000..0c69d532
--- /dev/null
+++ b/runner/testdata/TestRetryOnce.yaml
@@ -0,0 +1,21 @@
+---
+meta:
+  author: "tester"
+  enabled: true
+  name: "TestRetryOnce.yaml"
+  description: "Example Test"
+tests:
+  - test_id: 1
+    description: "used to simulate race condition in 980170"
+    stages:
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          expect_error: false
+          retry_once: true
+          log_contains: id "980170"
diff --git a/runner/testdata/TestRunMultipleMatches.yaml b/runner/testdata/TestRunMultipleMatches.yaml
index a9c64ab1..710750f2 100644
--- a/runner/testdata/TestRunMultipleMatches.yaml
+++ b/runner/testdata/TestRunMultipleMatches.yaml
@@ -1,22 +1,20 @@
 ---
 meta:
-    author: "tester"
-    enabled: true
-    name: "TestRunMultipleMatches.yaml"
-    description: "Example Test with multiple expected outputs per single rule"
+  author: "tester"
+  enabled: true
+  name: "TestRunMultipleMatches.yaml"
+  description: "Example Test with multiple expected outputs per single rule"
 tests:
-    - test_title: "001"
-      description: "access real external site"
-      stages:
-          - stage:
-                input:
-                    dest_addr: "{{ .TestAddr }}"
-                    # -1 designates port value must be replaced by test setup
-                    port: {{ .TestPort }}
-                    headers:
-                        User-Agent: "ModSecurity CRS 3 Tests"
-                        Accept: "*/*"
-                        Host: "{{ .TestAddr }}"
-                output:
-                    status: [200]
-                    response_contains: "Not contains this"
+  - test_id: 1
+    description: "access real external site"
+    stages:
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          status: 200
+          response_contains: "Not contains this"
diff --git a/runner/testdata/TestRunTests_Run.yaml b/runner/testdata/TestRunTests_Run.yaml
index 82b506fd..8582848a 100644
--- a/runner/testdata/TestRunTests_Run.yaml
+++ b/runner/testdata/TestRunTests_Run.yaml
@@ -5,71 +5,66 @@ meta:
   name: "TestRunTests_Run.yaml"
   description: "Example Test"
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "access real external site"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "{{ .TestAddr }}"
-          output:
-            expect_error: False
-            status: [200]
-  - test_title: "008"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "{{ .TestAddr }}"
+        output:
+          expect_error: False
+          status: 200
+  - test_id: 8
     description: "this test is number 8"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            status: [200]
-  - test_title: "010"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          status: 200
+  - test_id: 10
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: {{ .TestPort }}
-            version: "HTTP/1.1"
-            method: "OTHER"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            response_contains: "Hello, client"
-  - test_title: "101"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          version: "HTTP/1.1"
+          method: "OTHER"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          response_contains: "Hello, client"
+  - test_id: 101
     description: "this tests exceptions (connection timeout)"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: 8090
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "none.host"
-          output:
-            expect_error: True
-  - test_title: "102"
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: 8090
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "none.host"
+        output:
+          expect_error: True
+  - test_id: 102
     description: "this tests exceptions (connection timeout)"
     stages:
-      - stage:
-          input:
-            dest_addr: "{{ .TestAddr }}"
-            port: 8090
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Host: "none.host"
-              Accept: "*/*"
-            encoded_request: "UE9TVCAvaW5kZXguaHRtbCBIVFRQLzEuMQ0KSG9zdDogMTkyLjE2OC4xLjIzDQpVc2VyLUFnZW50OiBjdXJsLzcuNDMuMA0KQWNjZXB0OiAqLyoNCkNvbnRlbnQtTGVuZ3RoOiA2NA0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi94LXd3dy1mb3JtLXVybGVuY29kZWQNCkNvbm5lY3Rpb246IGNsb3NlDQoNCmQ9MTsyOzM7NDs1XG4xO0BTVU0oMSsxKSpjbWR8JyBwb3dlcnNoZWxsIElFWCh3Z2V0IDByLnBlL3ApJ1whQTA7Mw=="
-          output:
-            expect_error: True
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: 8090
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Host: "none.host"
+            Accept: "*/*"
+          encoded_request: "UE9TVCAvaW5kZXguaHRtbCBIVFRQLzEuMQ0KSG9zdDogMTkyLjE2OC4xLjIzDQpVc2VyLUFnZW50OiBjdXJsLzcuNDMuMA0KQWNjZXB0OiAqLyoNCkNvbnRlbnQtTGVuZ3RoOiA2NA0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi94LXd3dy1mb3JtLXVybGVuY29kZWQNCkNvbm5lY3Rpb246IGNsb3NlDQoNCmQ9MTsyOzM7NDs1XG4xO0BTVU0oMSsxKSpjbWR8JyBwb3dlcnNoZWxsIElFWCh3Z2V0IDByLnBlL3ApJ1whQTA7Mw=="
+        output:
+          expect_error: True
diff --git a/runner/types.go b/runner/types.go
index e6a01274..bbc0d02c 100644
--- a/runner/types.go
+++ b/runner/types.go
@@ -31,6 +31,8 @@ type RunnerConfig struct {
 	ReadTimeout time.Duration
 	// RateLimit is the rate limit for requests to the server. 0 is unlimited.
 	RateLimit time.Duration
+	// FailFast determines whether to stop running tests when the first failure is encountered.
+	FailFast bool
 }
 
 // TestRunContext carries information about the current test run.
@@ -38,6 +40,7 @@ type RunnerConfig struct {
 // and results.
 type TestRunContext struct {
 	Config         *config.FTWConfiguration
+	RunnerConfig   *RunnerConfig
 	Include        *regexp.Regexp
 	Exclude        *regexp.Regexp
 	ShowTime       bool
diff --git a/test/defaults.go b/test/defaults.go
index fb6a50c9..87ecf6c4 100644
--- a/test/defaults.go
+++ b/test/defaults.go
@@ -13,7 +13,10 @@ import (
 
 type Input schema.Input
 type Output schema.Output
-type FTWTest schema.FTWTest
+type FTWTest struct {
+	schema.FTWTest `yaml:",inline"`
+	FileName       string
+}
 
 // GetMethod returns the proper semantic when the field is empty
 func (i *Input) GetMethod() string {
@@ -77,7 +80,9 @@ func (i *Input) GetRawRequest() ([]byte, error) {
 		// if Encoded, first base64 decode, then dump
 		return base64.StdEncoding.DecodeString(i.EncodedRequest)
 	}
+	//nolint:staticcheck
 	if utils.IsNotEmpty(i.RAWRequest) {
+		//nolint:staticcheck
 		return []byte(i.RAWRequest), nil
 	}
 	return nil, nil
diff --git a/test/errors.go b/test/errors.go
deleted file mode 100644
index 6d4eccdd..00000000
--- a/test/errors.go
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2023 OWASP ModSecurity Core Rule Set Project
-// SPDX-License-Identifier: Apache-2.0
-
-package test
-
-import (
-	"bufio"
-	"fmt"
-	"os"
-	"regexp"
-
-	"github.com/rs/zerolog/log"
-)
-
-// GetLinesFromTest get the output lines from a test name, to show in errors
-func (f *FTWTest) GetLinesFromTest(testName string) (int, error) {
-	file, err := os.Open(f.FileName)
-	if err != nil {
-		log.Info().Msgf("yamlFile.Get err   #%v ", err)
-	}
-	match := fmt.Sprintf("test_title: %s", testName)
-	scanner := bufio.NewScanner(file)
-	scanner.Split(bufio.ScanLines)
-	line := 1
-
-	for scanner.Scan() {
-		log.Debug().Msgf("%d - %s\n", line, scanner.Text())
-		got, err := regexp.Match(match, []byte(scanner.Text()))
-		if err != nil {
-			log.Fatal().Msgf("ftw/test/error: bad regexp %s", err.Error())
-		}
-		if got {
-			log.Trace().Msgf("ftw/test/error: Found %s at %d", match, line)
-			break
-		}
-		line++
-	}
-
-	return line, err
-}
diff --git a/test/errors_test.go b/test/errors_test.go
deleted file mode 100644
index 6b485bed..00000000
--- a/test/errors_test.go
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright 2023 OWASP ModSecurity Core Rule Set Project
-// SPDX-License-Identifier: Apache-2.0
-
-package test
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/suite"
-
-	"github.com/coreruleset/go-ftw/utils"
-)
-
-var errorsTest = `---
-  meta:
-    author: "tester"
-    enabled: true
-    name: "911100.yaml"
-    description: "Description"
-  tests:
-    -
-      test_title: 911100-1
-      stages:
-        -
-          stage:
-            input:
-              dest_addr: "127.0.0.1"
-              port: 80
-              headers:
-                  User-Agent: "ModSecurity CRS 3 Tests"
-                  Host: "localhost"
-            output:
-              no_log_contains: "id \"911100\""
-    -
-      test_title: 911100-2
-      stages:
-        -
-          stage:
-            input:
-              dest_addr: "127.0.0.1"
-              port: 80
-              method: "OPTIONS"
-              headers:
-                  User-Agent: "ModSecurity CRS 3 Tests"
-                  Host: "localhost"
-            output:
-              no_log_contains: "id \"911100\""
-`
-
-type errorsTestSuite struct {
-	suite.Suite
-}
-
-func TestErrorsTestSuite(t *testing.T) {
-	suite.Run(t, new(errorsTestSuite))
-}
-
-func (s *errorsTestSuite) TestGetLinesFromTestName() {
-	filename, _ := utils.CreateTempFileWithContent(errorsTest, "test-yaml-*")
-	tests, _ := GetTestsFromFiles(filename)
-
-	for _, ft := range tests {
-		line, _ := ft.GetLinesFromTest("911100-2")
-		s.Equal(22, line, "Not getting the proper line.")
-	}
-}
diff --git a/test/files.go b/test/files.go
index 5bf4ffd1..26c78d03 100644
--- a/test/files.go
+++ b/test/files.go
@@ -6,6 +6,7 @@ package test
 import (
 	"errors"
 	"os"
+	"path"
 
 	"github.com/goccy/go-yaml"
 	"github.com/rs/zerolog/log"
@@ -28,20 +29,19 @@ func GetTestsFromFiles(globPattern string) ([]*FTWTest, error) {
 		return tests, err
 	}
 
-	for _, fileName := range testFiles {
-		yamlString, err := readFileContents(fileName)
+	for _, filePath := range testFiles {
+		yamlString, err := readFileContents(filePath)
 		if err != nil {
 			return tests, err
 		}
-		ftwTest, err := GetTestFromYaml(yamlString)
+		ftwTest, err := GetTestFromYaml(yamlString, path.Base(filePath))
 		if err != nil {
 			log.Error().Msgf("Problem detected in file %s:\n%s\n%s",
-				fileName, yaml.FormatError(err, true, true),
+				filePath, yaml.FormatError(err, true, true),
 				DescribeYamlError(err))
 			return tests, err
 		}
 
-		ftwTest.FileName = fileName
 		tests = append(tests, ftwTest)
 	}
 
diff --git a/test/files_test.go b/test/files_test.go
index e24bf1ea..e0097561 100644
--- a/test/files_test.go
+++ b/test/files_test.go
@@ -19,35 +19,31 @@ var yamlTest = `
     enabled: true
     name: "911100.yaml"
     description: "Description"
+  rule_id: 911100
   tests:
-    -
-      test_title: 911100-1
+    - test_id: 1
       stages:
-        -
-          stage:
-            input:
-              autocomplete_headers: false
-              dest_addr: "127.0.0.1"
-              port: 80
-              headers:
-                  User-Agent: "ModSecurity CRS 3 Tests"
-                  Host: "localhost"
-            output:
-              no_log_contains: "id \"911100\""
+        - input:
+            autocomplete_headers: false
+            dest_addr: "127.0.0.1"
+            port: 80
+            headers:
+              User-Agent: "ModSecurity CRS 3 Tests"
+              Host: "localhost"
+          output:
+            no_log_contains: "id \"911100\""
     -
-      test_title: 911100-2
+      test_id: 2
       stages:
-        -
-          stage:
-            input:
-              dest_addr: "127.0.0.1"
-              port: 80
-              method: "OPTIONS"
-              headers:
-                  User-Agent: "ModSecurity CRS 3 Tests"
-                  Host: "localhost"
-            output:
-              no_log_contains: "id \"911100\""
+        - input:
+            dest_addr: "127.0.0.1"
+            port: 80
+            method: "OPTIONS"
+            headers:
+              User-Agent: "ModSecurity CRS 3 Tests"
+              Host: "localhost"
+          output:
+            no_log_contains: "id \"911100\""
 `
 
 var wrongYamlTest = `
@@ -67,14 +63,13 @@ func (s *filesTestSuite) TestGetTestFromYAML() {
 	tests, _ := GetTestsFromFiles(filename)
 
 	for _, ft := range tests {
-		s.Equal(filename, ft.FileName)
 		s.Equal("tester", ft.Meta.Author)
 		s.Equal("911100.yaml", ft.Meta.Name)
 
-		re := regexp.MustCompile("911100*")
+		re := regexp.MustCompile("911100.*")
 
 		for _, test := range ft.Tests {
-			s.True(re.MatchString(test.TestTitle), "Can't read test title")
+			s.True(re.MatchString(test.IdString()), "Can't read test identifier")
 		}
 	}
 }
diff --git a/test/testdata/TestCheckBenchmarkCheckFiles.yaml b/test/testdata/TestCheckBenchmarkCheckFiles.yaml
index f4278e1f..18636ed5 100644
--- a/test/testdata/TestCheckBenchmarkCheckFiles.yaml
+++ b/test/testdata/TestCheckBenchmarkCheckFiles.yaml
@@ -4,6 +4,7 @@ meta:
   enabled: true
   name: "920420.yaml"
   description: "Description"
+rule_id: 920420
 tests:
   - test_title: 920420-1
     stages:
diff --git a/test/types.go b/test/types.go
index 112c6ebc..6a1e1fe5 100644
--- a/test/types.go
+++ b/test/types.go
@@ -4,40 +4,62 @@
 package test
 
 import (
-	"github.com/coreruleset/ftw-tests-schema/types"
+	"regexp"
+	"slices"
+	"strconv"
+
+	schema "github.com/coreruleset/ftw-tests-schema/types"
+	overridesSchema "github.com/coreruleset/ftw-tests-schema/types/overrides"
+	"github.com/coreruleset/go-ftw/config"
 	"github.com/coreruleset/go-ftw/ftwhttp"
+	"github.com/rs/zerolog/log"
 )
 
-// Overrides represents the overridden inputs that have to be applied to tests
-type Overrides struct {
-	DestAddr   *string        `yaml:"dest_addr,omitempty" koanf:"dest_addr,omitempty"`
-	Port       *int           `yaml:"port,omitempty" koanf:"port,omitempty"`
-	Protocol   *string        `yaml:"protocol,omitempty" koanf:"protocol,omitempty"`
-	URI        *string        `yaml:"uri,omitempty" koanf:"uri,omitempty"`
-	Version    *string        `yaml:"version,omitempty" koanf:"version,omitempty"`
-	Headers    ftwhttp.Header `yaml:"headers,omitempty" koanf:"headers,omitempty"`
-	Method     *string        `yaml:"method,omitempty" koanf:"method,omitempty"`
-	Data       *string        `yaml:"data,omitempty" koanf:"data,omitempty"`
-	SaveCookie *bool          `yaml:"save_cookie,omitempty" koanf:"save_cookie,omitempty"`
-	// Deprecated: replaced with AutocompleteHeaders
-	StopMagic               *bool   `yaml:"stop_magic" koanf:"stop_magic,omitempty"`
-	AutocompleteHeaders     *bool   `yaml:"autocomplete_headers" koanf:"autocomplete_headers,omitempty"`
-	EncodedRequest          *string `yaml:"encoded_request,omitempty" koanf:"encoded_request,omitempty"`
-	RAWRequest              *string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"`
-	OverrideEmptyHostHeader *bool   `yaml:"override_empty_host_header,omitempty" koanf:"override_empty_host_header,omitempty"`
-}
-
 // ApplyInputOverride will check if config had global overrides and write that into the test.
-func ApplyInputOverrides(overrides *Overrides, input *Input) {
+func ApplyInputOverrides(conf *config.FTWConfiguration, input *Input) {
+	overrides := &conf.TestOverride.Overrides
 	applySimpleOverrides(overrides, input)
 	applyDestAddrOverride(overrides, input)
 	applyHeadersOverride(overrides, input)
+	//nolint:staticcheck
 	if overrides.AutocompleteHeaders != nil || overrides.StopMagic != nil {
+		//nolint:staticcheck
 		postProcessAutocompleteHeaders(overrides.AutocompleteHeaders, overrides.StopMagic, input)
 	}
 }
 
-func applyDestAddrOverride(overrides *Overrides, input *Input) {
+func ApplyPlatformOverrides(conf *config.FTWConfiguration, testCase *schema.Test) {
+	platformOverrides := conf.PlatformOverrides
+	log.Debug().Msgf("Applying overrides for engine '%s', platform '%s", platformOverrides.Meta.Engine, platformOverrides.Meta.Platform)
+	overrides, ok := platformOverrides.OverridesMap[testCase.RuleId]
+	if !ok {
+		log.Trace().Msgf("no override found for rule %d", testCase.RuleId)
+		return
+	}
+
+	applyToAll := len(overrides) == 0
+	for _, override := range overrides {
+		for _, testId := range override.TestIds {
+			if applyToAll || testId == testCase.TestId {
+				basicApplyPlatformOverrides(override, testCase)
+			}
+		}
+	}
+}
+
+func basicApplyPlatformOverrides(override *overridesSchema.TestOverride, testCase *schema.Test) {
+	// Apply to all stages of the given test if the list of stage IDs is empty
+	applyToAll := len(override.StageIds) == 0
+
+	for index := 0; index < len(testCase.Stages); index++ {
+		stage := &testCase.Stages[index]
+		if applyToAll || slices.Contains(override.StageIds, uint(index)) {
+			stage.Output = override.Output
+		}
+	}
+}
+
+func applyDestAddrOverride(overrides *config.Overrides, input *Input) {
 	if overrides.DestAddr != nil {
 		input.DestAddr = overrides.DestAddr
 		if input.Headers == nil {
@@ -51,7 +73,7 @@ func applyDestAddrOverride(overrides *Overrides, input *Input) {
 	}
 }
 
-func applySimpleOverrides(overrides *Overrides, input *Input) {
+func applySimpleOverrides(overrides *config.Overrides, input *Input) {
 	if overrides.Port != nil {
 		input.Port = overrides.Port
 	}
@@ -85,11 +107,12 @@ func applySimpleOverrides(overrides *Overrides, input *Input) {
 	}
 
 	if overrides.RAWRequest != nil {
+		//nolint:staticcheck
 		input.RAWRequest = *overrides.RAWRequest
 	}
 }
 
-func applyHeadersOverride(overrides *Overrides, input *Input) {
+func applyHeadersOverride(overrides *config.Overrides, input *Input) {
 	if overrides.Headers != nil {
 		if input.Headers == nil {
 			input.Headers = ftwhttp.Header{}
@@ -100,23 +123,52 @@ func applyHeadersOverride(overrides *Overrides, input *Input) {
 	}
 }
 
-func postLoadTestFTWTest(ftwTest *FTWTest) {
-	for _, test := range ftwTest.Tests {
-		postLoadTest(&test)
+func postLoadTestFTWTest(ftwTest *FTWTest, fileName string) {
+	ftwTest.FileName = fileName
+	postLoadRuleId(ftwTest)
+	for index := 0; index < len(ftwTest.Tests); index++ {
+		postLoadTest(ftwTest.RuleId, uint(index+1), &ftwTest.Tests[index])
 	}
 }
 
-func postLoadTest(test *types.Test) {
+func postLoadRuleId(ftwTest *FTWTest) {
+	if ftwTest.RuleId > 0 {
+		return
+	}
+
+	if len(ftwTest.FileName) == 0 {
+		log.Fatal().Msg("The rule_id field is required for the top-level test structure")
+	} else {
+		ruleIdString := regexp.MustCompile(`\d+`).FindString(ftwTest.FileName)
+		if len(ruleIdString) == 0 {
+			log.Fatal().Msg("Failed to fall back on filename to find rule ID of test. The rule_id field is required for the top-level test structure")
+			return
+		}
+		ruleId, err := strconv.ParseUint(ruleIdString, 10, 0)
+		if err != nil {
+			log.Fatal().Msgf("failed to parse rule ID from filename '%s'", ftwTest.FileName)
+			return
+		}
+		ftwTest.RuleId = uint(ruleId)
+	}
+}
+func postLoadTest(ruleId uint, testId uint, test *schema.Test) {
+	test.RuleId = ruleId
+	// Retain explicitly defined test IDs
+	if test.TestId == 0 {
+		test.TestId = testId
+	}
 	for index := range test.Stages {
-		postLoadStage(&test.Stages[index].SD)
+		postLoadStage(&test.Stages[index])
 	}
 }
 
-func postLoadStage(stage *types.StageData) {
+func postLoadStage(stage *schema.Stage) {
 	postLoadInput((*Input)(&stage.Input))
 }
 
 func postLoadInput(input *Input) {
+	//nolint:staticcheck
 	postProcessAutocompleteHeaders(input.AutocompleteHeaders, input.StopMagic, input)
 }
 
@@ -134,5 +186,6 @@ func postProcessAutocompleteHeaders(autocompleteHeaders *bool, stopMagic *bool,
 	}
 	input.AutocompleteHeaders = &finalValue
 	// StopMagic has the inverse boolean logic
+	//nolint:staticcheck
 	input.StopMagic = func() *bool { b := !finalValue; return &b }()
 }
diff --git a/test/types_test.go b/test/types_test.go
index 94ffa206..9ef013f8 100644
--- a/test/types_test.go
+++ b/test/types_test.go
@@ -23,48 +23,46 @@ meta:
   enabled: true
   name: "gotest-ftw.yaml"
   description: "Example Test"
+rule_id: 123456
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "autocomplete headers by default"
     stages:
-      - stage:
-          input:
-            dest_addr: "localhost"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            expect_error: False
-            status: [200]
-  - test_title: "002"
+      - input:
+          dest_addr: "localhost"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          expect_error: False
+          status: 200
+  - test_id: 2
     description: "autocomplete headers by default"
     stages:
-      - stage:
-          input:
-            stop_magic: true
-            dest_addr: "localhost"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            expect_error: False
-            status: [200]
-  - test_title: "003"
+      - input:
+          stop_magic: true
+          dest_addr: "localhost"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          expect_error: False
+          status: 200
+  - test_id: 3
     description: "autocomplete headers by default"
     stages:
-      - stage:
-          input:
-            stop_magic: false
-            dest_addr: "localhost"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            expect_error: False
-            status: [200]
+      - input:
+          stop_magic: false
+          dest_addr: "localhost"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          expect_error: False
+          status: 200
 `
 
 var autocompleteHeadersFalseYaml = `---
@@ -73,51 +71,49 @@ meta:
   enabled: true
   name: "gotest-ftw.yaml"
   description: "Example Test"
+rule_id: 123456
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "autocomplete headers explicitly"
     stages:
-      - stage:
-          input:
-            autocomplete_headers: false
-            dest_addr: "localhost"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            expect_error: False
-            status: [200]
-  - test_title: "002"
+      - input:
+          autocomplete_headers: false
+          dest_addr: "localhost"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          expect_error: False
+          status: 200
+  - test_id: 2
     description: "autocomplete headers explicitly"
     stages:
-      - stage:
-          input:
-            autocomplete_headers: false
-            stop_magic: true
-            dest_addr: "localhost"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            expect_error: False
-            status: [200]
-  - test_title: "003"
+      - input:
+          autocomplete_headers: false
+          stop_magic: true
+          dest_addr: "localhost"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          expect_error: False
+          status: 200
+  - test_id: 3
     description: "autocomplete headers explicitly"
     stages:
-      - stage:
-          input:
-            autocomplete_headers: false
-            stop_magic: false
-            dest_addr: "localhost"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            expect_error: False
-            status: [200]
+      - input:
+          autocomplete_headers: false
+          stop_magic: false
+          dest_addr: "localhost"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          expect_error: False
+          status: 200
 `
 
 var autocompleteHeadersTrueYaml = `---
@@ -126,129 +122,136 @@ meta:
   enabled: true
   name: "gotest-ftw.yaml"
   description: "Example Test"
+rule_id: 123456
 tests:
-  - test_title: "001"
+  - test_id: 1
     description: "do not autocomplete headers explicitly"
     stages:
-      - stage:
-          input:
-            autocomplete_headers: true
-            dest_addr: "localhost"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            expect_error: False
-            status: [200]
-  - test_title: "002"
+      - input:
+          autocomplete_headers: true
+          dest_addr: "localhost"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          expect_error: False
+          status: 200
+  - test_id: 2
     description: "do not autocomplete headers explicitly"
     stages:
-      - stage:
-          input:
-            autocomplete_headers: true
-            stop_magic: true
-            dest_addr: "localhost"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            expect_error: False
-            status: [200]
-  - test_title: "003"
+      - input:
+          autocomplete_headers: true
+          stop_magic: true
+          dest_addr: "localhost"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          expect_error: False
+          status: 200
+  - test_id: 3
     description: "do not autocomplete headers explicitly"
     stages:
-      - stage:
-          input:
-            autocomplete_headers: true
-            stop_magic: false
-            dest_addr: "localhost"
-            headers:
-              User-Agent: "ModSecurity CRS 3 Tests"
-              Accept: "*/*"
-              Host: "localhost"
-          output:
-            expect_error: False
-            status: [200]
+      - input:
+          autocomplete_headers: true
+          stop_magic: false
+          dest_addr: "localhost"
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "localhost"
+        output:
+          expect_error: False
+          status: 200
 `
 
 func (s *typesTestSuite) TestAutocompleteHeadersDefault_StopMagicDefault() {
-	test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml))
+	test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml), "")
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[0].Stages[0].SD.Input
+	input := test.Tests[0].Stages[0].Input
 	s.True(*input.AutocompleteHeaders)
+	//nolint:staticcheck
 	s.False(*input.StopMagic)
 }
 
 func (s *typesTestSuite) TestAutocompleteHeadersDefault_StopMagicTrue() {
-	test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml))
+	test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml), "")
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[1].Stages[0].SD.Input
+	input := test.Tests[1].Stages[0].Input
 	s.False(*input.AutocompleteHeaders)
+	//nolint:staticcheck
 	s.True(*input.StopMagic)
 }
 func (s *typesTestSuite) TestAutocompleteHeadersDefault_StopMagicFalse() {
-	test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml))
+	test, err := GetTestFromYaml([]byte(autocompleteHeadersDefaultYaml), "")
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[2].Stages[0].SD.Input
+	input := test.Tests[2].Stages[0].Input
 	s.True(*input.AutocompleteHeaders)
+	//nolint:staticcheck
 	s.False(*input.StopMagic)
 }
 
 func (s *typesTestSuite) TestAutocompleteHeadersFalse_StopMagicDefault() {
-	test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml))
+	test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml), "")
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[0].Stages[0].SD.Input
+	input := test.Tests[0].Stages[0].Input
 	s.False(*input.AutocompleteHeaders)
+	//nolint:staticcheck
 	s.True(*input.StopMagic)
 }
 
 func (s *typesTestSuite) TestAutocompleteHeadersFalse_StopMagicTrue() {
-	test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml))
+	test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml), "")
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[1].Stages[0].SD.Input
+	input := test.Tests[1].Stages[0].Input
 	s.False(*input.AutocompleteHeaders)
+	//nolint:staticcheck
 	s.True(*input.StopMagic)
 }
 
 func (s *typesTestSuite) TestAutocompleteHeadersFalse_StopMagicFalse() {
-	test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml))
+	test, err := GetTestFromYaml([]byte(autocompleteHeadersFalseYaml), "")
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[2].Stages[0].SD.Input
+	input := test.Tests[2].Stages[0].Input
 	s.False(*input.AutocompleteHeaders)
+	//nolint:staticcheck
 	s.True(*input.StopMagic)
 }
 
 func (s *typesTestSuite) TestAutocompleteHeadersTrue_StopMagicDefault() {
-	test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml))
+	test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml), "")
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[0].Stages[0].SD.Input
+	input := test.Tests[0].Stages[0].Input
 	s.True(*input.AutocompleteHeaders)
+	//nolint:staticcheck
 	s.False(*input.StopMagic)
 }
 
 func (s *typesTestSuite) TestAutocompleteHeadersTrue_StopMagicTrue() {
-	test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml))
+	test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml), "")
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[1].Stages[0].SD.Input
+	input := test.Tests[1].Stages[0].Input
 	s.True(*input.AutocompleteHeaders)
+	//nolint:staticcheck
 	s.False(*input.StopMagic)
 }
 
 func (s *typesTestSuite) TestAutocompleteHeadersTrue_StopMagicFalse() {
-	test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml))
+	test, err := GetTestFromYaml([]byte(autocompleteHeadersTrueYaml), "")
 	s.NoError(err, "Parsing YAML shouldn't fail")
 
-	input := test.Tests[2].Stages[0].SD.Input
+	input := test.Tests[2].Stages[0].Input
 	s.True(*input.AutocompleteHeaders)
+	//nolint:staticcheck
 	s.False(*input.StopMagic)
 }
diff --git a/test/yaml.go b/test/yaml.go
index 8bf09d12..b8fd890f 100644
--- a/test/yaml.go
+++ b/test/yaml.go
@@ -10,14 +10,14 @@ import (
 )
 
 // GetTestFromYaml will get the tests to be processed from a YAML string.
-func GetTestFromYaml(testYaml []byte) (ftwTest *FTWTest, err error) {
+func GetTestFromYaml(testYaml []byte, fileName string) (ftwTest *FTWTest, err error) {
 	ftwTest = &FTWTest{}
 	err = yaml.Unmarshal(testYaml, ftwTest)
 	if err != nil {
 		return &FTWTest{}, err
 	}
 
-	postLoadTestFTWTest(ftwTest)
+	postLoadTestFTWTest(ftwTest, fileName)
 
 	return ftwTest, nil
 }
@@ -33,7 +33,7 @@ func DescribeYamlError(yamlError error) string {
 			"A simple example would be like this:\n\n" +
 			"status: 403\n" +
 			"needs to be changed to:\n\n" +
-			"status: [403]\n\n"
+			"status: 403\n\n"
 	}
 	matched, err = regexp.MatchString(`.*cannot unmarshal \[]interface {} into Go struct field FTWTest.Tests of type string.*`, yamlError.Error())
 	if err != nil {
diff --git a/waflog/read.go b/waflog/read.go
index fe2f3496..d8095af4 100644
--- a/waflog/read.go
+++ b/waflog/read.go
@@ -79,7 +79,7 @@ func (ll *FTWLogLines) getMarkedLines() [][]byte {
 // CheckLogForMarker reads the log file and searches for a marker line.
 // stageID is the ID of the current stage, which is part of the marker line
 // readLimit is the maximum numbers of lines to check
-func (ll *FTWLogLines) CheckLogForMarker(stageID string, readLimit int) []byte {
+func (ll *FTWLogLines) CheckLogForMarker(stageID string, readLimit uint) []byte {
 	offset, err := ll.logFile.Seek(0, io.SeekEnd)
 	if err != nil {
 		log.Error().Caller().Err(err).Msgf("failed to seek end of log file")
@@ -95,7 +95,7 @@ func (ll *FTWLogLines) CheckLogForMarker(stageID string, readLimit int) []byte {
 	crsHeaderBytes := bytes.ToLower([]byte(ll.LogMarkerHeaderName))
 
 	var line []byte
-	lineCounter := 0
+	lineCounter := uint(0)
 	// Look for the header until EOF or `readLimit` lines at most
 	for {
 		if lineCounter > readLimit {