QTest: add -[no]throwon{fail,skip} command line arguments

... to complement QTEST_THROW_ON_FAIl/SKIP environment variables.

This allows to conveniently test both modes by running each test
twice.

Task-number: QTBUG-66320
Change-Id: I8b2810e8345061c98472d846017de910a11e0657
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>

src/testlib/qtestcase.cpp

@@ -972,6 +972,10 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
          "                       repeated forever. This is intended as a developer tool, and\n"
          "                       is only supported with the plain text logger.\n"
          " -skipblacklisted    : Skip blacklisted tests. Useful for measuring test coverage.\n"
+         " -[no]throwonfail    : Enables/disables throwing on QCOMPARE()/QVERIFY()/etc.\n"
+         "                       Default: off,  unless QTEST_THROW_ON_FAIL is set."
+         " -[no]throwonskip    : Enables/disables throwing on QSKIP().\n"
+         "                       Default: off,  unless QTEST_THROW_ON_SKIP is set."
          "\n"
          " Benchmarking options:\n"
 #if QT_CONFIG(valgrind)
@@ -1133,6 +1137,14 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, const char *const argv[], bool
             QTest::noCrashHandler = true;
         } else if (strcmp(argv[i], "-skipblacklisted") == 0) {
             QTest::skipBlacklisted = true;
+        } else if (strcmp(argv[i], "-throwonfail") == 0) {
+            QTest::setThrowOnFail(true);
+        } else if (strcmp(argv[i], "-nothrowonfail") == 0) {
+            QTest::setThrowOnFail(false);
+        } else if (strcmp(argv[i], "-throwonskip") == 0) {
+            QTest::setThrowOnSkip(true);
+        } else if (strcmp(argv[i], "-nothrowonskip") == 0) {
+            QTest::setThrowOnSkip(false);
 #if QT_CONFIG(valgrind)
         } else if (strcmp(argv[i], "-callgrind") == 0) {
             if (!QBenchmarkValgrindUtils::haveValgrind()) {

tests/auto/testlib/selftests/tst_selftests.cpp

@@ -1014,10 +1014,12 @@ TestProcessResult runTestProcess(const QString &test, const QStringList &argumen
     return { process.exitCode(), standardOutput, standardError };
 }
 
+enum class Throw { OnFail = 1 };
+
 /*
     Runs a single test and verifies the output against the expected results.
 */
-void runTest(const QString &test, const TestLoggers &requestedLoggers)
+void runTest(const QString &test, const TestLoggers &requestedLoggers, Throw throwing = {})
 {
     TestLoggers loggers;
     for (auto logger : requestedLoggers) {
@@ -1031,6 +1033,10 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers)
     QStringList arguments;
     for (auto logger : loggers)
         arguments += logger.arguments(test);
+    if (throwing == Throw::OnFail) // don't distinguish between throwonfail/throwonskip
+        arguments += {"-throwonfail", "-throwonskip"};
+    else
+        arguments += {"-nothrowonfail", "-nothrowonskip"};
 
     CAPTURE(test);
     CAPTURE(arguments);
@@ -1057,9 +1063,9 @@ void runTest(const QString &test, const TestLoggers &requestedLoggers)
 /*
     Runs a single test and verifies the output against the expected result.
 */
-void runTest(const QString &test, const TestLogger &logger)
+void runTest(const QString &test, const TestLogger &logger, Throw t = {})
 {
-    runTest(test, TestLoggers{logger});
+    runTest(test, TestLoggers{logger}, t);
 }
 
 // ----------------------- Catch helpers -----------------------
@@ -1202,7 +1208,12 @@ SCENARIO("Test output of the loggers is as expected")
     GIVEN("The " << logger << " logger") {
         for (QString test : tests) {
             AND_GIVEN("The " << test << " subtest") {
-                runTest(test, TestLogger(logger, StdoutOutput));
+                WHEN("Throwing on failure or skip") {
+                    runTest(test, TestLogger(logger, StdoutOutput), Throw::OnFail);
+                }
+                WHEN("Returning on failure or skip") {
+                    runTest(test, TestLogger(logger, StdoutOutput));
+                }
             }
         }
     }