fix: exitcode for all setup failures (#150)

src/halmos/__main__.py

@@ -1111,7 +1111,7 @@ def _main(_args=None) -> MainResult:
                     )
 
                     num_passed = sum(r.exitcode == 0 for r in test_results)
-                    num_failed = len(test_results) - num_passed
+                    num_failed = len(funsigs) - num_passed
 
                     print(
                         f"Symbolic test result: {num_passed} passed; {num_failed} failed; time: {timer() - contract_start:0.2f}s"

tests/expected/all.json

@@ -842,6 +842,7 @@
                 "num_bounded_loops": null
             }
         ],
+        "test/Setup.t.sol:SetupFailTest": [],
         "test/Setup.t.sol:SetupTest": [
             {
                 "name": "check_True()",
@@ -1097,4 +1098,4 @@
             }
         ]
     }
-}
+}

tests/test/Setup.t.sol

@@ -21,3 +21,17 @@ contract SetupTest is Test {
         assertEq(users[2], address(0));
     }
 }
+
+contract SetupFailTest {
+    function setUp() public {
+        revert();
+    }
+
+    function check_setUp_Fail1() public {
+        assert(true);
+    }
+
+    function check_setUp_Fail2() public {
+        assert(true);
+    }
+}

tests/test_halmos.py

@@ -44,6 +44,22 @@ def test_main(cmd, expected_path, halmos_options):
     assert_eq(expected["test_results"], actual["test_results"])
 
 
+@pytest.mark.parametrize(
+    "cmd",
+    [
+        ["--root", "tests", "--contract", "SetupFailTest"],
+        ["--root", "tests", "--contract", "LibTest"],
+    ],
+    ids=(
+        "SetupFailTest",
+        "LibTest",
+    ),
+)
+def test_main_fail(cmd, halmos_options):
+    actual = asdict(_main(cmd + halmos_options.split()))
+    assert actual["exitcode"] != 0
+
+
 def assert_eq(m1: Dict, m2: Dict) -> int:
     assert list(m1.keys()) == list(m2.keys())
     for c in m1: