Do not retry crashed tests (#443)

Based on previous changes crashed tests are fatal and would always return a non-zero exit code. With this change, the crashed tests are NOT retried to make sure the app crash errors are surfaced without confusion. Also added new tests and fixed existing ones as per the new no-retry behavior.
MobileNativeFoundation · Jun 15, 2020 · 08d3951 · 08d3951
1 parent 91ecdd6
commit 08d3951
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 13 deletions.
diff --git a/bluepill/tests/BPIntegrationTests.m b/bluepill/tests/BPIntegrationTests.m
@@ -90,8 +90,9 @@ - (void)testClonedSimulators {
     config.failureTolerance = @0;
     config.cloneSimulator = TRUE;
     // need to validate the configuration to fill in simDevice and simRuntime
-    [config validateConfigWithError:nil];
-    NSError *err;
+    NSError *err = nil;
+    [config validateConfigWithError:&err];
+    XCTAssert(err == nil);
     BPApp *app = [BPApp appWithConfig:config
                             withError:&err];
 

diff --git a/bp/src/SimulatorMonitor.m b/bp/src/SimulatorMonitor.m
@@ -195,7 +195,8 @@ - (void)onOutputReceived:(NSString *)output {
             NSString *testClass = (__self.currentClassName ?: __self.previousClassName);
             NSString *testName = (__self.currentTestName ?: __self.previousTestName);
             if (__self.testsState == Running) {
-                [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app.", testClass, testName];
+                [self updateExecutedTestCaseList:testName inClass:testClass];
+                [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Not retrying it.", testClass, testName];
                 [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"];
             } else {
                 assert(__self.testsState == Idle);

diff --git a/bp/tests/BluepillTests.m b/bp/tests/BluepillTests.m
@@ -346,11 +346,12 @@ - (void)testReportFailureOnTimeoutCrashAndPass {
 }
 
 /**
- Execution plan: CRASH, TIMEOUT, PASS
+ Execution plan: CRASH
  */
-- (void)testReportFailureOnCrashTimeoutAndPass {
+- (void)testNoRetryOnCrash {
     self.config.stuckTimeout = @6;
-    self.config.testing_ExecutionPlan = @"CRASH TIMEOUT PASS";
+    self.config.testing_ExecutionPlan = @"CRASH";  // No retry
+    self.config.errorRetriesCount = @4;
     self.config.onlyRetryFailed = TRUE;
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
@@ -364,6 +365,49 @@ - (void)testReportFailureOnCrashTimeoutAndPass {
     XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
 }
 
+
+/**
+ Execution plan: One test CRASHes and another one TIMEs OUT and PASSes on retry
+ */
+- (void)testReportFailureOnCrashAndTimeoutTests {
+    self.config.stuckTimeout = @6;
+    self.config.testing_ExecutionPlan = @"CRASH; SKIP TIMEOUT PASS";
+    self.config.onlyRetryFailed = TRUE;
+    self.config.failureTolerance = @1;
+    self.config.errorRetriesCount = @2;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
+}
+
+/**
+ Execution plan: One test CRASHes and another one keeps timing out
+ */
+- (void)testReportBothCrashAndTimeout {
+    self.config.stuckTimeout = @6;
+    self.config.testing_ExecutionPlan = @"CRASH; SKIP TIMEOUT TIMEOUT";
+    self.config.onlyRetryFailed = TRUE;
+    self.config.failureTolerance = @1;
+    self.config.errorRetriesCount = @2;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == (BPExitStatusAppCrashed | BPExitStatusTestTimeout));
+}
+
 /**
  Execution plan: FAIL, TIMEOUT, PASS
  */

diff --git a/bp/tests/Resource Files/crash_tests_with_retry_attempt_2.xml b/bp/tests/Resource Files/crash_tests_with_retry_attempt_2.xml
@@ -1,11 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites name="Selected tests" tests="1" failures="1" errors="0">
-    <testsuite tests="1" failures="1" errors="0" name="BPSampleAppCrashingTests.xctest">
-        <testsuite tests="1" failures="1" errors="0" name="BPSampleAppCrashingTests">
-            <testcase classname="BPSampleAppCrashingTests" name="testAppCrash1">
-                <failure type="Failure" message="App Crashed"></failure>
-                <system-out></system-out>
-            </testcase>
+<testsuites name="Selected tests" tests="1" failures="0" errors="0">
+    <testsuite tests="1" failures="0" errors="0" name="BPSampleAppCrashingTests.xctest">
+        <testsuite tests="1" failures="0" errors="0" name="BPSampleAppCrashingTests">
+            <testcase classname="BPSampleAppCrashingTests" name="testAppCrash2"></testcase>
         </testsuite>
     </testsuite>
 </testsuites>