Release with Xcode 11.5 support and without app crash exit code fixes (…

…#448)

* Xcode 11.5 support

* Revert "Added a few more tests to mock test failure scenarios and fixed a few final Exit Status issues  (#430)"

This reverts commit 1cad980.

.github/workflows/PR.yml

@@ -10,8 +10,8 @@ jobs:
     steps:
       # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git
       - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598
-      - name: Select Xcode 11.3
-        run: sudo xcode-select -s /Applications/Xcode_11.3.app
+      - name: Select Xcode 11.5
+        run: sudo xcode-select -s /Applications/Xcode_11.5.app
       - name: Run Bluepill tests
         run: ./scripts/bluepill.sh instance_tests
 
@@ -22,8 +22,8 @@ jobs:
     steps:
       # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git
       - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598
-      - name: Select Xcode 11.3
-        run: sudo xcode-select -s /Applications/Xcode_11.3.app
+      - name: Select Xcode 11.5
+        run: sudo xcode-select -s /Applications/Xcode_11.5.app
       - name: Run BP tests
         run: ./scripts/bluepill.sh runner_tests
       - name: Build and Package

.github/workflows/master.yml

@@ -15,8 +15,8 @@ jobs:
     steps:
     # actions/checkout@v2 but we use the sha because tags can be rewritten in git
     - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598
-    - name: Select Xcode 11.3
-      run: sudo xcode-select -s /Applications/Xcode_11.3.app
+    - name: Select Xcode 11.5
+      run: sudo xcode-select -s /Applications/Xcode_11.5.app
     - name: Run Bluepill tests
       run: ./scripts/bluepill.sh instance_tests
 
@@ -27,8 +27,8 @@ jobs:
     steps:
     # actions/checkout@v2 but we use the sha because tags can be rewritten in git
     - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598
-    - name: Select Xcode 11.3
-      run: sudo xcode-select -s /Applications/Xcode_11.3.app
+    - name: Select Xcode 11.5
+      run: sudo xcode-select -s /Applications/Xcode_11.5.app
     - name: Run BP tests
       run: ./scripts/bluepill.sh runner_tests
     - name: Build and Package

.github/workflows/release.yml

@@ -13,8 +13,8 @@ jobs:
     steps:
     # actions/checkout@v2 but we use the sha because tags can be rewritten in git
     - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598
-    - name: Select Xcode 11.3
-      run: sudo xcode-select -s /Applications/Xcode_11.3.app
+    - name: Select Xcode 11.5
+      run: sudo xcode-select -s /Applications/Xcode_11.5.app
     - name: Run Bluepill tests
       run: ./scripts/bluepill.sh instance_tests
 
@@ -28,8 +28,8 @@ jobs:
     - name: Report event trigger data
       run: |
         echo "Event ${{ github.event_name }}, ref: ${{ github.ref }}"
-    - name: Select Xcode 11.3
-      run: sudo xcode-select -s /Applications/Xcode_11.3.app
+    - name: Select Xcode 11.5
+      run: sudo xcode-select -s /Applications/Xcode_11.5.app
     - name: Run Bluepill tests
       run: ./scripts/bluepill.sh runner_tests
     - name: Build Bluepill

BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m

@@ -15,100 +15,19 @@ @interface BPSampleAppHangingTests : XCTestCase
 
 @implementation BPSampleAppHangingTests
 
--(long)attemptFromSimulatorVersionInfo:(NSString *)simulatorVersionInfo {
-    // simulatorVersionInfo is something like
-    // CoreSimulator 587.35 - Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861) - Runtime: iOS 12.2 (16E226) - DeviceType: iPhone 7
-    NSLog(@"Dissecting version info %@ to extra attempt number.", simulatorVersionInfo);
-    NSArray<NSString *> *parts = [simulatorVersionInfo componentsSeparatedByString:@" - "];
-    NSString *deviceString = parts[1];
-    // Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861)
-    parts = [deviceString componentsSeparatedByString:@" "];
-    NSString *device = parts[1];
-    // BP93497-2-2
-    parts = [device componentsSeparatedByString:@"-"];
-    NSString *attempt = parts[1];
-    return [attempt longLongValue];
-}
-
--(void)extractPlanAndExecuteActions:(int)index {
-    NSDictionary *env = [[NSProcessInfo processInfo] environment];
-    NSString *simulatorVersionInfo = [env objectForKey:@"SIMULATOR_VERSION_INFO"];
-    long attempt = [self attemptFromSimulatorVersionInfo:simulatorVersionInfo];
-    NSString *executionPlan = [env objectForKey:@"_BP_TEST_EXECUTION_PLAN"];
-    if (!executionPlan) {
-        NSLog(@"No execution plan found in attempt#%ld. Failing the test.", attempt);
-        XCTAssert(NO);
-        return;
-    }
-    NSLog(@"Received execution plan %@ on attempt#%ld for this test.", executionPlan, attempt);
-    NSArray *setsOfPlans = [executionPlan componentsSeparatedByString:@";"];
-    if (index >= [setsOfPlans count]) {
-        NSLog(@"Not enough plans for test#%d in execution plan: '%@'.", index, executionPlan);
-        XCTAssert(YES);
-        return;
-    }
-    NSString *currentPlan = setsOfPlans[index];
-    currentPlan = [currentPlan stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
-    NSArray *array = [currentPlan componentsSeparatedByCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
-    if (attempt > [array count]) {
-        NSLog(@"Passing on attempt#%ld, by default, as there is no action defined in the execution plan", (long)attempt);
-        XCTAssert(YES);
-        return;
-    }
-    NSString *action = array[attempt - 1];
-    if ([action isEqualToString:@"TIMEOUT"]) {
-        NSLog(@"Entering into an infinite loop on attempt#%ld to timeout as per the execution plan", (long)attempt);
-        while(1) {
-        }
-        return;
-    } else if ([action isEqualToString:@"PASS"]) {
-        NSLog(@"Passing on attempt#%ld based on execution plan", (long)attempt);
-        XCTAssert(YES);
-        return;
-    } else if ([action isEqualToString:@"FAIL"]) {
-        NSLog(@"Failing on attempt#%ld based on execution plan", (long)attempt);
-        XCTAssert(NO);
-        return;
-    } else if ([action isEqualToString:@"CRASH"]) {
-        NSLog(@"Crashing on attempt#%ld based on execution plan", (long)attempt);
-        // ok, let's crash and burn
-        int *pointer = nil;
-        *pointer = 1;
-        return;
-    }
-    NSLog(@"Failing on attempt#%ld as an unidentified action is encountered in the execution plan", (long)attempt);
-    XCTAssert(NO);
-    return;
-}
-
 - (void)setUp {
     [super setUp];
     // Put setup code here. This method is called before the invocation of each test method in the class.
+
 }
 
 - (void)tearDown {
     // Put teardown code here. This method is called after the invocation of each test method in the class.
     [super tearDown];
 }
 
-- (void)testASimpleTest {
-    XCTAssert(YES);
-}
-
-- (void)testBasedOnExecutionPlan {
-    [self extractPlanAndExecuteActions:0];
-}
-
-- (void)testCaseFinal {
-    XCTAssert(YES);
-}
-
-- (void)testDoubleBasedOnExecutionPlan {
-    [self extractPlanAndExecuteActions:1];
-}
-
-- (void)testEndFinal {
-    XCTAssert(YES);
+- (void)testAppHanging {
+    while(TRUE){};
 }
 
 @end

README.md

@@ -91,34 +91,13 @@ A full list supported options are listed here.
 | unsafe-skip-xcode-version-check |               | Skip Xcode version check                                                            |     N    | NO               |
 
 
-## Exit Status
-
-Here is a list of Bluepill exit codes. If a Bluepill execution has multiple exit codes from same or different test bundles, the final exit code is a combination of all exit codes. Note that app crashes are fatal even if the test passes on retry.
-
-```shell
-  BPExitStatusAllTestsPassed          = 0,
-  BPExitStatusTestsFailed             = 1 << 0,
-  BPExitStatusSimulatorCreationFailed = 1 << 1,
-  BPExitStatusInstallAppFailed        = 1 << 2,
-  BPExitStatusInterrupted             = 1 << 3,
-  BPExitStatusSimulatorCrashed        = 1 << 4,
-  BPExitStatusLaunchAppFailed         = 1 << 5,
-  BPExitStatusTestTimeout             = 1 << 6,
-  BPExitStatusAppCrashed              = 1 << 7,
-  BPExitStatusSimulatorDeleted        = 1 << 8,
-  BPExitStatusUninstallAppFailed      = 1 << 9,
-  BPExitStatusSimulatorReuseFailed    = 1 << 10
-```
-**Note:** Please refer to `bp/src/BPExitStatus.h` for the latest/exact exit codes.
-
-
 ## Demo
 
 ![BluepillDemo](doc/img/demo.gif)
 
 ## Requirements
 
-Bluepill only works with **Xcode 11.3**. If you're looking for old Xcode support, please check out the other branches:
+Bluepill officially supports **Xcode 11.5**. If you're looking for old Xcode support, please check out the below branches:
 
 * [Xcode-8](https://github.com/linkedin/bluepill/tree/xcode8)
 * [Xcode-9.0](https://github.com/linkedin/bluepill/tree/xcode-9.0)
@@ -133,8 +112,10 @@ Bluepill only works with **Xcode 11.3**. If you're looking for old Xcode support
 * [Xcode-11.0](https://github.com/linkedin/bluepill/tree/xcode-11.0)
 * [Xcode-11.1](https://github.com/linkedin/bluepill/tree/xcode-11.1)
 * [Xcode-11.2](https://github.com/linkedin/bluepill/tree/xcode-11.2)
+* [Xcode-11.3](https://github.com/linkedin/bluepill/tree/xcode-11.3)
+* [Xcode-11.4](https://github.com/linkedin/bluepill/tree/xcode-11.4)
 
-**Note:** Refer to `unsafe-skip-xcode-version-check` flag introduced in `Bluepill v5.2.1` relaxing the Xcode version checks. Please make sure it is well tested and the underlying risks are understand.
+If you're looking for newer Xcode version support, try using Bluepill with `unsafe-skip-xcode-version-check` flag but make sure your app is tested with it and the underlying risks are understand.
 
 ## Acknowledgement
 
@@ -165,7 +146,7 @@ If you're using [Bitrise.io](https://bitrise.io) as your CI/CD, you can start us
 
   Latest [release](https://github.com/linkedin/bluepill/releases/).
 
-- How to test Bluepill in Xcode?
+- How to test Bluepill in Xcode
 
   Select BPSampleApp scheme and build it first. Then you can switch back to `bluepill` or `bluepill-cli` scheme to run their tests.
 

bluepill/tests/BPPackerTests.m

@@ -187,18 +187,17 @@ - (void)testPacking {
     // Make sure we don't split when we don't want to
     self.config.numSims = @4;
     self.config.noSplit = @[@"BPSampleAppTests"];
-    bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];  // withNoSplitList:@[@"BPSampleAppTests"] intoBundles:4 andError:nil];
+    bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];// withNoSplitList:@[@"BPSampleAppTests"] intoBundles:4 andError:nil];
     // When we prevent BPSampleTests from splitting, BPSampleAppFatalErrorTests and BPAppNegativeTests gets split in two
     want = [[want arrayByAddingObject:@"BPSampleAppFatalErrorTests"] sortedArrayUsingSelector:@selector(compare:)];
     XCTAssertEqual(bundles.count, app.testBundles.count + 2);
 
     XCTAssertEqual([bundles[0].skipTestIdentifiers count], 0);
     XCTAssertEqual([bundles[1].skipTestIdentifiers count], 0);
     XCTAssertEqual([bundles[2].skipTestIdentifiers count], 0);
-    XCTAssertEqual([bundles[3].skipTestIdentifiers count], 1);
-    XCTAssertEqual([bundles[4].skipTestIdentifiers count], 4);
+    XCTAssertEqual([bundles[3].skipTestIdentifiers count], 2);
+    XCTAssertEqual([bundles[4].skipTestIdentifiers count], 3);
     XCTAssertEqual([bundles[5].skipTestIdentifiers count], 1);
-    XCTAssertEqual([bundles[6].skipTestIdentifiers count], 4);
 
     self.config.numSims = @4;
     self.config.noSplit = nil;
@@ -210,17 +209,14 @@ - (void)testPacking {
     long testsPerBundle = [allTests count] / numSims;
     long skipTestsPerBundle = 0;
     long skipTestsInFinalBundle = 0;
-    long testCount = 0;
     for (int i = 0; i < bundles.count; ++i) {
         skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
+        skipTestsInFinalBundle = testsPerBundle * (numSims - 1);
         if (i < 4) {
             XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0);
-            testCount += [[bundles[i] allTestCases] count];
         } else if (i < bundles.count-1) {
             XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle);
-            testCount += testsPerBundle;
         } else {  /* last bundle */
-            skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([allTests count] - testCount);
             XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle);
         }
     }
@@ -235,19 +231,15 @@ - (void)testPacking {
 
     numSims = [self.config.numSims integerValue];
     testsPerBundle = [allTests count] / numSims;
-    testCount = 0;
     for (int i = 0; i < bundles.count; ++i) {
         skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
+        skipTestsInFinalBundle = testsPerBundle * (numSims - 1);
         if (i < 4) {
             XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0);
-            testCount += [[bundles[i] allTestCases] count];
         } else if (i < bundles.count-1) {
             XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle);
-            testCount += testsPerBundle;
         } else {  /* last bundle */
-            skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([allTests count] - testCount);
             XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle);
-
         }
     }
 

bp/bp.xcodeproj/xcshareddata/xcschemes/bp-tests.xcscheme

@@ -61,6 +61,12 @@
                ReferencedContainer = "container:bp.xcodeproj">
             </BuildableReference>
             <SkippedTests>
+               <Test
+                  Identifier = "BluepillTests/testAppThatCrashesOnLaunch">
+               </Test>
+               <Test
+                  Identifier = "BluepillTests/testAppThatHangsOnLaunch">
+               </Test>
                <Test
                   Identifier = "BluepillTests/testReportWithFailingTestsSet">
                </Test>
@@ -70,6 +76,12 @@
                <Test
                   Identifier = "BluepillTests/testRetryWithFailingTestsSet">
                </Test>
+               <Test
+                  Identifier = "BluepillTests/testRunningAndIgnoringCertainTestCases">
+               </Test>
+               <Test
+                  Identifier = "BluepillTests/testRunningOnlyCertainTestcases">
+               </Test>
                <Test
                   Identifier = "BluepillTests/testVoyager">
                </Test>

bp/src/BPConfiguration.h

@@ -119,7 +119,6 @@ typedef NS_ENUM(NSInteger, BPProgram) {
 @property (nonatomic) BOOL testing_HangAppOnLaunch;
 @property (nonatomic) BOOL testing_NoAppWillRun;
 @property (nonatomic) NSNumber *testing_crashOnAttempt;
-@property (nonatomic, strong) NSString *testing_ExecutionPlan;
 
 // Generated fields
 @property (nonatomic, strong) NSString *xcodePath;

bp/src/BPConfiguration.m

@@ -18,10 +18,10 @@
 
 
 typedef NS_OPTIONS(NSUInteger, BPOptionType) {
-    BP_VALUE   = 1 << 0, // Single value
-    BP_LIST    = 1 << 1, // List value
-    BP_PATH    = 1 << 2, // Single value, CWD will be prepended
-    BP_BOOL    = 1 << 3, // Boolean value
+    BP_VALUE = 1, // Single value
+    BP_LIST = 1 << 1, // List value
+    BP_PATH = 1 << 2, // Single value, CWD will be prepended
+    BP_BOOL = 1 << 3, // Boolean value
     BP_INTEGER = 1 << 4, // Integer value
 };
 
@@ -103,9 +103,9 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
     {'q', "quiet", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "quiet",
         "Turn off all output except fatal errors."},
     {'F', "only-retry-failed", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "onlyRetryFailed",
-        "Only retry failed tests instead of all. Also retry test that timed-out/crashed. Note that app crashes are fatal even if the test passes on retry."},
+        "If `failure-`tolerance` is > 0, only retry tests that failed."},
     {'l', "list-tests", BP_MASTER, NO, NO, no_argument, NULL, BP_VALUE | BP_BOOL, "listTestsOnly",
-        "Only list tests and exit without executing tests."},
+        "Only list tests in bundle"},
     {'v', "verbose", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "verboseLogging",
         "Enable verbose logging"},
     {'k', "keep-individual-test-reports", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepIndividualTestReports",
@@ -377,7 +377,6 @@ - (id)mutableCopyWithZone:(NSZone *)zone {
     newConfig.testing_HangAppOnLaunch = self.testing_HangAppOnLaunch;
     newConfig.testing_NoAppWillRun = self.testing_NoAppWillRun;
     newConfig.testing_crashOnAttempt = self.testing_crashOnAttempt;
-    newConfig.testing_ExecutionPlan = self.testing_ExecutionPlan;
     newConfig.xcTestRunPath = self.xcTestRunPath;
     newConfig.testPlanPath = self.testPlanPath;
     newConfig.xcTestRunDict = self.xcTestRunDict;

bp/src/BPConstants.h

@@ -10,9 +10,9 @@
 #import <Foundation/Foundation.h>
 
 #pragma mark - Version Constants
-#define BP_DEFAULT_XCODE_VERSION "11.3"
-#define BP_DEFAULT_RUNTIME "iOS 13.3"
-#define BP_DEFAULT_BASE_SDK "13.2"
+#define BP_DEFAULT_XCODE_VERSION "11.5"
+#define BP_DEFAULT_RUNTIME "iOS 13.5"
+#define BP_DEFAULT_BASE_SDK "13.5"
 
 #define BP_DEFAULT_DEVICE_TYPE "iPhone 8"