diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 1aae2bed..b616d1ce 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -6,12 +6,12 @@ jobs: # First machine, runs BP tests batch 1 integration_tests1: name: Instance Test 1 - runs-on: macos-12 + runs-on: macos-13 steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 14.0 - run: sudo xcode-select -s /Applications/Xcode_14.0.app + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app - name: Run Bluepill tests run: ./scripts/bluepill.sh instance_tests1 - name: Capture xcresult files @@ -25,12 +25,12 @@ jobs: # Second machine, runs BP tests batch 2 integration_tests2: name: Instance Test 2 - runs-on: macos-12 + runs-on: macos-13 steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 14.0 - run: sudo xcode-select -s /Applications/Xcode_14.0.app + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app - name: Run Bluepill tests run: ./scripts/bluepill.sh instance_tests2 - name: Capture xcresult files @@ -41,15 +41,34 @@ jobs: path: build/**/*.xcresult retention-days: 14 - # Third machine, runs Bluepill tests and makes release build + # Third machine, runs BP tests batch 3 + integration_tests3: + name: Instance Test 3 + runs-on: macos-13 + steps: + # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git + - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app + - name: Run Bluepill tests + run: ./scripts/bluepill.sh instance_tests3 + - name: Capture xcresult files + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: xcresults-bp-tests2 + path: build/**/*.xcresult + retention-days: 14 + + # Fourth machine, runs Bluepill tests and makes release build build: name: Bluepill Test and build - runs-on: macos-12 + runs-on: macos-13 steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 14.0 - run: sudo xcode-select -s /Applications/Xcode_14.0.app + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app - name: Run Bluepill tests run: ./scripts/bluepill.sh runner_tests - name: Capture xcresult files diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b28b613f..424adba2 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -11,12 +11,12 @@ jobs: # First machine, runs BP tests batch 1 integration_tests1: name: Instance Test 1 - runs-on: macos-12 + runs-on: macos-13 steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 14.0 - run: sudo xcode-select -s /Applications/Xcode_14.0.app + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app - name: Run Bluepill tests run: ./scripts/bluepill.sh instance_tests1 - name: Capture xcresult files @@ -30,12 +30,12 @@ jobs: # Second machine, runs BP tests batch 2 integration_tests2: name: Instance Test 2 - runs-on: macos-12 + runs-on: macos-13 steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 14.0 - run: sudo xcode-select -s /Applications/Xcode_14.0.app + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app - name: Run Bluepill tests run: ./scripts/bluepill.sh instance_tests2 - name: Capture xcresult files @@ -46,15 +46,34 @@ jobs: path: build/**/*.xcresult retention-days: 14 - # Third machine, runs Bluepill tests and makes release build + # Third machine, runs BP tests batch 3 + integration_tests3: + name: Instance Test 3 + runs-on: macos-13 + steps: + # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git + - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app + - name: Run Bluepill tests + run: ./scripts/bluepill.sh instance_tests3 + - name: Capture xcresult files + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: xcresults-bp-tests2 + path: build/**/*.xcresult + retention-days: 14 + + # Fourth machine, runs Bluepill tests and makes release build build: name: Bluepill Test and build - runs-on: macos-12 + runs-on: macos-13 steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 14.0 - run: sudo xcode-select -s /Applications/Xcode_14.0.app + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app - name: Run Bluepill tests run: ./scripts/bluepill.sh runner_tests - name: Capture xcresult files diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ab634754..73aedf04 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,12 +9,12 @@ jobs: # First machine, runs BP tests batch 1 integration_tests1: name: Instance Test 1 - runs-on: macos-12 + runs-on: macos-13 steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 14.0 - run: sudo xcode-select -s /Applications/Xcode_14.0.app + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app - name: Run Bluepill tests run: ./scripts/bluepill.sh instance_tests1 - name: Capture xcresult files @@ -28,12 +28,12 @@ jobs: # Second machine, runs BP tests batch 2 integration_tests2: name: Instance Test 2 - runs-on: macos-12 + runs-on: macos-13 steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 14.0 - run: sudo xcode-select -s /Applications/Xcode_14.0.app + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app - name: Run Bluepill tests run: ./scripts/bluepill.sh instance_tests2 - name: Capture xcresult files @@ -44,18 +44,37 @@ jobs: path: build/**/*.xcresult retention-days: 14 - # Third machine, runs Bluepill tests and makes release build + # Third machine, runs BP tests batch 3 + integration_tests3: + name: Instance Test 3 + runs-on: macos-13 + steps: + # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git + - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app + - name: Run Bluepill tests + run: ./scripts/bluepill.sh instance_tests3 + - name: Capture xcresult files + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: xcresults-bp-tests2 + path: build/**/*.xcresult + retention-days: 14 + + # Fourth machine, runs Bluepill tests and makes release build build: name: BP Test and build - runs-on: macos-12 + runs-on: macos-13 steps: # actions/checkout@v2 but we use the sha because tags can be rewritten in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - name: Report event trigger data run: | echo "Event ${{ github.event_name }}, ref: ${{ github.ref }}" - - name: Select Xcode 14.0 - run: sudo xcode-select -s /Applications/Xcode_14.0.app + - name: Select Xcode 15.1 + run: sudo xcode-select -s /Applications/Xcode_15.1.app - name: Run Bluepill tests run: ./scripts/bluepill.sh runner_tests - name: Capture xcresult files diff --git a/bluepill/bluepill.xcodeproj/project.pbxproj b/bluepill/bluepill.xcodeproj/project.pbxproj index 03ef4457..714f6093 100644 --- a/bluepill/bluepill.xcodeproj/project.pbxproj +++ b/bluepill/bluepill.xcodeproj/project.pbxproj @@ -359,9 +359,11 @@ buildSettings = { "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; DEVELOPMENT_TEAM = 57Y47U492U; + DSTROOT = ""; FRAMEWORK_SEARCH_PATHS = /Library/Developer/PrivateFrameworks/; HEADER_SEARCH_PATHS = "\"$(SRCROOT)/..\""; INFOPLIST_FILE = tests/Info.plist; + INSTALL_ROOT = /; IPHONEOS_DEPLOYMENT_TARGET = 12.0; OTHER_LDFLAGS = ""; PRODUCT_BUNDLE_IDENTIFIER = LI.BluepillRunnerTests; @@ -376,9 +378,11 @@ buildSettings = { "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; DEVELOPMENT_TEAM = 57Y47U492U; + DSTROOT = ""; FRAMEWORK_SEARCH_PATHS = /Library/Developer/PrivateFrameworks/; HEADER_SEARCH_PATHS = "\"$(SRCROOT)/..\""; INFOPLIST_FILE = tests/Info.plist; + INSTALL_ROOT = /; IPHONEOS_DEPLOYMENT_TARGET = 12.0; OTHER_LDFLAGS = ""; PRODUCT_BUNDLE_IDENTIFIER = LI.BluepillRunnerTests; diff --git a/bp/bp.xcodeproj/project.pbxproj b/bp/bp.xcodeproj/project.pbxproj index ccbf023c..eecbb53c 100644 --- a/bp/bp.xcodeproj/project.pbxproj +++ b/bp/bp.xcodeproj/project.pbxproj @@ -132,6 +132,7 @@ 018D5C1025B4FF4200B0314B /* BPIntTestCase.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = BPIntTestCase.h; sourceTree = ""; }; 018D5C1125B4FF4200B0314B /* BPIntTestCase.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPIntTestCase.m; sourceTree = ""; }; 018D5C1C25B6696000B0314B /* BPReportTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPReportTests.m; sourceTree = ""; }; + 71D4D0F82AEA1B4F00859482 /* SimDeviceBootInfo.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SimDeviceBootInfo.h; sourceTree = ""; }; 7A202A3D1DAED04900D935E3 /* BPExitStatus.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = BPExitStatus.h; sourceTree = ""; }; 7A202A3F1DB0066100D935E3 /* BPWriter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPWriter.h; sourceTree = ""; }; 7A202A401DB0066100D935E3 /* BPWriter.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPWriter.m; sourceTree = ""; }; @@ -527,6 +528,7 @@ BA954F651D6D1AB3007D011D /* NSString-SIMPackedVersion.h */, BA954F671D6D1AB3007D011D /* NSUserDefaults-SimDefaults.h */, BA954F681D6D1AB3007D011D /* SimDevice.h */, + 71D4D0F82AEA1B4F00859482 /* SimDeviceBootInfo.h */, BA954F691D6D1AB3007D011D /* SimDeviceIO.h */, BA954F6A1D6D1AB3007D011D /* SimDeviceIOClient.h */, BA954F6B1D6D1AB3007D011D /* SimDeviceIOInterface-Protocol.h */, @@ -1017,6 +1019,7 @@ buildSettings = { DEPLOYMENT_LOCATION = NO; "DEVELOPER_PRIVATE_FRAMEWORKS_DIR[arch=*]" = /Library/Developer/PrivateFrameworks/; + DSTROOT = ""; FRAMEWORK_SEARCH_PATHS = ( "\"/Library/Developer/PrivateFrameworks\"", "\"$(PRIVATE_FRAMEWORKS_DIR)\"", @@ -1031,6 +1034,7 @@ "$(SRCROOT)/src", ); INFOPLIST_FILE = "$(SRCROOT)/src/Info.plist"; + INSTALL_ROOT = /; MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_CFLAGS = "-DBP_USE_PRIVATE_FRAMEWORKS"; OTHER_LDFLAGS = ( @@ -1056,6 +1060,7 @@ buildSettings = { DEPLOYMENT_LOCATION = NO; "DEVELOPER_PRIVATE_FRAMEWORKS_DIR[arch=*]" = /Library/Developer/PrivateFrameworks/; + DSTROOT = ""; FRAMEWORK_SEARCH_PATHS = ( "\"/Library/Developer/PrivateFrameworks\"", "\"$(PRIVATE_FRAMEWORKS_DIR)\"", @@ -1070,6 +1075,7 @@ "$(SRCROOT)/src", ); INFOPLIST_FILE = "$(SRCROOT)/src/Info.plist"; + INSTALL_ROOT = /; MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_CFLAGS = "-DBP_USE_PRIVATE_FRAMEWORKS"; OTHER_LDFLAGS = ( @@ -1106,6 +1112,7 @@ CURRENT_PROJECT_VERSION = 1; DEFINES_MODULE = YES; "DEVELOPER_PRIVATE_FRAMEWORKS_DIR[arch=*]" = /Library/Developer/PrivateFrameworks; + DSTROOT = ""; DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; FRAMEWORK_VERSION = A; @@ -1115,6 +1122,7 @@ "$(SRCROOT)/src", ); INFOPLIST_FILE = "$(SRCROOT)/src/Info.plist"; + INSTALL_ROOT = /; MACH_O_TYPE = staticlib; MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_LDFLAGS = ""; @@ -1143,6 +1151,7 @@ CURRENT_PROJECT_VERSION = 1; DEFINES_MODULE = YES; "DEVELOPER_PRIVATE_FRAMEWORKS_DIR[arch=*]" = /Library/Developer/PrivateFrameworks; + DSTROOT = ""; DYLIB_COMPATIBILITY_VERSION = 1; DYLIB_CURRENT_VERSION = 1; FRAMEWORK_VERSION = A; @@ -1152,6 +1161,7 @@ "$(SRCROOT)/src", ); INFOPLIST_FILE = "$(SRCROOT)/src/Info.plist"; + INSTALL_ROOT = /; MACH_O_TYPE = staticlib; MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_LDFLAGS = ""; @@ -1169,17 +1179,17 @@ buildSettings = { COMBINE_HIDPI_IMAGES = YES; DEVELOPMENT_TEAM = 57Y47U492U; + DSTROOT = ""; HEADER_SEARCH_PATHS = ( "$(SRCROOT)/..", "$(SRCROOT)/src", ); INFOPLIST_FILE = tests/Info.plist; + INSTALL_ROOT = /; LD_RUNPATH_SEARCH_PATHS = "\"$(DEVELOPER_DIR)/Library/PrivateFrameworks\""; MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_CFLAGS = "-DBP_USE_PRIVATE_FRAMEWORKS"; OTHER_LDFLAGS = ( - "-weak_framework", - DVTFoundation, "-weak_framework", CoreSimulator, "-weak_framework", @@ -1201,17 +1211,17 @@ buildSettings = { COMBINE_HIDPI_IMAGES = YES; DEVELOPMENT_TEAM = 57Y47U492U; + DSTROOT = ""; HEADER_SEARCH_PATHS = ( "$(SRCROOT)/..", "$(SRCROOT)/src", ); INFOPLIST_FILE = tests/Info.plist; + INSTALL_ROOT = /; LD_RUNPATH_SEARCH_PATHS = "\"$(DEVELOPER_DIR)/Library/PrivateFrameworks\""; MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_CFLAGS = "-DBP_USE_PRIVATE_FRAMEWORKS"; OTHER_LDFLAGS = ( - "-weak_framework", - DVTFoundation, "-weak_framework", CoreSimulator, "-weak_framework", diff --git a/bp/src/BPConfiguration.h b/bp/src/BPConfiguration.h index 3e845ab9..423a5cb4 100644 --- a/bp/src/BPConfiguration.h +++ b/bp/src/BPConfiguration.h @@ -107,6 +107,7 @@ typedef NS_ENUM(NSInteger, BPProgram) { @property (nonatomic, strong) NSNumber *launchTimeout; @property (nonatomic, strong) NSNumber *deleteTimeout; @property (nonatomic) BOOL keepIndividualTestReports; +@property (nonatomic, strong) NSNumber *testBundleDisconnectTimeout; @property (nonatomic, strong) NSArray *commandLineArguments; // command line arguments for the app @property (nonatomic, strong) NSDictionary *environmentVariables; diff --git a/bp/src/BPConfiguration.m b/bp/src/BPConfiguration.m index 92c02ae8..b8539541 100644 --- a/bp/src/BPConfiguration.m +++ b/bp/src/BPConfiguration.m @@ -150,6 +150,8 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { "Directory where videos of test runs will be saved. If not provided, videos are not recorded."}, {368, "keep-passing-videos", BLUEPILL_BINARY | BP_BINARY, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepPassingVideos", "Whether recorded videos should be kept if the test passed. They are deleted by default."}, + {369, "test-bundle-disconnect-timeout", BLUEPILL_BINARY | BP_BINARY, NO, NO, required_argument, "60", BP_VALUE | BP_INTEGER, "testBundleDisconnectTimeout", + "The maximum amount of time, in seconds, to wait while a test bundle is disconnected but might still generate output."}, {0, 0, 0, 0, 0, 0, 0} }; diff --git a/bp/src/BPConstants.h b/bp/src/BPConstants.h index c5963e2a..2c7e0116 100644 --- a/bp/src/BPConstants.h +++ b/bp/src/BPConstants.h @@ -10,11 +10,11 @@ #import #pragma mark - Version Constants -#define BP_DEFAULT_XCODE_VERSION "14.0" -#define BP_DEFAULT_RUNTIME "iOS 16.0" -#define BP_DEFAULT_BASE_SDK "16.0" +#define BP_DEFAULT_XCODE_VERSION "15.1" +#define BP_DEFAULT_RUNTIME "iOS 17.2" +#define BP_DEFAULT_BASE_SDK "17.2" -#define BP_DEFAULT_DEVICE_TYPE "iPhone 8" +#define BP_DEFAULT_DEVICE_TYPE "iPhone SE (3rd generation)" #define BP_DAEMON_PROTOCOL_VERSION 26 #define BP_MAX_PROCESSES_PERCENT 0.75 diff --git a/bp/src/BPSimulator.m b/bp/src/BPSimulator.m index b9367258..7b55451f 100644 --- a/bp/src/BPSimulator.m +++ b/bp/src/BPSimulator.m @@ -18,6 +18,7 @@ #import "BPWaitTimer.h" #import "PrivateHeaders/CoreSimulator/CoreSimulator.h" #import "SimulatorHelper.h" +#import "PrivateHeaders/CoreSimulator/SimDeviceBootInfo.h" @interface BPSimulator() @@ -127,6 +128,8 @@ - (NSString *)installApplicationWithHost:(NSString *)testHost withError:(NSError }]; return nil; } else { + // wait for few seconds to make sure the app is being installed correctly + CFRunLoopRunInMode(kCFRunLoopDefaultMode, 15, NO); [self shutdownSimulator:simDevice withError:errPtr]; if(*errPtr) { [BPUtils printInfo:ERROR withString:@"Shutdown simulator failed with error: %@", [*errPtr localizedDescription]]; @@ -319,7 +322,6 @@ - (BOOL)shutdownSimulator:(SimDevice *)simDevice withError:(id *)error { *error = nil; return YES; } - return [simDevice shutdownWithError:error]; } @@ -336,10 +338,12 @@ - (void)openSimulatorHeadlessWithCompletion:(void (^)(NSError *))completion { [self.device bootAsyncWithOptions:options completionHandler:^(NSError *bootError){ NSError *error = [self waitForDeviceReady]; if (error) { - [self shutdownSimulator:self.device withError:&error]; - if (error) { + NSError *shutdownError; + [self shutdownSimulator:self.device withError:&shutdownError]; + if (shutdownError) { [BPUtils printInfo:ERROR withString:@"Shutting down Simulator failed: %@", [error localizedDescription]]; } + bootError = error; } completion(bootError); }]; @@ -347,7 +351,11 @@ - (void)openSimulatorHeadlessWithCompletion:(void (^)(NSError *))completion { - (NSError *)waitForDeviceReady { int attempts = 1200; - while (attempts > 0 && ![self.device.stateString isEqualToString:@"Booted"]) { + while (attempts > 0) { + SimDeviceBootInfo *bootStatus = self.device.bootStatus; + if (bootStatus.status == SimDeviceBootInfoStatusFinished) { + break; + } [NSThread sleepForTimeInterval:0.1]; --attempts; } @@ -477,7 +485,7 @@ - (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCom [commandLineArgs addObjectsFromArray:@[ @"-NSTreatUnknownArgumentsAsOpen", @"NO", @"-ApplePersistenceIgnoreState", @"YES", - @"-XCTIDEConnectionTimeout", @"180" + @"-XCTIDEConnectionTimeout", @"600" ]]; argsAndEnv[@"args"] = [commandLineArgs copy]; @@ -598,6 +606,11 @@ - (void)deleteSimulatorWithCompletion:(void (^)(NSError *error, BOOL success))co int attempts = 300; while (attempts > 0 && ![self.device.stateString isEqualToString:@"Shutdown"]) { [NSThread sleepForTimeInterval:1.0]; + if (!self.app && !self.device) { + [BPUtils printInfo:ERROR withString:@"device has been deleted already"]; + completion(nil, NO); + return; + } --attempts; } if (![self.device.stateString isEqualToString:@"Shutdown"]) { diff --git a/bp/src/BPTMDControlConnection.m b/bp/src/BPTMDControlConnection.m index ada7e654..ef6dda08 100644 --- a/bp/src/BPTMDControlConnection.m +++ b/bp/src/BPTMDControlConnection.m @@ -85,13 +85,13 @@ - (void)connect { DTXRemoteInvocationReceipt *receipt = [daemonProxy _IDE_initiateControlSessionForTestProcessID:@(self.testRunnerPid) protocolVersion:@(BP_TM_PROTOCOL_VERSION)]; [receipt handleCompletion:^(NSNumber *version, NSError *error) { + self.connected = TRUE; if (error) { [BPUtils printInfo:ERROR withString:@"Error with daemon connection: %@", error]; return; } NSInteger daemonProtocolVersion = version.integerValue; [BPUtils printInfo:INFO withString:@"Test manager daemon control session started (%ld)", (long)daemonProtocolVersion]; - self.connected = TRUE; }]; } @@ -118,6 +118,7 @@ - (void)connect { socklen_t length = (socklen_t)(strnlen(remote.sun_path, 1024) + sizeof(remote.sun_family) + sizeof(remote.sun_len)); if (connect(socketFD, (struct sockaddr *)&remote, length) == -1) { [BPUtils printInfo:ERROR withString:@"ERROR connecting socket"]; + close(socketFD); } DTXTransport *transport = [[objc_lookUpClass("DTXSocketTransport") alloc] initWithConnectedSocket:socketFD disconnectAction:^{ [BPUtils printInfo:INFO withString:@"DTXSocketTransport disconnected"]; diff --git a/bp/src/BPTMDRunnerConnection.h b/bp/src/BPTMDRunnerConnection.h index 474d7136..713db270 100644 --- a/bp/src/BPTMDRunnerConnection.h +++ b/bp/src/BPTMDRunnerConnection.h @@ -17,6 +17,7 @@ @end @interface BPTMDRunnerConnection : NSObject +@property (nonatomic, assign) BOOL disconnected; @property (nonatomic, strong) BPExecutionContext *context; @property (nonatomic, strong) BPSimulator *simulator; @property (nonatomic, copy) void (^completionBlock)(NSError *, pid_t); diff --git a/bp/src/BPTMDRunnerConnection.m b/bp/src/BPTMDRunnerConnection.m index c43835ef..cc6d9afe 100644 --- a/bp/src/BPTMDRunnerConnection.m +++ b/bp/src/BPTMDRunnerConnection.m @@ -21,6 +21,7 @@ #import "PrivateHeaders/XCTest/XCTMessagingChannel_RunnerToIDE-Protocol.h" #import "PrivateHeaders/XCTest/XCTMessagingChannel_DaemonToIDE-Protocol.h" #import "PrivateHeaders/XCTest/XCTMessagingChannel_IDEToDaemon-Protocol.h" +#import "PrivateHeaders/XCTest/XCTTestIdentifier.h" // DTX framework @@ -56,6 +57,7 @@ @interface BPTMDRunnerConnection() @property (nonatomic, strong) BPConfiguration *config; @@ -309,6 +311,7 @@ - (void)createSimulatorWithContext:(BPExecutionContext *)context { }; handler.onError = ^(NSError *error) { + [[BPStats sharedStats] startTimer:SIMULATOR_LIFETIME(context.runner.UDID) atTime:simStart]; [[BPStats sharedStats] addSimulatorCreateFailure]; [BPUtils printInfo:ERROR withString:@"%@", [error localizedDescription]]; // If we failed to create the simulator, there's no reason for us to try to delete it, which can just cause more issues @@ -445,10 +448,12 @@ - (void)connectTestBundleAndTestDaemonWithContext:(BPExecutionContext *)context [runnerConnection connectWithTimeout:180]; [runnerConnection startTestPlan]; - NEXT([self checkProcessWithContext:context]); + + checkDisconnectDelay = [self.config.testBundleDisconnectTimeout intValue]; + NEXT([self checkProcessWithContext:context conenction:runnerConnection]); } -- (void)checkProcessWithContext:(BPExecutionContext *)context { +- (void)checkProcessWithContext:(BPExecutionContext *)context conenction:(BPTMDRunnerConnection*)connection { BOOL isRunning = [self isProcessRunningWithContext:context]; if (!isRunning && [context.runner isFinished]) { [BPUtils printInfo:INFO withString:@"Finished"]; @@ -479,7 +484,17 @@ - (void)checkProcessWithContext:(BPExecutionContext *)context { return; } - NEXT_AFTER(1, [self checkProcessWithContext:context]); + if (connection.disconnected) { + // break early if possible + if (checkDisconnectDelay > 0) { + checkDisconnectDelay --; + } else { + [BPUtils printInfo:INFO withString:@"Connection disconnected, deleteing simulator"]; + [self deleteSimulatorWithContext:context andStatus:BPExitStatusLaunchAppFailed]; + return; + } + } + NEXT_AFTER(1, [self checkProcessWithContext:context conenction:connection]); } - (BOOL)isProcessRunningWithContext:(BPExecutionContext *)context { diff --git a/bp/src/PrivateHeaders/CoreSimulator/SimDeviceBootInfo.h b/bp/src/PrivateHeaders/CoreSimulator/SimDeviceBootInfo.h index 15898065..a7b7f59f 100644 --- a/bp/src/PrivateHeaders/CoreSimulator/SimDeviceBootInfo.h +++ b/bp/src/PrivateHeaders/CoreSimulator/SimDeviceBootInfo.h @@ -4,9 +4,17 @@ // class-dump is Copyright (C) 1997-1998, 2000-2001, 2004-2013 by Steve Nygard. // -#import "NSObject.h" +#import -#import "NSSecureCoding.h" + +typedef NS_ENUM(NSUInteger, SimDeviceBootInfoStatus) { + SimDeviceBootInfoStatusBooting = 0, + SimDeviceBootInfoStatusWaitingOnBackboard = 1, + SimDeviceBootInfoStatusWaitingOnDataMigration = 2, + SimDeviceBootInfoStatusDataMigrationFailed = 3, + SimDeviceBootInfoStatusWaitingOnSystemApp = 4, + SimDeviceBootInfoStatusFinished = 4294967295, +}; @class NSDictionary, NSString; @@ -23,7 +31,6 @@ @property(nonatomic) BOOL isTerminalStatus; // @synthesize isTerminalStatus=_isTerminalStatus; @property(nonatomic) double bootElapsedTime; // @synthesize bootElapsedTime=_bootElapsedTime; @property(nonatomic) unsigned int status; // @synthesize status=_status; -- (void).cxx_destruct; @property(readonly, nonatomic) double migrationElapsedTime; @property(readonly, nonatomic) NSString *migrationPhaseDescription; - (void)encodeWithCoder:(id)arg1; diff --git a/bp/tests/BPReportTests.m b/bp/tests/BPReportTests.m index 496eb3fd..646f511c 100644 --- a/bp/tests/BPReportTests.m +++ b/bp/tests/BPReportTests.m @@ -16,10 +16,61 @@ #import "BPUtils.h" @interface BPReportTests : BPIntTestCase +- (NSString *)sanitizeXMLFile:(NSString *)atPath; +- (void)assertGotReport:(NSString *)Got isEqualToWantReport:(NSString *)Want; @end @implementation BPReportTests +#pragma mark - Test helpers + +- (NSString *)sanitizeXMLFile:(NSString *)atPath { + NSString *XSLTemplate = @" \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + "; + NSError *error; + NSURL *sourceURL = [NSURL fileURLWithPath:atPath]; + NSXMLDocument *xmlDoc = [[NSXMLDocument alloc] initWithContentsOfURL:sourceURL options:0 error:&error]; + XCTAssert(xmlDoc != nil, @"%@", [error localizedDescription]); + NSData *XSLTransform = [XSLTemplate dataUsingEncoding:NSUTF8StringEncoding]; + NSXMLDocument *sanitizedXMLDoc = [xmlDoc objectByApplyingXSLT:XSLTransform arguments:nil error:&error]; + XCTAssert(sanitizedXMLDoc != nil, @"%@", [error localizedDescription]); + NSString *outFile = [BPUtils mkstemp:@"outXXX.xslt" withError:&error]; + XCTAssert(outFile != nil, @"%@", [error localizedDescription]); + NSData *outData = [sanitizedXMLDoc XMLDataWithOptions:NSXMLNodePrettyPrint]; + if (![outData writeToFile:outFile atomically:YES]) { + XCTAssert(false, @"Failed to write file: %@", outFile); + } + return outFile; +} + +- (void)assertGotReport:(NSString *)Got isEqualToWantReport:(NSString *)Want { + NSString *sanitizedGot = [self sanitizeXMLFile:Got]; + NSString *sanitizedWant = [self sanitizeXMLFile:Want]; + // we ignore white space (-b) just as a convenience to test writers + NSString *diffOutput = [BPUtils runShell:[NSString stringWithFormat:@"diff -u -b '%@' '%@'", sanitizedWant, sanitizedGot]]; + XCTAssert(diffOutput != nil && [diffOutput isEqualToString:@""], @"\ndiff -u -b '%@' '%@':\n%@", sanitizedWant, sanitizedGot, diffOutput); +} +@end + +@interface BPReportTests1 : BPReportTests +@end + +@implementation BPReportTests1 - (void)testReportWithAppCrashingTestsSet { @@ -261,6 +312,13 @@ - (void)testReportSuccessOnAppCrashTestPassesOnRetry { XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); } +@end + +@interface BPReportTests2 : BPReportTests +@end + +@implementation BPReportTests2 + /** Execution plan: One test CRASHes and another one keeps timing out */ @@ -464,48 +522,4 @@ - (void)testReportFailureOnTimeoutCrashAndPassWithDiagnostics { XCTAssertTrue(exitCode == BPExitStatusAppCrashed); } -#pragma mark - Test helpers - -- (NSString *)sanitizeXMLFile:(NSString *)atPath { - NSString *XSLTemplate = @" \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - "; - NSError *error; - NSURL *sourceURL = [NSURL fileURLWithPath:atPath]; - NSXMLDocument *xmlDoc = [[NSXMLDocument alloc] initWithContentsOfURL:sourceURL options:0 error:&error]; - XCTAssert(xmlDoc != nil, @"%@", [error localizedDescription]); - NSData *XSLTransform = [XSLTemplate dataUsingEncoding:NSUTF8StringEncoding]; - NSXMLDocument *sanitizedXMLDoc = [xmlDoc objectByApplyingXSLT:XSLTransform arguments:nil error:&error]; - XCTAssert(sanitizedXMLDoc != nil, @"%@", [error localizedDescription]); - NSString *outFile = [BPUtils mkstemp:@"outXXX.xslt" withError:&error]; - XCTAssert(outFile != nil, @"%@", [error localizedDescription]); - NSData *outData = [sanitizedXMLDoc XMLDataWithOptions:NSXMLNodePrettyPrint]; - if (![outData writeToFile:outFile atomically:YES]) { - XCTAssert(false, @"Failed to write file: %@", outFile); - } - return outFile; -} - -- (void)assertGotReport:(NSString *)Got isEqualToWantReport:(NSString *)Want { - NSString *sanitizedGot = [self sanitizeXMLFile:Got]; - NSString *sanitizedWant = [self sanitizeXMLFile:Want]; - // we ignore white space (-b) just as a convenience to test writers - NSString *diffOutput = [BPUtils runShell:[NSString stringWithFormat:@"diff -u -b '%@' '%@'", sanitizedWant, sanitizedGot]]; - XCTAssert(diffOutput != nil && [diffOutput isEqualToString:@""], @"\ndiff -u -b '%@' '%@':\n%@", sanitizedWant, sanitizedGot, diffOutput); -} - @end diff --git a/bp/tests/BluepillTests.m b/bp/tests/BluepillTests.m index 1c0be3f9..cd7f148f 100644 --- a/bp/tests/BluepillTests.m +++ b/bp/tests/BluepillTests.m @@ -29,8 +29,6 @@ @interface BluepillTests : BPIntTestCase @implementation BluepillTests - - - (void)tearDown { [super tearDown]; } @@ -195,6 +193,7 @@ - (void)testRunWithPassingTestsSet { NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath]; self.config.testBundlePath = testBundlePath; self.config.testCasesToSkip = @[@"BPSampleAppTests/testCase000"]; + self.config.errorRetriesCount = @1; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssert(exitCode == BPExitStatusAllTestsPassed); @@ -206,6 +205,7 @@ - (void)testRunWithFailingTestsSet { self.config.failureTolerance = @0; self.config.testCaseTimeout = @10; self.config.testCasesToRun = @[@"BPAppNegativeTests/testBPDoesNotHangWithBigOutput"]; + self.config.errorRetriesCount = @1; NSString *tempDir = NSTemporaryDirectory(); NSError *error; NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppFailingTestsSetTempDir", tempDir] withError:&error]; @@ -221,6 +221,7 @@ - (void)testKeepSimulatorWithAppCrashingTestsSet { NSString *testBundlePath = [BPTestHelper sampleAppCrashingTestsBundlePath]; self.config.testBundlePath = testBundlePath; self.config.keepSimulator = YES; + self.config.errorRetriesCount = @1; Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; @@ -276,6 +277,7 @@ - (void)testRunUITest { [BPUtils enableDebugOutput:YES]; // The delay of ui test bootstrapping is larger than 5s. self.config.testCaseTimeout = @300; + self.config.errorRetriesCount = @1; NSString *testBundlePath = [BPTestHelper sampleAppUITestBundlePath]; NSString *testRunnerPath = [BPTestHelper sampleAppUITestRunnerPath]; NSString *tempDir = NSTemporaryDirectory(); @@ -296,6 +298,7 @@ - (void)testCopySimulatorPreferencesFile { NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath]; self.config.testBundlePath = testBundlePath; self.config.keepSimulator = YES; + self.config.errorRetriesCount = @1; Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; @@ -305,7 +308,7 @@ - (void)testCopySimulatorPreferencesFile { NSURL *preferencesFile = bp.test_simulator.preferencesFile; NSDictionary *plist = [[NSDictionary alloc] initWithContentsOfURL:preferencesFile]; - XCTAssertEqualObjects(@"en_CH", plist[@"AppleLocale"]); + XCTAssertEqualObjects(@"en_CN", plist[@"AppleLocale"]); self.config.deleteSimUDID = bp.test_simulatorUDID; XCTAssertNotNil(self.config.deleteSimUDID); diff --git a/bp/tests/Resource Files/simulator-preferences.plist b/bp/tests/Resource Files/simulator-preferences.plist index 2b8fe9ac..f979f57f 100644 --- a/bp/tests/Resource Files/simulator-preferences.plist +++ b/bp/tests/Resource Files/simulator-preferences.plist @@ -3,7 +3,7 @@ AppleLocale - en_CH + en_CN AppleLanguages en diff --git a/scripts/bluepill.sh b/scripts/bluepill.sh index 3ab8bc00..5bda22a4 100755 --- a/scripts/bluepill.sh +++ b/scripts/bluepill.sh @@ -107,12 +107,17 @@ run_tests() { bluepill_instance_tests1() { - run_tests bp "-skip-testing bp-tests/BPReportTests" + run_tests bp "-skip-testing bp-tests/BPReportTests1 -skip-testing bp-tests/BPReportTests2" } bluepill_instance_tests2() { - run_tests bp "-only-testing bp-tests/BPReportTests" + run_tests bp "-only-testing bp-tests/BPReportTests1" +} + +bluepill_instance_tests3() +{ + run_tests bp "-only-testing bp-tests/BPReportTests2" } bluepill_runner_tests()