diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 5a6faa53..3626db01 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -10,8 +10,8 @@ jobs: steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 11.2 - run: sudo xcode-select -s /Applications/Xcode_11.2.app + - name: Select Xcode 12 + run: sudo xcode-select -s /Applications/Xcode_12.app - name: Run Bluepill tests run: ./scripts/bluepill.sh instance_tests @@ -22,8 +22,8 @@ jobs: steps: # actions/checkout@v2 but we use the SHA1 because tags can be re-written in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 11.2 - run: sudo xcode-select -s /Applications/Xcode_11.2.app + - name: Select Xcode 12 + run: sudo xcode-select -s /Applications/Xcode_12.app - name: Run BP tests run: ./scripts/bluepill.sh runner_tests - name: Build and Package diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 72ecb862..44ff2058 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -11,24 +11,24 @@ jobs: # First machine, runs Bluepill tests integration_tests: name: Bluepill Test - runs-on: macOS-latest + runs-on: macos-latest steps: # actions/checkout@v2 but we use the sha because tags can be rewritten in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 11.2 - run: sudo xcode-select -s /Applications/Xcode_11.2.app + - name: Select Xcode 12 + run: sudo xcode-select -s /Applications/Xcode_12.app - name: Run Bluepill tests run: ./scripts/bluepill.sh instance_tests # Second machine, runs BP tests and makes build build: name: BP Test and build - runs-on: macOS-latest + runs-on: macos-latest steps: # actions/checkout@v2 but we use the sha because tags can be rewritten in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 11.2 - run: sudo xcode-select -s /Applications/Xcode_11.2.app + - name: Select Xcode 12 + run: sudo xcode-select -s /Applications/Xcode_12.app - name: Run BP tests run: ./scripts/bluepill.sh runner_tests - name: Build and Package diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69ffe23e..30332410 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,27 +9,27 @@ jobs: # First machine, runs Bluepill tests integration_tests: name: Bluepill Test - runs-on: macOS-latest + runs-on: macos-latest steps: # actions/checkout@v2 but we use the sha because tags can be rewritten in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - - name: Select Xcode 11.2 - run: sudo xcode-select -s /Applications/Xcode_11.2.app + - name: Select Xcode 12 + run: sudo xcode-select -s /Applications/Xcode_12.app - name: Run Bluepill tests run: ./scripts/bluepill.sh instance_tests # Second machine, runs BP tests and makes build build: name: BP Test and build - runs-on: macOS-latest + runs-on: macos-latest steps: # actions/checkout@v2 but we use the sha because tags can be rewritten in git - uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 - name: Report event trigger data run: | echo "Event ${{ github.event_name }}, ref: ${{ github.ref }}" - - name: Select Xcode 11.2 - run: sudo xcode-select -s /Applications/Xcode_11.2.app + - name: Select Xcode 12 + run: sudo xcode-select -s /Applications/Xcode_12.app - name: Run Bluepill tests run: ./scripts/bluepill.sh runner_tests - name: Build Bluepill diff --git a/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m b/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m index 844146e0..0c990a08 100644 --- a/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m +++ b/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m @@ -15,10 +15,75 @@ @interface BPSampleAppHangingTests : XCTestCase @implementation BPSampleAppHangingTests +-(long)attemptFromSimulatorVersionInfo:(NSString *)simulatorVersionInfo { + // simulatorVersionInfo is something like + // CoreSimulator 587.35 - Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861) - Runtime: iOS 12.2 (16E226) - DeviceType: iPhone 7 + NSLog(@"Dissecting version info %@ to extra attempt number.", simulatorVersionInfo); + NSArray *parts = [simulatorVersionInfo componentsSeparatedByString:@" - "]; + NSString *deviceString = parts[1]; + // Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861) + parts = [deviceString componentsSeparatedByString:@" "]; + NSString *device = parts[1]; + // BP93497-2-2 + parts = [device componentsSeparatedByString:@"-"]; + NSString *attempt = parts[1]; + return [attempt longLongValue]; +} + +-(void)extractPlanAndExecuteActions:(int)index { + NSDictionary *env = [[NSProcessInfo processInfo] environment]; + NSString *simulatorVersionInfo = [env objectForKey:@"SIMULATOR_VERSION_INFO"]; + long attempt = [self attemptFromSimulatorVersionInfo:simulatorVersionInfo]; + NSString *executionPlan = [env objectForKey:@"_BP_TEST_EXECUTION_PLAN"]; + if (!executionPlan) { + NSLog(@"No execution plan found in attempt#%ld. Failing the test.", attempt); + XCTAssert(NO); + return; + } + NSLog(@"Received execution plan %@ on attempt#%ld for this test.", executionPlan, attempt); + NSArray *setsOfPlans = [executionPlan componentsSeparatedByString:@";"]; + if (index >= [setsOfPlans count]) { + NSLog(@"Not enough plans for test#%d in execution plan: '%@'.", index, executionPlan); + XCTAssert(YES); + return; + } + NSString *currentPlan = setsOfPlans[index]; + currentPlan = [currentPlan stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]]; + NSArray *array = [currentPlan componentsSeparatedByCharactersInSet:[NSCharacterSet whitespaceCharacterSet]]; + if (attempt > [array count]) { + NSLog(@"Passing on attempt#%ld, by default, as there is no action defined in the execution plan", (long)attempt); + XCTAssert(YES); + return; + } + NSString *action = array[attempt - 1]; + if ([action isEqualToString:@"TIMEOUT"]) { + NSLog(@"Entering into an infinite loop on attempt#%ld to timeout as per the execution plan", (long)attempt); + while(1) { + } + return; + } else if ([action isEqualToString:@"PASS"]) { + NSLog(@"Passing on attempt#%ld based on execution plan", (long)attempt); + XCTAssert(YES); + return; + } else if ([action isEqualToString:@"FAIL"]) { + NSLog(@"Failing on attempt#%ld based on execution plan", (long)attempt); + XCTAssert(NO); + return; + } else if ([action isEqualToString:@"CRASH"]) { + NSLog(@"Crashing on attempt#%ld based on execution plan", (long)attempt); + // ok, let's crash and burn + int *pointer = nil; + *pointer = 1; + return; + } + NSLog(@"Failing on attempt#%ld as an unidentified action is encountered in the execution plan", (long)attempt); + XCTAssert(NO); + return; +} + - (void)setUp { [super setUp]; // Put setup code here. This method is called before the invocation of each test method in the class. - } - (void)tearDown { @@ -26,8 +91,24 @@ - (void)tearDown { [super tearDown]; } -- (void)testAppHanging { - while(TRUE){}; +- (void)testASimpleTest { + XCTAssert(YES); +} + +- (void)testBasedOnExecutionPlan { + [self extractPlanAndExecuteActions:0]; +} + +- (void)testCaseFinal { + XCTAssert(YES); +} + +- (void)testDoubleBasedOnExecutionPlan { + [self extractPlanAndExecuteActions:1]; +} + +- (void)testEndFinal { + XCTAssert(YES); } @end diff --git a/README.md b/README.md index 86232ee2..d285cb3b 100644 --- a/README.md +++ b/README.md @@ -54,39 +54,65 @@ $ bluepill -c config.json A full list supported options are listed here. -| Config Arguments | Command Line Arguments | Explanation | Required | Default value | -|:----------------------:|:----------------------:|------------------------------------------------------------------------------------|:--------:|:----------------:| -| `app` | -a | The path to the host application to execute (your .app) | N | n/a | -| `xctestrun-path` | | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`. | Y | n/a | -| `output-dir` | -o | Directory where to put output log files (bluepill only) | Y | n/a | -| config | -c | Read options from the specified configuration file instead of the command line | N | n/a | -| device | -d | On which device to run the app. | N | iPhone 6 | -| exclude | -x | Exclude a testcase in the set of tests to run (takes priority over `include`). | N | empty | -| headless | -H | Run in headless mode (no GUI). | N | off | -| clone-simulator | -L | Spawn simulator by clone from simulator template. | N | off | -| xcode-path | -X | Path to xcode. | N | xcode-select -p | -| include | -i | Include a testcase in the set of tests to run (unless specified in `exclude`). | N | all tests | -| list-tests | -l | Only list tests in bundle | N | false | -| num-sims | -n | Number of simulators to run in parallel. (bluepill only) | N | 4 | -| printf-config | -P | Print a configuration file suitable for passing back using the `-c` option. | N | n/a | -| error-retries | -R | Number of times to recover from simulator/app crashing/hanging and continue running| N | 5 | -| failure-tolerance | -f | Number of times to retry on test failures | N | 0 | -| only-retry-failed | -F | When `failure-tolerance` > 0, only retry tests that failed | N | false | -| runtime | -r | What runtime to use. | N | iOS 11.1 | -| stuck-timeout | -S | Timeout in seconds for a test that seems stuck (no output). | N | 300s | -| test-timeout | -T | Timeout in seconds for a test that is producing output. | N | 300s | -| test-bundle-path | -t | The path to the test bundle to execute (single .xctest). | N | n/a | -| additional-unit-xctests| n/a | Additional XCTest bundles that is not Plugin folder | N | n/a | -| additional-ui-xctests | n/a | Additional XCTUITest bundles that is not Plugin folder | N | n/a | -| repeat-count | -C | Number of times we'll run the entire test suite (used for load testing). | N | 1 | -| no-split | -N | Test bundles you don't want to be packed into different groups to run in parallel. | N | n/a | -| quiet | -q | Turn off all output except fatal errors. | N | YES | -| diagnostics | n/a | Enable collection of diagnostics in outputDir in case of test failures | N | NO | -| help | -h | Help. | N | n/a | -| runner-app-path | -u | The test runner for UI tests. | N | n/a | -| screenshots-directory | n/a | Directory where simulator screenshots for failed ui tests will be stored | N | n/a | -| video-paths | -V | A list of videos that will be saved in the simulators | N | n/a | -| image-paths | -I | A list of images that will be saved in the simulators | N | n/a | +| Config Arguments | Command Line Arguments | Explanation | Required | Default value | +|:----------------------:|:----------------------:|----------------------------------------------------------------------------------------------|:--------:|:----------------:| +| app | -a | The path to the host application to execute (your `.app`) | N | n/a | +| xctestrun-path | | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`. | Y | n/a | +| test-plan-path | | The path of a json file which describes the test plan. It is equivalent to the `.xctestrun` file generated by Xcode, but it can be generated by a different build system, e.g. Bazel | Y | n/a | +| output-dir | -o | Directory where to put output log files. **(bluepill only)** | Y | n/a | +| config | -c | Read options from the specified configuration file instead of the command line. | N | n/a | +| device | -d | On which device to run the app. | N | iPhone 8 | +| exclude | -x | Exclude a testcase in the set of tests to run (takes priority over `include`). | N | empty | +| headless | -H | Run in headless mode (no GUI). | N | off | +| clone-simulator | -L | Spawn simulator by clone from simulator template. | N | off | +| xcode-path | -X | Path to xcode. | N | xcode-select -p | +| include | -i | Include a testcase in the set of tests to run (unless specified in `exclude`). | N | all tests | +| list-tests | -l | Only list tests and exit without executing tests. | N | false | +| num-sims | -n | Number of simulators to run in parallel. **(bluepill only)** | N | 4 | +| printf-config | -P | Print a configuration file suitable for passing back using the `-c` option. | N | n/a | +| error-retries | -R | Number of times to recover from simulator/app crashing/hanging and continue running.| N | 4 | +| failure-tolerance | -f | Number of times to retry on test failures | N | 0 | +| only-retry-failed | -F | Only retry failed tests instead of all. Also retry test that timed-out/crashed. | N | false | +| runtime | -r | What runtime to use. | N | iOS 13.3 | +| stuck-timeout | -S | Timeout in seconds for a test that seems stuck (no output). | N | 300s | +| test-timeout | -T | Timeout in seconds for a test that is producing output. | N | 300s | +| test-bundle-path | -t | The path to the test bundle to execute (single `.xctest`). | N | n/a | +| additional-unit-xctests| n/a | Additional XCTest bundles that is not Plugin folder | N | n/a | +| additional-ui-xctests | n/a | Additional XCTUITest bundles that is not Plugin folder | N | n/a | +| repeat-count | -C | Number of times we'll run the entire test suite (used for load testing). | N | 1 | +| no-split | -N | Test bundles you don't want to be packed into different groups to run in parallel. | N | n/a | +| quiet | -q | Turn off all output except fatal errors. | N | YES | +| diagnostics | n/a | Enable collection of diagnostics in output directory in case of test failures. | N | NO | +| help | -h | Help. | N | n/a | +| runner-app-path | -u | The test runner for UI tests. | N | n/a | +| screenshots-directory | n/a | Directory where simulator screenshots for failed ui tests will be stored. | N | n/a | +| videos-directory | n/a | Directory where videos of test runs will be saved. If not provided, videos are not recorded. | N | n/a | +| video-paths | -V | A list of videos that will be saved in the simulators. | N | n/a | +| image-paths | -I | A list of images that will be saved in the simulators. | N | n/a | +| unsafe-skip-xcode-version-check | | Skip Xcode version check | N | NO | +| retry-app-crash-tests | | Retry tests that crashed app and consider it non-fatal if it passes on retry. | N | false | + + +## Exit Status + +Here is a list of Bluepill exit codes. If a Bluepill execution has multiple exit codes from same or different test bundles, the final exit code is a combination of all exit codes. Note that app crashes are fatal even if the test passes on retry. + +```shell + BPExitStatusAllTestsPassed = 0, + BPExitStatusTestsFailed = 1 << 0, + BPExitStatusSimulatorCreationFailed = 1 << 1, + BPExitStatusInstallAppFailed = 1 << 2, + BPExitStatusInterrupted = 1 << 3, + BPExitStatusSimulatorCrashed = 1 << 4, + BPExitStatusLaunchAppFailed = 1 << 5, + BPExitStatusTestTimeout = 1 << 6, + BPExitStatusAppCrashed = 1 << 7, + BPExitStatusSimulatorDeleted = 1 << 8, + BPExitStatusUninstallAppFailed = 1 << 9, + BPExitStatusSimulatorReuseFailed = 1 << 10 +``` +**Note:** Please refer to `bp/src/BPExitStatus.h` for the latest/exact exit codes. + ## Demo @@ -94,7 +120,7 @@ A full list supported options are listed here. ## Requirements -Bluepill only works with **Xcode 11.2**. If you're looking for old Xcode support, please check out the other branches: +Bluepill officially supports **Xcode 12.0**. If you're looking for old Xcode support, please checkout the following branches: * [Xcode-8](https://github.com/linkedin/bluepill/tree/xcode8) * [Xcode-9.0](https://github.com/linkedin/bluepill/tree/xcode-9.0) @@ -108,6 +134,12 @@ Bluepill only works with **Xcode 11.2**. If you're looking for old Xcode support * [Xcode-10.3](https://github.com/linkedin/bluepill/tree/xcode-10.3) * [Xcode-11.0](https://github.com/linkedin/bluepill/tree/xcode-11.0) * [Xcode-11.1](https://github.com/linkedin/bluepill/tree/xcode-11.1) +* [Xcode-11.2](https://github.com/linkedin/bluepill/tree/xcode-11.2) +* [Xcode-11.3](https://github.com/linkedin/bluepill/tree/xcode-11.3) +* [Xcode-11.4](https://github.com/linkedin/bluepill/tree/xcode-11.4) +* [Xcode-11.5](https://github.com/linkedin/bluepill/tree/xcode-11.5) + +If you're looking for newer Xcode version support, try using Bluepill with `unsafe-skip-xcode-version-check` flag but make sure your app is tested with it and the underlying risks are understand. ## Acknowledgement @@ -138,7 +170,7 @@ If you're using [Bitrise.io](https://bitrise.io) as your CI/CD, you can start us Latest [release](https://github.com/linkedin/bluepill/releases/). -- How to test Bluepill in Xcode +- How to test Bluepill in Xcode? Select BPSampleApp scheme and build it first. Then you can switch back to `bluepill` or `bluepill-cli` scheme to run their tests. diff --git a/bluepill/src/BPApp.m b/bluepill/src/BPApp.m index 0d0b419c..ac134386 100644 --- a/bluepill/src/BPApp.m +++ b/bluepill/src/BPApp.m @@ -106,6 +106,8 @@ + (instancetype)appWithConfig:(BPConfiguration *)config for (NSString *testName in config.tests) { BPTestPlan *testPlan = [config.tests objectForKey:testName]; BPXCTestFile *xcTestFile = [BPXCTestFile BPXCTestFileFromBPTestPlan:testPlan withName:testName andError:errPtr]; + if (*errPtr) + return nil; [loadedTests addObject:xcTestFile]; } diff --git a/bluepill/src/BPRunner.m b/bluepill/src/BPRunner.m index 1436a3ab..54268669 100644 --- a/bluepill/src/BPRunner.m +++ b/bluepill/src/BPRunner.m @@ -93,6 +93,7 @@ - (NSTask *)newTaskWithBundle:(BPXCTestFile *)bundle } else { cfg.environmentVariables = bundle.environmentVariables; } + cfg.dependencies = bundle.dependencies; if (self.config.cloneSimulator) { cfg.templateSimUDID = self.testHostSimTemplates[bundle.testHostPath]; } diff --git a/bluepill/tests/BPIntegrationTests.m b/bluepill/tests/BPIntegrationTests.m index 00f715e7..5018bd0c 100644 --- a/bluepill/tests/BPIntegrationTests.m +++ b/bluepill/tests/BPIntegrationTests.m @@ -21,7 +21,7 @@ @interface BPIntegrationTests : XCTestCase @implementation BPIntegrationTests -- (BPConfiguration *)generateConfig { +- (BPConfiguration *)generateConfigWithVideoDir:(NSString *)videoDir { NSString *hostApplicationPath = [BPTestHelper sampleAppPath]; NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath]; BPConfiguration *config = [[BPConfiguration alloc] initWithProgram:BP_MASTER]; @@ -36,9 +36,16 @@ - (BPConfiguration *)generateConfig { config.deviceType = @BP_DEFAULT_DEVICE_TYPE; config.headlessMode = YES; config.quiet = [BPUtils isBuildScript]; + if (videoDir != nil) { + config.videosDirectory = videoDir; + } return config; } +- (BPConfiguration *)generateConfig { + return [self generateConfigWithVideoDir: nil]; +} + - (void)setUp { [super setUp]; // Put setup code here. This method is called before the invocation of each test method in the class. @@ -90,8 +97,9 @@ - (void)testClonedSimulators { config.failureTolerance = @0; config.cloneSimulator = TRUE; // need to validate the configuration to fill in simDevice and simRuntime - [config validateConfigWithError:nil]; - NSError *err; + NSError *err = nil; + [config validateConfigWithError:&err]; + XCTAssert(err == nil); BPApp *app = [BPApp appWithConfig:config withError:&err]; @@ -131,8 +139,8 @@ - (void)testTwoBPInstancesWithXCTestRunFile { config.numSims = @2; config.testBundlePath = nil; config.testRunnerAppPath = nil; - NSString *runtime = [[NSString stringWithUTF8String:BP_DEFAULT_RUNTIME] stringByReplacingOccurrencesOfString:@"iOS " withString:@""]; - NSString *xcTestRunFile = [NSString stringWithFormat:@"Build/Products/BPSampleApp_iphonesimulator%@-x86_64.xctestrun", runtime]; + NSString *baseSDK = [[NSString stringWithUTF8String:BP_DEFAULT_BASE_SDK] stringByReplacingOccurrencesOfString:@"iOS " withString:@""]; + NSString *xcTestRunFile = [NSString stringWithFormat:@"Build/Products/BPSampleApp_iphonesimulator%@-arm64-i386-x86_64.xctestrun", baseSDK]; config.xcTestRunPath = [[[BPTestHelper derivedDataPath] stringByDeletingLastPathComponent] stringByAppendingPathComponent:xcTestRunFile]; NSError *err; [config validateConfigWithError:&err]; @@ -215,4 +223,59 @@ - (void)writeTestPlan { [jsonData writeToFile:[BPTestHelper testPlanPath] atomically:YES]; } +- (void)testTwoBPInstancesWithVideo { + NSFileManager *fileManager = [NSFileManager defaultManager]; + NSError *mkdtempError; + NSString *path = [BPUtils mkdtemp:@"bpout" withError:&mkdtempError]; + XCTAssertNil(mkdtempError); + + NSString* videoDirName = @"my_videos"; + NSString *videoPath = [path stringByAppendingPathComponent:videoDirName]; + BPConfiguration *config = [self generateConfigWithVideoDir:videoPath]; + config.numSims = @2; + config.errorRetriesCount = @1; + config.failureTolerance = @0; + // This looks backwards but we want the main app to be the runner + // and the sampleApp is launched from the callback. + config.testBundlePath = [BPTestHelper sampleAppUITestBundlePath]; + config.testRunnerAppPath = [BPTestHelper sampleAppPath]; + config.appBundlePath = [BPTestHelper sampleAppUITestRunnerPath]; + + NSError *err; + BPApp *app = [BPApp appWithConfig:config + withError:&err]; + NSString *bpPath = [BPTestHelper bpExecutablePath]; + + // Run the tests through one time to flush out any weird errors that happen with video recording + BPRunner *dryRunRunner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath]; + XCTAssert(dryRunRunner != nil); + int dryRunRC = [dryRunRunner runWithBPXCTestFiles:app.testBundles]; + XCTAssert(dryRunRC == 0); + XCTAssert([dryRunRunner.nsTaskList count] == 0); + [fileManager removeItemAtPath:videoPath error:nil]; + NSArray *dryRunOutputContents = [fileManager contentsOfDirectoryAtPath:videoPath error:nil]; + XCTAssertEqual(dryRunOutputContents.count, 0); + + // Start the real test now + BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath]; + XCTAssert(runner != nil); + int rc = [runner runWithBPXCTestFiles:app.testBundles]; + XCTAssert(rc == 0); + XCTAssert([runner.nsTaskList count] == 0); + + NSError *dirContentsError; + NSArray *directoryContent = [fileManager contentsOfDirectoryAtPath:videoPath error:&dirContentsError]; + XCTAssertNil(dirContentsError); + XCTAssertNotNil(directoryContent); + XCTAssertEqual(directoryContent.count, 2); + + NSString *testClass = @"BPSampleAppUITests"; + NSSet *filenameSet = [NSSet setWithArray: directoryContent]; + XCTAssertEqual(filenameSet.count, 2); + BOOL hasTest1 = [filenameSet containsObject: [NSString stringWithFormat:@"%@__%@__1.mp4", testClass, @"testExample"]]; + XCTAssertTrue(hasTest1); + BOOL hasTest2 = [filenameSet containsObject: [NSString stringWithFormat:@"%@__%@__1.mp4", testClass, @"testExample2"]]; + XCTAssertTrue(hasTest2); +} + @end diff --git a/bluepill/tests/BPPackerTests.m b/bluepill/tests/BPPackerTests.m index 0f5c3326..44149cd2 100644 --- a/bluepill/tests/BPPackerTests.m +++ b/bluepill/tests/BPPackerTests.m @@ -187,7 +187,7 @@ - (void)testPacking { // Make sure we don't split when we don't want to self.config.numSims = @4; self.config.noSplit = @[@"BPSampleAppTests"]; - bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];// withNoSplitList:@[@"BPSampleAppTests"] intoBundles:4 andError:nil]; + bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil]; // withNoSplitList:@[@"BPSampleAppTests"] intoBundles:4 andError:nil]; // When we prevent BPSampleTests from splitting, BPSampleAppFatalErrorTests and BPAppNegativeTests gets split in two want = [[want arrayByAddingObject:@"BPSampleAppFatalErrorTests"] sortedArrayUsingSelector:@selector(compare:)]; XCTAssertEqual(bundles.count, app.testBundles.count + 2); @@ -195,9 +195,10 @@ - (void)testPacking { XCTAssertEqual([bundles[0].skipTestIdentifiers count], 0); XCTAssertEqual([bundles[1].skipTestIdentifiers count], 0); XCTAssertEqual([bundles[2].skipTestIdentifiers count], 0); - XCTAssertEqual([bundles[3].skipTestIdentifiers count], 2); - XCTAssertEqual([bundles[4].skipTestIdentifiers count], 3); + XCTAssertEqual([bundles[3].skipTestIdentifiers count], 1); + XCTAssertEqual([bundles[4].skipTestIdentifiers count], 4); XCTAssertEqual([bundles[5].skipTestIdentifiers count], 1); + XCTAssertEqual([bundles[6].skipTestIdentifiers count], 4); self.config.numSims = @4; self.config.noSplit = nil; @@ -209,14 +210,17 @@ - (void)testPacking { long testsPerBundle = [allTests count] / numSims; long skipTestsPerBundle = 0; long skipTestsInFinalBundle = 0; + long testCount = 0; for (int i = 0; i < bundles.count; ++i) { skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle); - skipTestsInFinalBundle = testsPerBundle * (numSims - 1); if (i < 4) { XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0); + testCount += [[bundles[i] allTestCases] count]; } else if (i < bundles.count-1) { XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle); + testCount += testsPerBundle; } else { /* last bundle */ + skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([allTests count] - testCount); XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle); } } @@ -231,15 +235,19 @@ - (void)testPacking { numSims = [self.config.numSims integerValue]; testsPerBundle = [allTests count] / numSims; + testCount = 0; for (int i = 0; i < bundles.count; ++i) { skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle); - skipTestsInFinalBundle = testsPerBundle * (numSims - 1); if (i < 4) { XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0); + testCount += [[bundles[i] allTestCases] count]; } else if (i < bundles.count-1) { XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle); + testCount += testsPerBundle; } else { /* last bundle */ + skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([allTests count] - testCount); XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle); + } } diff --git a/bp/bp.xcodeproj/xcshareddata/xcschemes/bp-tests.xcscheme b/bp/bp.xcodeproj/xcshareddata/xcschemes/bp-tests.xcscheme index ca5a1f5d..56188853 100644 --- a/bp/bp.xcodeproj/xcshareddata/xcschemes/bp-tests.xcscheme +++ b/bp/bp.xcodeproj/xcshareddata/xcschemes/bp-tests.xcscheme @@ -61,12 +61,6 @@ ReferencedContainer = "container:bp.xcodeproj"> - - - - @@ -76,12 +70,6 @@ - - - - diff --git a/bp/src/BPConfiguration.h b/bp/src/BPConfiguration.h index 0b0f8a79..0e9da12b 100644 --- a/bp/src/BPConfiguration.h +++ b/bp/src/BPConfiguration.h @@ -22,6 +22,7 @@ @property (nonatomic, strong) NSString *testHost; @property (nonatomic, strong) NSDictionary *environment; @property (nonatomic, strong) NSDictionary *arguments; +@property (nonatomic, strong) NSDictionary *dependencies; @property (nonatomic, strong) NSString *testBundlePath; @property (nonatomic, strong) NSString *testHostBundleIdentifier; @property (nonatomic, strong) NSString *uiTargetAppPath; @@ -78,6 +79,7 @@ typedef NS_ENUM(NSInteger, BPProgram) { @property (nonatomic) BOOL saveDiagnosticsOnError; @property (nonatomic, strong) NSNumber *failureTolerance; @property (nonatomic) BOOL onlyRetryFailed; +@property (nonatomic) BOOL retryAppCrashTests; @property (nonatomic, strong) NSArray *testCasesToSkip; @property (nonatomic, strong) NSArray *testCasesToRun; @property (nonatomic, strong) NSArray *allTestCases; @@ -85,10 +87,12 @@ typedef NS_ENUM(NSInteger, BPProgram) { @property (nonatomic, strong) NSString *outputDirectory; @property (nonatomic, strong) NSString *testTimeEstimatesJsonFile; @property (nonatomic, strong) NSString *screenshotsDirectory; +@property (nonatomic, strong) NSString *videosDirectory; @property (nonatomic, strong) NSString *simulatorPreferencesFile; @property (nonatomic, strong) NSString *scriptFilePath; @property (nonatomic) BOOL headlessMode; @property (nonatomic) BOOL cloneSimulator; +@property (nonatomic) BOOL unsafeSkipXcodeVersionCheck; @property (nonatomic, strong) NSNumber *numSims; @property (nonatomic) BOOL listTestsOnly; @property (nonatomic) BOOL quiet; @@ -106,6 +110,7 @@ typedef NS_ENUM(NSInteger, BPProgram) { @property (nonatomic, strong) NSArray *commandLineArguments; // command line arguments for the app @property (nonatomic, strong) NSDictionary *environmentVariables; @property (nonatomic, strong) NSDictionary *tests; +@property (nonatomic, strong) NSDictionary *dependencies; // Media Assets @property (nonatomic, strong) NSArray *videoPaths; // The videos to be pushed into each simulator. @@ -116,6 +121,7 @@ typedef NS_ENUM(NSInteger, BPProgram) { @property (nonatomic) BOOL testing_HangAppOnLaunch; @property (nonatomic) BOOL testing_NoAppWillRun; @property (nonatomic) NSNumber *testing_crashOnAttempt; +@property (nonatomic, strong) NSString *testing_ExecutionPlan; // Generated fields @property (nonatomic, strong) NSString *xcodePath; diff --git a/bp/src/BPConfiguration.m b/bp/src/BPConfiguration.m index 8d85b7c3..713e05ab 100644 --- a/bp/src/BPConfiguration.m +++ b/bp/src/BPConfiguration.m @@ -18,10 +18,10 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { - BP_VALUE = 1, // Single value - BP_LIST = 1 << 1, // List value - BP_PATH = 1 << 2, // Single value, CWD will be prepended - BP_BOOL = 1 << 3, // Boolean value + BP_VALUE = 1 << 0, // Single value + BP_LIST = 1 << 1, // List value + BP_PATH = 1 << 2, // Single value, CWD will be prepended + BP_BOOL = 1 << 3, // Boolean value BP_INTEGER = 1 << 4, // Integer value }; @@ -103,9 +103,9 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { {'q', "quiet", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "quiet", "Turn off all output except fatal errors."}, {'F', "only-retry-failed", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "onlyRetryFailed", - "If `failure-`tolerance` is > 0, only retry tests that failed."}, + "Only retry failed tests instead of all. Also retry test that timed-out/crashed."}, {'l', "list-tests", BP_MASTER, NO, NO, no_argument, NULL, BP_VALUE | BP_BOOL, "listTestsOnly", - "Only list tests in bundle"}, + "Only list tests and exit without executing tests."}, {'v', "verbose", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "verboseLogging", "Enable verbose logging"}, {'k', "keep-individual-test-reports", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepIndividualTestReports", @@ -142,7 +142,12 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { "A script that will be called after the simulator is booted, but before tests are run. Can be used to do any setup (e.g. installing certs). The environment will contain $BP_DEVICE_ID with the ID of the simulator and $BP_DEVICE_PATH with its full path. Exit with zero for success and non-zero for failure."}, {364, "test-plan-path", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "testPlanPath", "The path of a json file which describes the test plan. It is equivalent to the .xctestrun file generated by Xcode, but it can be generated by a different build system, e.g. Bazel"}, - + {365, "unsafe-skip-xcode-version-check", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL , "unsafeSkipXcodeVersionCheck", + "Skip Xcode version check if using an Xcode version that is not officially supported the Bluepill version being used. Not safe/recommended and has a limited support."}, + {366, "retry-app-crash-tests", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "retryAppCrashTests", + "Retry the tests after an app crash and if it passes on retry, consider them non-fatal."}, + {367, "videos-directory", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "videosDirectory", + "Directory where videos of test runs will be saved. If not provided, videos are not recorded."}, {0, 0, 0, 0, 0, 0, 0} }; @@ -177,6 +182,7 @@ - (nonnull id)copyWithZone:(nullable NSZone *)zone { BPTestPlan *c = [[BPTestPlan alloc] init]; c.arguments = [self.arguments copy]; c.environment = [self.environment copy]; + c.dependencies = [self.dependencies copy]; c.testBundlePath = [self.testBundlePath copy]; c.testHost = [self.testHost copy]; return c; @@ -332,6 +338,9 @@ - (NSString *)configString { if (self.environmentVariables) { [dict setValue:self.environmentVariables forKey:@"environmentVariables"]; } + if (self.dependencies) { + [dict setValue:self.dependencies forKey:@"dependencies"]; + } if ([NSJSONSerialization isValidJSONObject:dict]) { NSError *err; NSData *json = [NSJSONSerialization dataWithJSONObject:dict @@ -371,11 +380,13 @@ - (id)mutableCopyWithZone:(NSZone *)zone { newConfig.testing_HangAppOnLaunch = self.testing_HangAppOnLaunch; newConfig.testing_NoAppWillRun = self.testing_NoAppWillRun; newConfig.testing_crashOnAttempt = self.testing_crashOnAttempt; + newConfig.testing_ExecutionPlan = self.testing_ExecutionPlan; newConfig.xcTestRunPath = self.xcTestRunPath; newConfig.testPlanPath = self.testPlanPath; newConfig.xcTestRunDict = self.xcTestRunDict; newConfig.commandLineArguments = self.commandLineArguments; newConfig.environmentVariables = self.environmentVariables; + newConfig.dependencies = self.dependencies; newConfig.tests = self.tests; return newConfig; @@ -521,9 +532,10 @@ - (BOOL)loadConfigFile:(NSString *)file withError:(NSError **)errPtr{ BP_SET_ERROR(errPtr, @"Number of simulators set to %lu but there cannot be fewer than one simulator.", self.numSims.integerValue); return NO; } - // Pull out two keys that are undocumented but needed for supporting xctest + // Pull out three keys that are undocumented but needed for supporting xctest self.commandLineArguments = [configDict objectForKey:@"commandLineArguments"]; self.environmentVariables = [configDict objectForKey:@"environmentVariables"]; + self.dependencies = [configDict objectForKey:@"dependencies"]; return YES; } @@ -554,6 +566,7 @@ - (BOOL)loadTestPlan:(NSString *)file withError:(NSError **)errPtr{ plan.uiTargetAppPath = [planDictionary objectForKey:@"ui_target_app_path"]; plan.environment = [planDictionary objectForKey:@"environment"]; plan.arguments = [planDictionary objectForKey:@"arguments"]; + plan.dependencies = [planDictionary objectForKey:@"dependencies"]; if (![plan isValid:errPtr]) { BP_SET_ERROR(errPtr, @"Invalid BPTestPlan configuration: %@", plan); @@ -666,18 +679,21 @@ - (BOOL)validateConfigWithError:(NSError *__autoreleasing *)errPtr { //Check if xcode version running on the host match the intended Bluepill branch: Xcode 9 branch is not backward compatible NSString *xcodeVersion = [BPUtils runShell:@"xcodebuild -version"]; [BPUtils printInfo:DEBUGINFO withString:@"xcode build version: %@", xcodeVersion]; - if ([xcodeVersion rangeOfString:@BP_DEFAULT_XCODE_VERSION].location == NSNotFound) { - BP_SET_ERROR(errPtr, @"ERROR: Invalid Xcode version:\n%s;\nOnly %s is supported\n", [xcodeVersion UTF8String], BP_DEFAULT_XCODE_VERSION); - return NO; - } - //Check if Bluepill compile time Xcode version is matched with Bluepill runtime Xcode version - //Senario to prevent: Bluepill is compiled with Xcode 8, but runs with host installed with Xcode 9 - //Only compare major and minor version version Exg. 9.1 == 9.1 - if (![[[BPUtils getXcodeRuntimeVersion] substringToIndex:4] isEqualToString:@BP_DEFAULT_XCODE_VERSION]) { - BP_SET_ERROR(errPtr, @"ERROR: Bluepill runtime version %s and compile time version %s are mismatched\n", - [[[BPUtils getXcodeRuntimeVersion] substringToIndex:4] UTF8String], [@BP_DEFAULT_XCODE_VERSION UTF8String]); - return NO; + if (!self.unsafeSkipXcodeVersionCheck) { + if ([xcodeVersion rangeOfString:@BP_DEFAULT_XCODE_VERSION].location == NSNotFound) { + BP_SET_ERROR(errPtr, @"ERROR: Invalid Xcode version:\n%s;\nOnly %s is supported\n", [xcodeVersion UTF8String], BP_DEFAULT_XCODE_VERSION); + return NO; + } + + // Check if Bluepill compile time Xcode version is matched with Bluepill runtime Xcode version + // This check prevents Bluepill compiled with Xcode 8 running on host installed with Xcode 9 + // Only compare major and minor version version Eg. 11.2 ~ 11.2.1 but 11.2 <> 11.3 + if (![[[BPUtils getXcodeRuntimeVersion] substringToIndex:4] isEqualToString:@BP_DEFAULT_XCODE_VERSION]) { + BP_SET_ERROR(errPtr, @"ERROR: Bluepill runtime version %s and compile time version %s are mismatched\n", + [[[BPUtils getXcodeRuntimeVersion] substringToIndex:4] UTF8String], [@BP_DEFAULT_XCODE_VERSION UTF8String]); + return NO; + } } if (self.deleteSimUDID) { @@ -697,23 +713,25 @@ - (BOOL)validateConfigWithError:(NSError *__autoreleasing *)errPtr { return NO; } - if (self.appBundlePath && (![[NSFileManager defaultManager] fileExistsAtPath: self.appBundlePath isDirectory:&isdir] || !isdir)) { - BP_SET_ERROR(errPtr, @"%@ not found.", self.appBundlePath); - return NO; - } - NSString *path = [self.appBundlePath stringByAppendingPathComponent:@"Info.plist"]; - NSDictionary *dic = [NSDictionary dictionaryWithContentsOfFile:path]; + if (self.appBundlePath) { + if (![[NSFileManager defaultManager] fileExistsAtPath: self.appBundlePath isDirectory:&isdir] || !isdir) { + BP_SET_ERROR(errPtr, @"%@ not found.", self.appBundlePath); + return NO; + } + NSString *path = [self.appBundlePath stringByAppendingPathComponent:@"Info.plist"]; + NSDictionary *dic = [NSDictionary dictionaryWithContentsOfFile:path]; - if (!dic) { - BP_SET_ERROR(errPtr, @"Could not read %@, perhaps you forgot to run xcodebuild build-for-testing?", path); - } + if (!dic) { + BP_SET_ERROR(errPtr, @"Could not read %@, perhaps you forgot to run xcodebuild build-for-testing?", path); + } - NSString *platform = [dic objectForKey:@"DTPlatformName"]; - if (platform && ![platform isEqualToString:@"iphonesimulator"]) { - BP_SET_ERROR(errPtr, @"Wrong platform in %@. Expected 'iphonesimulator', found '%@'", self.appBundlePath, platform); - return NO; + NSString *platform = [dic objectForKey:@"DTPlatformName"]; + if (platform && ![platform isEqualToString:@"iphonesimulator"]) { + BP_SET_ERROR(errPtr, @"Wrong platform in %@. Expected 'iphonesimulator', found '%@'", self.appBundlePath, platform); + return NO; + } } - + if (self.outputDirectory) { if ([[NSFileManager defaultManager] fileExistsAtPath:self.outputDirectory isDirectory:&isdir]) { if (!isdir) { @@ -759,6 +777,23 @@ - (BOOL)validateConfigWithError:(NSError *__autoreleasing *)errPtr { } } + if (self.videosDirectory) { + if ([[NSFileManager defaultManager] fileExistsAtPath:self.videosDirectory isDirectory:&isdir]) { + if (!isdir) { + BP_SET_ERROR(errPtr, @"%@ is not a directory.", self.videosDirectory); + return NO; + } + } else { + // create the directory + if (![[NSFileManager defaultManager] createDirectoryAtPath:self.videosDirectory + withIntermediateDirectories:YES + attributes:nil + error:errPtr]) { + return NO; + } + } + } + if (self.simulatorPreferencesFile) { if ([[NSFileManager defaultManager] fileExistsAtPath:self.simulatorPreferencesFile isDirectory:&isdir]) { if (isdir) { diff --git a/bp/src/BPConstants.h b/bp/src/BPConstants.h index fc2e2453..853ef55e 100644 --- a/bp/src/BPConstants.h +++ b/bp/src/BPConstants.h @@ -9,15 +9,17 @@ #import -#define BP_DEFAULT_RUNTIME "iOS 13.2" +#pragma mark - Version Constants +#define BP_DEFAULT_XCODE_VERSION "12.0" +#define BP_DEFAULT_RUNTIME "iOS 14.0" +#define BP_DEFAULT_BASE_SDK "14.0" + #define BP_DEFAULT_DEVICE_TYPE "iPhone 8" -#define BP_TM_PROTOCOL_VERSION 17 + #define BP_DAEMON_PROTOCOL_VERSION 26 -#define BP_DEFAULT_XCODE_VERSION "11.2" #define BP_MAX_PROCESSES_PERCENT 0.75 #define BP_TM_PROTOCOL_VERSION 17 - extern NSString * const kCFBundleIdentifier; extern NSString * const kOptionsArgumentsKey; extern NSString * const kOptionsEnvironmentKey; diff --git a/bp/src/BPExitStatus.h b/bp/src/BPExitStatus.h index 47573461..41f6110b 100644 --- a/bp/src/BPExitStatus.h +++ b/bp/src/BPExitStatus.h @@ -10,18 +10,18 @@ #import typedef NS_ENUM(NSInteger, BPExitStatus) { - BPExitStatusTestsAllPassed = 0, - BPExitStatusTestsFailed = 1, - BPExitStatusSimulatorCreationFailed = 2, - BPExitStatusSimulatorCrashed = 3, - BPExitStatusInstallAppFailed = 4, - BPExitStatusLaunchAppFailed = 5, - BPExitStatusTestTimeout = 6, - BPExitStatusAppCrashed = 7, - BPExitStatusInterrupted = 8, - BPExitStatusSimulatorDeleted = 9, - BPExitStatusUninstallAppFailed = 10, - BPExitStatusSimulatorReuseFailed = 11, + BPExitStatusAllTestsPassed = 0, + BPExitStatusTestsFailed = 1 << 0, + BPExitStatusSimulatorCreationFailed = 1 << 1, + BPExitStatusInstallAppFailed = 1 << 2, + BPExitStatusInterrupted = 1 << 3, + BPExitStatusSimulatorCrashed = 1 << 4, + BPExitStatusLaunchAppFailed = 1 << 5, + BPExitStatusTestTimeout = 1 << 6, + BPExitStatusAppCrashed = 1 << 7, + BPExitStatusSimulatorDeleted = 1 << 8, + BPExitStatusUninstallAppFailed = 1 << 9, + BPExitStatusSimulatorReuseFailed = 1 << 10 }; @protocol BPExitStatusProtocol diff --git a/bp/src/BPExitStatus.m b/bp/src/BPExitStatus.m index 1519254f..c066086e 100644 --- a/bp/src/BPExitStatus.m +++ b/bp/src/BPExitStatus.m @@ -13,15 +13,18 @@ @implementation BPExitStatusHelper -// Exit status to string -+ (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { ++ (NSString *)simpleExitStatus:(BPExitStatus)exitStatus { switch (exitStatus) { - case BPExitStatusTestsAllPassed: - return @"BPExitStatusTestsAllPassed"; + case BPExitStatusAllTestsPassed: + return @"BPExitStatusAllTestsPassed"; case BPExitStatusTestsFailed: return @"BPExitStatusTestsFailed"; case BPExitStatusSimulatorCreationFailed: return @"BPExitStatusSimulatorCreationFailed"; + case BPExitStatusInstallAppFailed: + return @"BPExitStatusInstallAppFailed"; + case BPExitStatusInterrupted: + return @"BPExitStatusInterrupted"; case BPExitStatusSimulatorCrashed: return @"BPExitStatusSimulatorCrashed"; case BPExitStatusLaunchAppFailed: @@ -30,17 +33,30 @@ + (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { return @"BPExitStatusTestTimeout"; case BPExitStatusAppCrashed: return @"BPExitStatusAppCrashed"; - case BPExitStatusInstallAppFailed: - return @"BPExitStatusInstallAppFailed"; - case BPExitStatusInterrupted: - return @"BPExitStatusInterrupted"; case BPExitStatusSimulatorDeleted: return @"BPExitStatusSimulatorDeleted"; + case BPExitStatusUninstallAppFailed: + return @"BPExitStatusUninstallAppFailed"; case BPExitStatusSimulatorReuseFailed: return @"BPExitStatusSimulatorReuseFailed"; default: - return @"UNKNOWN_BPEXITSTATUS"; + return [NSString stringWithFormat:@"UNKNOWN_BPEXITSTATUS - %ld", (long)exitStatus]; + } +} + +// Exit status to string ++ (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { + if (exitStatus == BPExitStatusAllTestsPassed) + return @"BPExitStatusAllTestsPassed"; + + NSString *exitStatusString = @""; + while (exitStatus > 0) { + BPExitStatus prevExitStatus = exitStatus; + exitStatus = exitStatus & (exitStatus - 1); + exitStatusString = [exitStatusString stringByAppendingFormat:@"%@ ", [self simpleExitStatus:(prevExitStatus - exitStatus)]]; } + + return [exitStatusString stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]]; } @end diff --git a/bp/src/BPSimulator.m b/bp/src/BPSimulator.m index c65860e0..6af1d1a1 100644 --- a/bp/src/BPSimulator.m +++ b/bp/src/BPSimulator.m @@ -498,6 +498,9 @@ - (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCom if (self.config.testing_HangAppOnLaunch) { mutableAppLaunchEnv[@"_BP_TEST_HANG_ON_LAUNCH"] = @"YES"; } + if (self.config.testing_ExecutionPlan) { + mutableAppLaunchEnv[@"_BP_TEST_EXECUTION_PLAN"] = self.config.testing_ExecutionPlan; + } appLaunchEnvironment = [mutableAppLaunchEnv copy]; NSDictionary *options = @{ kOptionsArgumentsKey: argsAndEnv[@"args"], @@ -622,7 +625,7 @@ - (BOOL)didTestStart { - (BOOL)checkFinished { if ([self.monitor isExecutionComplete]) { switch ([self.monitor exitStatus]) { - case BPExitStatusTestsAllPassed: + case BPExitStatusAllTestsPassed: case BPExitStatusTestsFailed: return YES; default: diff --git a/bp/src/BPStats.m b/bp/src/BPStats.m index 90f84ab8..5bda6529 100644 --- a/bp/src/BPStats.m +++ b/bp/src/BPStats.m @@ -240,7 +240,7 @@ -(unsigned long)bundleID { } - (NSString *)resultToCname:(NSString *)result { - if ([result isEqualToString:@"PASSED"] || [result isEqualToString:@"BPExitStatusTestsAllPassed"]) { + if ([result isEqualToString:@"PASSED"] || [result isEqualToString:@"BPExitStatusAllTestsPassed"]) { return @"good"; } else if ([result isEqualToString:@"FAILED"] || [result isEqualToString:@"BPExitStatusTestsFailed"]) { return @"bad"; diff --git a/bp/src/BPTestBundleConnection.h b/bp/src/BPTestBundleConnection.h index 7169c3bd..5bca100a 100644 --- a/bp/src/BPTestBundleConnection.h +++ b/bp/src/BPTestBundleConnection.h @@ -7,6 +7,7 @@ // #import +#import "BPExecutionContext.h" #import "BPSimulator.h" // This is a small subset of XCTestManager_IDEInterface protocol @@ -16,9 +17,10 @@ @interface BPTestBundleConnection : NSObject @property (nonatomic, strong) BPConfiguration *config; +@property (nonatomic, strong) BPExecutionContext *context; @property (nonatomic, strong) BPSimulator *simulator; @property (nonatomic, copy) void (^completionBlock)(NSError *, pid_t); -- (instancetype)initWithDevice:(BPSimulator *)device andInterface:(id)interface; +- (instancetype)initWithContext:(BPExecutionContext *)context andInterface:(id)interface; - (void)connectWithTimeout:(NSTimeInterval)timeout; - (void)startTestPlan; @end diff --git a/bp/src/BPTestBundleConnection.m b/bp/src/BPTestBundleConnection.m index b32ca80e..32da6d7a 100644 --- a/bp/src/BPTestBundleConnection.m +++ b/bp/src/BPTestBundleConnection.m @@ -49,15 +49,19 @@ @interface BPTestBundleConnection() @property (nonatomic, strong) dispatch_queue_t queue; @property (nonatomic, strong) NSString *bundleID; @property (nonatomic, assign) pid_t appProcessPID; +@property (nonatomic, nullable) NSTask *recordVideoTask; +//@property (nonatomic, nullable) NSPipe *recordVideoPipe; + @end @implementation BPTestBundleConnection -- (instancetype)initWithDevice:(BPSimulator *)simulator andInterface:(id)interface { +- (instancetype)initWithContext:(BPExecutionContext *)context andInterface:(id)interface { self = [super init]; if (self) { - self.simulator = simulator; + self.context = context; + self.simulator = context.runner; self.interface = interface; self.queue = dispatch_queue_create("com.linkedin.bluepill.connection.queue", DISPATCH_QUEUE_PRIORITY_DEFAULT); } @@ -82,6 +86,8 @@ - (void)connect { DTXTransport *transport = [self connectTransport]; DTXConnection *connection = [[objc_lookUpClass("DTXConnection") alloc] initWithTransport:transport]; [connection registerDisconnectHandler:^{ + // This is called when the task is abruptly terminated (e.g. if the test times out) + [self stopVideoRecording:YES]; [BPUtils printInfo:INFO withString:@"DTXConnection disconnected."]; }]; [connection @@ -212,6 +218,63 @@ - (id)_XCT_initializationForUITestingDidFailWithError:(NSError *__strong)errPtr return nil; } +#pragma mark - Video Recording + +static inline NSString* getVideoPath(NSString *directory, NSString *testClass, NSString *method, NSInteger attemptNumber) +{ + return [NSString stringWithFormat:@"%@/%@__%@__%ld.mp4", directory, testClass, method, (long)attemptNumber]; +} + +- (BOOL)shouldRecordVideo { + return self.context.config.videosDirectory.length > 0; +} + +- (void)startVideoRecordingForTestClass:(NSString *)testClass method:(NSString *)method +{ + [self stopVideoRecording:YES]; + NSString *videoFileName = getVideoPath(self.context.config.videosDirectory, testClass, method, self.context.attemptNumber); + NSString *command = [NSString stringWithFormat:@"xcrun simctl io %@ recordVideo --force %@", [self.simulator UDID], videoFileName]; + NSTask *task = [BPUtils buildShellTaskForCommand:command]; + self.recordVideoTask = task; + [task launch]; + [BPUtils printInfo:INFO withString:@"Started recording video to %@", videoFileName]; + [BPUtils printInfo:DEBUGINFO withString:@"Started recording video task with pid %d and command: %@", [task processIdentifier], [BPUtils getCommandStringForTask:task]]; +} + +- (void)stopVideoRecording:(BOOL)forced +{ + NSTask *task = self.recordVideoTask; + if (task == nil) { + if (!forced) { + [BPUtils printInfo:ERROR withString: @"Tried to end video recording task normally, but there was no task."]; + } + return; + } + + if (forced) { + [BPUtils printInfo:ERROR withString: @"Found dangling video recording task. Stopping it."]; + } + + if (![task isRunning]) { + [BPUtils printInfo:ERROR withString:@"Video task exists but it was already terminated with status %d", [task terminationStatus]]; + } + + [BPUtils printInfo:INFO withString:@"Stopping recording video."]; + [BPUtils printInfo:DEBUGINFO withString:@"Stopping video recording task with pid %d and command: %@", [task processIdentifier], [BPUtils getCommandStringForTask:task]]; + [task interrupt]; + [task waitUntilExit]; + + if ([task terminationStatus] != 0) { + [BPUtils printInfo:ERROR withString:@"Video task was interrupted, but exited with non-zero status %d", [task terminationStatus]]; + } + + NSString *filePath = [[task arguments].lastObject componentsSeparatedByString:@" "].lastObject; + if (![[NSFileManager defaultManager] fileExistsAtPath:filePath]) { + [BPUtils printInfo:ERROR withString:@"Video recording file missing, expected at path %@!", filePath]; + } + self.recordVideoTask = nil; +} + #pragma mark - XCTestManager_IDEInterface protocol #pragma mark Process Launch Delegation @@ -255,7 +318,7 @@ - (id)_XCT_getProgressForLaunch:(id)token { - (id)_XCT_terminateProcess:(id)token { NSError *error; - kill(self.appProcessPID, SIGTERM); + kill(self.appProcessPID, SIGINT); DTXRemoteInvocationReceipt *receipt = [objc_lookUpClass("DTXRemoteInvocationReceipt") new]; [receipt invokeCompletionWithReturnValue:token error:error]; [BPUtils printInfo:DEBUGINFO withString:@"BPTestBundleConnection_XCT_terminateProcess with token %@", token]; @@ -277,6 +340,9 @@ - (id)_XCT_testSuite:(NSString *)tests didStartAt:(NSString *)time { - (id)_XCT_testCaseDidStartForTestClass:(NSString *)testClass method:(NSString *)method { [BPUtils printInfo:DEBUGINFO withString:@"BPTestBundleConnection_XCT_testCaseDidStartForTestClass: %@ and method: %@", testClass, method]; + if ([self shouldRecordVideo]) { + [self startVideoRecordingForTestClass:testClass method:method]; + } return nil; } @@ -298,12 +364,18 @@ - (id)_XCT_logMessage:(NSString *)message { - (id)_XCT_testCaseDidFinishForTestClass:(NSString *)testClass method:(NSString *)method withStatus:(NSString *)statusString duration:(NSNumber *)duration { [BPUtils printInfo:DEBUGINFO withString: @"BPTestBundleConnection_XCT_testCaseDidFinishForTestClass: %@, method: %@, withStatus: %@, duration: %@", testClass, method, statusString, duration]; + if ([self shouldRecordVideo]) { + [self stopVideoRecording:NO]; + } return nil; } - (id)_XCT_testSuite:(NSString *)arg1 didFinishAt:(NSString *)time runCount:(NSNumber *)count withFailures:(NSNumber *)failureCount unexpected:(NSNumber *)unexpectedCount testDuration:(NSNumber *)testDuration totalDuration:(NSNumber *)totalTime { [BPUtils printInfo:DEBUGINFO withString: @"BPTestBundleConnection_XCT_testSuite: %@, didFinishAt: %@, runCount: %@, withFailures: %@, unexpectedCount: %@, testDuration: %@, totalDuration: %@", arg1, time, count, failureCount, unexpectedCount, testDuration, totalTime]; + if ([self shouldRecordVideo]) { + [self stopVideoRecording:YES]; + } return nil; } diff --git a/bp/src/BPUtils.h b/bp/src/BPUtils.h index f80b3342..ef250c9a 100644 --- a/bp/src/BPUtils.h +++ b/bp/src/BPUtils.h @@ -125,6 +125,29 @@ typedef NS_ENUM(int, BPKind) { * @return return the shell output */ + (NSString *)runShell:(NSString *)command; + +/*! + * @discussion builds a task to run a shell command + * @param command the shell command the task should run + * @return an NSTask that will run the provided command. + */ ++ (NSTask *)buildShellTaskForCommand:(NSString *)command; + +/*! + * @discussion builds a task to run a shell command, pointing stdout and stderr to the provided pipe + * @param command the shell command the task should run + * @param pipe the pipe that stdout and stderr will be pointed to, so the caller can handle the output. + * @return an NSTask that will run the provided command. + */ ++ (NSTask *)buildShellTaskForCommand:(NSString *)command withPipe:(NSPipe *)pipe; + +/*! + * @discussion builds a user readable representation of the command that a task is configured to run + * @param task to get command from + * @return a user readable string of the task's command +*/ ++ (NSString *)getCommandStringForTask:(NSTask *)task; + + (NSString *)getXcodeRuntimeVersion; typedef BOOL (^BPRunBlock)(void); diff --git a/bp/src/BPUtils.m b/bp/src/BPUtils.m index c6e36e56..c9fbf957 100644 --- a/bp/src/BPUtils.m +++ b/bp/src/BPUtils.m @@ -189,29 +189,40 @@ + (BOOL)isStdOut:(NSString *)fileName { + (NSString *)runShell:(NSString *)command { NSAssert(command, @"Command should not be nil"); - NSTask *task = [[NSTask alloc] init]; - NSData *data; - task.launchPath = @"/bin/sh"; - task.arguments = @[@"-c", command]; NSPipe *pipe = [[NSPipe alloc] init]; - task.standardError = pipe; - task.standardOutput = pipe; + NSTask *task = [BPUtils buildShellTaskForCommand:command withPipe:pipe]; + NSAssert(task, @"task should not be nil"); NSFileHandle *fh = pipe.fileHandleForReading; - if (task) { - [task launch]; - } else { - NSAssert(task, @"task should not be nil"); - } - if (fh) { - data = [fh readDataToEndOfFile]; - } else { - NSAssert(task, @"fh should not be nil"); - } + NSAssert(fh, @"fh should not be nil"); + + [task launch]; + NSData *data = [fh readDataToEndOfFile]; [task waitUntilExit]; NSString *result = [[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] stringByTrimmingCharactersInSet:[NSCharacterSet newlineCharacterSet]]; return result; } ++ (NSTask *)buildShellTaskForCommand:(NSString *)command { + return [BPUtils buildShellTaskForCommand:command withPipe: nil]; +} + ++ (NSTask *)buildShellTaskForCommand:(NSString *)command withPipe:(NSPipe *)pipe { + NSAssert(command, @"Command should not be nil"); + NSTask *task = [[NSTask alloc] init]; + task.launchPath = @"/bin/sh"; + task.arguments = @[@"-c", command]; + if (pipe != nil) { + task.standardError = pipe; + task.standardOutput = pipe; + } + NSAssert(task, @"task should not be nil"); + return task; +} + ++ (NSString *)getCommandStringForTask:(NSTask *)task { + return [NSString stringWithFormat:@"%@ %@", [task launchPath], [[task arguments] componentsJoinedByString:@" "]]; +} + + (BOOL)runWithTimeOut:(NSTimeInterval)timeout until:(BPRunBlock)block { if (!block) { return NO; @@ -379,7 +390,6 @@ + (NSDictionary *)loadSimpleJsonFile:(NSString *)filePath NSDictionary *testsToRunByFilePath = [BPUtils getTestsToRunByFilePathWithConfig:config andXCTestFiles:xcTestFiles]; for(NSString *filePath in testsToRunByFilePath) { - NSLog(@"filePath=%@ Tests to run in this filePath=%@", filePath, [testsToRunByFilePath objectForKey:filePath]); NSSet *bundleTestsToRun = [testsToRunByFilePath objectForKey:filePath]; double __block testBundleExecutionTime = 0.0; [bundleTestsToRun enumerateObjectsUsingBlock:^(id _Nonnull test, BOOL * _Nonnull stop) { @@ -419,7 +429,6 @@ + (double)getTotalTimeWithConfig:(BPConfiguration *)config NSDictionary *testsToRunByFilePath = [BPUtils getTestsToRunByFilePathWithConfig:config andXCTestFiles:xcTestFiles]; for(NSString *filePath in testsToRunByFilePath) { - NSLog(@"filePath=%@ Tests to run in this filePath=%@", filePath, [testsToRunByFilePath objectForKey:filePath]); NSSet *bundleTestsToRun = [testsToRunByFilePath objectForKey:filePath]; double __block testBundleExecutionTime = 0.0; [bundleTestsToRun enumerateObjectsUsingBlock:^(id _Nonnull test, BOOL * _Nonnull stop) { diff --git a/bp/src/BPXCTestFile.h b/bp/src/BPXCTestFile.h index 84c373a9..336be782 100644 --- a/bp/src/BPXCTestFile.h +++ b/bp/src/BPXCTestFile.h @@ -21,6 +21,7 @@ @property (nonatomic, strong) NSString *UITargetAppPath; @property (nonatomic, strong) NSArray *skipTestIdentifiers; @property (nonatomic, strong) NSNumber *estimatedExecutionTime; +@property (nonatomic, strong) NSDictionary *dependencies; // All test classes in the test bundle @property (nonatomic, strong) NSArray *testClasses; diff --git a/bp/src/BPXCTestFile.m b/bp/src/BPXCTestFile.m index 56db07c9..7bcaf7e9 100644 --- a/bp/src/BPXCTestFile.m +++ b/bp/src/BPXCTestFile.m @@ -11,6 +11,7 @@ #import "BPConstants.h" #import "BPTestClass.h" #import "BPUtils.h" +#import "SimulatorHelper.h" @implementation BPXCTestFile @@ -162,6 +163,16 @@ + (instancetype)BPXCTestFileFromDictionary:(NSDictionary *)dict if (skipTestIdentifiers) { xcTestFile.skipTestIdentifiers = [[NSArray alloc] initWithArray:skipTestIdentifiers]; } + NSArray *dependencies = [dict objectForKey:@"DependentProductPaths"]; + if (dependencies) { + NSMutableDictionary *dependenciesWithBundleIDs = [NSMutableDictionary dictionary]; + for (NSString *dependency in dependencies) { + NSString *expandedDependency = [dependency stringByReplacingOccurrencesOfString:TESTROOT withString:testRoot]; + NSString *bundleID = [SimulatorHelper bundleIdForPath:expandedDependency]; + dependenciesWithBundleIDs[bundleID] = expandedDependency; + } + xcTestFile.dependencies = dependenciesWithBundleIDs; + } return xcTestFile; } @@ -176,6 +187,7 @@ + (instancetype)BPXCTestFileFromBPTestPlan:(BPTestPlan*)testPlan withError:errPtr]; xcTestFile.name = name; xcTestFile.environmentVariables = testPlan.environment; + xcTestFile.dependencies = testPlan.dependencies; NSMutableArray *args = [[NSMutableArray alloc] initWithCapacity:testPlan.arguments.count * 2]; for (NSString *key in xcTestFile.commandLineArguments) { @@ -231,6 +243,7 @@ - (id)copyWithZone:(NSZone *)zone { copy.testClasses = self.testClasses; copy.commandLineArguments = self.commandLineArguments; copy.environmentVariables = self.environmentVariables; + copy.dependencies = self.dependencies; copy.testHostPath = self.testHostPath; copy.testHostBundleIdentifier = self.testHostBundleIdentifier; copy.testBundlePath= self.testBundlePath; diff --git a/bp/src/Bluepill.m b/bp/src/Bluepill.m index ac88364f..d9aa58af 100644 --- a/bp/src/Bluepill.m +++ b/bp/src/Bluepill.m @@ -117,14 +117,17 @@ - (void)beginWithContext:(BPExecutionContext *)context { // Retry from the beginning (default) or failed tests only if onlyRetryFailed is true - (void)retry { - // There were test failures. If our failure tolerance is 0, then we're good with that. - if (self.failureTolerance == 0) { + // There were test failures. Check if it can be retried. + if (![self canRetryOnError] || self.failureTolerance <= 0) { // If there is no more retries, set the final exitCode to current context's exitCode - self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus; + self.finalExitStatus |= self.context.finalExitStatus; + [BPUtils printInfo:ERROR withString:@"No retries left. Giving up."]; [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; return; } + // Resetting the failed bit since the test is being retried + self.context.finalExitStatus &= ~self.context.exitStatus; [self.context.parser cleanup]; // Otherwise, reduce our failure tolerance count and retry self.failureTolerance -= 1; @@ -153,40 +156,40 @@ - (void)retry { // - BPExitStatusUninstallAppFailed // - BPExitStatusLaunchAppFailed - (void)recover { - // If error retry reach to the max, then return - if (self.retries == [self.config.errorRetriesCount integerValue]) { - self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus; - [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; - self.exitLoop = YES; - [BPUtils printInfo:ERROR withString:@"Too many retries have occurred. Giving up."]; - return; - } - - [self.context.parser cleanup]; - // If we're not retrying only failed tests, we need to get rid of our saved tests, so that we re-execute everything. Recopy config. - if (self.executionConfigCopy.onlyRetryFailed == NO) { - self.executionConfigCopy = [self.config copy]; - } - // Increment the retry count - self.retries += 1; - - // Log some useful information to the log - [BPUtils printInfo:INFO withString:@"Exit Status: %@", [BPExitStatusHelper stringFromExitStatus:self.context.exitStatus]]; - [BPUtils printInfo:INFO withString:@"Failure Tolerance: %lu", self.failureTolerance]; - [BPUtils printInfo:INFO withString:@"Retry count: %lu", self.retries]; - - // Then start again from the beginning - [BPUtils printInfo:INFO withString:@"Recovering from tooling problem"]; - NEXT([self begin]); + // If error retry reach to the max, then return + if (![self canRetryOnError]) { + self.finalExitStatus |= self.context.finalExitStatus; + [BPUtils printInfo:ERROR withString:@"No retries left. Giving up."]; + [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; + self.exitLoop = YES; + return; + } + + [self.context.parser cleanup]; + // If we're not retrying only failed tests, we need to get rid of our saved tests, so that we re-execute everything. Recopy config. + if (self.executionConfigCopy.onlyRetryFailed == NO) { + self.executionConfigCopy = [self.config copy]; + } + // Increment the retry count + self.retries += 1; + + // Log some useful information to the log + [BPUtils printInfo:INFO withString:@"Exit Status: %@", [BPExitStatusHelper stringFromExitStatus:self.context.exitStatus]]; + [BPUtils printInfo:INFO withString:@"Failure Tolerance: %lu", self.failureTolerance]; + [BPUtils printInfo:INFO withString:@"Retry count: %lu", self.retries]; + + // Then start again from the beginning + [BPUtils printInfo:INFO withString:@"Recovering from tooling problem"]; + NEXT([self begin]); } // Proceed to next test case - (void)proceed { - if (self.retries == [self.config.errorRetriesCount integerValue]) { - self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus; + if (![self canRetryOnError]) { + self.finalExitStatus |= self.context.finalExitStatus; + [BPUtils printInfo:ERROR withString:@"No retries left. Giving up."]; [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; - [BPUtils printInfo:ERROR withString:@"Too many retries have occurred. Giving up."]; return; } self.retries += 1; @@ -194,7 +197,8 @@ - (void)proceed { [BPUtils printInfo:INFO withString:@"Failure Tolerance: %lu", self.failureTolerance]; [BPUtils printInfo:INFO withString:@"Retry count: %lu", self.retries]; self.context.attemptNumber = self.retries + 1; // set the attempt number - self.context.exitStatus = BPExitStatusTestsAllPassed; // reset exitStatus + self.context.exitStatus = BPExitStatusAllTestsPassed; // reset exitStatus + [BPUtils printInfo:INFO withString:@"Proceeding to next test"]; NEXT([self beginWithContext:self.context]); } @@ -428,7 +432,7 @@ - (void)connectTestBundleAndTestDaemonWithContext:(BPExecutionContext *)context // If the isTestRunnerContext is flipped on, don't connect testbundle again. return; } - BPTestBundleConnection *bConnection = [[BPTestBundleConnection alloc] initWithDevice:context.runner andInterface:self]; + BPTestBundleConnection *bConnection = [[BPTestBundleConnection alloc] initWithContext:context andInterface:self]; bConnection.simulator = context.runner; bConnection.config = self.config; @@ -502,13 +506,13 @@ - (void)runnerCompletedWithContext:(BPExecutionContext *)context { // If we crashed, we need to retry [self deleteSimulatorWithContext:context andStatus:BPExitStatusSimulatorCrashed]; } else if (self.config.keepSimulator - && (context.runner.exitStatus == BPExitStatusTestsAllPassed + && (context.runner.exitStatus == BPExitStatusAllTestsPassed || context.runner.exitStatus == BPExitStatusTestsFailed)) { context.exitStatus = [context.runner exitStatus]; NEXT([self finishWithContext:context]); } else { // If the tests failed, save as much debugging info as we can. XXX: Put this behind a flag - if (context.runner.exitStatus != BPExitStatusTestsAllPassed && _config.saveDiagnosticsOnError) { + if (context.runner.exitStatus != BPExitStatusAllTestsPassed && _config.saveDiagnosticsOnError) { [BPUtils printInfo:INFO withString:@"Saving Diagnostics for Debugging"]; [BPUtils saveDebuggingDiagnostics:_config.outputDirectory]; } @@ -578,36 +582,19 @@ - (void)deleteSimulatorOnlyTaskWithContext:(BPExecutionContext *)context { } } -- (BOOL)hasRemainingTestsInContext:(BPExecutionContext *)context { - // Make sure we're not doing unnecessary work on the next run. - NSMutableSet *testsRemaining = [[NSMutableSet alloc] initWithArray:context.config.allTestCases]; - NSSet *testsToSkip = [[NSSet alloc] initWithArray:context.config.testCasesToSkip]; - [testsRemaining minusSet:testsToSkip]; - return ([testsRemaining count] > 0); -} - /** Scenarios: - 1. crash/time out and proceed passes -> Crash/Timeout - 2. crash/time out and retry passes -> AllPass + 1. crash and proceed passes -> Crash + 2. time out and retry passes -> AllPass 3. failure and retry passes -> AllPass 4. happy all pass -> AllPassed 5. failure and still fails -> TestFailed */ - (void)finishWithContext:(BPExecutionContext *)context { - - // Because BPExitStatusTestsAllPassed is 0, we must check it explicitly against - // the run rather than the aggregate bitmask built with finalExitStatus - - if (![self hasRemainingTestsInContext:context] && (context.attemptNumber <= [context.config.errorRetriesCount integerValue])) { - [BPUtils printInfo:INFO withString:@"No more tests to run."]; - [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; - // TODO: Temporarily disabling the fix from PR#338 while the issue is being investigated - // self.finalExitStatus = context.exitStatus; - self.finalExitStatus = context.finalExitStatus | context.exitStatus; - self.exitLoop = YES; - return; - } + context.finalExitStatus |= context.exitStatus; + [BPUtils printInfo:INFO withString:@"Attempt's Exit Status: %@, Bundle exit status: %@", + [BPExitStatusHelper stringFromExitStatus:context.exitStatus], + [BPExitStatusHelper stringFromExitStatus:context.finalExitStatus]]; switch (context.exitStatus) { // BP exit handler @@ -615,23 +602,15 @@ - (void)finishWithContext:(BPExecutionContext *)context { self.exitLoop = YES; return; - // MARK: Test suite completed - // If there is no test crash/time out, we retry from scratch case BPExitStatusTestsFailed: NEXT([self retry]); return; - case BPExitStatusTestsAllPassed: - // Check previous result - if (context.finalExitStatus != BPExitStatusTestsAllPassed) { - // If there is a test crashed/timed out before, retry from scratch - NEXT([self retry]); - } else { - // If it is a real all pass, exit - self.exitLoop = YES; - return; - } + case BPExitStatusAllTestsPassed: + // Time to exit + self.finalExitStatus |= BPExitStatusAllTestsPassed; + self.exitLoop = YES; return; // Recover from scratch if there is tooling failure. @@ -645,21 +624,33 @@ - (void)finishWithContext:(BPExecutionContext *)context { // If it is test hanging or crashing, we set final exit code of current context and proceed. case BPExitStatusTestTimeout: - context.finalExitStatus = BPExitStatusTestTimeout; + if (!self.config.onlyRetryFailed) { + self.finalExitStatus |= context.exitStatus; + } NEXT([self proceed]); return; + case BPExitStatusAppCrashed: - context.finalExitStatus = BPExitStatusAppCrashed; + if (!self.config.retryAppCrashTests) { + // Crashed test is considered fatal when retry is disabled + self.finalExitStatus |= context.exitStatus; + } NEXT([self proceed]); return; + case BPExitStatusSimulatorDeleted: case BPExitStatusSimulatorReuseFailed: - self.finalExitStatus = context.exitStatus; - [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; + self.finalExitStatus |= context.finalExitStatus; + [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", + __FILE__, __LINE__, + [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; return; } - + [BPUtils printInfo:ERROR withString:@"%s:%d YOU SHOULDN'T BE HERE. exitStatus = %@, finalExitStatus = %@", + __FILE__, __LINE__, + [BPExitStatusHelper stringFromExitStatus:context.exitStatus], + [BPExitStatusHelper stringFromExitStatus:context.finalExitStatus]]; } // MARK: Helpers @@ -676,6 +667,22 @@ - (BPSimulator *)test_simulator { return self.context.runner; } +- (BOOL)canRetryOnError { + NSInteger maxErrorRetryCount = [self.config.errorRetriesCount integerValue]; + if (self.retries < maxErrorRetryCount) { + return true; + } + + if (self.retries > maxErrorRetryCount) { + // If retries strictly exceeds the max error retry, then we must have incremented it beyond the limit somehow. + // It is safe to halt retries here, but log to alert unexpected behavior. + [BPUtils printInfo:ERROR withString:@"Current retry count (%d) exceeded maximum retry count (%d)!", + (int) self.retries, + (int) maxErrorRetryCount]; + } + return false; +} + int __line; NSString *__function; NSString *__from; diff --git a/bp/src/SimulatorHelper.m b/bp/src/SimulatorHelper.m index ae688f69..0731d20a 100644 --- a/bp/src/SimulatorHelper.m +++ b/bp/src/SimulatorHelper.m @@ -63,13 +63,20 @@ + (NSDictionary *)appLaunchEnvironmentWithBundleID:(NSString *)hostBundleID device:(SimDevice *)device config:(BPConfiguration *)config { NSString *hostAppExecPath = [SimulatorHelper executablePathforPath:config.appBundlePath]; - NSString *testSimulatorFrameworkPath = [[hostAppExecPath stringByDeletingLastPathComponent] stringByDeletingLastPathComponent]; + NSString *hostAppPath = [hostAppExecPath stringByDeletingLastPathComponent]; + NSString *testSimulatorFrameworkPath = [hostAppPath stringByDeletingLastPathComponent]; + NSString *libXCTestBundleInjectPath = [[hostAppPath stringByAppendingPathComponent:@"Frameworks"] stringByAppendingPathComponent:@"libXCTestBundleInject.dylib"]; + NSString *libXCTestBundleInjectValue = libXCTestBundleInjectPath; + if (![NSFileManager.defaultManager fileExistsAtPath:libXCTestBundleInjectPath]) { + [BPUtils printInfo:DEBUGINFO withString:@"Not injecting libXCTestBundleInject dylib because it was not found in the app host bundle at path: %@", libXCTestBundleInjectValue]; + libXCTestBundleInjectValue = @""; + } NSMutableDictionary *environment = [@{ @"DYLD_FALLBACK_FRAMEWORK_PATH" : [NSString stringWithFormat:@"%@/Library/Frameworks:%@/Platforms/iPhoneSimulator.platform/Developer/Library/Frameworks", config.xcodePath, config.xcodePath], @"DYLD_FALLBACK_LIBRARY_PATH" : [NSString stringWithFormat:@"%@/Platforms/iPhoneSimulator.platform/Developer/usr/lib", config.xcodePath], - @"DYLD_INSERT_LIBRARIES" : [NSString stringWithFormat:@"%@/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/Developer/usr/lib/libXCTTargetBootstrapInject.dylib", config.xcodePath], + @"DYLD_INSERT_LIBRARIES" : [NSString stringWithFormat:@"%@/Platforms/iPhoneSimulator.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/Developer/usr/lib/libXCTTargetBootstrapInject.dylib", config.xcodePath], @"DYLD_LIBRARY_PATH" : [NSString stringWithFormat:@"%@/Platforms/iPhoneSimulator.platform/Developer/Library/Frameworks", config.xcodePath], - @"DYLD_ROOT_PATH" : [NSString stringWithFormat:@"%@/Platforms/iPhoneOS.platform/Developer/Library/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot", config.xcodePath], + @"DYLD_ROOT_PATH" : [NSString stringWithFormat:@"%@/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot", config.xcodePath], @"NSUnbufferedIO" : @"1", @"OS_ACTIVITY_DT_MODE" : @"1", @"XCODE_DBG_XPC_EXCLUSIONS" : @"com.apple.dt.xctestSymbolicator", @@ -106,7 +113,7 @@ + (NSString *)testEnvironmentWithConfiguration:(BPConfiguration *)config { testHostPath = config.appBundlePath; NSString *bundleID = [self bundleIdForPath:config.appBundlePath]; - xctConfig.testApplicationDependencies = @{bundleID: config.appBundlePath}; + xctConfig.testApplicationDependencies = config.dependencies.count > 0 ? config.dependencies : @{bundleID: config.appBundlePath}; if (config.testRunnerAppPath) { xctConfig.targetApplicationBundleID = bundleID; diff --git a/bp/src/SimulatorMonitor.m b/bp/src/SimulatorMonitor.m index 6d42f11b..c8a5fad7 100644 --- a/bp/src/SimulatorMonitor.m +++ b/bp/src/SimulatorMonitor.m @@ -68,7 +68,7 @@ - (void)onAllTestsEnded { if (self.failureCount) { self.exitStatus = BPExitStatusTestsFailed; } else { - self.exitStatus = BPExitStatusTestsAllPassed; + self.exitStatus = BPExitStatusAllTestsPassed; } [[BPStats sharedStats] endTimer:ALL_TESTS withResult:[BPExitStatusHelper stringFromExitStatus: self.exitStatus]]; [BPUtils printInfo:INFO withString:@"All Tests Completed."]; @@ -195,7 +195,12 @@ - (void)onOutputReceived:(NSString *)output { NSString *testClass = (__self.currentClassName ?: __self.previousClassName); NSString *testName = (__self.currentTestName ?: __self.previousTestName); if (__self.testsState == Running) { - [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app.", testClass, testName]; + if (self.config.retryAppCrashTests) { + [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Configured to retry.", testClass, testName]; + } else { + [self updateExecutedTestCaseList:testName inClass:testClass]; + [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Retry disabled.", testClass, testName]; + } [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"]; } else { assert(__self.testsState == Idle); diff --git a/bp/tests/BPUtilsTests.m b/bp/tests/BPUtilsTests.m index 5a0b2165..9b0adf33 100644 --- a/bp/tests/BPUtilsTests.m +++ b/bp/tests/BPUtilsTests.m @@ -8,6 +8,7 @@ // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #import +#import "BPExitStatus.h" #import "BPUtils.h" #import "BPXCTestFile.h" #import "BPTestHelper.h" @@ -110,4 +111,64 @@ - (void) testTrailingParanthesesInTestNames { XCTAssert([testCasesWithParantheses count] == 0); } +- (void) testExitStatus { + BPExitStatus exitCode; + + exitCode = 0; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusAllTestsPassed"]); + exitCode = 1; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsFailed"]); + exitCode = 2; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCreationFailed"]); + exitCode = 4; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusInstallAppFailed"]); + exitCode = 8; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusInterrupted"]); + exitCode = 16; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCrashed"]); + exitCode = 32; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusLaunchAppFailed"]); + exitCode = 64; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestTimeout"]); + exitCode = 128; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusAppCrashed"]); + exitCode = 256; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorDeleted"]); + exitCode = 512; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusUninstallAppFailed"]); + exitCode = 1024; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorReuseFailed"]); + exitCode = 3; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsFailed BPExitStatusSimulatorCreationFailed"]); + exitCode = 192; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestTimeout BPExitStatusAppCrashed"]); + exitCode = 2048; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"UNKNOWN_BPEXITSTATUS - 2048"]); + exitCode = 2050; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCreationFailed UNKNOWN_BPEXITSTATUS - 2048"]); +} + +- (void) testBuildShellTaskForCommand_withoutPipe { + NSString *command = @"ls -al"; + NSTask *task = [BPUtils buildShellTaskForCommand:command]; + XCTAssertEqual(task.launchPath, @"/bin/sh"); + XCTAssertEqual(task.arguments.count, 2); + XCTAssertEqual(task.arguments[0], @"-c"); + XCTAssertEqual(task.arguments[1], command); + XCTAssertFalse(task.isRunning); +} + +- (void) testBuildShellTaskForCommand_withPipe { + NSString *command = @"ls -al"; + NSPipe *pipe = [[NSPipe alloc] init]; + NSTask *task = [BPUtils buildShellTaskForCommand:command withPipe: pipe]; + XCTAssertEqual(task.standardError, pipe); + XCTAssertEqual(task.standardOutput, pipe); + XCTAssertEqual(task.launchPath, @"/bin/sh"); + XCTAssertEqual(task.arguments.count, 2); + XCTAssertEqual(task.arguments[0], @"-c"); + XCTAssertEqual(task.arguments[1], command); + XCTAssertFalse(task.isRunning); +} + @end diff --git a/bp/tests/BluepillTests.m b/bp/tests/BluepillTests.m index f91cc2ca..e63d2ed0 100644 --- a/bp/tests/BluepillTests.m +++ b/bp/tests/BluepillTests.m @@ -35,19 +35,19 @@ @implementation BluepillTests - (void)setUp { [super setUp]; - + self.continueAfterFailure = NO; NSString *hostApplicationPath = [BPTestHelper sampleAppPath]; NSString *testBundlePath = [BPTestHelper sampleAppNegativeTestsBundlePath]; self.config = [[BPConfiguration alloc] initWithProgram:BP_SLAVE]; self.config.testBundlePath = testBundlePath; self.config.appBundlePath = hostApplicationPath; - self.config.stuckTimeout = @80; + self.config.stuckTimeout = @40; self.config.xcodePath = [BPUtils runShell:@"/usr/bin/xcode-select -print-path"]; self.config.runtime = @BP_DEFAULT_RUNTIME; self.config.repeatTestsCount = @1; self.config.errorRetriesCount = @0; - self.config.testCaseTimeout = @40; + self.config.testCaseTimeout = @20; self.config.deviceType = @BP_DEFAULT_DEVICE_TYPE; self.config.headlessMode = YES; self.config.videoPaths = @[[BPTestHelper sampleVideoPath]]; @@ -90,7 +90,10 @@ - (void)testAppThatCrashesOnLaunch { self.config.testing_CrashAppOnLaunch = YES; self.config.stuckTimeout = @3; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusAppCrashed, @"Expected: %ld Got: %ld", (long)BPExitStatusAppCrashed, (long)exitCode); + XCTAssert(exitCode == BPExitStatusAppCrashed, + @"Expected: %@ Got: %@", + [BPExitStatusHelper stringFromExitStatus:BPExitStatusAppCrashed], + [BPExitStatusHelper stringFromExitStatus:exitCode]); } - (void)testAppThatHangsOnLaunch { @@ -99,7 +102,10 @@ - (void)testAppThatHangsOnLaunch { self.config.testing_HangAppOnLaunch = YES; self.config.stuckTimeout = @3; BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusSimulatorCrashed); + XCTAssert(exitCode == BPExitStatusSimulatorCrashed, + @"Expected: %@ Got: %@", + [BPExitStatusHelper stringFromExitStatus:BPExitStatusSimulatorCrashed], + [BPExitStatusHelper stringFromExitStatus:exitCode]); } - (void)testRecoverSimulatorOnCrash { @@ -141,7 +147,7 @@ - (void)testRunningOnlyCertainTestcases { ]; BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); NSString *reportPath = [outputDir stringByAppendingPathComponent:@"TEST-BPSampleAppTests-1-results.xml"]; NSError *error; @@ -171,7 +177,7 @@ - (void)testRunningAndIgnoringCertainTestCases { self.config.testCasesToSkip = @[@"BPSampleAppTests/testCase173"]; BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); NSString *reportPath = [outputDir stringByAppendingPathComponent:@"TEST-BPSampleAppTests-1-results.xml"]; NSError *error; @@ -213,15 +219,14 @@ - (void)testReportWithAppCrashingAndRetryOnlyFailedTestsSet { NSString *tempDir = NSTemporaryDirectory(); NSError *error; NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppCrashingTestsSetTempDir", tempDir] withError:&error]; - // NSLog(@"output directory is %@", outputDir); self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @1; self.config.failureTolerance = @1; - self.config.onlyRetryFailed = YES; - + self.config.onlyRetryFailed = TRUE; + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssertTrue(exitCode == BPExitStatusAppCrashed); - + NSString *junitReportPath = [outputDir stringByAppendingPathComponent:@"TEST-BPSampleAppCrashingTests-1-results.xml"]; NSLog(@"JUnit file: %@", junitReportPath); NSString *expectedFilePath = [[[NSBundle bundleForClass:[self class]] resourcePath] stringByAppendingPathComponent:@"crash_tests_with_retry_attempt_1.xml"]; @@ -243,10 +248,10 @@ - (void)DISABLE_testAppCrashingAndRetryReportsCorrectExitCode { self.config.testing_crashOnAttempt = @1; self.config.errorRetriesCount = @2; self.config.failureTolerance = @1; - self.config.onlyRetryFailed = YES; + self.config.onlyRetryFailed = TRUE; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; - XCTAssertTrue(exitCode == BPExitStatusTestsAllPassed); + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); } - (void)testReportWithFatalErrorTestsSet { @@ -258,7 +263,7 @@ - (void)testReportWithFatalErrorTestsSet { NSLog(@"output directory is %@", outputDir); self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @2; - self.config.testCaseTimeout = @60; // make sure we don't time-out + self.config.testCaseTimeout = @30; // make sure we don't time-out BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssertTrue(exitCode == BPExitStatusAppCrashed); @@ -275,8 +280,10 @@ - (void)testReportWithFatalErrorTestsSet { - (void)testReportWithAppHangingTestsSet { // Testcase timeout should be set larger than the stuck timeout - self.config.stuckTimeout = @40; + self.config.testCaseTimeout = @20; + self.config.stuckTimeout = @15; self.config.errorRetriesCount = @0; + self.config.testing_ExecutionPlan = @"TIMEOUT"; NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; NSString *tempDir = NSTemporaryDirectory(); @@ -301,6 +308,7 @@ - (void)testReportWithAppHangingTestsShouldReturnFailure { self.config.stuckTimeout = @6; self.config.failureTolerance = @0; self.config.errorRetriesCount = @4; + self.config.testing_ExecutionPlan = @"TIMEOUT"; NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; NSString *tempDir = NSTemporaryDirectory(); @@ -317,6 +325,279 @@ - (void)testReportWithAppHangingTestsShouldReturnFailure { [self assertGotReport:junitReportPath isEqualToWantReport:expectedFilePath]; } +/** + Execution plan: TIMEOUT, CRASH (not retried) + */ +- (void)testReportFailureOnTimeoutCrashAndPass { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"TIMEOUT CRASH"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAppCrashed); +} + +/** + Execution plan: TIMEOUT, CRASH, CRASH w/ flag to retry crashes and consider them non-fatal + */ +- (void)testReportFailureOnTimeoutCrashAndCrashOnRetry { + self.config.stuckTimeout = @6; + self.config.retryAppCrashTests = TRUE; + self.config.testing_ExecutionPlan = @"TIMEOUT CRASH CRASH"; + self.config.errorRetriesCount = @2; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == (BPExitStatusTestTimeout | BPExitStatusAppCrashed)); +} + +/** + Execution plan: TIMEOUT, CRASH, PASS w/ flag to retry crashes and consider them non-fatal + */ +- (void)testReportSuccessOnTimeoutCrashAndPassOnRetry { + self.config.stuckTimeout = @6; + self.config.retryAppCrashTests = TRUE; + self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + +/** + Execution plan: CRASH + */ +- (void)testNoRetryOnCrash { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"CRASH"; // No retry + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAppCrashed); +} + + +/** + Execution plan: One test CRASHes and another one TIMEs OUT and PASSes on retry + */ +- (void)testReportFailureOnCrashAndTimeoutTests { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"CRASH; SKIP TIMEOUT PASS"; + self.config.onlyRetryFailed = TRUE; + self.config.failureTolerance = @1; + self.config.errorRetriesCount = @2; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAppCrashed); +} + +/** + Execution plan: Test crashes but passes on retry w/ retry app crash tests flag set + */ +- (void)testReportSuccessOnAppCrashTestPassesOnRetry { + self.config.stuckTimeout = @6; + self.config.retryAppCrashTests = TRUE; + self.config.testing_ExecutionPlan = @"CRASH PASS; SKIP PASS"; + self.config.onlyRetryFailed = TRUE; + self.config.failureTolerance = @1; + self.config.errorRetriesCount = @2; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + +/** + Execution plan: One test CRASHes and another one keeps timing out + */ +- (void)testReportBothCrashAndTimeout { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"CRASH; SKIP TIMEOUT TIMEOUT"; + self.config.onlyRetryFailed = TRUE; + self.config.failureTolerance = @1; + self.config.errorRetriesCount = @2; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == (BPExitStatusAppCrashed | BPExitStatusTestTimeout)); +} + +/** + Execution plan: FAIL, TIMEOUT, PASS + */ +- (void)testReportSuccessOnFailTimeoutAndPass { + self.config.stuckTimeout = @6; + self.config.failureTolerance = @1; + self.config.testing_ExecutionPlan = @"FAIL TIMEOUT PASS"; + self.config.errorRetriesCount = @3; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + +/** + Execution plan: FAIL, TIMEOUT, PASS + */ +- (void)testReportFailureOnFailTimeoutAndPass { + self.config.stuckTimeout = @6; + self.config.failureTolerance = @0; + self.config.testing_ExecutionPlan = @"FAIL TIMEOUT PASS"; + self.config.errorRetriesCount = @3; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusTestsFailed); +} + +/** + Execution plan: TIMEOUT, PASS + */ +- (void)testReportSuccessOnTimeoutAndPassOnRetry { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"TIMEOUT PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + self.config.failureTolerance = @0; // Not relevant + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + +/** + Execution plan: TIMEOUT (NO RETRY)) + */ +- (void)testReportFailureOnTimeoutAndNoRetry { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"TIMEOUT"; + self.config.errorRetriesCount = @2; + self.config.onlyRetryFailed = FALSE; + self.config.failureTolerance = @1; // Not relevant since it's not a test failure + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusTestTimeout); +} + +/** + Execution plan: FAIL and PASS on retry all + */ +- (void)testReportSuccessOnFailedTestAndPassOnRetryAll { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"FAIL PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = NO; // Indicates to retry all tests when a test fails + self.config.failureTolerance = @1; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + +/** + Execution plan: FAIL, PASS + */ +- (void)testReportSuccessOnTestFailedAndPassOnRetry { + self.config.stuckTimeout = @6; + self.config.failureTolerance = @1; + self.config.testing_ExecutionPlan = @"FAIL PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed); +} + - (void)testReportWithFailingTestsSetAndDiagnostics { NSString *tempDir = NSTemporaryDirectory(); NSError *error; @@ -361,7 +642,7 @@ - (void)testRetryOnlyFailures { self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @100; self.config.failureTolerance = @1; - self.config.onlyRetryFailed = YES; + self.config.onlyRetryFailed = TRUE; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; XCTAssert(exitCode == BPExitStatusTestsFailed); // Make sure all tests started on the first run @@ -385,7 +666,7 @@ - (void)testRunWithPassingTestsSet { self.config.testCasesToSkip = @[@"BPSampleAppTests/testCase000"]; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); } - (void)testRunWithFailingTestsSet { @@ -409,7 +690,7 @@ - (void)testKeepSimulatorWithAppCrashingTestsSet { NSString *testBundlePath = [BPTestHelper sampleAppCrashingTestsBundlePath]; self.config.testBundlePath = testBundlePath; self.config.keepSimulator = YES; - + Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; XCTAssert(exitCode == BPExitStatusAppCrashed); @@ -421,7 +702,8 @@ - (void)testKeepSimulatorWithAppHangingTestsSet { NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; self.config.testBundlePath = testBundlePath; self.config.keepSimulator = YES; - + self.config.testing_ExecutionPlan = @"TIMEOUT"; + Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; XCTAssert(exitCode == BPExitStatusTestTimeout); @@ -431,15 +713,15 @@ - (void)testDeleteSimulatorOnly { NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath]; self.config.testBundlePath = testBundlePath; self.config.keepSimulator = YES; - + Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); XCTAssertNotNil(bp.test_simulatorUDID); - + self.config.deleteSimUDID = bp.test_simulatorUDID; XCTAssertNotNil(self.config.deleteSimUDID); - + Bluepill *bp2 = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode2 = [bp2 run]; XCTAssert(exitCode2 == BPExitStatusSimulatorDeleted); @@ -472,7 +754,7 @@ - (void)testRunUITest { self.config.outputDirectory = outputDir; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; self.config.testRunnerAppPath = nil; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); } @@ -485,7 +767,7 @@ - (void)testCopySimulatorPreferencesFile { Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); XCTAssertNotNil(bp.test_simulatorUDID); NSURL *preferencesFile = bp.test_simulator.preferencesFile; @@ -513,7 +795,7 @@ - (void)testRunScript { Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config]; BPExitStatus exitCode = [bp run]; - XCTAssert(exitCode == BPExitStatusTestsAllPassed); + XCTAssert(exitCode == BPExitStatusAllTestsPassed); XCTAssertNotNil(bp.test_simulatorUDID); NSString *devicePath = bp.test_simulator.device.devicePath; diff --git a/bp/tests/Resource Files/BPAppNegativeTests-results.xml b/bp/tests/Resource Files/BPAppNegativeTests-results.xml index 93a9449e..73e36824 100644 --- a/bp/tests/Resource Files/BPAppNegativeTests-results.xml +++ b/bp/tests/Resource Files/BPAppNegativeTests-results.xml @@ -13,7 +13,7 @@ - + /Users/obonilla/o/bluepill/BPSampleApp/BPAppNegativeTests/BPAppNegativeTests.m:48 diff --git a/bp/tests/Resource Files/crash_tests_with_retry_attempt_2.xml b/bp/tests/Resource Files/crash_tests_with_retry_attempt_2.xml index 620f1c98..3b9a6876 100644 --- a/bp/tests/Resource Files/crash_tests_with_retry_attempt_2.xml +++ b/bp/tests/Resource Files/crash_tests_with_retry_attempt_2.xml @@ -1,11 +1,8 @@ - - - - - - - + + + + diff --git a/bp/tests/Resource Files/hanging_tests.xml b/bp/tests/Resource Files/hanging_tests.xml index 79e78efd..54c45afe 100644 --- a/bp/tests/Resource Files/hanging_tests.xml +++ b/bp/tests/Resource Files/hanging_tests.xml @@ -1,8 +1,9 @@ - - - - + + + + + diff --git a/bp/tests/Resource Files/testConfig-busted.json b/bp/tests/Resource Files/testConfig-busted.json index e919cfe4..e3c5ba72 100644 --- a/bp/tests/Resource Files/testConfig-busted.json +++ b/bp/tests/Resource Files/testConfig-busted.json @@ -4,6 +4,8 @@ "VoyagerTests" ], "error-retries": 0, + "num-sims": 2, + "unsafe-skip-xcode-version-check": true, "test": "/tmp", "output-dir": "/tmp/simulator", "test-bundle-path": "/usr/bin", diff --git a/bp/tests/SimulatorHelperTests.m b/bp/tests/SimulatorHelperTests.m index ca718a62..fe260b46 100644 --- a/bp/tests/SimulatorHelperTests.m +++ b/bp/tests/SimulatorHelperTests.m @@ -42,6 +42,7 @@ - (void)testAppLaunchEnvironment { XCTAssert([appLaunchEnvironment[@"DYLD_FALLBACK_FRAMEWORK_PATH"] containsString:@"Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/Library/Frameworks"]); XCTAssert([appLaunchEnvironment[@"DYLD_FALLBACK_LIBRARY_PATH"] containsString:@"Platforms/iPhoneSimulator.platform/Developer/usr/lib"]); XCTAssert([appLaunchEnvironment[@"DYLD_INSERT_LIBRARIES"] containsString:@"Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/Developer/usr/lib/libXCTTargetBootstrapInject.dylib"]); + XCTAssert([appLaunchEnvironment[@"DYLD_INSERT_LIBRARIES"] containsString:@"libXCTestBundleInject.dylib"]); XCTAssert([appLaunchEnvironment[@"DYLD_LIBRARY_PATH"] containsString:@"/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/Library/Frameworks"]); XCTAssert([appLaunchEnvironment[@"XCTestConfigurationFilePath"] containsString:@"T/BPSampleAppTests-"]); XCTAssertEqualObjects(appLaunchEnvironment[@"LLVM_PROFILE_FILE"], @"/Users/test/output/%p.profraw"); diff --git a/bptestrunner/bluepill_batch_test.bzl b/bptestrunner/bluepill_batch_test.bzl index 1d2be0cd..7e2bcdc2 100644 --- a/bptestrunner/bluepill_batch_test.bzl +++ b/bptestrunner/bluepill_batch_test.bzl @@ -20,11 +20,11 @@ def _bluepill_batch_test_impl(ctx): test_host = test_info.test_host test_bundle = test_info.test_bundle if test_bundle: - test_bundle_paths.append("\"{}\"".format(test_bundle.basename)) + test_bundle_paths.append("\"{}\"".format(test_bundle.short_path)) runfiles.append(test_bundle) if test_host and test_host not in runfiles: - test_host_paths.append("\"{}\"".format(test_host.basename)) + test_host_paths.append("\"{}\"".format(test_host.short_path)) runfiles.append(test_host) #test_plan diff --git a/scripts/bluepill.sh b/scripts/bluepill.sh index 0ddf7431..b4f3b64c 100755 --- a/scripts/bluepill.sh +++ b/scripts/bluepill.sh @@ -1,25 +1,18 @@ -# // Copyright 2016 LinkedIn Corporation -# // Licensed under the BSD 2-Clause License (the "License"); -# // you may not use this file except in compliance with the License. -# // You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause -# // -# // Unless required by applicable law or agreed to in writing, software -# // distributed under the License is distributed on an "AS IS" BASIS, -# // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - #!/bin/bash +# Copyright 2016 LinkedIn Corporation +# Licensed under the BSD 2-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. XCPRETTY='xcpretty --report junit' -command -v $XCPRETTY >/dev/null 2>&1 || { - XCPRETTY=cat +command -v "$XCPRETTY" >/dev/null 2>&1 || { + XCPRETTY="cat" } -if [ "$1" == "-v" ] -then - VERBOSE=1 - shift -fi - if [[ $# -ne 1 ]]; then echo "$0: usage: bluepill.sh " exit 1 @@ -60,18 +53,18 @@ bluepill_build() # package bluepill TAG=$(git describe --always --tags) DST="Bluepill-$TAG" - mkdir -p build/$DST/bin - cp build/Build/Products/Release/{bp,bluepill} build/$DST/bin + mkdir -p "build/$DST/bin" + cp build/Build/Products/Release/{bp,bluepill} "build/$DST/bin" ## build the man page - mkdir -p build/$DST/man/man1 - /usr/bin/python scripts/man.py build/$DST/man/man1/bluepill.1 + mkdir -p "build/$DST/man/man1" + /usr/bin/python scripts/man.py "build/$DST/man/man1/bluepill.1" # License - cp LICENSE build/$DST + cp LICENSE "build/$DST" # bptestrunner - cp bptestrunner/* build/$DST + cp bptestrunner/* "build/$DST" - (cd build && zip -qr $DST.zip $DST) - echo Release in build/$DST.zip + (cd build && zip -qr "$DST.zip" "$DST") + echo Release in "build/$DST.zip" } bluepill_build_sample_app() @@ -142,6 +135,6 @@ then bluepill_build_sample_app fi -bluepill_$conf +"bluepill_$conf" exit 0