Record video using simctl (#441)

Use the XCTest listener protocol to add hooks for `xcrun simctl io [sim-id] recordVideo`.

README.md

@@ -54,10 +54,10 @@ $ bluepill -c config.json
 A full list supported options are listed here.
 
 
-|   Config Arguments     | Command Line Arguments | Explanation                                                                         | Required | Default value    |
-|:----------------------:|:----------------------:|-------------------------------------------------------------------------------------|:--------:|:----------------:|
-|          app           |           -a           | The path to the host application to execute (your `.app`)                           |     N    | n/a              |
-|     xctestrun-path     |                        | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`.   |     Y    | n/a              |
+|   Config Arguments     | Command Line Arguments | Explanation                                                                                  | Required | Default value    |
+|:----------------------:|:----------------------:|----------------------------------------------------------------------------------------------|:--------:|:----------------:|
+|          app           |           -a           | The path to the host application to execute (your `.app`)                                    |     N    | n/a              |
+|     xctestrun-path     |                        | The path to the `.xctestrun` file that xcode leaves when you `build-for-testing`.            |     Y    | n/a              |
 |     test-plan-path     |                        | The path of a json file which describes the test plan. It is equivalent to the `.xctestrun` file generated by Xcode, but it can be generated by a different build system, e.g. Bazel |     Y    | n/a              |
 |       output-dir       |           -o           | Directory where to put output log files. **(bluepill only)**                        |     Y    | n/a              |
 |         config         |           -c           | Read options from the specified configuration file instead of the command line.     |     N    | n/a              |
@@ -86,6 +86,7 @@ A full list supported options are listed here.
 |          help          |           -h           | Help.                                                                               |     N    | n/a              |
 |     runner-app-path    |           -u           | The test runner for UI tests.                                                       |     N    | n/a              |
 | screenshots-directory  |           n/a          | Directory where simulator screenshots for failed ui tests will be stored.           |     N    | n/a              |
+|    videos-directory    |           n/a          | Directory where videos of test runs will be saved. If not provided, videos are not recorded. | N | n/a            |
 |       video-paths      |           -V           | A list of videos that will be saved in the simulators.                              |     N    | n/a              |
 |       image-paths      |           -I           | A list of images that will be saved in the simulators.                              |     N    | n/a              |
 | unsafe-skip-xcode-version-check |               | Skip Xcode version check                                                            |     N    | NO               |

bluepill/tests/BPIntegrationTests.m

@@ -21,7 +21,7 @@ @interface BPIntegrationTests : XCTestCase
 
 @implementation BPIntegrationTests
 
-- (BPConfiguration *)generateConfig {
+- (BPConfiguration *)generateConfigWithVideoDir:(NSString *)videoDir {
     NSString *hostApplicationPath = [BPTestHelper sampleAppPath];
     NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
     BPConfiguration *config = [[BPConfiguration alloc] initWithProgram:BP_MASTER];
@@ -36,9 +36,16 @@ - (BPConfiguration *)generateConfig {
     config.deviceType = @BP_DEFAULT_DEVICE_TYPE;
     config.headlessMode = YES;
     config.quiet = [BPUtils isBuildScript];
+    if (videoDir != nil) {
+        config.videosDirectory = videoDir;
+    }
     return config;
 }
 
+- (BPConfiguration *)generateConfig {
+    return [self generateConfigWithVideoDir: nil];
+}
+
 - (void)setUp {
     [super setUp];
     // Put setup code here. This method is called before the invocation of each test method in the class.
@@ -216,4 +223,59 @@ - (void)writeTestPlan {
     [jsonData writeToFile:[BPTestHelper testPlanPath] atomically:YES];
 }
 
+- (void)testTwoBPInstancesWithVideo {
+    NSFileManager *fileManager = [NSFileManager defaultManager];
+    NSError *mkdtempError;
+    NSString *path = [BPUtils mkdtemp:@"bpout" withError:&mkdtempError];
+    XCTAssertNil(mkdtempError);
+
+    NSString* videoDirName = @"my_videos";
+    NSString *videoPath = [path stringByAppendingPathComponent:videoDirName];
+    BPConfiguration *config = [self generateConfigWithVideoDir:videoPath];
+    config.numSims = @2;
+    config.errorRetriesCount = @1;
+    config.failureTolerance = @0;
+    // This looks backwards but we want the main app to be the runner
+    // and the sampleApp is launched from the callback.
+    config.testBundlePath = [BPTestHelper sampleAppUITestBundlePath];
+    config.testRunnerAppPath = [BPTestHelper sampleAppPath];
+    config.appBundlePath = [BPTestHelper sampleAppUITestRunnerPath];
+
+    NSError *err;
+    BPApp *app = [BPApp appWithConfig:config
+                            withError:&err];
+    NSString *bpPath = [BPTestHelper bpExecutablePath];
+
+    // Run the tests through one time to flush out any weird errors that happen with video recording
+    BPRunner *dryRunRunner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
+    XCTAssert(dryRunRunner != nil);
+    int dryRunRC = [dryRunRunner runWithBPXCTestFiles:app.testBundles];
+    XCTAssert(dryRunRC == 0);
+    XCTAssert([dryRunRunner.nsTaskList count] == 0);
+    [fileManager removeItemAtPath:videoPath error:nil];
+    NSArray *dryRunOutputContents  = [fileManager  contentsOfDirectoryAtPath:videoPath error:nil];
+    XCTAssertEqual(dryRunOutputContents.count, 0);
+
+    // Start the real test now
+    BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath];
+    XCTAssert(runner != nil);
+    int rc = [runner runWithBPXCTestFiles:app.testBundles];
+    XCTAssert(rc == 0);
+    XCTAssert([runner.nsTaskList count] == 0);
+
+    NSError *dirContentsError;
+    NSArray *directoryContent  = [fileManager contentsOfDirectoryAtPath:videoPath error:&dirContentsError];
+    XCTAssertNil(dirContentsError);
+    XCTAssertNotNil(directoryContent);
+    XCTAssertEqual(directoryContent.count, 2);
+
+    NSString *testClass = @"BPSampleAppUITests";
+    NSSet *filenameSet = [NSSet setWithArray: directoryContent];
+    XCTAssertEqual(filenameSet.count, 2);
+    BOOL hasTest1 = [filenameSet containsObject: [NSString stringWithFormat:@"%@__%@__1.mp4", testClass, @"testExample"]];
+    XCTAssertTrue(hasTest1);
+    BOOL hasTest2 = [filenameSet containsObject: [NSString stringWithFormat:@"%@__%@__1.mp4", testClass, @"testExample2"]];
+    XCTAssertTrue(hasTest2);
+}
+
 @end

bp/src/BPConfiguration.h

@@ -87,6 +87,7 @@ typedef NS_ENUM(NSInteger, BPProgram) {
 @property (nonatomic, strong) NSString *outputDirectory;
 @property (nonatomic, strong) NSString *testTimeEstimatesJsonFile;
 @property (nonatomic, strong) NSString *screenshotsDirectory;
+@property (nonatomic, strong) NSString *videosDirectory;
 @property (nonatomic, strong) NSString *simulatorPreferencesFile;
 @property (nonatomic, strong) NSString *scriptFilePath;
 @property (nonatomic) BOOL headlessMode;

bp/src/BPConfiguration.m

@@ -146,7 +146,8 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
         "Skip Xcode version check if using an Xcode version that is not officially supported the Bluepill version being used. Not safe/recommended and has a limited support."},
     {366, "retry-app-crash-tests", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "retryAppCrashTests",
         "Retry the tests after an app crash and if it passes on retry, consider them non-fatal."},
-
+    {367, "videos-directory", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "videosDirectory",
+        "Directory where videos of test runs will be saved. If not provided, videos are not recorded."},
     {0, 0, 0, 0, 0, 0, 0}
 };
 
@@ -776,6 +777,23 @@ - (BOOL)validateConfigWithError:(NSError *__autoreleasing *)errPtr {
         }
     }
 
+    if (self.videosDirectory) {
+        if ([[NSFileManager defaultManager] fileExistsAtPath:self.videosDirectory isDirectory:&isdir]) {
+            if (!isdir) {
+                BP_SET_ERROR(errPtr, @"%@ is not a directory.", self.videosDirectory);
+                return NO;
+            }
+        } else {
+            // create the directory
+            if (![[NSFileManager defaultManager] createDirectoryAtPath:self.videosDirectory
+                                           withIntermediateDirectories:YES
+                                                            attributes:nil
+                                                                 error:errPtr]) {
+                return NO;
+            }
+        }
+    }
+
     if (self.simulatorPreferencesFile) {
         if ([[NSFileManager defaultManager] fileExistsAtPath:self.simulatorPreferencesFile isDirectory:&isdir]) {
             if (isdir) {

bp/src/BPTestBundleConnection.h

@@ -7,6 +7,7 @@
 //
 
 #import <Foundation/Foundation.h>
+#import "BPExecutionContext.h"
 #import "BPSimulator.h"
 
 // This is a small subset of XCTestManager_IDEInterface protocol
@@ -16,9 +17,10 @@
 
 @interface BPTestBundleConnection : NSObject
 @property (nonatomic, strong) BPConfiguration *config;
+@property (nonatomic, strong) BPExecutionContext *context;
 @property (nonatomic, strong) BPSimulator *simulator;
 @property (nonatomic, copy) void (^completionBlock)(NSError *, pid_t);
-- (instancetype)initWithDevice:(BPSimulator *)device andInterface:(id<BPTestBundleConnectionDelegate>)interface;
+- (instancetype)initWithContext:(BPExecutionContext *)context andInterface:(id<BPTestBundleConnectionDelegate>)interface;
 - (void)connectWithTimeout:(NSTimeInterval)timeout;
 - (void)startTestPlan;
 @end

bp/src/BPTestBundleConnection.m

@@ -49,15 +49,19 @@ @interface BPTestBundleConnection()<XCTestManager_IDEInterface>
 @property (nonatomic, strong) dispatch_queue_t queue;
 @property (nonatomic, strong) NSString *bundleID;
 @property (nonatomic, assign) pid_t appProcessPID;
+@property (nonatomic, nullable) NSTask *recordVideoTask;
+//@property (nonatomic, nullable) NSPipe *recordVideoPipe;
+
 
 @end
 
 @implementation BPTestBundleConnection
 
-- (instancetype)initWithDevice:(BPSimulator *)simulator andInterface:(id<BPTestBundleConnectionDelegate>)interface {
+- (instancetype)initWithContext:(BPExecutionContext *)context andInterface:(id<BPTestBundleConnectionDelegate>)interface {
     self = [super init];
     if (self) {
-        self.simulator = simulator;
+        self.context = context;
+        self.simulator = context.runner;
         self.interface = interface;
         self.queue = dispatch_queue_create("com.linkedin.bluepill.connection.queue", DISPATCH_QUEUE_PRIORITY_DEFAULT);
     }
@@ -82,6 +86,8 @@ - (void)connect {
         DTXTransport *transport = [self connectTransport];
         DTXConnection *connection = [[objc_lookUpClass("DTXConnection") alloc] initWithTransport:transport];
         [connection registerDisconnectHandler:^{
+            // This is called when the task is abruptly terminated (e.g. if the test times out)
+            [self stopVideoRecording:YES];
             [BPUtils printInfo:INFO withString:@"DTXConnection disconnected."];
         }];
         [connection
@@ -212,6 +218,63 @@ - (id)_XCT_initializationForUITestingDidFailWithError:(NSError *__strong)errPtr
     return nil;
 }
 
+#pragma mark - Video Recording
+
+static inline NSString* getVideoPath(NSString *directory, NSString *testClass, NSString *method, NSInteger attemptNumber)
+{
+    return [NSString stringWithFormat:@"%@/%@__%@__%ld.mp4", directory, testClass, method, (long)attemptNumber];
+}
+
+- (BOOL)shouldRecordVideo {
+    return self.context.config.videosDirectory.length > 0;
+}
+
+- (void)startVideoRecordingForTestClass:(NSString *)testClass method:(NSString *)method
+{
+    [self stopVideoRecording:YES];
+    NSString *videoFileName = getVideoPath(self.context.config.videosDirectory, testClass, method, self.context.attemptNumber);
+    NSString *command = [NSString stringWithFormat:@"xcrun simctl io %@ recordVideo --force %@", [self.simulator UDID], videoFileName];
+    NSTask *task = [BPUtils buildShellTaskForCommand:command];
+    self.recordVideoTask = task;
+    [task launch];
+    [BPUtils printInfo:INFO withString:@"Started recording video to %@", videoFileName];
+    [BPUtils printInfo:DEBUGINFO withString:@"Started recording video task with pid %d and command: %@",  [task processIdentifier], [BPUtils getCommandStringForTask:task]];
+}
+
+- (void)stopVideoRecording:(BOOL)forced
+{
+    NSTask *task = self.recordVideoTask;
+    if (task == nil) {
+        if (!forced) {
+            [BPUtils printInfo:ERROR withString: @"Tried to end video recording task normally, but there was no task."];
+        }
+        return;
+    }
+
+    if (forced) {
+        [BPUtils printInfo:ERROR withString: @"Found dangling video recording task. Stopping it."];
+    }
+
+    if (![task isRunning]) {
+        [BPUtils printInfo:ERROR withString:@"Video task exists but it was already terminated with status %d", [task terminationStatus]];
+    }
+
+    [BPUtils printInfo:INFO withString:@"Stopping recording video."];
+    [BPUtils printInfo:DEBUGINFO withString:@"Stopping video recording task with pid %d and command: %@", [task processIdentifier], [BPUtils getCommandStringForTask:task]];
+    [task interrupt];
+    [task waitUntilExit];
+
+    if ([task terminationStatus] != 0) {
+        [BPUtils printInfo:ERROR withString:@"Video task was interrupted, but exited with non-zero status %d", [task terminationStatus]];
+    }
+
+    NSString *filePath = [[task arguments].lastObject componentsSeparatedByString:@" "].lastObject;
+    if (![[NSFileManager defaultManager] fileExistsAtPath:filePath]) {
+        [BPUtils printInfo:ERROR withString:@"Video recording file missing, expected at path %@!", filePath];
+    }
+    self.recordVideoTask = nil;
+}
+
 #pragma mark - XCTestManager_IDEInterface protocol
 
 #pragma mark Process Launch Delegation
@@ -255,7 +318,7 @@ - (id)_XCT_getProgressForLaunch:(id)token {
 
 - (id)_XCT_terminateProcess:(id)token {
     NSError *error;
-    kill(self.appProcessPID, SIGTERM);
+    kill(self.appProcessPID, SIGINT);
     DTXRemoteInvocationReceipt *receipt = [objc_lookUpClass("DTXRemoteInvocationReceipt") new];
     [receipt invokeCompletionWithReturnValue:token error:error];
     [BPUtils printInfo:DEBUGINFO withString:@"BPTestBundleConnection_XCT_terminateProcess with token %@", token];
@@ -277,6 +340,9 @@ - (id)_XCT_testSuite:(NSString *)tests didStartAt:(NSString *)time {
 
 - (id)_XCT_testCaseDidStartForTestClass:(NSString *)testClass method:(NSString *)method {
     [BPUtils printInfo:DEBUGINFO withString:@"BPTestBundleConnection_XCT_testCaseDidStartForTestClass: %@ and method: %@", testClass, method];
+    if ([self shouldRecordVideo]) {
+        [self startVideoRecordingForTestClass:testClass method:method];
+    }
     return nil;
 }
 
@@ -298,12 +364,18 @@ - (id)_XCT_logMessage:(NSString *)message {
 
 - (id)_XCT_testCaseDidFinishForTestClass:(NSString *)testClass method:(NSString *)method withStatus:(NSString *)statusString duration:(NSNumber *)duration {
     [BPUtils printInfo:DEBUGINFO withString: @"BPTestBundleConnection_XCT_testCaseDidFinishForTestClass: %@, method: %@, withStatus: %@, duration: %@", testClass, method, statusString, duration];
+    if ([self shouldRecordVideo]) {
+        [self stopVideoRecording:NO];
+    }
     return nil;
 }
 
 - (id)_XCT_testSuite:(NSString *)arg1 didFinishAt:(NSString *)time runCount:(NSNumber *)count withFailures:(NSNumber *)failureCount unexpected:(NSNumber *)unexpectedCount testDuration:(NSNumber *)testDuration totalDuration:(NSNumber *)totalTime {
     [BPUtils printInfo:DEBUGINFO withString: @"BPTestBundleConnection_XCT_testSuite: %@, didFinishAt: %@, runCount: %@, withFailures: %@, unexpectedCount: %@, testDuration: %@, totalDuration: %@", arg1, time, count, failureCount, unexpectedCount, testDuration, totalTime];
 
+    if ([self shouldRecordVideo]) {
+        [self stopVideoRecording:YES];
+    }
     return nil;
 }
 

bp/src/BPUtils.h

@@ -125,6 +125,29 @@ typedef NS_ENUM(int, BPKind) {
  * @return return the shell output
  */
 + (NSString *)runShell:(NSString *)command;
+
+/*!
+ * @discussion builds a task to run a shell command
+ * @param command the shell command the task should run
+ * @return an NSTask that will run the provided command.
+ */
++ (NSTask *)buildShellTaskForCommand:(NSString *)command;
+
+/*!
+ * @discussion builds a task to run a shell command, pointing stdout and stderr to the provided pipe
+ * @param command the shell command the task should run
+ * @param pipe the pipe that stdout and stderr will be pointed to, so the caller can handle the output.
+ * @return an NSTask that will run the provided command.
+ */
++ (NSTask *)buildShellTaskForCommand:(NSString *)command withPipe:(NSPipe *)pipe;
+
+/*!
+ * @discussion builds a user readable representation of the command that a task is configured to run
+ * @param task to get command from
+ * @return a user readable string of the task's command
+*/
++ (NSString *)getCommandStringForTask:(NSTask *)task;
+
 + (NSString *)getXcodeRuntimeVersion;
 
 typedef BOOL (^BPRunBlock)(void);

bp/src/BPUtils.m

@@ -189,29 +189,40 @@ + (BOOL)isStdOut:(NSString *)fileName {
 
 + (NSString *)runShell:(NSString *)command {
     NSAssert(command, @"Command should not be nil");
-    NSTask *task = [[NSTask alloc] init];
-    NSData *data;
-    task.launchPath = @"/bin/sh";
-    task.arguments = @[@"-c", command];
     NSPipe *pipe = [[NSPipe alloc] init];
-    task.standardError = pipe;
-    task.standardOutput = pipe;
+    NSTask *task = [BPUtils buildShellTaskForCommand:command withPipe:pipe];
+    NSAssert(task, @"task should not be nil");
     NSFileHandle *fh = pipe.fileHandleForReading;
-    if (task) {
-        [task launch];
-    } else {
-        NSAssert(task, @"task should not be nil");
-    }
-    if (fh) {
-        data = [fh readDataToEndOfFile];
-    } else {
-        NSAssert(task, @"fh should not be nil");
-    }
+    NSAssert(fh, @"fh should not be nil");
+
+    [task launch];
+    NSData *data = [fh readDataToEndOfFile];
     [task waitUntilExit];
     NSString *result = [[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] stringByTrimmingCharactersInSet:[NSCharacterSet newlineCharacterSet]];
     return result;
 }
 
++ (NSTask *)buildShellTaskForCommand:(NSString *)command {
+    return [BPUtils buildShellTaskForCommand:command withPipe: nil];
+}
+
++ (NSTask *)buildShellTaskForCommand:(NSString *)command withPipe:(NSPipe *)pipe {
+    NSAssert(command, @"Command should not be nil");
+    NSTask *task = [[NSTask alloc] init];
+    task.launchPath = @"/bin/sh";
+    task.arguments = @[@"-c", command];
+    if (pipe != nil) {
+        task.standardError = pipe;
+        task.standardOutput = pipe;
+    }
+    NSAssert(task, @"task should not be nil");
+    return task;
+}
+
++ (NSString *)getCommandStringForTask:(NSTask *)task {
+    return [NSString stringWithFormat:@"%@ %@", [task launchPath], [[task arguments] componentsJoinedByString:@" "]];
+}
+
 + (BOOL)runWithTimeOut:(NSTimeInterval)timeout until:(BPRunBlock)block {
     if (!block) {
         return NO;

bp/src/Bluepill.m

@@ -432,7 +432,7 @@ - (void)connectTestBundleAndTestDaemonWithContext:(BPExecutionContext *)context
         // If the isTestRunnerContext is flipped on, don't connect testbundle again.
         return;
     }
-    BPTestBundleConnection *bConnection = [[BPTestBundleConnection alloc] initWithDevice:context.runner andInterface:self];
+    BPTestBundleConnection *bConnection = [[BPTestBundleConnection alloc] initWithContext:context andInterface:self];
     bConnection.simulator = context.runner;
     bConnection.config = self.config;