Tests with adam server stub (#508)

* feat(adam): AndroidAppInstallerTest * feat(adam): AndroidAppInstallerTest reworked * feat(android): simplify test server stubbing * feat(android): fix bug with am instrument args * feat(android): fully test AndroidAppInstaller * feat(android): add RemoteFileManagerTest * feat(android): add VideoConfigurationTest * feat(android): add more tests * feat(android): add test for Test extension * feat(android): remove unused temp from RemoteFileManagerTest * feat(android): add more tests + fix surfaced issues * fix(all): always system exit * feat(cli): print marathon version on start * feat(android): more tests for screen capturing + saving last screen capture when batch times out * feat(android): more tests + coverage * fix(adam): close vertx on termination * chore(deps): update adam to 0.3.0 * fix(android): fix ScreenCapturerTestRunListenerTest * feat(android): rename ScreenRecorderTestRunListener to ScreenRecorderTestBatchListener Co-authored-by: Ivan Balaksha <[email protected]>
MarathonLabs · Jun 5, 2021 · 7f582c3 · 7f582c3
1 parent 62e1efb
commit 7f582c3
Show file tree

Hide file tree

Showing 50 changed files with 2,498 additions and 156 deletions.
diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt
@@ -5,7 +5,7 @@ object Versions {
     val coroutines = "1.3.9"
 
     val ddmlib = "27.1.2"
-    val adam = "0.2.5"
+    val adam = "0.3.0"
     val dexTestParser = "2.1.1"
     val kotlinLogging = "1.7.6"
     val slf4jAPI = "1.0.0"
@@ -16,7 +16,7 @@ object Versions {
     val junitGradle = "1.0.0"
     val androidGradleVersion = "4.0.0"
 
-    val junit5 = "5.6.0"
+    val junit5 = "5.7.2"
     val kluent = "1.64"
 
     val kakao = "1.4.0"
@@ -62,7 +62,7 @@ object BuildPlugins {
 
 object Libraries {
     val ddmlib = "com.android.tools.ddms:ddmlib:${Versions.ddmlib}"
-    val adam = "com.malinskiy:adam:${Versions.adam}"
+    val adam = "com.malinskiy.adam:adam:${Versions.adam}"
     val androidCommon = "com.android.tools:common:${Versions.ddmlib}"
     val dexTestParser = "com.linkedin.dextestparser:parser:${Versions.dexTestParser}"
     val kotlinStdLib = "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${Versions.kotlin}"
@@ -111,6 +111,7 @@ object TestLibraries {
 
     val testContainers = "org.testcontainers:testcontainers:${Versions.testContainers}"
     val testContainersInflux = "org.testcontainers:influxdb:${Versions.testContainers}"
+    val adamServerStubJunit5 = "com.malinskiy.adam:server-stub-junit5:${Versions.adam}"
 }
 
 object Analytics {

diff --git a/cli/src/main/kotlin/com/malinskiy/marathon/cli/ApplicationView.kt b/cli/src/main/kotlin/com/malinskiy/marathon/cli/ApplicationView.kt
@@ -30,7 +30,7 @@ fun main(args: Array<String>): Unit = mainBody(
     programName = "marathon v${BuildConfig.VERSION}"
 ) {
     ArgParser(args).parseInto(::MarathonCliConfiguration).run {
-        logger.info { "Starting marathon" }
+        logger.info { "Starting marathon v${BuildConfig.VERSION}" }
         val bugsnagExceptionsReporter = ExceptionsReporterFactory.get(bugsnagReporting)
         try {
             bugsnagExceptionsReporter.start(AppType.CLI)
@@ -52,7 +52,9 @@ fun main(args: Array<String>): Unit = mainBody(
 
             val shouldReportFailure = !configuration.ignoreFailures
             if (!success && shouldReportFailure) {
-                throw SystemExitException("Build failed", 1)
+                throw SystemExitException("Test run failed", 1)
+            } else {
+                throw SystemExitException("Test run finished", 0)
             }
         } finally {
             stopKoin()

diff --git a/core/src/main/kotlin/com/malinskiy/marathon/io/FileManager.kt b/core/src/main/kotlin/com/malinskiy/marathon/io/FileManager.kt
@@ -17,6 +17,12 @@ class FileManager(private val output: File) {
         return createFile(directory, filename)
     }
 
+    fun createFile(fileType: FileType, pool: DevicePoolId, device: DeviceInfo, testBatchId: String): File {
+        val directory = createDirectory(fileType, pool, device)
+        val filename = createFilename(fileType, testBatchId)
+        return createFile(directory, filename)
+    }
+
     fun createFile(fileType: FileType, pool: DevicePoolId, device: DeviceInfo): File {
         val directory = createDirectory(fileType, pool)
         val filename = createFilename(device, fileType)
@@ -56,6 +62,9 @@ class FileManager(private val output: File) {
 
     private fun createFile(directory: Path, filename: String): File = File(directory.toFile(), filename)
 
+    private fun createFilename(fileType: FileType, testBatchId: String): String =
+        "$testBatchId.${fileType.suffix}"
+
     private fun createFilename(test: Test, fileType: FileType, testBatchId: String? = null): String =
         "${test.toTestName()}${testBatchId?.let { "-$it" } ?: ""}.${fileType.suffix}"
 

diff --git a/vendor/vendor-android/adam/build.gradle.kts b/vendor/vendor-android/adam/build.gradle.kts
@@ -22,6 +22,7 @@ dependencies {
     testImplementation(TestLibraries.koin)
     testImplementation(TestLibraries.junit5)
     testImplementation(TestLibraries.jupiterEngine)
+    testImplementation(TestLibraries.adamServerStubJunit5)
 }
 
 Deployment.initialize(project)

diff --git a/...dor-android/adam/src/main/kotlin/com/malinskiy/marathon/android/adam/AdamAndroidDevice.kt b/...dor-android/adam/src/main/kotlin/com/malinskiy/marathon/android/adam/AdamAndroidDevice.kt
@@ -22,6 +22,7 @@ import com.malinskiy.adam.request.sync.AndroidFileType
 import com.malinskiy.adam.request.sync.ListFilesRequest
 import com.malinskiy.adam.request.sync.compat.CompatPullFileRequest
 import com.malinskiy.adam.request.sync.compat.CompatPushFileRequest
+import com.malinskiy.adam.request.sync.compat.CompatStatFileRequest
 import com.malinskiy.adam.request.testrunner.TestEvent
 import com.malinskiy.adam.request.testrunner.TestRunnerRequest
 import com.malinskiy.marathon.analytics.internal.pub.Track
@@ -123,18 +124,23 @@ class AdamAndroidDevice(
             val local = File(localFilePath)
 
             measureFileTransfer(local) {
-                val channel = client.execute(
-                    CompatPullFileRequest(remoteFilePath, local, supportedFeatures, coroutineScope = this),
-                    serial = adbSerial
-                )
-                for (update in channel) {
-                    progress = update
+                val stat = client.execute(CompatStatFileRequest(remoteFilePath, supportedFeatures), serialNumber)
+                if (stat.exists()) {
+                    val channel = client.execute(
+                        CompatPullFileRequest(remoteFilePath, local, supportedFeatures, coroutineScope = this, size = stat.size().toLong()),
+                        serial = adbSerial
+                    )
+                    for (update in channel) {
+                        progress = update
+                    }
+                } else {
+                    throw TransferException("Couldn't pull file $remoteFilePath from device $serialNumber because it doesn't exist")
                 }
             }
         } catch (e: PullFailedException) {
-            throw TransferException("Couldn't pull file $remoteFilePath from device $serialNumber")
+            throw TransferException("Couldn't pull file $remoteFilePath from device $serialNumber", e)
         } catch (e: UnsupportedSyncProtocolException) {
-            throw TransferException("Device $serialNumber does not support sync: file transfer")
+            throw TransferException("Device $serialNumber does not support sync: file transfer", e)
         }
 
         if (progress != 1.0) {
@@ -228,7 +234,7 @@ class AdamAndroidDevice(
         }
     }
 
-    override suspend fun installPackage(absolutePath: String, reinstall: Boolean, optionalParams: String): String? {
+    override suspend fun installPackage(absolutePath: String, reinstall: Boolean, optionalParams: List<String>): String? {
         val file = File(absolutePath)
         val remotePath = "/data/local/tmp/${file.name}"
 
@@ -245,7 +251,7 @@ class AdamAndroidDevice(
                 InstallRemotePackageRequest(
                     remotePath,
                     reinstall = reinstall,
-                    extraArgs = optionalParams.split(" ").toList() + " "
+                    extraArgs = optionalParams.filter { it.isNotBlank() }
                 ), serial = adbSerial
             )
         } ?: throw InstallException("Timeout transferring $absolutePath")

diff --git a/...or-android/adam/src/main/kotlin/com/malinskiy/marathon/android/adam/AdamDeviceProvider.kt b/...or-android/adam/src/main/kotlin/com/malinskiy/marathon/android/adam/AdamDeviceProvider.kt
@@ -155,6 +155,7 @@ class AdamDeviceProvider(
             deviceEventsChannel.cancel()
         }
         logcatManager.close()
+        client.socketFactory.close()
     }
 
     override fun subscribe() = channel

diff --git a/vendor/vendor-android/base/build.gradle.kts b/vendor/vendor-android/base/build.gradle.kts
@@ -5,6 +5,7 @@ plugins {
     id("org.jetbrains.kotlin.jvm")
     id("org.jetbrains.dokka")
     id("org.junit.platform.gradle.plugin")
+    jacoco
 }
 
 dependencies {
@@ -25,6 +26,8 @@ dependencies {
     testImplementation(TestLibraries.junit5)
     testRuntimeOnly(TestLibraries.jupiterEngine)
     testImplementation(TestLibraries.koin)
+    testImplementation(TestLibraries.adamServerStubJunit5)
+    testImplementation(project(":vendor:vendor-android:adam"))
 }
 
 Deployment.initialize(project)

diff --git a/...vendor-android/base/src/main/kotlin/com/malinskiy/marathon/android/AndroidAppInstaller.kt b/...vendor-android/base/src/main/kotlin/com/malinskiy/marathon/android/AndroidAppInstaller.kt
@@ -65,17 +65,14 @@ class AndroidAppInstaller(configuration: Configuration) {
         return lines.any { it == "package:$appPackage" }
     }
 
-    private fun optionalParams(device: AndroidDevice): String {
-        val options = if (device.apiLevel >= MARSHMALLOW_VERSION_CODE && androidConfiguration.autoGrantPermission) {
-            "-g -r"
-        } else {
-            "-r"
-        }
-
-        return if (androidConfiguration.installOptions.isNotEmpty()) {
-            "$options ${androidConfiguration.installOptions}"
-        } else {
-            options
-        }
+    private fun optionalParams(device: AndroidDevice): List<String> {
+        return mutableListOf<String>().apply {
+            if (device.apiLevel >= MARSHMALLOW_VERSION_CODE && androidConfiguration.autoGrantPermission) {
+                add("-g")
+            }
+            if (androidConfiguration.installOptions.isNotEmpty()) {
+                addAll(androidConfiguration.installOptions.split(" "))
+            }
+        }.toList()
     }
 }
diff --git a/vendor/vendor-android/base/src/main/kotlin/com/malinskiy/marathon/android/AndroidDevice.kt b/vendor/vendor-android/base/src/main/kotlin/com/malinskiy/marathon/android/AndroidDevice.kt
@@ -60,7 +60,7 @@ interface AndroidDevice : Device {
     /**
      * @throws com.malinskiy.marathon.android.exception.InstallException in case of failure to push the apk
      */
-    suspend fun installPackage(absolutePath: String, reinstall: Boolean, optionalParams: String): String?
+    suspend fun installPackage(absolutePath: String, reinstall: Boolean, optionalParams: List<String>): String?
     suspend fun safeUninstallPackage(appPackage: String, keepData: Boolean = false): String?
     suspend fun safeClearPackage(packageName: String): String?
 

diff --git a/...r/vendor-android/base/src/main/kotlin/com/malinskiy/marathon/android/BaseAndroidDevice.kt b/...r/vendor-android/base/src/main/kotlin/com/malinskiy/marathon/android/BaseAndroidDevice.kt
@@ -15,7 +15,7 @@ import com.malinskiy.marathon.android.executor.listeners.ProgressTestRunListener
 import com.malinskiy.marathon.android.executor.listeners.TestRunResultsListener
 import com.malinskiy.marathon.android.executor.listeners.filesync.FileSyncTestRunListener
 import com.malinskiy.marathon.android.executor.listeners.screenshot.ScreenCapturerTestRunListener
-import com.malinskiy.marathon.android.executor.listeners.video.ScreenRecorderTestRunListener
+import com.malinskiy.marathon.android.executor.listeners.video.ScreenRecorderTestBatchListener
 import com.malinskiy.marathon.android.model.Rotation
 import com.malinskiy.marathon.device.DeviceFeature
 import com.malinskiy.marathon.device.DevicePoolId
@@ -242,7 +242,7 @@ abstract class BaseAndroidDevice(
 
         val recordConfiguration = this@BaseAndroidDevice.androidConfiguration.screenRecordConfiguration
         val screenRecordingPolicy = configuration.screenRecordingPolicy
-        val recorderListener = selectRecorderType(features, recordConfiguration)?.let { feature ->
+        val recorderListener = RecorderTypeSelector.selectRecorderType(features, recordConfiguration)?.let { feature ->
             prepareRecorderListener(feature, fileManager, devicePoolId, testBatch.id, screenRecordingPolicy, attachmentProviders)
         } ?: NoOpTestRunListener()
 
@@ -271,7 +271,7 @@ abstract class BaseAndroidDevice(
     ): NoOpTestRunListener =
         when (feature) {
             DeviceFeature.VIDEO -> {
-                ScreenRecorderTestRunListener(
+                ScreenRecorderTestBatchListener(
                     fileManager,
                     devicePoolId,
                     testBatchId,
@@ -298,30 +298,6 @@ abstract class BaseAndroidDevice(
             }
         }
 
-    private fun selectRecorderType(supportedFeatures: Collection<DeviceFeature>, configuration: ScreenRecordConfiguration): DeviceFeature? {
-        val preferred = configuration.preferableRecorderType
-        val screenshotEnabled = recorderEnabled(DeviceFeature.SCREENSHOT, configuration)
-        val videoEnabled = recorderEnabled(DeviceFeature.VIDEO, configuration)
-
-        if (preferred != null && supportedFeatures.contains(preferred) && recorderEnabled(preferred, configuration)) {
-            return preferred
-        }
-
-        return when {
-            supportedFeatures.contains(DeviceFeature.VIDEO) && videoEnabled -> DeviceFeature.VIDEO
-            supportedFeatures.contains(DeviceFeature.SCREENSHOT) && screenshotEnabled -> DeviceFeature.SCREENSHOT
-            else -> null
-        }
-    }
-
-    private fun recorderEnabled(
-        type: DeviceFeature,
-        configuration: ScreenRecordConfiguration
-    ) = when (type) {
-        DeviceFeature.VIDEO -> configuration.videoConfiguration.enabled
-        DeviceFeature.SCREENSHOT -> configuration.screenshotConfiguration.enabled
-    }
-
     private suspend fun detectMd5Binary(): String {
         for (path in listOf("/system/bin/md5", "/system/bin/md5sum")) {
             if (hasBinary(path)) return path.split("/").last()

diff --git a/...endor-android/base/src/main/kotlin/com/malinskiy/marathon/android/RecorderTypeSelector.kt b/...endor-android/base/src/main/kotlin/com/malinskiy/marathon/android/RecorderTypeSelector.kt
@@ -0,0 +1,30 @@
+package com.malinskiy.marathon.android
+
+import com.malinskiy.marathon.device.DeviceFeature
+
+object RecorderTypeSelector {
+    fun selectRecorderType(supportedFeatures: Collection<DeviceFeature>, configuration: ScreenRecordConfiguration): DeviceFeature? {
+        val preferred = configuration.preferableRecorderType
+        val screenshotEnabled = recorderEnabled(DeviceFeature.SCREENSHOT, configuration)
+        val videoEnabled = recorderEnabled(DeviceFeature.VIDEO, configuration)
+
+        if (preferred != null && supportedFeatures.contains(preferred) && recorderEnabled(preferred, configuration)) {
+            return preferred
+        }
+
+        return when {
+            supportedFeatures.contains(DeviceFeature.VIDEO) && videoEnabled -> DeviceFeature.VIDEO
+            supportedFeatures.contains(DeviceFeature.SCREENSHOT) && screenshotEnabled -> DeviceFeature.SCREENSHOT
+            else -> null
+        }
+    }
+
+    private fun recorderEnabled(
+        type: DeviceFeature,
+        configuration: ScreenRecordConfiguration
+    ) = when (type) {
+        DeviceFeature.VIDEO -> configuration.videoConfiguration.enabled
+        DeviceFeature.SCREENSHOT -> configuration.screenshotConfiguration.enabled
+    }
+
+}
diff --git a/...r/vendor-android/base/src/main/kotlin/com/malinskiy/marathon/android/RemoteFileManager.kt b/...r/vendor-android/base/src/main/kotlin/com/malinskiy/marathon/android/RemoteFileManager.kt
@@ -7,7 +7,7 @@ class RemoteFileManager(private val device: AndroidDevice) {
 
     suspend fun removeRemotePath(remotePath: String, recursive: Boolean = false) {
         val errorMessage = "Could not delete remote file(s): $remotePath"
-        device.criticalExecuteShellCommand("rm ${if (recursive) "-r" else ""} $remotePath", errorMessage)
+        device.criticalExecuteShellCommand("rm ${if (recursive) "-r " else ""}$remotePath", errorMessage)
     }
 
     suspend fun createRemoteDirectory(remoteDir: String = outputDir) {

diff --git a/...-android/base/src/main/kotlin/com/malinskiy/marathon/android/ScreenRecordConfiguration.kt b/...-android/base/src/main/kotlin/com/malinskiy/marathon/android/ScreenRecordConfiguration.kt
@@ -17,6 +17,9 @@ data class ScreenshotConfiguration(
     @JsonProperty("delayMs") val delayMs: Int = 500
 )
 
+/**
+ * See https://android.googlesource.com/platform/frameworks/av/+/master/cmds/screenrecord/screenrecord.cpp for a list of latest defaults
+ */
 data class VideoConfiguration(
     @JsonProperty("enabled") val enabled: Boolean = true,
     @JsonProperty("width") val width: Int = 720,
@@ -41,7 +44,14 @@ data class VideoConfiguration(
 
         if (bitrateMbps > 0) {
             sb.append("--bit-rate ")
-            sb.append(bitrateMbps * 1_000_000)
+            var bitrate = bitrateMbps * 1_000_000
+            /**
+             * screenrecord supports bitrate up to 200Mbps
+             */
+            if (bitrate > 200_000_000) {
+                bitrate = 200_000_000
+            }
+            sb.append(bitrate)
             sb.append(' ')
         }
 

diff --git a/...base/src/main/kotlin/com/malinskiy/marathon/android/configuration/TimeoutConfiguration.kt b/...base/src/main/kotlin/com/malinskiy/marathon/android/configuration/TimeoutConfiguration.kt
@@ -10,7 +10,7 @@ data class TimeoutConfiguration(
     var pullFile: Duration = Duration.ofSeconds(30),
     var uninstall: Duration = shell,
     var install: Duration = shell,
-    var screenrecorder: Duration = Duration.ofMinutes(10),
+    var screenrecorder: Duration = Duration.ofSeconds(200),
     var screencapturer: Duration = Duration.ofMillis(300),
     var socketIdleTimeout: Duration = Duration.ofSeconds(30)
 )
diff --git a/...ndroid/base/src/main/kotlin/com/malinskiy/marathon/android/exception/TransferException.kt b/...ndroid/base/src/main/kotlin/com/malinskiy/marathon/android/exception/TransferException.kt
@@ -2,5 +2,6 @@ package com.malinskiy.marathon.android.exception
 
 class TransferException : RuntimeException {
     constructor(message: String) : super(message)
+    constructor(message: String, reason: Throwable) : super(message, reason)
     constructor(t: Throwable) : super(t)
 }
diff --git a/...kotlin/com/malinskiy/marathon/android/executor/listeners/AbstractTestRunResultListener.kt b/...kotlin/com/malinskiy/marathon/android/executor/listeners/AbstractTestRunResultListener.kt
@@ -2,9 +2,10 @@ package com.malinskiy.marathon.android.executor.listeners
 
 import com.malinskiy.marathon.android.model.TestIdentifier
 import com.malinskiy.marathon.android.model.TestRunResultsAccumulator
+import com.malinskiy.marathon.time.Timer
 
-abstract class AbstractTestRunResultListener : NoOpTestRunListener() {
-    private val runResult = TestRunResultsAccumulator()
+abstract class AbstractTestRunResultListener(timer: Timer) : NoOpTestRunListener() {
+    private val runResult = TestRunResultsAccumulator(timer)
 
     override suspend fun testRunStarted(runName: String, testCount: Int) {
         runResult.testRunStarted(runName, testCount)

diff --git a/.../base/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/LogCatListener.kt b/.../base/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/LogCatListener.kt
@@ -26,18 +26,20 @@ class LogCatListener(
     private val stringBuffer = StringBuffer(4096)
 
     override fun onLine(line: String) {
-        stringBuffer.appendln(line)
+        stringBuffer.appendLine(line)
     }
 
     override suspend fun testRunStarted(runName: String, testCount: Int) {
         device.addLogcatListener(this)
     }
 
-    override suspend fun testEnded(test: TestIdentifier, testMetrics: Map<String, String>) {
-        device.removeLogcatListener(this)
+    override suspend fun testStarted(test: TestIdentifier) {
+        super.testStarted(test)
+        stringBuffer.reset()
+    }
 
+    override suspend fun testEnded(test: TestIdentifier, testMetrics: Map<String, String>) {
         val file = logWriter.saveLogs(test.toTest(), devicePoolId, testBatchId, device.toDeviceInfo(), listOf(stringBuffer.toString()))
-
         attachmentListeners.forEach { it.onAttachment(test.toTest(), Attachment(file, AttachmentType.LOG)) }
     }
 
@@ -48,4 +50,8 @@ class LogCatListener(
     override suspend fun testRunFailed(errorMessage: String) {
         device.removeLogcatListener(this)
     }
+
+    private fun StringBuffer.reset() {
+        delete(0, length)
+    }
 }