Add support for bugsnagReporting in gradle config. (#820)

cli/src/main/kotlin/com/malinskiy/marathon/cli/ApplicationView.kt

@@ -3,6 +3,7 @@ package com.malinskiy.marathon.cli
 import com.github.ajalt.clikt.core.PrintMessage
 import com.github.ajalt.clikt.core.subcommands
 import com.malinskiy.marathon.Marathon
+import com.malinskiy.marathon.exceptions.ExceptionsReporter
 import com.malinskiy.marathon.android.AndroidVendor
 import com.malinskiy.marathon.android.adam.di.adamModule
 import com.malinskiy.marathon.cli.args.CliConfiguration
@@ -45,10 +46,8 @@ private fun execute(cliConfiguration: CliConfiguration) {
         else -> throw IllegalArgumentException("Please handle the new format of cliConfiguration=$cliConfiguration")
     }
 
-    val bugsnagExceptionsReporter = ExceptionsReporterFactory.get(marathonStartConfiguration.bugsnagReporting)
+    var bugsnagExceptionsReporter: ExceptionsReporter? = null;
     try {
-        bugsnagExceptionsReporter.start(AppType.CLI)
-
         logger.info { "Checking ${marathonStartConfiguration.marathonfile} config" }
         if (!marathonStartConfiguration.marathonfile.isFile) {
             logger.error { "No config ${marathonStartConfiguration.marathonfile.absolutePath} present" }
@@ -58,7 +57,12 @@ private fun execute(cliConfiguration: CliConfiguration) {
         val configuration = ConfigurationFactory(
             marathonfileDir = marathonStartConfiguration.marathonfile.canonicalFile.parentFile,
             analyticsTracking = marathonStartConfiguration.analyticsTracking,
+            bugsnagReporting = marathonStartConfiguration.bugsnagReporting,
         ).parse(marathonStartConfiguration.marathonfile)
+
+        bugsnagExceptionsReporter = ExceptionsReporterFactory.get(configuration.bugsnagReporting)
+        bugsnagExceptionsReporter.start(AppType.CLI)
+
         val vendorConfiguration = configuration.vendorConfiguration
         val modules = when (vendorConfiguration) {
             is VendorConfiguration.IOSConfiguration -> {
@@ -87,6 +91,8 @@ private fun execute(cliConfiguration: CliConfiguration) {
         }
     } finally {
         stopKoin()
-        bugsnagExceptionsReporter.end()
+        if (bugsnagExceptionsReporter != null) {
+            bugsnagExceptionsReporter.end()
+        }
     }
 }

cli/src/main/kotlin/com/malinskiy/marathon/cli/args/CliCommands.kt

@@ -13,7 +13,7 @@ class MarathonRunCommonOptions : OptionGroup() {
     val marathonfile by option("--marathonfile", "-m", help="Marathonfile file path")
         .file()
         .default(File("Marathonfile"))
-    val analyticsTracking by option("--analyticsTracking", help="Enable anonymous analytics tracking")
+    val analyticsTracking by option("--analyticsTracking", help="Enable / Disable anonymous analytics tracking. Enabled by default.")
         .convert { it.toBoolean() }
         .default(true)
     val bugsnagReporting by option("--bugsnag", help="Enable/Disable anonymous crash reporting. Enabled by default")

configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt

@@ -49,6 +49,7 @@ data class Configuration private constructor(
     val vendorConfiguration: VendorConfiguration,
 
     val analyticsTracking: Boolean,
+    val bugsnagReporting: Boolean,
     val deviceInitializationTimeoutMillis: Long,
 ) {
     fun toMap() =
@@ -109,6 +110,7 @@ data class Configuration private constructor(
         if (screenRecordingPolicy != other.screenRecordingPolicy) return false
         if (vendorConfiguration != other.vendorConfiguration) return false
         if (analyticsTracking != other.analyticsTracking) return false
+        if (bugsnagReporting != other.bugsnagReporting) return false
         if (deviceInitializationTimeoutMillis != other.deviceInitializationTimeoutMillis) return false
 
         return true
@@ -139,6 +141,7 @@ data class Configuration private constructor(
         result = 31 * result + screenRecordingPolicy.hashCode()
         result = 31 * result + vendorConfiguration.hashCode()
         result = 31 * result + analyticsTracking.hashCode()
+        result = 31 * result + bugsnagReporting.hashCode()
         result = 31 * result + deviceInitializationTimeoutMillis.hashCode()
         return result
     }
@@ -170,7 +173,8 @@ data class Configuration private constructor(
 
          var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE,
 
-         var analyticsTracking: Boolean = false,
+         var analyticsTracking: Boolean = true,
+         var bugsnagReporting: Boolean = true,
          var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS,
 
          var outputConfiguration: OutputConfiguration = OutputConfiguration(),
@@ -202,6 +206,7 @@ data class Configuration private constructor(
                 screenRecordingPolicy = screenRecordingPolicy,
                 vendorConfiguration = vendorConfiguration,
                 analyticsTracking = analyticsTracking,
+                bugsnagReporting = bugsnagReporting,
                 deviceInitializationTimeoutMillis = deviceInitializationTimeoutMillis
             )
         }

configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt

@@ -44,6 +44,7 @@ class ConfigurationFactory(
     },
     private val environmentVariableSubstitutor: StringSubstitutor = StringSubstitutor(StringLookupFactory.INSTANCE.environmentVariableStringLookup()),
     private val analyticsTracking: Boolean? = null,
+    private val bugsnagReporting: Boolean? = null,
 ) {
     fun parse(marathonfile: File): Configuration {
         val configWithEnvironmentVariablesReplaced = environmentVariableSubstitutor.replace(marathonfile.readText())
@@ -106,10 +107,13 @@ class ConfigurationFactory(
                 VendorConfiguration.StubVendorConfiguration -> configuration.vendorConfiguration
                 is VendorConfiguration.EmptyVendorConfiguration -> throw ConfigurationException("No vendor configuration specified")
             }
-
             return configuration.copy(
                 vendorConfiguration = vendorConfiguration,
-                analyticsTracking = analyticsTracking ?: configuration.analyticsTracking
+                // Default value for analyticsTracking / bugsnagReporting in both CLI and marathon / gradle config is true.
+                // If analyticsTracking / bugsnagReporting is set to false in either CLI or marathon / gradle config, it will be disabled.
+                // Note that it's not possible to set the CLI value when running marathon via gradle.
+                analyticsTracking = if (analyticsTracking == false || configuration.analyticsTracking == false) false else true,
+                bugsnagReporting = if (bugsnagReporting == false || configuration.bugsnagReporting == false) false else true
             )
         } catch (e: JsonProcessingException) {
             throw ConfigurationException("Error parsing config file ${marathonfile.absolutePath}", e)

configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt

@@ -410,4 +410,42 @@ class ConfigurationFactoryTest {
             )
         )
     }
+
+    @Test
+    fun `on configuration with tracking enabled by default`() {
+        val file = File(ConfigurationFactoryTest::class.java.getResource("/fixture/config/sample_1.yaml").file)
+        parser = ConfigurationFactory(
+            file.parentFile,
+        )
+        val configuration = parser.parse(file)
+
+        configuration.analyticsTracking shouldBeEqualTo true
+        configuration.bugsnagReporting shouldBeEqualTo true
+    }
+
+    @Test
+    fun `on configuration with tracking disabled in cli`() {
+        val file = File(ConfigurationFactoryTest::class.java.getResource("/fixture/config/sample_1.yaml").file)
+        parser = ConfigurationFactory(
+            file.parentFile,
+            analyticsTracking = false,
+            bugsnagReporting = false,
+        )
+        val configuration = parser.parse(file)
+
+        configuration.analyticsTracking shouldBeEqualTo false
+        configuration.bugsnagReporting shouldBeEqualTo false
+    }
+
+    @Test
+    fun `on configuration with tracking disabled in config file`() {
+        val file = File(ConfigurationFactoryTest::class.java.getResource("/fixture/config/sample_1_no_tracking.yaml").file)
+        parser = ConfigurationFactory(
+            file.parentFile,
+        )
+        val configuration = parser.parse(file)
+
+        configuration.analyticsTracking shouldBeEqualTo false
+        configuration.bugsnagReporting shouldBeEqualTo false
+    }
 }

configuration/src/test/resources/fixture/config/sample_1_no_tracking.yaml

@@ -0,0 +1,107 @@
+analyticsTracking: false
+bugsnagReporting: false
+name: "sample-app tests"
+outputDir: "./marathon"
+analyticsConfiguration:
+  type: "influxdb"
+  url: "http://influx.svc.cluster.local:8086"
+  user: "root"
+  password: "root"
+  dbName: "marathon"
+poolingStrategy:
+  type: "combo"
+  list:
+    - type: "omni"
+    - type: "device-model"
+    - type: "os-version"
+    - type: "manufacturer"
+    - type: "abi"
+shardingStrategy:
+  type: "count"
+  count: 5
+sortingStrategy:
+  type: "success-rate"
+  timeLimit: "2015-03-14T09:26:53.590Z"
+batchingStrategy:
+  type: "fixed-size"
+  size: 5
+flakinessStrategy:
+  type: "probability"
+  minSuccessRate: 0.7
+  maxCount: 3
+  timeLimit: "2015-03-14T09:26:53.590Z"
+retryStrategy:
+  type: "fixed-quota"
+  totalAllowedRetryQuota: 100
+  retryPerTestQuota: 2
+filteringConfiguration:
+  allowlist:
+    - type: "simple-class-name"
+      regex: ".*"
+    - type: "simple-class-name"
+      values:
+        - "SimpleTest"
+    - type: "fully-qualified-class-name"
+      file: "filterfile"
+    - type: "fully-qualified-class-name"
+      regex: ".*"
+    - type: "method"
+      regex: ".*"
+    - type: "composition"
+      filters:
+        - type: "package"
+          regex: ".*"
+        - type: "method"
+          regex: ".*"
+      op: "UNION"
+  blocklist:
+    - type: "package"
+      regex: ".*"
+    - type: "annotation"
+      regex: ".*"
+    - type: "annotationData"
+      nameRegex: ".*"
+      valueRegex: ".*"
+testClassRegexes:
+  - "^((?!Abstract).)*Test$"
+includeSerialRegexes:
+  - "emulator-500[2,4]"
+excludeSerialRegexes:
+  - "emulator-5002"
+ignoreFailures: false
+isCodeCoverageEnabled: false
+executionStrategy:
+  mode: ANY_SUCCESS
+  fast: true
+testBatchTimeoutMillis: 20000
+testOutputTimeoutMillis: 30000
+debug: true
+screenRecordingPolicy: "ON_ANY"
+deviceInitializationTimeoutMillis: 300000
+vendorConfiguration:
+  type: "Android"
+  androidSdk: "/local/android"
+  applicationApk: "kotlin-buildscript/build/outputs/apk/debug/kotlin-buildscript-debug.apk"
+  testApplicationApk: "kotlin-buildscript/build/outputs/apk/androidTest/debug/kotlin-buildscript-debug-androidTest.apk"
+  splitApks:
+    - "kotlin-buildscript/build/outputs/apk/androidTest/debug/kotlin-buildscript-split-debug.apk"
+  autoGrantPermission: true
+  applicationPmClear: true
+  testApplicationPmClear: true
+  instrumentationArgs:
+    debug: "false"
+  installOptions: "-d"
+  screenRecordConfiguration:
+    preferableRecorderType: "screenshot"
+    videoConfiguration:
+      enabled: false
+      width: 1080
+      height: 1920
+      bitrateMbps: 2
+      timeLimit: 300
+    screenshotConfiguration:
+      enabled: false
+      width: 1080
+      height: 1920
+      delayMs: 200
+  waitForDevicesTimeoutMillis: 15000

docs/docs/intro/configure.md

@@ -312,6 +312,7 @@ marathon {
 </Tabs>
 
 ### Analytics tracking
+
 To better understand the use-cases that marathon is used for we're asking you to provide us with anonymised information about your usage. By
 default, this is enabled. Use **false** to disable.
 
@@ -343,6 +344,52 @@ marathon {
 </TabItem>
 </Tabs>
 
+:::note
+
+analyticsTracking can also be enabled (default value) / disabled directly from the CLI. It is disabled if it's set to be disabled in either the config or the CLI.
+
+:::
+
+
+### BugSnag reporting
+
+To better understand crashes, we report crashes with anonymised info. By default, this is enabled. Use **false** to disable.
+
+<Tabs>
+<TabItem value="YAML" label="Marathonfile">
+
+```yaml
+bugsnagReporting: false
+```
+
+</TabItem>
+<TabItem value="kts" label="Kotlin DSL">
+
+```kotlin
+marathon {
+  bugsnagReporting = false
+}
+```
+
+</TabItem>
+<TabItem value="groovy" label="Groovy DSL">
+
+```groovy
+marathon {
+  bugsnagReporting = false
+}
+```
+
+</TabItem>
+</Tabs>
+
+:::note
+
+bugsnagReporting can also be enabled (default value) / disabled directly from the CLI. It is disabled if it's set to be disabled in either the config or the CLI.
+
+:::
+
+
 ### Uncompleted test retry quota
 By default, tests that don't have any status reported after execution (for example a device disconnected during the execution) retry
 indefinitely. You can limit the number of total execution for such cases using this option.

marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonExtension.kt

@@ -221,6 +221,11 @@ open class MarathonExtension {
      */
     var analyticsTracking: Boolean = true
 
+    /**
+     * Whether to report crashes to Bugsnag. By default, this is enabled. Use **false** to disable.
+     */
+    var bugsnagReporting: Boolean = true
+
     /**
      * When the test run starts device provider is expected to provide some devices. This should not take more than 3 minutes by default. If your
      * setup requires this to be changed please override using this parameter

marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt

@@ -183,6 +183,7 @@ class MarathonPlugin : Plugin<Project> {
                 config.debug?.let { debug = it }
                 config.screenRecordingPolicy?.let { screenRecordingPolicy = it }
                 config.analyticsTracking?.let { analyticsTracking = it }
+                config.bugsnagReporting?.let { bugsnagReporting = it }
                 config.deviceInitializationTimeoutMillis?.let {
                     deviceInitializationTimeoutMillis = deviceInitializationTimeoutMillis
                 }

marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/task/GenerateMarathonfileTask.kt

@@ -69,7 +69,8 @@ open class GenerateMarathonfileTask @Inject constructor(objects: ObjectFactory)
         // Write a Marathonfile
         val configurationFactory = ConfigurationFactory(
             marathonfileDir = temporaryDir,
-            analyticsTracking = cnf.analyticsTracking
+            analyticsTracking = cnf.analyticsTracking,
+            bugsnagReporting = cnf.bugsnagReporting
         )
         val yaml = configurationFactory.serialize(cnf)
         marathonfile.get().asFile.writeText(yaml)