From 0354fc915bf9293c58194b9c30be9d8bb96b55ff Mon Sep 17 00:00:00 2001 From: Aaron Wang Date: Sat, 29 Jul 2023 23:40:30 -0700 Subject: [PATCH] Add support for bugsnagReporting in gradle config. (#820) --- .../malinskiy/marathon/cli/ApplicationView.kt | 14 ++- .../marathon/cli/args/CliCommands.kt | 2 +- .../marathon/config/Configuration.kt | 7 +- .../serialization/ConfigurationFactory.kt | 8 +- .../serialization/ConfigurationFactoryTest.kt | 38 +++++++ .../fixture/config/sample_1_no_tracking.yaml | 107 ++++++++++++++++++ docs/docs/intro/configure.md | 47 ++++++++ .../marathon/gradle/MarathonExtension.kt | 5 + .../marathon/gradle/MarathonPlugin.kt | 1 + .../gradle/task/GenerateMarathonfileTask.kt | 3 +- 10 files changed, 223 insertions(+), 9 deletions(-) create mode 100644 configuration/src/test/resources/fixture/config/sample_1_no_tracking.yaml diff --git a/cli/src/main/kotlin/com/malinskiy/marathon/cli/ApplicationView.kt b/cli/src/main/kotlin/com/malinskiy/marathon/cli/ApplicationView.kt index 0c935b2d7..1789c9a89 100644 --- a/cli/src/main/kotlin/com/malinskiy/marathon/cli/ApplicationView.kt +++ b/cli/src/main/kotlin/com/malinskiy/marathon/cli/ApplicationView.kt @@ -3,6 +3,7 @@ package com.malinskiy.marathon.cli import com.github.ajalt.clikt.core.PrintMessage import com.github.ajalt.clikt.core.subcommands import com.malinskiy.marathon.Marathon +import com.malinskiy.marathon.exceptions.ExceptionsReporter import com.malinskiy.marathon.android.AndroidVendor import com.malinskiy.marathon.android.adam.di.adamModule import com.malinskiy.marathon.cli.args.CliConfiguration @@ -45,10 +46,8 @@ private fun execute(cliConfiguration: CliConfiguration) { else -> throw IllegalArgumentException("Please handle the new format of cliConfiguration=$cliConfiguration") } - val bugsnagExceptionsReporter = ExceptionsReporterFactory.get(marathonStartConfiguration.bugsnagReporting) + var bugsnagExceptionsReporter: ExceptionsReporter? = null; try { - bugsnagExceptionsReporter.start(AppType.CLI) - logger.info { "Checking ${marathonStartConfiguration.marathonfile} config" } if (!marathonStartConfiguration.marathonfile.isFile) { logger.error { "No config ${marathonStartConfiguration.marathonfile.absolutePath} present" } @@ -58,7 +57,12 @@ private fun execute(cliConfiguration: CliConfiguration) { val configuration = ConfigurationFactory( marathonfileDir = marathonStartConfiguration.marathonfile.canonicalFile.parentFile, analyticsTracking = marathonStartConfiguration.analyticsTracking, + bugsnagReporting = marathonStartConfiguration.bugsnagReporting, ).parse(marathonStartConfiguration.marathonfile) + + bugsnagExceptionsReporter = ExceptionsReporterFactory.get(configuration.bugsnagReporting) + bugsnagExceptionsReporter.start(AppType.CLI) + val vendorConfiguration = configuration.vendorConfiguration val modules = when (vendorConfiguration) { is VendorConfiguration.IOSConfiguration -> { @@ -87,6 +91,8 @@ private fun execute(cliConfiguration: CliConfiguration) { } } finally { stopKoin() - bugsnagExceptionsReporter.end() + if (bugsnagExceptionsReporter != null) { + bugsnagExceptionsReporter.end() + } } } diff --git a/cli/src/main/kotlin/com/malinskiy/marathon/cli/args/CliCommands.kt b/cli/src/main/kotlin/com/malinskiy/marathon/cli/args/CliCommands.kt index 269253413..fff70003e 100644 --- a/cli/src/main/kotlin/com/malinskiy/marathon/cli/args/CliCommands.kt +++ b/cli/src/main/kotlin/com/malinskiy/marathon/cli/args/CliCommands.kt @@ -13,7 +13,7 @@ class MarathonRunCommonOptions : OptionGroup() { val marathonfile by option("--marathonfile", "-m", help="Marathonfile file path") .file() .default(File("Marathonfile")) - val analyticsTracking by option("--analyticsTracking", help="Enable anonymous analytics tracking") + val analyticsTracking by option("--analyticsTracking", help="Enable / Disable anonymous analytics tracking. Enabled by default.") .convert { it.toBoolean() } .default(true) val bugsnagReporting by option("--bugsnag", help="Enable/Disable anonymous crash reporting. Enabled by default") diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt index 5dc0f7939..388583d26 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt @@ -49,6 +49,7 @@ data class Configuration private constructor( val vendorConfiguration: VendorConfiguration, val analyticsTracking: Boolean, + val bugsnagReporting: Boolean, val deviceInitializationTimeoutMillis: Long, ) { fun toMap() = @@ -109,6 +110,7 @@ data class Configuration private constructor( if (screenRecordingPolicy != other.screenRecordingPolicy) return false if (vendorConfiguration != other.vendorConfiguration) return false if (analyticsTracking != other.analyticsTracking) return false + if (bugsnagReporting != other.bugsnagReporting) return false if (deviceInitializationTimeoutMillis != other.deviceInitializationTimeoutMillis) return false return true @@ -139,6 +141,7 @@ data class Configuration private constructor( result = 31 * result + screenRecordingPolicy.hashCode() result = 31 * result + vendorConfiguration.hashCode() result = 31 * result + analyticsTracking.hashCode() + result = 31 * result + bugsnagReporting.hashCode() result = 31 * result + deviceInitializationTimeoutMillis.hashCode() return result } @@ -170,7 +173,8 @@ data class Configuration private constructor( var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE, - var analyticsTracking: Boolean = false, + var analyticsTracking: Boolean = true, + var bugsnagReporting: Boolean = true, var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS, var outputConfiguration: OutputConfiguration = OutputConfiguration(), @@ -202,6 +206,7 @@ data class Configuration private constructor( screenRecordingPolicy = screenRecordingPolicy, vendorConfiguration = vendorConfiguration, analyticsTracking = analyticsTracking, + bugsnagReporting = bugsnagReporting, deviceInitializationTimeoutMillis = deviceInitializationTimeoutMillis ) } diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt index a89a5a265..aa1086566 100644 --- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt +++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt @@ -44,6 +44,7 @@ class ConfigurationFactory( }, private val environmentVariableSubstitutor: StringSubstitutor = StringSubstitutor(StringLookupFactory.INSTANCE.environmentVariableStringLookup()), private val analyticsTracking: Boolean? = null, + private val bugsnagReporting: Boolean? = null, ) { fun parse(marathonfile: File): Configuration { val configWithEnvironmentVariablesReplaced = environmentVariableSubstitutor.replace(marathonfile.readText()) @@ -106,10 +107,13 @@ class ConfigurationFactory( VendorConfiguration.StubVendorConfiguration -> configuration.vendorConfiguration is VendorConfiguration.EmptyVendorConfiguration -> throw ConfigurationException("No vendor configuration specified") } - return configuration.copy( vendorConfiguration = vendorConfiguration, - analyticsTracking = analyticsTracking ?: configuration.analyticsTracking + // Default value for analyticsTracking / bugsnagReporting in both CLI and marathon / gradle config is true. + // If analyticsTracking / bugsnagReporting is set to false in either CLI or marathon / gradle config, it will be disabled. + // Note that it's not possible to set the CLI value when running marathon via gradle. + analyticsTracking = if (analyticsTracking == false || configuration.analyticsTracking == false) false else true, + bugsnagReporting = if (bugsnagReporting == false || configuration.bugsnagReporting == false) false else true ) } catch (e: JsonProcessingException) { throw ConfigurationException("Error parsing config file ${marathonfile.absolutePath}", e) diff --git a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt index 79d07dd3a..4ff9f6346 100644 --- a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt +++ b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt @@ -410,4 +410,42 @@ class ConfigurationFactoryTest { ) ) } + + @Test + fun `on configuration with tracking enabled by default`() { + val file = File(ConfigurationFactoryTest::class.java.getResource("/fixture/config/sample_1.yaml").file) + parser = ConfigurationFactory( + file.parentFile, + ) + val configuration = parser.parse(file) + + configuration.analyticsTracking shouldBeEqualTo true + configuration.bugsnagReporting shouldBeEqualTo true + } + + @Test + fun `on configuration with tracking disabled in cli`() { + val file = File(ConfigurationFactoryTest::class.java.getResource("/fixture/config/sample_1.yaml").file) + parser = ConfigurationFactory( + file.parentFile, + analyticsTracking = false, + bugsnagReporting = false, + ) + val configuration = parser.parse(file) + + configuration.analyticsTracking shouldBeEqualTo false + configuration.bugsnagReporting shouldBeEqualTo false + } + + @Test + fun `on configuration with tracking disabled in config file`() { + val file = File(ConfigurationFactoryTest::class.java.getResource("/fixture/config/sample_1_no_tracking.yaml").file) + parser = ConfigurationFactory( + file.parentFile, + ) + val configuration = parser.parse(file) + + configuration.analyticsTracking shouldBeEqualTo false + configuration.bugsnagReporting shouldBeEqualTo false + } } diff --git a/configuration/src/test/resources/fixture/config/sample_1_no_tracking.yaml b/configuration/src/test/resources/fixture/config/sample_1_no_tracking.yaml new file mode 100644 index 000000000..a1d93d7e0 --- /dev/null +++ b/configuration/src/test/resources/fixture/config/sample_1_no_tracking.yaml @@ -0,0 +1,107 @@ +analyticsTracking: false +bugsnagReporting: false +name: "sample-app tests" +outputDir: "./marathon" +analyticsConfiguration: + type: "influxdb" + url: "http://influx.svc.cluster.local:8086" + user: "root" + password: "root" + dbName: "marathon" +poolingStrategy: + type: "combo" + list: + - type: "omni" + - type: "device-model" + - type: "os-version" + - type: "manufacturer" + - type: "abi" +shardingStrategy: + type: "count" + count: 5 +sortingStrategy: + type: "success-rate" + timeLimit: "2015-03-14T09:26:53.590Z" +batchingStrategy: + type: "fixed-size" + size: 5 +flakinessStrategy: + type: "probability" + minSuccessRate: 0.7 + maxCount: 3 + timeLimit: "2015-03-14T09:26:53.590Z" +retryStrategy: + type: "fixed-quota" + totalAllowedRetryQuota: 100 + retryPerTestQuota: 2 +filteringConfiguration: + allowlist: + - type: "simple-class-name" + regex: ".*" + - type: "simple-class-name" + values: + - "SimpleTest" + - type: "fully-qualified-class-name" + file: "filterfile" + - type: "fully-qualified-class-name" + regex: ".*" + - type: "method" + regex: ".*" + - type: "composition" + filters: + - type: "package" + regex: ".*" + - type: "method" + regex: ".*" + op: "UNION" + blocklist: + - type: "package" + regex: ".*" + - type: "annotation" + regex: ".*" + - type: "annotationData" + nameRegex: ".*" + valueRegex: ".*" +testClassRegexes: + - "^((?!Abstract).)*Test$" +includeSerialRegexes: + - "emulator-500[2,4]" +excludeSerialRegexes: + - "emulator-5002" +ignoreFailures: false +isCodeCoverageEnabled: false +executionStrategy: + mode: ANY_SUCCESS + fast: true +testBatchTimeoutMillis: 20000 +testOutputTimeoutMillis: 30000 +debug: true +screenRecordingPolicy: "ON_ANY" +deviceInitializationTimeoutMillis: 300000 +vendorConfiguration: + type: "Android" + androidSdk: "/local/android" + applicationApk: "kotlin-buildscript/build/outputs/apk/debug/kotlin-buildscript-debug.apk" + testApplicationApk: "kotlin-buildscript/build/outputs/apk/androidTest/debug/kotlin-buildscript-debug-androidTest.apk" + splitApks: + - "kotlin-buildscript/build/outputs/apk/androidTest/debug/kotlin-buildscript-split-debug.apk" + autoGrantPermission: true + applicationPmClear: true + testApplicationPmClear: true + instrumentationArgs: + debug: "false" + installOptions: "-d" + screenRecordConfiguration: + preferableRecorderType: "screenshot" + videoConfiguration: + enabled: false + width: 1080 + height: 1920 + bitrateMbps: 2 + timeLimit: 300 + screenshotConfiguration: + enabled: false + width: 1080 + height: 1920 + delayMs: 200 + waitForDevicesTimeoutMillis: 15000 diff --git a/docs/docs/intro/configure.md b/docs/docs/intro/configure.md index a3b3dab1d..a9e02f5d6 100644 --- a/docs/docs/intro/configure.md +++ b/docs/docs/intro/configure.md @@ -312,6 +312,7 @@ marathon { ### Analytics tracking + To better understand the use-cases that marathon is used for we're asking you to provide us with anonymised information about your usage. By default, this is enabled. Use **false** to disable. @@ -343,6 +344,52 @@ marathon { +:::note + +analyticsTracking can also be enabled (default value) / disabled directly from the CLI. It is disabled if it's set to be disabled in either the config or the CLI. + +::: + + +### BugSnag reporting + +To better understand crashes, we report crashes with anonymised info. By default, this is enabled. Use **false** to disable. + + + + +```yaml +bugsnagReporting: false +``` + + + + +```kotlin +marathon { + bugsnagReporting = false +} +``` + + + + +```groovy +marathon { + bugsnagReporting = false +} +``` + + + + +:::note + +bugsnagReporting can also be enabled (default value) / disabled directly from the CLI. It is disabled if it's set to be disabled in either the config or the CLI. + +::: + + ### Uncompleted test retry quota By default, tests that don't have any status reported after execution (for example a device disconnected during the execution) retry indefinitely. You can limit the number of total execution for such cases using this option. diff --git a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonExtension.kt b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonExtension.kt index 050de1ca1..380721bb6 100644 --- a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonExtension.kt +++ b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonExtension.kt @@ -221,6 +221,11 @@ open class MarathonExtension { */ var analyticsTracking: Boolean = true + /** + * Whether to report crashes to Bugsnag. By default, this is enabled. Use **false** to disable. + */ + var bugsnagReporting: Boolean = true + /** * When the test run starts device provider is expected to provide some devices. This should not take more than 3 minutes by default. If your * setup requires this to be changed please override using this parameter diff --git a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt index c3c950a98..b4b682bb8 100644 --- a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt +++ b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/MarathonPlugin.kt @@ -183,6 +183,7 @@ class MarathonPlugin : Plugin { config.debug?.let { debug = it } config.screenRecordingPolicy?.let { screenRecordingPolicy = it } config.analyticsTracking?.let { analyticsTracking = it } + config.bugsnagReporting?.let { bugsnagReporting = it } config.deviceInitializationTimeoutMillis?.let { deviceInitializationTimeoutMillis = deviceInitializationTimeoutMillis } diff --git a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/task/GenerateMarathonfileTask.kt b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/task/GenerateMarathonfileTask.kt index e3e70d7cb..c63096018 100644 --- a/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/task/GenerateMarathonfileTask.kt +++ b/marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/gradle/task/GenerateMarathonfileTask.kt @@ -69,7 +69,8 @@ open class GenerateMarathonfileTask @Inject constructor(objects: ObjectFactory) // Write a Marathonfile val configurationFactory = ConfigurationFactory( marathonfileDir = temporaryDir, - analyticsTracking = cnf.analyticsTracking + analyticsTracking = cnf.analyticsTracking, + bugsnagReporting = cnf.bugsnagReporting ) val yaml = configurationFactory.serialize(cnf) marathonfile.get().asFile.writeText(yaml)