Ensure leak test are running

android/scripts/run-instrumented-tests.sh

@@ -132,8 +132,8 @@ case "$TEST_TYPE" in
         echo "Error: The variable PARTNER_AUTH or VALID_TEST_ACCOUNT_NUMBER must be set."
         exit 1
     fi
-    OPTIONAL_TEST_ARGUMENTS+=" -e ENABLE_HIGHLY_RATE_LIMITED_TESTS $ENABLE_HIGHLY_RATE_LIMITED_TESTS"
-    OPTIONAL_TEST_ARGUMENTS+=" -e ENABLE_ACCESS_TO_LOCAL_API_TESTS $ENABLE_ACCESS_TO_LOCAL_API_TESTS"
+    OPTIONAL_TEST_ARGUMENTS+=" -e enable_access_to_local_api_tests $ENABLE_ACCESS_TO_LOCAL_API_TESTS"
+    OPTIONAL_TEST_ARGUMENTS+=" -e enable_highly_rate_limited_tests $ENABLE_HIGHLY_RATE_LIMITED_TESTS"
     USE_ORCHESTRATOR="true"
     PACKAGE_NAME="net.mullvad.mullvadvpn"
     if [[ "$INFRA_FLAVOR" =~ ^(devmole|stagemole)$ ]]; then

android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/extension/UiAutomatorExtensions.kt

@@ -17,6 +17,9 @@ fun UiObject2.findObjectByCaseInsensitiveText(text: String): UiObject2 {
     return findObjectWithTimeout(By.text(Pattern.compile(text, Pattern.CASE_INSENSITIVE)))
 }
 
+fun UiDevice.hasObjectWithTimeout(selector: BySelector, timeout: Long = DEFAULT_TIMEOUT): Boolean =
+    wait(Until.hasObject(selector), timeout)
+
 fun UiDevice.findObjectWithTimeout(
     selector: BySelector,
     timeout: Long = DEFAULT_TIMEOUT,

android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/rule/ForgetAllVpnAppsInSettingsTestRule.kt

@@ -5,9 +5,13 @@ import android.provider.Settings
 import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.uiautomator.By
 import androidx.test.uiautomator.UiDevice
+import androidx.test.uiautomator.UiObject2
+import androidx.test.uiautomator.Until
 import java.util.regex.Pattern
+import net.mullvad.mullvadvpn.test.common.constant.DEFAULT_TIMEOUT
 import net.mullvad.mullvadvpn.test.common.extension.findObjectByCaseInsensitiveText
 import net.mullvad.mullvadvpn.test.common.extension.findObjectWithTimeout
+import net.mullvad.mullvadvpn.test.common.extension.hasObjectWithTimeout
 import org.junit.jupiter.api.extension.BeforeTestExecutionCallback
 import org.junit.jupiter.api.extension.ExtensionContext
 
@@ -20,22 +24,40 @@ class ForgetAllVpnAppsInSettingsTestRule : BeforeTestExecutionCallback {
                 flags = Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TASK
             }
         )
-        val vpnSettingsButtons =
-            device.findObjects(By.res(SETTINGS_PACKAGE, VPN_SETTINGS_BUTTON_ID))
-        vpnSettingsButtons.forEach { button ->
-            button.click()
-
-            try {
-                device.findObjectWithTimeout(By.text(FORGET_VPN_VPN_BUTTON_TEXT)).click()
-                device.findObjectByCaseInsensitiveText(FORGET_VPN_VPN_CONFIRM_BUTTON_TEXT).click()
-            } catch (_: Exception) {
-                device.findObjectWithTimeout(By.text(DELETE_VPN_PROFILE_TEXT)).click()
-                device.findObjectWithTimeout(By.text(DELETE_VPN_CONFIRM_BUTTON_TEXT_REGEXP)).click()
+
+        val vpnSettingsSelector = By.res(SETTINGS_PACKAGE, VPN_SETTINGS_BUTTON_ID)
+        device.wait(Until.hasObject(vpnSettingsSelector), DEFAULT_TIMEOUT)
+        val vpnSettingsButtons = device.findObjects(vpnSettingsSelector)
+
+        vpnSettingsButtons
+            .filter { !it.isHardcodedVpn() }
+            .forEach { button ->
+                button.click()
+
+                if (device.hasObjectWithTimeout(By.text(FORGET_VPN_VPN_BUTTON_TEXT))) {
+                    device.findObjectWithTimeout(By.text(FORGET_VPN_VPN_BUTTON_TEXT)).click()
+                    device
+                        .findObjectByCaseInsensitiveText(FORGET_VPN_VPN_CONFIRM_BUTTON_TEXT)
+                        .click()
+                } else if (device.hasObjectWithTimeout(By.text(DELETE_VPN_PROFILE_TEXT))) {
+                    device.findObjectWithTimeout(By.text(DELETE_VPN_PROFILE_TEXT)).click()
+                    device
+                        .findObjectWithTimeout(By.text(DELETE_VPN_CONFIRM_BUTTON_TEXT_REGEXP))
+                        .click()
+                } else {
+                    error("Unable to find forget or delete button")
+                }
             }
-        }
     }
 
+    private fun UiObject2.isHardcodedVpn(): Boolean =
+        parent.parent.children.any { uiObject ->
+            HARDCODED_VPN_PROFILE_NAMES.any { uiObject.hasObject(By.text(it)) }
+        }
+
     companion object {
+        private val HARDCODED_VPN_PROFILE_NAMES = listOf("VPN by Google")
+
         private const val FORGET_VPN_VPN_BUTTON_TEXT = "Forget VPN"
         private const val DELETE_VPN_PROFILE_TEXT = "Delete VPN profile"
         private const val FORGET_VPN_VPN_CONFIRM_BUTTON_TEXT = "Forget"

android/test/e2e/build.gradle.kts

@@ -24,15 +24,17 @@ android {
         targetProjectPath = ":app"
 
         fun Properties.addRequiredPropertyAsBuildConfigField(name: String) {
-            val value = getProperty(name) ?: throw GradleException("Missing property: $name")
+            val value =
+                System.getenv(name)
+                    ?: getProperty(name)
+                    ?: throw GradleException("Missing property: $name")
+
             buildConfigField(type = "String", name = name, value = "\"$value\"")
         }
 
         Properties().apply {
             load(project.file("e2e.properties").inputStream())
             addRequiredPropertyAsBuildConfigField("API_VERSION")
-            addRequiredPropertyAsBuildConfigField("ENABLE_HIGHLY_RATE_LIMITED_TESTS")
-            addRequiredPropertyAsBuildConfigField("ENABLE_ACCESS_TO_LOCAL_API_TESTS")
             addRequiredPropertyAsBuildConfigField("TRAFFIC_GENERATION_IP_ADDRESS")
             addRequiredPropertyAsBuildConfigField("PACKET_CAPTURE_API_HOST")
         }
@@ -50,6 +52,8 @@ android {
         testInstrumentationRunnerArguments +=
             mutableMapOf<String, String>().apply {
                 put("clearPackageData", "true")
+                addOptionalPropertyAsArgument("enable_access_to_local_api_tests")
+                addOptionalPropertyAsArgument("enable_highly_rate_limited_tests")
                 addOptionalPropertyAsArgument("valid_test_account_number")
                 addOptionalPropertyAsArgument("invalid_test_account_number")
             }

android/test/e2e/e2e.properties

@@ -1,5 +1,3 @@
 API_VERSION=v1
-ENABLE_HIGHLY_RATE_LIMITED_TESTS=false
-ENABLE_ACCESS_TO_LOCAL_API_TESTS=false
 TRAFFIC_GENERATION_IP_ADDRESS=45.83.223.209
 PACKET_CAPTURE_API_HOST=192.168.105.1

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LeakTest.kt

@@ -62,9 +62,9 @@ class LeakTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) {
             val targetPort = 80
 
             device.findObjectWithTimeout(By.res(SELECT_LOCATION_BUTTON_TEST_TAG)).click()
-            clickLocationExpandButton((EndToEndTest.DEFAULT_COUNTRY))
-            clickLocationExpandButton((EndToEndTest.DEFAULT_CITY))
-            device.findObjectWithTimeout(By.text(EndToEndTest.DEFAULT_RELAY)).click()
+            clickLocationExpandButton(DEFAULT_COUNTRY)
+            clickLocationExpandButton(DEFAULT_CITY)
+            device.findObjectWithTimeout(By.text(DEFAULT_RELAY)).click()
             device.findObjectWithTimeout(By.text("OK")).click()
             device.findObjectWithTimeout(By.text("CONNECTED"), VERY_LONG_TIMEOUT)
 
@@ -100,9 +100,9 @@ class LeakTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) {
 
             device.findObjectWithTimeout(By.res(SELECT_LOCATION_BUTTON_TEST_TAG)).click()
             delay(1000.milliseconds)
-            clickLocationExpandButton((EndToEndTest.DEFAULT_COUNTRY))
-            clickLocationExpandButton((EndToEndTest.DEFAULT_CITY))
-            device.findObjectWithTimeout(By.text(EndToEndTest.DEFAULT_RELAY)).click()
+            clickLocationExpandButton(DEFAULT_COUNTRY)
+            clickLocationExpandButton(DEFAULT_CITY)
+            device.findObjectWithTimeout(By.text(DEFAULT_RELAY)).click()
             device.findObjectWithTimeout(By.text("OK")).click()
             device.findObjectWithTimeout(By.text("CONNECTED"), VERY_LONG_TIMEOUT)
 

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/annotations/HasDependencyOnLocalAPI.kt

@@ -1,6 +1,8 @@
 package net.mullvad.mullvadvpn.test.e2e.annotations
 
-import net.mullvad.mullvadvpn.test.e2e.BuildConfig
+import androidx.test.platform.app.InstrumentationRegistry
+import net.mullvad.mullvadvpn.test.e2e.constant.ENABLE_ACCESS_TO_LOCAL_API_TESTS
+import net.mullvad.mullvadvpn.test.e2e.extension.getRequiredArgument
 import org.junit.jupiter.api.extension.ConditionEvaluationResult
 import org.junit.jupiter.api.extension.ExecutionCondition
 import org.junit.jupiter.api.extension.ExtendWith
@@ -17,7 +19,11 @@ annotation class HasDependencyOnLocalAPI {
         override fun evaluateExecutionCondition(
             context: ExtensionContext?
         ): ConditionEvaluationResult {
-            val enable = BuildConfig.ENABLE_ACCESS_TO_LOCAL_API_TESTS.toBoolean() ?: false
+
+            val enable =
+                InstrumentationRegistry.getArguments()
+                    .getRequiredArgument(ENABLE_ACCESS_TO_LOCAL_API_TESTS)
+                    .toBoolean()
 
             return if (enable) {
                 ConditionEvaluationResult.enabled(

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/annotations/HighlyRateLimited.kt

@@ -1,6 +1,8 @@
 package net.mullvad.mullvadvpn.test.e2e.annotations
 
-import net.mullvad.mullvadvpn.test.e2e.BuildConfig
+import androidx.test.platform.app.InstrumentationRegistry
+import net.mullvad.mullvadvpn.test.e2e.constant.ENABLE_HIGHLY_RATE_LIMITED_TESTS
+import net.mullvad.mullvadvpn.test.e2e.extension.getRequiredArgument
 import org.junit.jupiter.api.extension.ConditionEvaluationResult
 import org.junit.jupiter.api.extension.ExecutionCondition
 import org.junit.jupiter.api.extension.ExtendWith
@@ -17,10 +19,12 @@ annotation class HighlyRateLimited {
         override fun evaluateExecutionCondition(
             context: ExtensionContext?
         ): ConditionEvaluationResult {
-            val enableHighlyRateLimited =
-                BuildConfig.ENABLE_HIGHLY_RATE_LIMITED_TESTS.toBoolean() ?: false
+            val enable =
+                InstrumentationRegistry.getArguments()
+                    .getRequiredArgument(ENABLE_HIGHLY_RATE_LIMITED_TESTS)
+                    .toBoolean()
 
-            return if (enableHighlyRateLimited) {
+            return if (enable) {
                 ConditionEvaluationResult.enabled("Running test highly affected by rate limiting.")
             } else {
                 ConditionEvaluationResult.disabled(

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt

@@ -4,3 +4,6 @@ const val LOG_TAG = "mullvad-e2e"
 const val PARTNER_AUTH = "partner_auth"
 const val VALID_TEST_ACCOUNT_NUMBER_ARGUMENT_KEY = "valid_test_account_number"
 const val INVALID_TEST_ACCOUNT_NUMBER_ARGUMENT_KEY = "invalid_test_account_number"
+
+const val ENABLE_ACCESS_TO_LOCAL_API_TESTS = "enable_access_to_local_api_tests"
+const val ENABLE_HIGHLY_RATE_LIMITED_TESTS = "enable_highly_rate_limited_tests"