feat: introduced retry rules for flaky android push tests

Author: ttypic

.github/workflows/emulate.yml

@@ -35,7 +35,9 @@ jobs:
           api-level: ${{ matrix.android-api-level }}
           emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           disable-animations: true
-          script: ./gradlew :android:connectedAndroidTest
+          script: |
+            ./gradlew installDebugAndroidTest
+            adb shell am instrument -w io.ably.lib.test.android/android.support.test.runner.AndroidJUnitRunner
 
       - uses: actions/upload-artifact@v3
         if: always()

android/build.gradle.kts

@@ -35,7 +35,7 @@ android {
         abortOnError = false
     }
 
-    testOptions.targetSdk = 30
+    testOptions.targetSdk = 34
 
     sourceSets {
         getByName("main") {

android/src/androidTest/java/io/ably/lib/test/RetryTestRule.java

@@ -0,0 +1,49 @@
+package io.ably.lib.test;
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+
+public class RetryTestRule implements TestRule {
+
+    private final int timesToRunTestCount;
+
+    /**
+     * If `times` is 0, then we should run the test once.
+     */
+    public RetryTestRule(int times) {
+        this.timesToRunTestCount = times + 1;
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description) {
+        return statement(base, description);
+    }
+
+    private Statement statement(Statement base, Description description) {
+        return new Statement() {
+
+            @Override
+            public void evaluate() throws Throwable {
+                Throwable latestException = null;
+
+                for (int runCount = 0; runCount < timesToRunTestCount; runCount++) {
+                    try {
+                        base.evaluate();
+                        return;
+                    } catch (Throwable t) {
+                        latestException = t;
+                        System.err.printf("%s: test failed on run: `%d`. Will run a maximum of `%d` times.%n", description.getDisplayName(), runCount, timesToRunTestCount);
+                        t.printStackTrace();
+                    }
+                }
+
+                if (latestException != null) {
+                    System.err.printf("%s: giving up after `%d` failures%n", description.getDisplayName(), timesToRunTestCount);
+                    throw latestException;
+                }
+            }
+        };
+    }
+}

android/src/androidTest/java/io/ably/lib/test/android/AndroidPushTest.java

@@ -6,6 +6,7 @@
 import android.content.IntentFilter;
 import android.os.Build;
 import android.preference.PreferenceManager;
+import android.support.test.InstrumentationRegistry;
 import android.support.test.runner.AndroidJUnit4;
 import android.util.Log;
 import androidx.localbroadcastmanager.content.LocalBroadcastManager;
@@ -41,6 +42,7 @@
 import io.ably.lib.rest.Auth;
 import io.ably.lib.rest.Channel;
 import io.ably.lib.rest.DeviceDetails;
+import io.ably.lib.test.RetryTestRule;
 import io.ably.lib.test.common.Helpers;
 import io.ably.lib.test.common.Helpers.AsyncWaiter;
 import io.ably.lib.test.common.Helpers.CompletionWaiter;
@@ -59,7 +61,9 @@
 import java9.util.stream.StreamSupport;
 
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
@@ -86,6 +90,9 @@
 public class AndroidPushTest {
     private static final int TIMEOUT_SECONDS = 30;
 
+    @Rule
+    public RetryTestRule retryRule = new RetryTestRule(2);
+
     private class TestActivation {
         private Helpers.RawHttpTracker httpTracker;
         private AblyRest rest;
@@ -181,6 +188,11 @@ private void moveToAfterRegistrationUpdateFailed() throws AblyException {
         }
     }
 
+    @Before
+    public void cleanUp() {
+        PreferenceManager.getDefaultSharedPreferences(InstrumentationRegistry.getTargetContext()).edit().clear().commit();
+    }
+
     // RSH2a
     @Test
     public void push_activate() throws InterruptedException, AblyException {

gradle/libs.versions.toml

@@ -12,7 +12,7 @@ concurrentunit = "0.4.2"
 slf4j = "1.7.30"
 build-config = "5.4.0"
 firebase-messaging = "22.0.0"
-android-test = "0.5"
+android-test = "1.0.2"
 dexmaker = "1.4"
 android-retrostreams = "1.7.4"
 maven-publish = "0.29.0"