Make tests and infrastructure less flaky

.github/actions/start-runtime/action.yml

@@ -14,40 +14,38 @@ runs:
       uses: actions/setup-python@v4.5.0
       with:
         python-version: "3.x"
+
     - name: "Install Python dependencies"
       run: pip install pyaml httplib2
       shell: bash
+
     - name: "Setup Java 21"
       id: setup-java
       uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 #v4
       with:
         distribution: "temurin"
         java-version: "21"
-    - name: "Extract deployment package"
-      run: |
-        mkdir project
-        unzip -qq ${{ inputs.mda-file }} -d project
-        cp configs/e2e/m2ee-native.yml project/m2ee-native.yml
-        sed -i -- 's=$ROOT_PATH=${{ github.workspace }}=g' project/m2ee-native.yml
-        sed -i -- 's=$JAVA_HOME=${{ steps.setup-java.outputs.path }}=g' project/m2ee-native.yml
+
+    - name: "Make setup-runtime.sh executable"
+      run: chmod +x .github/scripts/setup-runtime.sh
       shell: bash
-    - name: "Setup m2ee"
+
+    - name: "Initial setup"
       run: |
-        mkdir -p var/log var/opt/m2ee var/run bin tmp
-        git clone https://github.com/KevinVlaanderen/m2ee-tools.git tmp/m2ee
-        mv tmp/m2ee/src/* var/opt/m2ee
-        chmod a=rwx var/log/ var/run/
-        echo "#!/bin/bash -x" > bin/m2ee
-        echo "python3 var/opt/m2ee/m2ee.py \$@" >>bin/m2ee
-        chmod +x bin/m2ee
+        .github/scripts/setup-runtime.sh "${{ inputs.mda-file }}" "${{ inputs.mendix-version }}" "${{ steps.setup-java.outputs.path }}" "${{ github.workspace }}"
       shell: bash
-    - name: "Setup mxruntime"
+
+    - name: "Start mxruntime with retries"
       run: |
-        mkdir -p ${{ github.workspace }}/project/runtimes ${{ github.workspace }}/project/data/model-upload ${{ github.workspace }}/project/data/database ${{ github.workspace }}/project/data/files ${{ github.workspace }}/project/data/tmp
-        wget -q https://cdn.mendix.com/runtime/mendix-${{ inputs.mendix-version }}.tar.gz -O tmp/runtime.tar.gz
-        tar xfz tmp/runtime.tar.gz --directory ${{ github.workspace }}/project/runtimes
-        rm tmp/runtime.tar.gz
-      shell: bash
-    - name: "Start mxruntime"
-      run: bin/m2ee -c ${{ github.workspace }}/project/m2ee-native.yml --verbose --yolo start
-      shell: bash
+        MAX_RETRIES=3
+        RETRY=0
+        until bin/m2ee -c ${{ github.workspace }}/project/m2ee-native.yml --verbose --yolo start; do
+          RETRY=$((RETRY+1))
+          if [ $RETRY -ge $MAX_RETRIES ]; then
+            echo "mxruntime failed after $MAX_RETRIES attempts."
+            exit 1
+          fi
+          echo "mxruntime failed, retrying ($RETRY/$MAX_RETRIES)..."
+          .github/scripts/setup-runtime.sh "${{ inputs.mda-file }}" "${{ inputs.mendix-version }}" "${{ steps.setup-java.outputs.path }}" "${{ github.workspace }}"
+        done
+      shell: bash

.github/scripts/determine-nt-version.py

@@ -37,51 +37,48 @@
 if best_match:
     print(f"Best matching range: {best_match}")
 
-    # Get the major version to look for in tags
+    # Get the major version to look for in releases
     max_pattern = version_data[best_match].get("max", "")
-    major_version = max_pattern.split('.')[0]
-
-    print(f"Looking for latest version with major version: {major_version}")
+    if max_pattern == "*":
+        major_version = None
+        print("Looking for latest available release (no major version restriction)")
+    else:
+        major_version = max_pattern.split('.')[0]
+        print(f"Looking for latest release with major version: {major_version}")
 
-    # Get available tags from native-template repository
-    response = requests.get('https://api.github.com/repos/mendix/native-template/tags')
+    # Get available releases from native-template repository
+    response = requests.get('https://api.github.com/repos/mendix/native-template/releases')
     if response.status_code == 200:
-        all_tags = response.json()
-        tag_names = [tag['name'] for tag in all_tags]
-        print(f"Available tags: {tag_names}")
+        all_releases = response.json()
+        release_names = [release['tag_name'] for release in all_releases]
+        print(f"Available releases: {release_names}")
 
-        # Find the latest version matching the major version
-        matching_tags = []
+        # Find the latest release matching the major version
+        matching_releases = []
 
-        for tag in tag_names:
-            # Remove 'v' prefix if present for comparison
+        for release in all_releases:
+            tag = release['tag_name']
             clean_tag = tag[1:] if tag.startswith('v') else tag
-
-            # Check if the tag starts with the major version
-            if clean_tag.startswith(f"{major_version}."):
-                try:
-                    # Try to parse as a version (this handles proper version numbers)
-                    tag_version = version.parse(clean_tag)
-                    matching_tags.append((tag, tag_version))
-                except:
-                    # Skip tags that don't parse as proper versions
-                    pass
+            try:
+                tag_version = version.parse(clean_tag)
+                if major_version is None or clean_tag.startswith(f"{major_version}."):
+                    matching_releases.append((tag, tag_version))
+            except:
+                pass
 
-        if matching_tags:
-            # Sort by version (highest last) and get the last one
-            matching_tags.sort(key=lambda x: x[1])
-            latest_version = matching_tags[-1][0]
-            print(f"Selected Native Template version: {latest_version}")
+        if matching_releases:
+            matching_releases.sort(key=lambda x: x[1])
+            latest_tag = matching_releases[-1][0]
+            print(f"Selected Native Template release: {latest_tag}")
             with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
-                f.write(f"nt_branch={latest_version}\n")
+                f.write(f"nt_branch={latest_tag}\n")
         else:
-            # Fallback to min version if no matching tag found
             min_version = version_data[best_match].get("min", "")
-            print(f"No matching tag found, using minimum version: {min_version}")
+            print(f"No matching release found, using minimum version: {min_version}")
             with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
                 f.write(f"nt_branch={min_version}\n")
     else:
-        print(f"Failed to get tags: {response.status_code}")
+        print(f"Failed to get releases: {response.status_code}")
         print("Using master as fallback")
         with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
             f.write("nt_branch=master\n")

.github/scripts/setup-runtime.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+MDA_FILE="$1"
+MENDIX_VERSION="$2"
+JAVA_PATH="$3"
+WORKSPACE="$4"
+
+rm -rf project var tmp bin
+
+mkdir project
+unzip -qq "$MDA_FILE" -d project
+cp configs/e2e/m2ee-native.yml project/m2ee-native.yml
+sed -i -- "s=\$ROOT_PATH=$WORKSPACE=g" project/m2ee-native.yml
+sed -i -- "s=\$JAVA_HOME=$JAVA_PATH=g" project/m2ee-native.yml
+
+mkdir -p var/log var/opt/m2ee var/run bin tmp
+git clone https://github.com/KevinVlaanderen/m2ee-tools.git tmp/m2ee
+mv tmp/m2ee/src/* var/opt/m2ee
+chmod a=rwx var/log/ var/run/
+echo "#!/bin/bash -x" > bin/m2ee
+echo "python3 var/opt/m2ee/m2ee.py \$@" >>bin/m2ee
+chmod +x bin/m2ee
+
+mkdir -p "$WORKSPACE/project/runtimes" "$WORKSPACE/project/data/model-upload" "$WORKSPACE/project/data/database" "$WORKSPACE/project/data/files" "$WORKSPACE/project/data/tmp"
+wget -q "https://cdn.mendix.com/runtime/mendix-$MENDIX_VERSION.tar.gz" -O tmp/runtime.tar.gz
+tar xfz tmp/runtime.tar.gz --directory "$WORKSPACE/project/runtimes"
+rm tmp/runtime.tar.gz

.github/workflows/NativePipeline.yml

@@ -439,6 +439,7 @@ jobs:
   android-tests:
     needs: [scope, mendix-version, project, android-app]
     runs-on: ubuntu-22.04
+    timeout-minutes: 60
     strategy:
       matrix:
         widget: ${{ fromJson(needs.scope.outputs.widgets) }}
@@ -557,6 +558,7 @@ jobs:
   ios-tests:
     needs: [scope, mendix-version, project, ios-app]
     runs-on: macos-15
+    timeout-minutes: 60
     strategy:
       matrix:
         widget: ${{ fromJson(needs.scope.outputs.widgets) }}

configs/e2e/mendix-versions.json

@@ -1,4 +1,3 @@
 {
-    "latest": "10.22.0.68245",
-    "8": "8.18.23.62193"
+    "latest": "10.24.0.73019"
 }

maestro/helpers/compare_screenshots.js

@@ -57,7 +57,8 @@ fs.readdirSync(actualDir).forEach(file => {
       ];
     } else if (platform === 'android') {
       ignoredAreas = [
-        { x: 0, y: 0, width, height: 50 } // Ignore top 40 pixels on Android
+        { x: 0, y: 0, width, height: 50 }, // Ignore top 50 pixels on Android
+        { x: width - 15, y: 0, width: 15, height } // Ignore right 15 pixels on Android
       ];
     }
 

maestro/run_maestro_tests_local.sh

@@ -4,14 +4,12 @@
 # Use command like: ./run_maestro_tests_local.sh android/ios proggress-circle-native
 
 if [ "$1" == "android" ]; then
-  #APP_ID="myapp.nativecomponentstestproject"
   APP_ID="com.mendix.nativetemplate"
   DEVICE_ID="emulator-5554"
   PLATFORM="android"
 elif [ "$1" == "ios" ]; then
-  APP_ID="myapp.nativecomponentstestproject"
-  # APP_ID="com.mendix.native.template"
-  DEVICE_ID="4293ECC5-CA5A-4820-9433-40562F7F6F5E"
+  APP_ID="com.mendix.native.template"
+  DEVICE_ID="8CDB91BF-9D4A-4F1D-8952-A7ABBED10078"
   PLATFORM="ios"
 else
   echo "Usage: $0 [android|ios] [widget]"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_dynamic_image.yaml

@@ -8,9 +8,10 @@ appId: "${APP_ID}"
     text: "Background image"
 - tapOn:
     text: "Dynamic image"
-- extendedWaitUntil:
-    visible:
-        id: "dynamicBgImage"
-    timeout: 30000
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 30000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_dynamic_image"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_dynamic_svg.yaml

@@ -8,9 +8,10 @@ appId: "${APP_ID}"
     text: "Background image"
 - tapOn:
     text: "Dynamic SVG image"
-- extendedWaitUntil:
-    visible: "Dynamic SVG image"
-    timeout: 30000
-    optional: true
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_dynamic_svg"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_layout_grid.yaml

@@ -9,5 +9,10 @@ appId: "${APP_ID}"
 - tapOn:
     text: "Layout grid"
 - assertVisible: "Layout grid"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_layout_grid"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_nested.yaml

@@ -9,5 +9,10 @@ appId: "${APP_ID}"
 - tapOn:
     text: "Nested"
 - assertVisible: "Nested"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_nested"

packages/pluggableWidgets/background-image-native/e2e/specs/maestro/BgImage_static_images.yaml

@@ -10,5 +10,10 @@ appId: "${APP_ID}"
     text: "Static images"
 - assertVisible:
     text: "Static images"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 10000 # 10 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/bg_image_static_images"

packages/pluggableWidgets/bottom-sheet-native/e2e/specs/maestro/Bottom_sheet_expanding.yaml

@@ -8,5 +8,10 @@ appId: "${APP_ID}"
     text: "Bottom Sheet"
 - tapOn:
     text: "Expanding"
-- assertVisible:
-    text: "Header"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 10000 # 10 seconds
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/bottom_sheet_expanding"

packages/pluggableWidgets/bottom-sheet-native/e2e/specs/maestro/Botton_sheet_expanding_fullscreen.yaml

@@ -8,5 +8,10 @@ appId: "${APP_ID}"
     text: "Bottom Sheet"
 - tapOn:
     text: "Expanding fullscreen"
-- assertVisible:
-    text: "Header"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 10000 # 10 seconds
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/bottom_sheet_expanding_fullscreen"

packages/pluggableWidgets/bottom-sheet-native/e2e/specs/maestro/Modal_basic_non_native.yaml

@@ -10,5 +10,10 @@ appId: "${APP_ID}"
     text: "Modal basic non native"
 - tapOn:
     text: "Open"
-- tapOn:
-    text: "Diego"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 10000 # 10 seconds
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/bottom_sheet_modal_basic_non_native"

packages/pluggableWidgets/bottom-sheet-native/e2e/specs/maestro/Modal_custom.yaml

@@ -10,7 +10,10 @@ appId: "${APP_ID}"
     text: "Modal custom"
 - tapOn:
     text: "Open"
-- tapOn:
-    text: "test"
-- assertVisible:
-    text: "Successfully opened"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 5000 # 5 seconds
+- takeScreenshot:
+    path: "maestro/images/actual/${PLATFORM}/bottom_sheet_modal_custom"

packages/pluggableWidgets/image-native/e2e/specs/maestro/Image_dynamic.yaml

@@ -8,5 +8,10 @@ appId: "${APP_ID}"
     text: "Image"
 - tapOn:
     text: "Image dynamic"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 30000 # 30 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/image_dynamic"

packages/pluggableWidgets/image-native/e2e/specs/maestro/Image_static.yaml

@@ -8,7 +8,12 @@ appId: "${APP_ID}"
     text: "Image"
 - tapOn:
     text: "Image static"
-- assertVisible:
-    text: "Image Static"
+- assertVisible: 
+    text: "Main as Static Image; Default as Static; Background as Contain"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/image_static"

packages/pluggableWidgets/image-native/e2e/specs/maestro/Image_url.yaml

@@ -9,6 +9,11 @@ appId: "${APP_ID}"
 - tapOn:
     text: "Image url"
 - assertVisible: "Main as Image URL(svg); Background as No; Action as Open dialog box"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/image_url"
 

packages/pluggableWidgets/line-chart-native/e2e/specs/maestro/LineChart.yaml

@@ -10,5 +10,10 @@ appId: "${APP_ID}"
     text: "Line chart"
 - assertVisible:
     id: "container1"
+# Because the image loading can take some time due to resources on emulator, we need to wait for the image to be visible
+- extendedWaitUntil: 
+     visible: randText  # Any random text that does not exist in the UI
+     optional: true  # This should be true so that the test won't fail
+     timeout: 15000 # 15 seconds
 - takeScreenshot:
     path: "maestro/images/actual/${PLATFORM}/line_chart"