From 51acd0fd8e8059d2daba67a20dfa749ef95b980b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20Vasconcellos?= Date: Mon, 25 Mar 2024 16:42:37 -0300 Subject: [PATCH] Make mobile tests more resilient (#2236) * Attempt at making mobile tests more resilient * Forgot shell property * Remove debug flag from run-maestro-tests.sh * Fix kill command * Improve device emulator launch code - Remove maestro version lock (expo is not using it anymore on their repo) - Use some of the same command expo uses to ensure device emulator launched * ... * ... * Make grep timeout less strict * Increase timeouts * Downgrade macOS and Xcode Following Expo changes: https://github.com/expo/expo/commit/cb7c90dac47a46848f406e78e3903aea7fc24f5e --- .github/actions/setup-system/action.yml | 5 ++ .github/workflows/mobile-ci.yml | 25 +++--- apps/mobile/scripts/run-maestro-tests.sh | 98 +++++++++++++++++++++--- 3 files changed, 103 insertions(+), 25 deletions(-) diff --git a/.github/actions/setup-system/action.yml b/.github/actions/setup-system/action.yml index a440e8662f12..7c6ef9541fcb 100644 --- a/.github/actions/setup-system/action.yml +++ b/.github/actions/setup-system/action.yml @@ -42,6 +42,11 @@ runs: key: ${{ steps.cache-llvm-restore.outputs.cache-primary-key }} path: C:/Program Files/LLVM + - name: Install current Bash on macOS + shell: bash + if: runner.os == 'macOS' + run: brew install bash + - name: Install Nasm if: ${{ runner.os != 'Linux' }} uses: ilammy/setup-nasm@v1 diff --git a/.github/workflows/mobile-ci.yml b/.github/workflows/mobile-ci.yml index 32c190c5d718..198f593e4153 100644 --- a/.github/workflows/mobile-ci.yml +++ b/.github/workflows/mobile-ci.yml @@ -138,15 +138,15 @@ jobs: ios: name: iOS - runs-on: macos-14 + runs-on: macos-12 steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Xcode - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: latest-stable + # - name: Install Xcode + # uses: maxim-lobanov/setup-xcode@v1 + # with: + # xcode-version: latest-stable - name: Setup System and Rust uses: ./.github/actions/setup-system @@ -189,9 +189,6 @@ jobs: run: xcodebuild -workspace ./Spacedrive.xcworkspace -scheme Spacedrive -configuration Release -sdk iphonesimulator -derivedDataPath build -arch "$(uname -m)" - name: Install Maestro - env: - # Workaround: https://github.com/mobile-dev-inc/maestro/issues/1585 - MAESTRO_VERSION: '1.33.1' run: | curl -Ls "https://get.maestro.mobile.dev" | bash brew tap facebook/fb @@ -199,16 +196,12 @@ jobs: echo "${HOME}/.maestro/bin" >> $GITHUB_PATH - name: Run Simulator + id: run_simulator uses: futureware-tech/simulator-action@v3 with: - model: 'iPhone 15' - os_version: 17 + model: 'iPhone 14' + os_version: 16 erase_before_boot: false - name: Run Tests - env: - # https://github.com/expo/expo/blob/339fa68/apps/bare-expo/scripts/start-ios-e2e-test.ts#L12 - MAESTRO_DRIVER_STARTUP_TIMEOUT: 120000 - run: | - xcrun simctl install booted apps/mobile/ios/build/Build/Products/Release-iphonesimulator/Spacedrive.app - ./apps/mobile/scripts/run-maestro-tests.sh ios + run: ./apps/mobile/scripts/run-maestro-tests.sh ios ${{ steps.run_simulator.outputs.udid }} diff --git a/apps/mobile/scripts/run-maestro-tests.sh b/apps/mobile/scripts/run-maestro-tests.sh index b190521ce772..d695819f11ff 100755 --- a/apps/mobile/scripts/run-maestro-tests.sh +++ b/apps/mobile/scripts/run-maestro-tests.sh @@ -2,23 +2,70 @@ set -eEuo pipefail -if [ "${CI:-}" = "true" ]; then - set -x -fi - # Script root _root="$(CDPATH='' cd -- "$(dirname "$0")" && pwd -P)" _test_dir="$(CDPATH='' cd -- "${_root}/../tests" && pwd -P)" -PLATFORM=${1:-} +PLATFORM="${1:-}" +DEVICE_ID="" +IOS_APP_BIN_PATH="${_root}/../ios/build/Build/Products/Release-iphonesimulator/Spacedrive.app" case $PLATFORM in - ios | android) ;; + ios) + DEVICE_ID="${2:-}" + if [ -z "$DEVICE_ID" ]; then + echo "Empty IOS emulator UUID" >&2 + exit 1 + fi + + if ! [ -e "$IOS_APP_BIN_PATH" ]; then + echo "Invalid IOS app binary path" >&2 + exit 1 + fi + ;; + android) + echo 'Android tests are not implemented yet' >&2 + exit 1 + ;; *) echo "Usage: run-maestro-tests.sh " >&2 exit 1 ;; esac +start_app() { + case $PLATFORM in + ios) + xcrun simctl bootstatus "$DEVICE_ID" -b + open -a Simulator --args -CurrentDeviceUDID "$DEVICE_ID" + xcrun simctl install "$DEVICE_ID" "${_root}/../ios/build/Build/Products/Release-iphonesimulator/Spacedrive.app" + # ¯\_(ツ)_/¯ + sleep 10 + ;; + android) + echo 'Android tests are not implemented yet' >&2 + exit 1 + ;; + esac +} + +# https://stackoverflow.com/q/11027679#answer-59592881 +# SYNTAX: +# catch STDOUT_VARIABLE STDERR_VARIABLE COMMAND [ARG1[ ARG2[ ...[ ARGN]]]] +catch() { + { + IFS=$'\n' read -r -d '' "${1}" + IFS=$'\n' read -r -d '' "${2}" + ( + IFS=$'\n' read -r -d '' _ERRNO_ + return "$_ERRNO_" + ) + } < <((printf '\0%s\0%d\0' "$( ( ( ({ + shift 2 + "${@}" + echo "${?}" 1>&3- + } | tr -d '\0' 1>&4-) 4>&2- 2>&1- | tr -d '\0' 1>&4-) 3>&1- | exit "$(cat)") 4>&1-)" "${?}" 1>&2) 2>&1) +} + run_maestro_test() { if [ $# -ne 1 ]; then echo "Usage: run_maestro_test " >&2 @@ -28,13 +75,43 @@ run_maestro_test() { local i local retry_seconds for i in {1..6}; do - if maestro test "$1"; then + _maestro_out='' + _maestro_err='' + + # https://github.com/expo/expo/blob/339fa68/apps/bare-expo/scripts/start-ios-e2e-test.ts#L12 + if catch _maestro_out _maestro_err \ + env MAESTRO_DRIVER_STARTUP_TIMEOUT=120000 maestro --device "$DEVICE_ID" test "$1"; then # Test succeeded + printf '%s' "$_maestro_out" + printf '%s' "$_maestro_err" >&2 return - else + elif echo "$_maestro_err" | grep 'TimeoutException'; then + # Test timed out + # Kill maestro processes + pgrep -fi maestro | xargs kill -KILL + + # Restart app if necessary + case $PLATFORM in + ios) + if ! { xcrun simctl listapps booted | grep CFBundleIdentifier | grep Spacedrive; }; then + start_app + fi + ;; + android) + echo 'Android tests are not implemented yet' >&2 + exit 1 + ;; + esac + + # Retry retry_seconds=$((20 * i)) - echo "Test $1 failed. Retrying in $retry_seconds seconds..." + echo "Test $1 timed out. Retrying in $retry_seconds seconds..." sleep $retry_seconds + else + # Test failed + printf '%s' "$_maestro_out" + printf '%s' "$_maestro_err" >&2 + return 1 fi done @@ -57,6 +134,9 @@ else ) fi +# Start Spacedrive in the device emulator +start_app + # Run onboarding first onboardingFile="${_test_dir}/onboarding.yml" if ! run_maestro_test "$onboardingFile"; then