[Test] Run iOS E2E tests in macOS CI agent

.buildkite/commands/build-ios.sh

@@ -1,16 +1,13 @@
 #!/bin/bash -eu
 
-echo '--- :node: Setup Node depenendencies'
+echo '--- :node: Setup Node dependencies'
 npm ci --unsafe-perm --prefer-offline --no-audit --no-progress
 
 echo '--- :ios: Set env var for iOS E2E testing'
 set -x
-export TEST_RN_PLATFORM=ios
-export TEST_ENV=sauce
 CONFIG_FILE="$(pwd)/gutenberg/packages/react-native-editor/__device-tests__/helpers/device-config.json"
-# Uses the local deviceName since SauceLabs uses a different one.
 IOS_DEVICE_NAME=$(jq -r '.ios.local.deviceName' "$CONFIG_FILE")
-IOS_PLATFORM_VERSION=$(jq -r '.ios.buildkite.platformVersion' "$CONFIG_FILE")
+export IOS_PLATFORM_VERSION=$(jq -r '.ios.buildkite.platformVersion' "$CONFIG_FILE")
 # Set a destination different from the hardcoded one which only works in the
 # older Xcode-setup used by CircleCI
 export RN_EDITOR_E2E_IOS_DESTINATION="platform=iOS Simulator,name=$IOS_DEVICE_NAME,OS=$IOS_PLATFORM_VERSION"
@@ -22,17 +19,23 @@ npm run core test:e2e:build-app:ios
 echo '--- :react: Build iOS bundle for E2E testing'
 npm run test:e2e:bundle:ios
 
-echo '--- :compression: Prepare artifact for SauceLabs upload'
+echo "--- :react: Prepare tests setup"
+npm run core test:e2e:setup
+
+echo '--- :react: Build WDA for E2E testing'
+npm run core test:e2e:build-wda
+
+echo '--- :compression: Prepare artifacts'
 WORK_DIR=$(pwd) \
   && pushd ./gutenberg/packages/react-native-editor/ios/build/GutenbergDemo/Build/Products/Release-iphonesimulator \
   && zip -r "$WORK_DIR/gutenberg/packages/react-native-editor/ios/GutenbergDemo.app.zip" GutenbergDemo.app \
   && popd
 
-echo '--- :saucelabs: Upload app artifact to SauceLabs'
-SAUCE_FILENAME=${BUILDKITE_BRANCH//[\/]/-}
-curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" \
-  --location \
-  --request POST 'https://api.us-west-1.saucelabs.com/v1/storage/upload' \
-  --form 'payload=@"./gutenberg/packages/react-native-editor/ios/GutenbergDemo.app.zip"' \
-  --form "name=Gutenberg-$SAUCE_FILENAME.app.zip" \
-  --form 'description="Gutenberg"'
+WORK_DIR=$(pwd) \
+  && pushd ./gutenberg/packages/react-native-editor/ios/build/WDA \
+  && zip -r "$WORK_DIR/gutenberg/packages/react-native-editor/ios/WDA.zip" ./* \
+  && popd
+
+echo "--- :arrow_up: Upload Build Products"
+upload_artifact "./gutenberg/packages/react-native-editor/ios/GutenbergDemo.app.zip"
+upload_artifact "./gutenberg/packages/react-native-editor/ios/WDA.zip"

.buildkite/commands/test-ios.sh

@@ -1,5 +1,9 @@
 #!/bin/bash -eu
 
+CONFIG_FILE="$(pwd)/gutenberg/packages/react-native-editor/__device-tests__/helpers/device-config.json"
+DEVICE_NAME=$(jq -r '.ios.local.deviceName' "$CONFIG_FILE")
+DEVICE_TABLET_NAME=$(jq -r '.ios.local.deviceTabletName' "$CONFIG_FILE")
+
 MODE="iphone"
 INPUT="${1-}"
 while [ "$INPUT" != "" ]; do
@@ -19,33 +23,46 @@ while [ "$INPUT" != "" ]; do
     INPUT="${1-}"
 done
 
-echo '--- :node: Setup Node depenendencies'
+if [ "$MODE" == 'canary' ]; then
+    SECTION='--- :react: Test iOS Canary Pages'
+    TESTS_CMD='device-tests-canary'
+elif [ "$MODE" == "ipad" ]; then
+    SECTION='--- :react: Test iOS iPad'
+    DEVICE_NAME=$DEVICE_TABLET_NAME
+    TESTS_CMD='device-tests-ipad'
+else
+    SECTION='--- :react: Test iOS iPhone'
+    TESTS_CMD='device-tests'
+fi
+
+echo "--- :apple_logo: Start booting up simulator"
+xcrun simctl boot "$DEVICE_NAME" &
+
+echo "--- 📦 Downloading Build Artifacts"
+export IOS_APP_PATH=./gutenberg/packages/react-native-editor/ios/GutenbergDemo.app.zip
+download_artifact "GutenbergDemo.app.zip" "$IOS_APP_PATH"
+
+export WDA_PATH=./gutenberg/packages/react-native-editor/ios/build/WDA
+download_artifact "WDA.zip" "$WDA_PATH/WDA.zip"
+unzip "$WDA_PATH/WDA.zip" -d "$WDA_PATH"
+
+echo '--- :node: Setup Node dependencies'
 npm ci --prefer-offline --no-audit --ignore-scripts
 npm ci --prefix gutenberg --prefer-offline --no-audit
 
 echo '--- :ios: Set env var for iOS E2E testing'
 set -x
 export TEST_RN_PLATFORM=ios
-export TEST_ENV=sauce
+export TEST_ENV=buildkite
 export JEST_JUNIT_OUTPUT_FILE="reports/test-results/ios-test-results.xml"
-# This is a relic of the CircleCI setup.
-# It should be removed once the migration to Buildkite is completed.
-export CIRCLE_BRANCH=${BUILDKITE_BRANCH}
 set +x
 
-if [ "$MODE" == 'canary' ]; then
-    SECTION='--- :saucelabs: Test iOS Canary Pages'
-    TESTS_CMD='device-tests-canary'
-elif [ "$MODE" == "ipad" ]; then
-    SECTION='--- :saucelabs: Test iOS iPad'
-    TESTS_CMD='device-tests-ipad'
-else
-    SECTION='--- :saucelabs: Test iOS iPhone'
-    TESTS_CMD='device-tests'
-fi
+echo "--- :react: Prepare tests setup"
+npm run core test:e2e:setup
 
 set +e
 echo "$SECTION"
+
 npm run "$TESTS_CMD"
 TESTS_EXIT_CODE=$?
 set -e

.buildkite/pipeline.yml

@@ -22,7 +22,9 @@ x-common-params:
         - 'AWS_ACCESS_KEY'
         - 'AWS_SECRET_KEY'
   - &nvm_plugin
-    automattic/nvm#0.2.1
+    automattic/nvm#0.3.0:
+      # This is an attempt to fix curl error (92) during installing nodejs.
+      curlrc: --http1.1
   - &ci_toolkit_plugin
     automattic/a8c-ci-toolkit#2.18.2
   - &xcode_agent_env
@@ -175,11 +177,11 @@ steps:
       queue: mac
     env: *xcode_agent_env
 
-  - label: iOS Build and Sauce Labs
-    key: ios-build-and-saucelabs
-    depends_on:
-      - lint
-      - ios-unit-tests
+  - label: iOS Build
+    key: ios-build
+    # depends_on:
+    #   - lint
+    #   - ios-unit-tests
     command: .buildkite/commands/build-ios.sh
     plugins:
       - *ci_toolkit_plugin
@@ -192,14 +194,15 @@ steps:
     env: *xcode_agent_env
 
   - label: Test iOS on Device – Canary Pages
-    depends_on: ios-build-and-saucelabs
+    depends_on: ios-build
     command: .buildkite/commands/test-ios.sh --canary
     plugins:
       - *ci_toolkit_plugin
       - *nvm_plugin
       - *git-cache-plugin
     artifact_paths:
       - reports/test-results/ios-test-results.xml
+      - ios-screen-recordings/*
     agents:
       queue: mac
     env: *xcode_agent_env
@@ -212,13 +215,13 @@ steps:
     if: *is_branch_for_quick_ui_tests
     key: run-full-ui-test
     prompt: "Run full UI tests suites?"
-    depends_on: ios-build-and-saucelabs
+    depends_on: ios-build
 
   - label: Test iOS on Device – Full iPhone
     # The quick UI tests suite version depends on the block step being unblocked
     if: *is_branch_for_quick_ui_tests
     depends_on:
-      - ios-build-and-saucelabs
+      - ios-build
       - run-full-ui-test
     command: .buildkite/commands/test-ios.sh
     plugins:
@@ -227,6 +230,7 @@ steps:
       - *git-cache-plugin
     artifact_paths:
       - reports/test-results/ios-test-results.xml
+      - ios-screen-recordings/*
     agents:
       queue: mac
     env: *xcode_agent_env
@@ -236,7 +240,7 @@ steps:
     # The full UI tests suite version depends only on the ios-build step, meaning it has no manual step that triggers it
     if: *is_branch_for_full_ui_tests
     depends_on:
-      - ios-build-and-saucelabs
+      - ios-build
     command: .buildkite/commands/test-ios.sh
     plugins:
       - *ci_toolkit_plugin
@@ -252,7 +256,7 @@ steps:
     # The quick UI tests suite version depends on the block step being unblocked
     if: *is_branch_for_quick_ui_tests
     depends_on:
-      - ios-build-and-saucelabs
+      - ios-build
       - run-full-ui-test
     command: .buildkite/commands/test-ios.sh --ipad
     plugins:
@@ -261,6 +265,7 @@ steps:
       - *git-cache-plugin
     artifact_paths:
       - reports/test-results/ios-test-results.xml
+      - ios-screen-recordings/*
     agents:
       queue: mac
     env: *xcode_agent_env
@@ -269,14 +274,15 @@ steps:
     # The full UI tests suite version depends only on the ios-build step, meaning it has no manual step that triggers it
     if: *is_branch_for_full_ui_tests
     depends_on:
-      - ios-build-and-saucelabs
+      - ios-build
     command: .buildkite/commands/test-ios.sh --ipad
     plugins:
       - *ci_toolkit_plugin
       - *nvm_plugin
       - *git-cache-plugin
     artifact_paths:
       - reports/test-results/ios-test-results.xml
+      - ios-screen-recordings/*
     agents:
       queue: mac
     env: *xcode_agent_env

bin/test-e2e.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+
+set -o pipefail
+
+# Using the ':' operator with ':=' to check if a variable is set.
+# If the variable is unset or null, the value after ':=' is assigned to it.
+: ${IOS_APP_PATH:='./gutenberg/packages/react-native-editor/ios/build/GutenbergDemo/Build/Products/Release-iphonesimulator/GutenbergDemo.app'}
+: ${WDA_PATH:='./gutenberg/packages/react-native-editor/ios/build/WDA'}
+: ${ANDROID_APP_PATH:='./gutenberg/packages/react-native-editor/android/app/build/outputs/apk/debug/app-debug.apk'}
+
+export IOS_APP_PATH
+export WDA_PATH
+export ANDROID_APP_PATH
+
+export APPIUM_HOME=~/.appium
+export NODE_ENV=test 
+
+# Check for debug mode
+if [ "$1" == "--debug" ]; then
+    shift # Remove first argument
+    node $NODE_DEBUG_OPTION --inspect-brk node_modules/jest/bin/jest --runInBand --detectOpenHandles --verbose --config jest_ui.config.js "$@"
+else
+    cross-env jest --config ./jest_ui.config.js --maxWorkers 1 --forceExit "$@"
+fi

package.json

@@ -84,11 +84,11 @@
 		"test": "cross-env NODE_ENV=test jest --config ./jest.config.js",
 		"test:update": "cross-env NODE_ENV=test jest --config ./jest.config.js --updateSnapshot",
 		"test:debug": "cross-env NODE_ENV=test node --inspect-brk node_modules/.bin/jest --runInBand --verbose --config jest.config.js",
-		"device-tests": "cross-env NODE_ENV=test jest --maxWorkers=2 --testPathIgnorePatterns='canary|gutenberg-editor-rendering|ipad' --config jest_ui.config.js",
-		"device-tests-canary": "cross-env NODE_ENV=test jest --maxWorkers=2 --testPathPattern=@canary --config jest_ui.config.js",
-		"device-tests-ipad": "cross-env NODE_ENV=test IPAD=true jest --maxWorkers=2 --testPathPattern=@ipad --config jest_ui.config.js",
-		"device-tests:local": "APPIUM_HOME=~/.appium IOS_APP_PATH='./gutenberg/packages/react-native-editor/ios/build/GutenbergDemo/Build/Products/Release-iphonesimulator/GutenbergDemo.app' WDA_PATH='./gutenberg/packages/react-native-editor/ios/build/WDA' ANDROID_APP_PATH='./gutenberg/packages/react-native-editor/android/app/build/outputs/apk/debug/app-debug.apk' NODE_ENV=test jest --maxWorkers=2 --detectOpenHandles --config jest_ui.config.js",
-		"device-tests:debug": "IOS_APP_PATH='./gutenberg/packages/react-native-editor/ios/build/GutenbergDemo/Build/Products/Release-iphonesimulator/GutenbergDemo.app' WDA_PATH='./gutenberg/packages/react-native-editor/ios/build/WDA' ANDROID_APP_PATH='./gutenberg/packages/react-native-editor/android/app/build/outputs/apk/debug/app-debug.apk' cross-env NODE_ENV=test node $NODE_DEBUG_OPTION --inspect-brk node_modules/jest/bin/jest --runInBand --detectOpenHandles --verbose --config jest_ui.config.js",
+		"device-tests": "./bin/test-e2e.sh --testPathIgnorePatterns='canary|gutenberg-editor-rendering|ipad'",
+		"device-tests-canary": "./bin/test-e2e.sh --testPathPattern=@canary",
+		"device-tests-ipad": "IPAD=true ./bin/test-e2e.sh --testPathPattern=@ipad",
+		"device-tests:local": "./bin/test-e2e.sh --detectOpenHandles",
+		"device-tests:debug": "./bin/test-e2e.sh --debug --runInBand --detectOpenHandles --verbose",
 		"test:e2e:bundle:android": "npm run test:e2e:bundle:android:text && npm run test:e2e:bundle:android:bytecode",
 		"test:e2e:bundle:android:text": "mkdir -p gutenberg/packages/react-native-editor/android/app/src/main/assets && npm run rn-bundle -- --reset-cache --platform android --dev false --minify false --entry-file index.js --bundle-output gutenberg/packages/react-native-editor/android/app/src/main/assets/index.android.text.bundle --assets-dest gutenberg/packages/react-native-editor/android/app/src/main/res",
 		"test:e2e:bundle:android:bytecode": "./gutenberg/node_modules/react-native/sdks/hermesc/`node -e \"const platform=require('os').platform();console.log(platform === 'darwin' ? 'osx-bin' : (platform === 'linux' ? 'linux64-bin' : (platform === 'win32' ? 'win64-bin' : 'unsupported-os')));\"`/hermesc -emit-binary -O -out gutenberg/packages/react-native-editor/android/app/src/main/assets/index.android.bundle gutenberg/packages/react-native-editor/android/app/src/main/assets/index.android.text.bundle -output-source-map",