Add all current UI tests to maestro

.github/workflows/eval-maestro-tests.yml

@@ -136,4 +136,11 @@ jobs:
         run: |
           xcrun simctl install booted ${{ matrix.app}}
 
-          maestro test ${{matrix.test_suit}} --format junit
+          maestro test ${{matrix.test_suit}} --format junit --debug-output /tmp/MaestroLogs
+
+      - name: Store Maestro Logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: maestro-logs-${{matrix.sample-name}}-${{matrix.device}}-${{matrix.os-version}}
+          path: /tmp/MaestroLogs

Samples/iOS-Swift/.maestro/breadcrumbData.yaml

@@ -0,0 +1,5 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Extra"
+- assertVisible: "{ category: ui.lifecycle, parentViewController: UITabBarController, beingPresented: false, window_isKeyWindow: true, is_window_rootViewController: false }"

Samples/iOS-Swift/.maestro/captureError.yaml

@@ -0,0 +1,4 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Capture Error"

Samples/iOS-Swift/.maestro/captureException.yaml

@@ -0,0 +1,4 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Capture NSException"

Samples/iOS-Swift/.maestro/checkFramesCount.yaml

@@ -0,0 +1,8 @@
+appId: io.sentry.sample.iOS-Swift
+env:
+  MAESTRO_FLOW_NAME: "Check Frames Proportions"
+---
+- launchApp
+- tapOn: "Extra"
+- tapOn: "Frames Count"
+- assertVisible: "Frames Count OK"

Samples/iOS-Swift/.maestro/checkFramesProportions.yaml

@@ -0,0 +1,8 @@
+appId: io.sentry.sample.iOS-Swift
+env:
+  MAESTRO_FLOW_NAME: "Check Frames Proportions"
+---
+- launchApp
+- tapOn: "Extra"
+- tapOn: "Frames Proportions"
+- assertVisible: "Frames Proportions OK"

Samples/iOS-Swift/.maestro/showImage.yaml

@@ -0,0 +1,6 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Load Image Controller"
+- assertVisible: "ASSERT: SUCCESS"

Samples/iOS-Swift/.maestro/showNib.yaml

@@ -0,0 +1,7 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Nib Controller"
+- assertVisible: "a lonely button"
+- assertVisible: "ASSERT: SUCCESS"

Samples/iOS-Swift/.maestro/showSplitView.yaml

@@ -0,0 +1,6 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Split Controller"
+- assertVisible: "ASSERT: SUCCESS"

Samples/iOS-Swift/.maestro/showTableView.yaml

@@ -0,0 +1,7 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Table Controller"
+- assertVisible: "Gradient Table View"
+- assertVisible: "ASSERT: SUCCESS"

Samples/iOS-Swift/.maestro/showTextView.yaml

@@ -0,0 +1,7 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Lorem Ipsum Controller"
+- assertVisible:
+    id: "LoremIpsumTextView"

Samples/iOS-Swift/.maestro/uiClickTransaction.yaml

@@ -0,0 +1,5 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "UIClick Transaction"

Samples/iOS-Swift/.maestro/useAfterFreeAfterUIImageNamedEmptyString.yaml

@@ -0,0 +1,10 @@
+appId: io.sentry.sample.iOS-Swift
+---
+# We received a customer report that ASAN reports a use-after-free error after
+# calling UIImage(named:) with an empty string argument. Recording another
+# transaction leads to the ASAN error.
+- launchApp
+# this primes the state required according to the customer report, by setting a UIImageView.image property to a UIImage(named: "")
+- tapOn: "Use-after-free"
+# this causes another transaction to be recorded which hits the codepath necessary for the ASAN to trip
+- tapOn: "Extra"

Samples/iOS-Swift/iOS-Swift/ExtraViewController.swift

@@ -5,6 +5,7 @@ import UIKit
 class ExtraViewController: UIViewController {
 
     @IBOutlet weak var framesLabel: UILabel!
+    @IBOutlet weak var framesCheckLabel: UILabel!
     @IBOutlet weak var breadcrumbLabel: UILabel!
     @IBOutlet weak var uiTestNameLabel: UILabel!
     @IBOutlet weak var anrFullyBlockingButton: UIButton!
@@ -18,7 +19,6 @@ class ExtraViewController: UIViewController {
         if let uiTestName = ProcessInfo.processInfo.environment["--io.sentry.ui-test.test-name"] {
             uiTestNameLabel.text = uiTestName
         }
-
         Timer.scheduledTimer(withTimeInterval: 0.5, repeats: true) { _ in
             self.framesLabel?.text = "Frames Total:\(PrivateSentrySDKOnly.currentScreenFrames.total) Slow:\(PrivateSentrySDKOnly.currentScreenFrames.slow) Frozen:\(PrivateSentrySDKOnly.currentScreenFrames.frozen)"
         }
@@ -193,6 +193,56 @@ class ExtraViewController: UIViewController {
         }
     }
 
+    @IBAction func checkFramesProportions(_ sender: UIButton) {
+        let slowFramesPercentage = Double(PrivateSentrySDKOnly.currentScreenFrames.slow) / Double(PrivateSentrySDKOnly.currentScreenFrames.total)
+        if slowFramesPercentage > 0.5 {
+            fatalError("Too many slow frames.")
+        }
+
+        let frozenFramesPercentage = Double(PrivateSentrySDKOnly.currentScreenFrames.frozen) / Double(PrivateSentrySDKOnly.currentScreenFrames.total)
+        if frozenFramesPercentage > 0.5 {
+            fatalError("Too many frozen frames.")
+        }
+
+        framesCheckLabel.text = "Frames Proportions OK"
+    }
+
+    @IBAction func checkFramesCount(_ sender: UIButton) {
+        let startDate = Date()
+        let startSlowFrames = PrivateSentrySDKOnly.currentScreenFrames.slow
+        let startTotalFrames = PrivateSentrySDKOnly.currentScreenFrames.total
+
+        Timer.scheduledTimer(withTimeInterval: 1, repeats: false) { _ in
+            let endDate = Date()
+            let endSlowFrames = PrivateSentrySDKOnly.currentScreenFrames.slow
+            let endTotalFrames = PrivateSentrySDKOnly.currentScreenFrames.total
+
+            let secondsBetween = endDate.timeIntervalSince(startDate)
+
+            // We don't calculate the min and max values based on the frame rate, as it could have changed while waiting for them to render.
+            // Instead, we pick the minimum value based on 60fps and the maximum value based on 120fps.
+            let slowFrames = endSlowFrames - startSlowFrames
+            let slowFramesBuffer = Double(slowFrames) * 0.2
+
+            let expectedMinimumTotalFrames = (secondsBetween - 0.5 - slowFramesBuffer) * 60
+            let expectedMaximumTotalFrames = (secondsBetween + 0.5) * 120
+
+            let actualTotalFrames = Double(endTotalFrames - startTotalFrames)
+
+            if actualTotalFrames < expectedMinimumTotalFrames {
+                print("Actual frames:\(actualTotalFrames) should be greater than or equal to \(expectedMinimumTotalFrames)")
+                return
+            }
+
+            if actualTotalFrames > expectedMaximumTotalFrames {
+                print("Actual frames:\(actualTotalFrames) should be less than or equal to \(expectedMaximumTotalFrames)")
+                return
+            }
+
+            self.framesCheckLabel?.text = "Frames Count OK"
+        }
+    }
+
     private func calcPi() -> Double {
         var denominator = 1.0
         var pi = 0.0

Samples/iOS-SwiftUI/.maestro/noNewTransactionForSecondCallToBody.yaml

@@ -0,0 +1,13 @@
+appId: io.sentry.iOS-SwiftUI
+---
+- launchApp
+
+- tapOn: "Form Screen"
+- copyTextFrom:
+    id: "SPAN_ID"
+- assertTrue: ${maestro.copiedText != "NO SPAN"}
+
+- tapOn: "Test"
+- copyTextFrom:
+    id: "SPAN_ID"
+- assertTrue: ${maestro.copiedText == "NO SPAN"}

Samples/iOS-SwiftUI/.maestro/transactionSpan.yaml

@@ -0,0 +1,24 @@
+appId: io.sentry.iOS-SwiftUI
+---
+- launchApp
+
+- copyTextFrom:
+    id: "TRANSACTION_NAME"
+- assertTrue: ${maestro.copiedText == "Content View Body"}
+- copyTextFrom:
+    id: "CHILD_NAME"
+- assertTrue: ${maestro.copiedText == "Child Span"}
+
+- copyTextFrom:
+    id: "CHILD_PARENT_SPANID"
+- evalScript: ${output.childParentSpanId = maestro.copiedText}
+- copyTextFrom:
+    id: "TRANSACTION_ID"
+- assertTrue: ${maestro.copiedText == output.childParentSpanId}
+
+- copyTextFrom:
+    id: "TRACE_ORIGIN"
+- assertTrue: ${maestro.copiedText == "auto.ui.swift_ui"}
+- copyTextFrom:
+    id: "CHILD_TRACE_ORIGIN"
+- assertTrue: ${maestro.copiedText == "auto.ui.swift_ui"}