Skip to content

Commit

Permalink
Delay snapshotting various flakey UI tests
Browse files Browse the repository at this point in the history
  • Loading branch information
stefanceriu committed Jan 8, 2025
1 parent 082b4b2 commit fc719d9
Show file tree
Hide file tree
Showing 8 changed files with 29 additions and 29 deletions.
4 changes: 2 additions & 2 deletions UITests/Sources/AppLockSetupUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class AppLockSetupUITests: XCTestCase {
try await Task.sleep(for: .seconds(0.5))

// Create PIN screen.
try await app.assertScreenshot(.appLockSetupFlow, step: Step.createPIN)
try await app.assertScreenshot(.appLockSetupFlow, step: Step.createPIN, delay: .seconds(0.5))

enterPIN()

Expand Down Expand Up @@ -107,7 +107,7 @@ class AppLockSetupUITests: XCTestCase {
enterPIN()

// Settings screen.
try await app.assertScreenshot(.appLockSetupFlow, step: Step.settings)
try await app.assertScreenshot(.appLockSetupFlow, step: Step.settings, delay: .seconds(0.5))

app.buttons[A11yIdentifiers.appLockSetupSettingsScreen.removePIN].tap()
app.alerts.element.buttons[A11yIdentifiers.alertInfo.primaryButton].tap()
Expand Down
16 changes: 8 additions & 8 deletions UITests/Sources/AppLockUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -57,21 +57,21 @@ class AppLockUITests: XCTestCase {
await client.waitForApp()

// Blank form representing an unlocked app.
try await app.assertScreenshot(.appLockFlow, step: Step.unlocked)
try await app.assertScreenshot(.appLockFlow, step: Step.unlocked, delay: .seconds(0.5))

// When backgrounding the app.
try client.send(.notification(name: UIApplication.didEnterBackgroundNotification))

// Then the app should remain unlocked.
try await app.assertScreenshot(.appLockFlow, step: Step.unlocked)
try await app.assertScreenshot(.appLockFlow, step: Step.unlocked, delay: .seconds(0.5))

// When foregrounding the app.
try client.send(.notification(name: UIApplication.willEnterForegroundNotification))
try? await Task.sleep(for: .milliseconds(100))
try client.send(.notification(name: UIApplication.didBecomeActiveNotification))

// Then the app should still remain unlocked.
try await app.assertScreenshot(.appLockFlow, step: Step.unlocked)
try await app.assertScreenshot(.appLockFlow, step: Step.unlocked, delay: .seconds(0.5))
}

func testWrongPIN() async throws {
Expand All @@ -80,30 +80,30 @@ class AppLockUITests: XCTestCase {
app = Application.launch(.appLockFlow)
await client.waitForApp()

try await app.assertScreenshot(.appLockFlow, step: Step.unlocked)
try await app.assertScreenshot(.appLockFlow, step: Step.unlocked, delay: .seconds(0.5))
try client.send(.notification(name: UIApplication.didEnterBackgroundNotification))
try await Task.sleep(for: .milliseconds(500)) // Don't overwrite the previous signal immediately.

try client.send(.notification(name: UIApplication.willEnterForegroundNotification))
try? await Task.sleep(for: .milliseconds(100))
try client.send(.notification(name: UIApplication.didBecomeActiveNotification))

try await app.assertScreenshot(.appLockFlow, step: Step.lockScreen)
try await app.assertScreenshot(.appLockFlow, step: Step.lockScreen, delay: .seconds(0.5))

// When entering an incorrect PIN
enterWrongPIN()

// Then the app should remain locked with a warning.
try await app.assertScreenshot(.appLockFlow, step: Step.failedUnlock)
try await app.assertScreenshot(.appLockFlow, step: Step.failedUnlock, delay: .seconds(0.5))

// When entering it incorrectly twice more.
enterWrongPIN()
enterWrongPIN()

// Then then the app should sign the user out.
try await app.assertScreenshot(.appLockFlow, step: Step.logoutAlert)
try await app.assertScreenshot(.appLockFlow, step: Step.logoutAlert, delay: .seconds(0.5))
app.alerts.element.buttons[A11yIdentifiers.alertInfo.primaryButton].tap()
try await app.assertScreenshot(.appLockFlow, step: Step.forcedLogout)
try await app.assertScreenshot(.appLockFlow, step: Step.forcedLogout, delay: .seconds(0.5))
}

func testResignActive() async throws {
Expand Down
2 changes: 1 addition & 1 deletion UITests/Sources/EncryptionResetUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,6 @@ class EncryptionResetUITests: XCTestCase {
let passwordField = app.secureTextFields[A11yIdentifiers.encryptionResetPasswordScreen.passwordField]
passwordField.clearAndTypeText("supersecurepassword", app: app)
app.buttons[A11yIdentifiers.encryptionResetPasswordScreen.submit].tap()
try await app.assertScreenshot(.encryptionReset, step: Step.resetingEncryption)
try await app.assertScreenshot(.encryptionReset, step: Step.resetingEncryption, delay: .seconds(0.5))
}
}
4 changes: 2 additions & 2 deletions UITests/Sources/PollFormScreenUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class PollFormScreenUITests: XCTestCase {
let createButton = app.buttons[A11yIdentifiers.pollFormScreen.submit]
XCTAssertTrue(createButton.isEnabled)

try await app.assertScreenshot(.createPoll, step: 1)
try await app.assertScreenshot(.createPoll, step: 1, delay: .seconds(0.5))
}

func testMaxOptions() async throws {
Expand All @@ -57,6 +57,6 @@ class PollFormScreenUITests: XCTestCase {
XCTAssertFalse(addOption.exists)
XCTAssertFalse(createButton.isEnabled)

try await app.assertScreenshot(.createPoll, step: 2)
try await app.assertScreenshot(.createPoll, step: 2, delay: .seconds(0.5))
}
}
4 changes: 2 additions & 2 deletions UITests/Sources/RoomMembersListScreenUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class RoomMembersListScreenUITests: XCTestCase {
let searchBar = app.searchFields.firstMatch
searchBar.clearAndTypeText("alice\n", app: app)

try await app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 1)
try await app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 1, delay: .seconds(0.5))
}

func testSearchJoinedMember() async throws {
Expand All @@ -30,6 +30,6 @@ class RoomMembersListScreenUITests: XCTestCase {
let searchBar = app.searchFields.firstMatch
searchBar.clearAndTypeText("bob\n", app: app)

try await app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 2)
try await app.assertScreenshot(.roomMembersListScreenPendingInvites, step: 2, delay: .seconds(0.5))
}
}
20 changes: 10 additions & 10 deletions UITests/Sources/RoomScreenUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@ class RoomScreenUITests: XCTestCase {
XCTAssert(app.staticTexts[A11yIdentifiers.roomScreen.name].exists)
XCTAssert(app.staticTexts[A11yIdentifiers.roomScreen.avatar].exists)

try await app.assertScreenshot(.roomPlainNoAvatar)
try await app.assertScreenshot(.roomPlainNoAvatar, delay: .seconds(0.5))
}

func testSmallTimelineLayout() async throws {
let app = Application.launch(.roomSmallTimeline)

// The messages should be bottom aligned.
try await app.assertScreenshot(.roomSmallTimeline)
try await app.assertScreenshot(.roomSmallTimeline, delay: .seconds(0.5))
}

func testSmallTimelineWithIncomingAndPagination() async throws {
Expand All @@ -38,7 +38,7 @@ class RoomScreenUITests: XCTestCase {
try await performOperation(.paginate, using: client)

// Then the 4 visible messages should stay aligned to the bottom.
try await app.assertScreenshot(.roomSmallTimelineIncomingAndSmallPagination)
try await app.assertScreenshot(.roomSmallTimelineIncomingAndSmallPagination, delay: .seconds(0.5))
}

func testSmallTimelineWithLargePagination() async throws {
Expand All @@ -53,7 +53,7 @@ class RoomScreenUITests: XCTestCase {
try await performOperation(.paginate, using: client)

// The bottom of the timeline should remain visible with more items added above.
try await app.assertScreenshot(.roomSmallTimelineLargePagination)
try await app.assertScreenshot(.roomSmallTimelineLargePagination, delay: .seconds(0.5))
}

func testTimelineLayoutAtTop() async throws {
Expand All @@ -68,13 +68,13 @@ class RoomScreenUITests: XCTestCase {
for _ in 0...5 {
app.swipeDown()
}
try await app.assertScreenshot(.roomLayoutTop) // Assert initial state for comparison.
try await app.assertScreenshot(.roomLayoutTop, delay: .seconds(0.5)) // Assert initial state for comparison.

// When a back pagination occurs.
try await performOperation(.paginate, using: client)

// Then the bottom of the timeline should remain unchanged (with new items having been added above).
try await app.assertScreenshot(.roomLayoutTop)
try await app.assertScreenshot(.roomLayoutTop, delay: .seconds(0.5))
}

func testTimelineLayoutAtBottom() async throws {
Expand All @@ -93,12 +93,12 @@ class RoomScreenUITests: XCTestCase {
try await Task.sleep(for: .seconds(1))

// Then the timeline should scroll down to reveal the message.
try await app.assertScreenshot(.roomLayoutBottom, step: 0)
try await app.assertScreenshot(.roomLayoutBottom, step: 0, delay: .seconds(0.5))

// When the keyboard appears for the message composer.
try await tapMessageComposer(in: app)

try await app.assertScreenshot(.roomLayoutBottom, step: 1)
try await app.assertScreenshot(.roomLayoutBottom, step: 1, delay: .seconds(0.5))
}

func testTimelineLayoutHighlightExisting() async throws {
Expand All @@ -113,7 +113,7 @@ class RoomScreenUITests: XCTestCase {
try await performOperation(.focusOnEvent("$5"), using: client)

// Then the item should become highlighted.
try await app.assertScreenshot(.roomLayoutHighlight, step: 0)
try await app.assertScreenshot(.roomLayoutHighlight, step: 0, delay: .seconds(0.5))

guard UIDevice.current.userInterfaceIdiom == .phone else { return }

Expand All @@ -130,7 +130,7 @@ class RoomScreenUITests: XCTestCase {
let app = Application.launch(.roomSmallTimelineWithReadReceipts)

// The messages should be bottom aligned.
try await app.assertScreenshot(.roomSmallTimelineWithReadReceipts)
try await app.assertScreenshot(.roomSmallTimelineWithReadReceipts, delay: .seconds(0.5))
}

func testTimelineDisclosedPolls() async throws {
Expand Down
4 changes: 2 additions & 2 deletions UITests/Sources/ServerSelectionUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class ServerSelectionUITests: XCTestCase {
app.textFields[A11yIdentifiers.changeServerScreen.server].buttons.element.tap()

// Then the screen should not allow the user to continue.
try await app.assertScreenshot(.serverSelection, step: 1)
try await app.assertScreenshot(.serverSelection, step: 1, delay: .seconds(0.5))
}

func testInvalidAddress() async throws {
Expand All @@ -37,7 +37,7 @@ class ServerSelectionUITests: XCTestCase {
app.textFields[A11yIdentifiers.changeServerScreen.server].clearAndTypeText("thisisbad\n", app: app) // The tests only accept an address from LoginHomeserver.mockXYZ

// Then an error should be shown and the confirmation button disabled.
try await app.assertScreenshot(.serverSelection, step: 2)
try await app.assertScreenshot(.serverSelection, step: 2, delay: .seconds(0.5))
XCTAssertFalse(app.buttons[A11yIdentifiers.changeServerScreen.continue].isEnabled, "The continue button should be disabled when there is an error.")
}
}
4 changes: 2 additions & 2 deletions UITests/Sources/StartChatScreenUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ class StartChatScreenUITests: XCTestCase {
let searchField = app.searchFields.firstMatch
searchField.clearAndTypeText("None\n", app: app)
XCTAssert(app.staticTexts[A11yIdentifiers.startChatScreen.searchNoResults].waitForExistence(timeout: 1.0))
try await app.assertScreenshot(.startChat, step: 1)
try await app.assertScreenshot(.startChat, step: 1, delay: .seconds(0.5))
}

func testSearchWithResults() async throws {
Expand All @@ -28,6 +28,6 @@ class StartChatScreenUITests: XCTestCase {
searchField.clearAndTypeText("Bob\n", app: app)
XCTAssertFalse(app.staticTexts[A11yIdentifiers.startChatScreen.searchNoResults].waitForExistence(timeout: 1.0))
XCTAssertEqual(app.collectionViews.firstMatch.cells.count, 2)
try await app.assertScreenshot(.startChat, step: 2)
try await app.assertScreenshot(.startChat, step: 2, delay: .seconds(0.5))
}
}

0 comments on commit fc719d9

Please sign in to comment.