Test UI test on Cirrus Labs (#6576)

itaybre · web-flow · commit 2fc1ea52a4f8 · 2025-11-02T11:27:40.000-03:00
* Add cirrus lab tests

* Update name

* Test new parameters

* Fix open simulator

* Change OS versions

* Set device for tvOS

* Add device

* Bump xcode version

* Print failed job

* Add missing install platforms to iOS17

* Update UI tests for Cirrus Runners to run only on commits to main

* Use a single ui tests file

* Use macos 15

* Modify matrixes

* Use macos-15 and fix fastlane command

* Define specific xcode version for cirrus labs

* Disable fail fast

* Bump iOS version used

* Use iOS 18.6

* Cleanup comments

* Skip cirrus labs if the job is not a commit on main
diff --git a/.github/file-filters.yml b/.github/file-filters.yml
@@ -434,6 +434,7 @@ run_ui_tests_for_prs: &run_ui_tests_for_prs
 
   # GH Actions
   - ".github/workflows/ui-tests.yml"
+  - ".github/workflows/ui-tests-cirrus-labs.yml"
   - ".github/workflows/ui-tests-common.yml"
   - ".github/workflows/build-xcframework-variant-slices.yml"
   - ".github/workflows/assemble-xcframework-variant.yml"
diff --git a/.github/workflows/ui-tests-common.yml b/.github/workflows/ui-tests-common.yml
@@ -24,6 +24,7 @@ on:
       macos_version:
         description: "macOS version"
         required: true
+        default: macos-26
         type: string
       needs_xcframework:
         description: "Whether the workflow needs to download the XCFramework."
@@ -60,14 +61,25 @@ on:
         required: false
         default: ""
         type: string
+      run_on_cirrus_labs:
+        description: "Whether to run the tests on Cirrus Labs"
+        required: false
+        default: false
+        type: boolean
+      should_skip:
+        description: "Whether to skip the job"
+        required: false
+        default: false
+        type: boolean
     secrets:
       CODECOV_TOKEN:
         required: true
 
 jobs:
   common-ui-tests:
+    if: ${{ !inputs.should_skip }}
     name: UI Tests Common
-    runs-on: ${{ inputs.macos_version }}
+    runs-on: ${{ inputs.run_on_cirrus_labs && fromJSON(format('["ghcr.io/cirruslabs/macos-runner:{0}", "runner_group_id:10"]', inputs.macos_version)) || inputs.macos_version }}
     timeout-minutes: 40
     steps:
       - uses: actions/checkout@v5
diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml
@@ -36,106 +36,165 @@ jobs:
   ui-tests:
     if: github.event_name != 'pull_request' || needs.files-changed.outputs.run_ui_tests_for_prs == 'true'
     needs: files-changed
-    name: Test ${{matrix.name}} V3 # Up the version with every change to keep track of flaky tests
+    name: Test ${{matrix.name}} on ${{matrix.runner}} V3 # Up the version with every change to keep track of flaky tests
     uses: ./.github/workflows/ui-tests-common.yml
     strategy:
+      fail-fast: false
       matrix:
+        runner:
+          - GitHub
+          - Cirrus Labs
+        job_id:
+          - ios_objc
+          - tvos_swift
         include:
-          - name: iOS Objective-C
+          - job_id: ios_objc
+            name: iOS Objective-C
             target: ios_objc
-          - name: tvOS Swift
+            test_destination_os: "18.6"
+            device: iPhone 16 Pro
+          - job_id: tvos_swift
+            name: tvOS Swift
             target: tvos_swift
+            test_destination_os: "18.5"
+            device: "Apple TV"
     with:
       fastlane_command: ui_tests_${{matrix.target}}
-      xcode_version: 16.2
+      xcode_version: 16.4
       build_with_make: true
-      macos_version: macos-14
+      macos_version: ${{ matrix.runner == 'Cirrus Labs' && 'sequoia' || 'macos-15' }}
+      run_on_cirrus_labs: ${{ matrix.runner == 'Cirrus Labs' }}
       codecov_test_analytics: true
+      test-destination-os: ${{matrix.test_destination_os}}
+      device: ${{matrix.device}}
+      files_suffix: _${{matrix.target}}_${{matrix.runner}}
+      # Github doesn't allow using matrix variables in if conditions, so we pass it to the common workflow and skip there.
+      # This lets us skip the job if it's a push to the main branch and the runner is Cirrus Labs.
+      should_skip: ${{ ( github.event_name != 'push' || github.ref != 'refs/heads/main' ) && matrix.runner == 'Cirrus Labs' }}
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
-  # SwiftUI only supports iOS 14+ so we run it in a separate matrix here
   ui-tests-swift-ui:
     if: github.event_name != 'pull_request' || needs.files-changed.outputs.run_ui_tests_for_prs == 'true'
     needs: files-changed
-    name: Test SwiftUI V4 # Up the version with every change to keep track of flaky tests
+    name: Test SwiftUI on ${{matrix.runner}} V4 # Up the version with every change to keep track of flaky tests
     uses: ./.github/workflows/ui-tests-common.yml
     with:
       fastlane_command: ui_tests_ios_swiftui
       build_with_make: true
       codecov_test_analytics: true
       xcode_version: 16.4
-      macos_version: macos-15
-      test-destination-os: "18.4"
+      macos_version: ${{ matrix.runner == 'Cirrus Labs' && 'sequoia' || 'macos-15' }}
+      run_on_cirrus_labs: ${{ matrix.runner == 'Cirrus Labs' }}
+      test-destination-os: "18.6"
       device: iPhone 16 Pro
+      files_suffix: _swiftui_-${{matrix.runner}}
+      # Github doesn't allow using matrix variables in if conditions, so we pass it to the common workflow and skip there.
+      # This lets us skip the job if it's a push to the main branch and the runner is Cirrus Labs.
+      should_skip: ${{ ( github.event_name != 'push' || github.ref != 'refs/heads/main' ) && matrix.runner == 'Cirrus Labs' }}
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
+    strategy:
+      fail-fast: false
+      matrix:
+        runner:
+          - GitHub
+          - Cirrus Labs
   ui-tests-swift:
     if: github.event_name != 'pull_request' || needs.files-changed.outputs.run_ui_tests_for_prs == 'true'
     needs: files-changed
-    name: Test Swift ${{matrix.name}} V5 # Up the version with every change to keep track of flaky tests
+    name: Test Swift ${{matrix.name}} on ${{matrix.runner}} V5 # Up the version with every change to keep track of flaky tests
     uses: ./.github/workflows/ui-tests-common.yml
     strategy:
       fail-fast: false
       matrix:
+        runner:
+          - GitHub
+          - Cirrus Labs
+        job_id:
+          - ios_17
+          - ios_18
+          - ios_26
         include:
-          # iOS 17 - Use pre-installed iOS 17.5 runtime on macOS-14 with Xcode 15.4
-          - name: iOS 17
-            runs-on: macos-14
+          # iOS 17 - Use iOS 17.5 runtime on macOS-15 with Xcode 16.4
+          - job_id: ios_17
+            name: iOS 17
+            runs_on: macos-14
+            runs_on_cirrus: sequoia
             xcode: "15.4"
-            test-destination-os: "17.5"
+            xcode_cirrus: "16.4"
+            test_destination_os: "17.5"
             platform: "iOS"
+            install_platforms: false
             create_device: true
             device: iPhone 15 Pro
 
           # iOS 18 - Use pre-installed iOS 18.4 runtime on macOS-15
-          - name: iOS 18
-            runs-on: macos-15
+          - job_id: ios_18
+            name: iOS 18
+            runs_on: macos-15
+            runs_on_cirrus: sequoia
             xcode: "16.4"
-            test-destination-os: "18.4"
+            test_destination_os: "18.4"
             platform: "iOS"
             device: iPhone 16 Pro
             install_platforms: false
             create_device: false
 
           # iOS 26 - Download iOS 26.1 beta runtime on macOS-26
-          - name: iOS 26
-            runs-on: macos-26
+          - job_id: ios_26
+            name: iOS 26
+            runs_on: macos-26
+            runs_on_cirrus: tahoe
             xcode: "26.1"
-            test-destination-os: "26.1"
+            test_destination_os: "26.1"
             install_platforms: true
             platform: "iOS"
             create_device: true
             device: iPhone 17 Pro
     with:
       fastlane_command: ui_tests_ios_swift
-      files_suffix: _xcode_${{matrix.xcode}}-${{matrix.device}}
+      files_suffix: _xcode_${{matrix.xcode}}-${{matrix.device}}-${{matrix.runner}}
       build_with_make: true
-      macos_version: ${{matrix.runs-on}}
+      macos_version: ${{matrix.runner == 'Cirrus Labs' && matrix.runs_on_cirrus || matrix.runs_on}}
+      run_on_cirrus_labs: ${{ matrix.runner == 'Cirrus Labs' }}
       codecov_test_analytics: true
-      xcode_version: ${{matrix.xcode}}
-      test-destination-os: ${{matrix.test-destination-os}}
+      xcode_version: ${{ matrix.runner == 'Cirrus Labs' && matrix.xcode_cirrus || matrix.xcode}}
+      test-destination-os: ${{matrix.test_destination_os}}
       install_platforms: ${{matrix.install_platforms}}
       platform: ${{matrix.platform}}
       create_device: ${{matrix.create_device}}
       device: ${{matrix.device}}
+      # Github doesn't allow using matrix variables in if conditions, so we pass it to the common workflow and skip there.
+      # This lets us skip the job if it's a push to the main branch and the runner is Cirrus Labs.
+      should_skip: ${{ ( github.event_name != 'push' || github.ref != 'refs/heads/main' ) && matrix.runner == 'Cirrus Labs' }}
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   ui-tests-swift6:
     if: github.event_name != 'pull_request' || needs.files-changed.outputs.run_ui_tests_for_prs == 'true'
     needs: files-changed
-    name: Test iOS Swift6 V3 # Up the version with every change to keep track of flaky tests
+    name: Test iOS Swift6 on ${{matrix.runner}} V3 # Up the version with every change to keep track of flaky tests
     uses: ./.github/workflows/ui-tests-common.yml
+    strategy:
+      fail-fast: false
+      matrix:
+        runner:
+          - GitHub
+          - Cirrus Labs
     with:
       fastlane_command: ui_tests_ios_swift6
       xcode_version: 16.4
       build_with_make: true
-      macos_version: macos-15
+      macos_version: ${{matrix.runner == 'Cirrus Labs' && 'sequoia' || 'macos-15'}}
+      run_on_cirrus_labs: ${{matrix.runner == 'Cirrus Labs'}}
       codecov_test_analytics: true
       device: iPhone 16 Pro
       test-destination-os: "18.4"
+      files_suffix: _swift6_-${{matrix.runner}}
+      # Github doesn't allow using matrix variables in if conditions, so we pass it to the common workflow and skip there.
+      # This lets us skip the job if it's a push to the main branch and the runner is Cirrus Labs.
+      should_skip: ${{ ( github.event_name != 'push' || github.ref != 'refs/heads/main' ) && matrix.runner == 'Cirrus Labs' }}
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
diff --git a/fastlane/Fastfile b/fastlane/Fastfile
@@ -271,9 +271,10 @@ platform :ios do
     )
   end
 
-  lane :ui_tests_ios_objc do
+  lane :ui_tests_ios_objc do |options|
     run_ui_tests(
       scheme: "iOS-ObjectiveC",
+      device: options[:device],
       result_bundle_name: "ui_tests_ios_objc"
     )
   end
@@ -287,9 +288,10 @@ platform :ios do
     )
   end
 
-  lane :ui_tests_tvos_swift do
+  lane :ui_tests_tvos_swift do |options|
     run_ui_tests(
       scheme: "tvOS-Swift",
+      device: options[:device],
       result_bundle_name: "ui_tests_tvos_swift"
     )
   end
diff --git a/scripts/ci-boot-simulator.sh b/scripts/ci-boot-simulator.sh
@@ -174,8 +174,9 @@ for attempt in $(seq 1 $MAX_BOOT_ATTEMPTS); do
     # Open Simulator app UI (only on first attempt)
     if [ "$attempt" -eq 1 ]; then
         log_notice "Opening Simulator app UI"
-        if ! open -a Simulator; then
-            log_error "Failed to open Simulator app"
+        SIMULATOR_APP_PATH="$(xcode-select -p)/Applications/Simulator.app"
+        if ! open "$SIMULATOR_APP_PATH"; then
+            log_error "Failed to open Simulator app at $SIMULATOR_APP_PATH"
             exit 1
         fi
         log_notice "Simulator app opened successfully"

Original file line number	Diff line number	Diff line change
`@@ -271,9 +271,10 @@ platform :ios do`
`271`	`271`	`)`
`272`	`272`	`end`
`273`	`273`
`274`		`- lane :ui_tests_ios_objc do`
	`274`	`+ lane :ui_tests_ios_objc do \|options\|`
`275`	`275`	`run_ui_tests(`
`276`	`276`	`scheme: "iOS-ObjectiveC",`
	`277`	`+ device: options[:device],`
`277`	`278`	`result_bundle_name: "ui_tests_ios_objc"`
`278`	`279`	`)`
`279`	`280`	`end`
`@@ -287,9 +288,10 @@ platform :ios do`
`287`	`288`	`)`
`288`	`289`	`end`
`289`	`290`
`290`		`- lane :ui_tests_tvos_swift do`
	`291`	`+ lane :ui_tests_tvos_swift do \|options\|`
`291`	`292`	`run_ui_tests(`
`292`	`293`	`scheme: "tvOS-Swift",`
	`294`	`+ device: options[:device],`
`293`	`295`	`result_bundle_name: "ui_tests_tvos_swift"`
`294`	`296`	`)`
`295`	`297`	`end`