E2E matrix #1948
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E matrix | |
| on: | |
| schedule: | |
| - cron: "0 5 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| type: boolean | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: false | |
| env: | |
| CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress | |
| permissions: { } | |
| jobs: | |
| preinstall: | |
| if: ${{ github.repository_owner == 'nrwl' }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 20 | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| # - windows-latest Windows fails to build gradle wrapper which always runs when we build nx. | |
| ## https://staging.nx.app/runs/LgD4vxGn8w?utm_source=pull-request&utm_medium=comment | |
| node_version: | |
| - 20 | |
| - 22 | |
| # - 23 | |
| exclude: | |
| # run just node v20 on macos and windows | |
| - os: macos-latest | |
| node_version: 22 | |
| # - os: macos-latest | |
| # node_version: 23 | |
| # - os: windows-latest TODO (emily): Windows fails to build gradle wrapper which always runs when we build nx. Re-enable when we fix this. | |
| # node_version: 22 | |
| # - os: windows-latest TODO (emily): Windows fails to build gradle wrapper which always runs when we build nx. Re-enable when we fix this. | |
| # node_version: 23 | |
| name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| filter: tree:0 | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| version: 10.11.1 | |
| run_install: false | |
| - name: Set node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node_version }} | |
| cache: 'pnpm' | |
| - name: Get pnpm store directory | |
| id: pnpm-cache | |
| run: echo "path=$(pnpm store path)" >> $GITHUB_OUTPUT | |
| - name: Cache pnpm store | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.pnpm-cache.outputs.path }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pnpm-store- | |
| - name: Ensure Python setuptools Installed on Macos | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| id: brew-install-python-setuptools | |
| run: brew install python-setuptools | |
| - name: Install packages | |
| run: | | |
| pnpm install --frozen-lockfile | |
| pnpm playwright install --with-deps | |
| - name: Homebrew cache directory path | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| id: homebrew-cache-dir-path | |
| run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT | |
| - name: Cache Homebrew | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| uses: actions/cache@v4 | |
| with: | |
| lookup-only: true | |
| path: ${{ steps.homebrew-cache-dir-path.outputs.dir }} | |
| key: brew-${{ matrix.node_version }} | |
| restore-keys: | | |
| brew- | |
| - name: Cache Cypress | |
| id: cache-cypress | |
| uses: actions/cache@v4 | |
| with: | |
| lookup-only: true | |
| path: '${{ github.workspace }}/.cypress' | |
| key: ${{ runner.os }}-cypress | |
| - name: Install Cypress | |
| if: steps.cache-cypress.outputs.cache-hit != 'true' | |
| run: npx cypress install | |
| prepare-matrix: | |
| name: Prepare matrix combinations | |
| if: ${{ github.repository_owner == 'nrwl' }} | |
| timeout-minutes: 5 | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.process-json.outputs.MATRIX }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| filter: tree:0 | |
| - name: Process matrix data | |
| id: process-json | |
| run: | |
| echo "MATRIX=$(npx tsx .github/workflows/nightly/process-matrix.ts | jq -c .)" >> $GITHUB_OUTPUT | |
| e2e: | |
| if: ${{ github.repository_owner == 'nrwl' }} | |
| needs: | |
| - preinstall | |
| - prepare-matrix | |
| permissions: | |
| contents: read | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 200 # <- cap each job to 200 minutes | |
| strategy: | |
| matrix: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} # Load matrix from previous job | |
| fail-fast: false | |
| name: ${{ matrix.os_name }}/${{ matrix.package_manager }}/${{ matrix.node_version }} ${{ join(matrix.project) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| filter: tree:0 | |
| - name: Prepare dir for output | |
| run: mkdir -p outputs | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| version: 10.11.1 | |
| run_install: false | |
| - name: Use Node.js ${{ matrix.node_version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node_version }} | |
| cache: 'pnpm' | |
| - name: Install Rust | |
| if: ${{ matrix.os != 'windows-latest' }} | |
| run: | | |
| curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y | |
| source "$HOME/.cargo/env" | |
| rustup toolchain install 1.70.0 | |
| - name: Load Cargo Env | |
| if: ${{ matrix.os != 'windows-latest' }} | |
| run: echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV | |
| - name: Install bun | |
| if: ${{ matrix.os != 'windows-latest' }} | |
| uses: oven-sh/setup-bun@v1 | |
| with: | |
| bun-version: latest | |
| - name: Install packages | |
| run: | | |
| pnpm install --frozen-lockfile | |
| pnpm playwright install --with-deps | |
| - name: Cleanup | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| # Workaround to provide additional free space for testing. | |
| # https://github.com/actions/virtual-environments/issues/2840 | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo apt-get install lsof | |
| echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p | |
| - name: Homebrew cache directory path | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| id: homebrew-cache-dir-path | |
| run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT | |
| - name: Cache Homebrew | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.homebrew-cache-dir-path.outputs.dir }} | |
| key: brew-${{ matrix.node_version }} | |
| restore-keys: | | |
| brew- | |
| - name: Cache Cypress | |
| id: cache-cypress | |
| uses: actions/cache@v4 | |
| with: | |
| path: '${{ github.workspace }}/.cypress' | |
| key: ${{ runner.os }}-cypress | |
| - name: Install Cypress | |
| if: steps.cache-cypress.outputs.cache-hit != 'true' | |
| run: npx cypress install | |
| - name: Configure Detox Environment, Install applesimutils | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| run: | | |
| # Ensure Xcode command line tools are installed and configured | |
| xcode-select --print-path || sudo xcode-select --reset | |
| sudo xcode-select -s /Applications/Xcode.app | |
| # Install or update applesimutils with error handling | |
| if ! brew list applesimutils &>/dev/null; then | |
| echo "Installing applesimutils..." | |
| HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null | |
| HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null || { | |
| echo "Failed to install applesimutils, retrying with update..." | |
| brew update | |
| HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils | |
| } | |
| else | |
| echo "Updating applesimutils..." | |
| HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade applesimutils || true | |
| fi | |
| # Verify applesimutils installation | |
| applesimutils --version || (echo "applesimutils installation failed" && exit 1) | |
| # Configure environment for M-series Mac | |
| echo "DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer" >> $GITHUB_ENV | |
| echo "PLATFORM_NAME=iOS Simulator" >> $GITHUB_ENV | |
| # Set additional environment variables for better debugging | |
| echo "DETOX_DISABLE_TELEMETRY=1" >> $GITHUB_ENV | |
| echo "DETOX_LOG_LEVEL=trace" >> $GITHUB_ENV | |
| # Verify Xcode installation | |
| xcodebuild -version | |
| timeout-minutes: 10 | |
| continue-on-error: false | |
| - name: Reset iOS Simulators | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| id: reset-simulators | |
| run: | | |
| echo "Resetting iOS Simulators..." | |
| # Kill simulator processes | |
| sudo killall -9 com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true | |
| killall "Simulator" 2>/dev/null || true | |
| killall "iOS Simulator" 2>/dev/null || true | |
| # Wait for processes to terminate | |
| sleep 3 | |
| # Shutdown and erase all simulators (ignore failures) | |
| xcrun simctl shutdown all 2>/dev/null || true | |
| sleep 5 | |
| xcrun simctl erase all 2>/dev/null || true | |
| # If erase failed, try the nuclear option | |
| if xcrun simctl list devices | grep -q "Booted" 2>/dev/null; then | |
| echo "Standard reset failed, using nuclear option..." | |
| rm -rf ~/Library/Developer/CoreSimulator/Devices/* 2>/dev/null || true | |
| launchctl remove com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true | |
| sleep 3 | |
| fi | |
| # Clean up additional directories | |
| rm -rf ~/Library/Developer/CoreSimulator/Caches/* 2>/dev/null || true | |
| rm -rf ~/Library/Logs/CoreSimulator/* 2>/dev/null || true | |
| rm -rf ~/Library/Developer/Xcode/DerivedData/* 2>/dev/null || true | |
| echo "Simulator reset completed" | |
| timeout-minutes: 5 | |
| continue-on-error: true | |
| - name: Verify Simulator Reset | |
| if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }} | |
| run: | | |
| # Verify CoreSimulator service restarted | |
| pgrep -fl "CoreSimulator" || (echo "CoreSimulator service not running" && exit 1) | |
| # Verify simulator runtime paths exist and are writable | |
| test -d ~/Library/Developer/CoreSimulator/Devices || (echo "Simulator devices directory missing" && exit 1) | |
| touch ~/Library/Developer/CoreSimulator/Devices/test || (echo "Simulator devices directory not writable" && exit 1) | |
| rm ~/Library/Developer/CoreSimulator/Devices/test | |
| timeout-minutes: 5 | |
| - name: Diagnose Simulator Reset Failure | |
| if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'failure' }} | |
| run: | | |
| echo "Simulator reset failed. Collecting diagnostic information..." | |
| xcrun simctl list | |
| echo "Checking simulator logs..." | |
| ls -la ~/Library/Logs/CoreSimulator/ || echo "No simulator logs found" | |
| - name: Configure git metadata (needed for lerna smoke tests) | |
| if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }} | |
| run: | | |
| git config --global user.email test@test.com | |
| git config --global user.name "Test Test" | |
| - name: Set starting timestamp | |
| if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }} | |
| id: before-e2e | |
| shell: bash | |
| run: | | |
| echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT | |
| - name: Run e2e tests with pnpm (Linux/Windows) | |
| id: e2e-run-pnpm | |
| if: ${{ matrix.os != 'macos-latest' }} | |
| run: pnpm nx run ${{ matrix.project }}:e2e-local | |
| shell: bash | |
| timeout-minutes: ${{ matrix.os_timeout }} | |
| env: | |
| NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }} | |
| NX_DAEMON: 'true' | |
| NX_PERF_LOGGING: 'false' | |
| NX_E2E_VERBOSE_LOGGING: 'true' | |
| NX_NATIVE_LOGGING: 'false' | |
| NX_E2E_RUN_E2E: 'true' | |
| NX_CLOUD_NO_TIMEOUTS: 'true' | |
| NX_E2E_SKIP_CLEANUP: 'true' | |
| NODE_OPTIONS: --max_old_space_size=8192 | |
| SELECTED_PM: ${{ matrix.package_manager }} | |
| npm_config_registry: http://localhost:4872 | |
| YARN_REGISTRY: http://localhost:4872 | |
| CI: true | |
| - name: Run e2e tests with npm (macOS) | |
| id: e2e-run-npm | |
| if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }} | |
| run: | | |
| # Run the tests | |
| if [[ "${{ matrix.project }}" == "e2e-detox" ]] || [[ "${{ matrix.project }}" == "e2e-react-native" ]] || [[ "${{ matrix.project }}" == "e2e-expo" ]]; then | |
| NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-macos-local | |
| else | |
| NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-local | |
| fi | |
| env: | |
| NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }} | |
| NX_PERF_LOGGING: 'false' | |
| NX_CI_EXECUTION_ENV: 'macos' | |
| NX_E2E_VERBOSE_LOGGING: 'true' | |
| NX_NATIVE_LOGGING: 'false' | |
| NX_E2E_RUN_E2E: 'true' | |
| NX_E2E_SKIP_CLEANUP: 'true' | |
| NODE_OPTIONS: --max_old_space_size=8192 | |
| SELECTED_PM: 'npm' | |
| npm_config_registry: http://localhost:4872 | |
| YARN_REGISTRY: http://localhost:4872 | |
| DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer' | |
| CI: true | |
| - name: Save matrix config in file | |
| if: ${{ always() }} | |
| id: save-matrix | |
| shell: bash | |
| run: | | |
| before=${{ steps.before-e2e.outputs.timestamp }} | |
| now=$(date +%s) | |
| delta=$(($now - $before)) | |
| # Determine the outcome based on which step ran | |
| outcome="${{ matrix.os == 'macos-latest' && steps.e2e-run-npm.outcome || steps.e2e-run-pnpm.outcome }}" | |
| matrix=$(( | |
| echo '${{ toJSON(matrix) }}' | |
| ) | jq --argjson delta $delta -c '. + { "status": "'"$outcome"'", "duration": $delta }') | |
| echo "$matrix" > 'outputs/matrix.json' | |
| - name: Upload matrix config | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: ${{ matrix.os_name}}-${{ matrix.node_version}}-${{ matrix.package_manager}}-${{ matrix.project }} | |
| overwrite: true | |
| if-no-files-found: 'ignore' | |
| path: 'outputs/matrix.json' | |
| - name: Setup tmate session | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }} | |
| uses: mxschmitt/action-tmate@v3.8 | |
| timeout-minutes: 15 | |
| with: | |
| sudo: ${{ matrix.os != 'windows-latest' }} # disable sudo for windows debugging | |
| process-result: | |
| if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| runs-on: ubuntu-latest | |
| needs: e2e | |
| timeout-minutes: 10 | |
| outputs: | |
| message: ${{ steps.process-json.outputs.slack_message }} | |
| proj_duration: ${{ steps.process-json.outputs.slack_proj_duration }} | |
| pm_duration: ${{ steps.process-json.outputs.slack_pm_duration }} | |
| codeowners: ${{ steps.process-json.outputs.codeowners }} | |
| has_golden_failures: ${{ steps.process-json.outputs.has_golden_failures }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| filter: tree:0 | |
| - name: Prepare dir for output | |
| run: mkdir -p outputs | |
| - name: Load outputs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: outputs | |
| - name: Join and stringify matrix configs | |
| id: combine-json | |
| run: | | |
| combined=$(jq -sc . outputs/*/matrix.json) | |
| echo "combined=$combined" >> $GITHUB_OUTPUT | |
| - name: Process results with TypeScript script | |
| id: process-json | |
| run: | | |
| echo '${{ steps.combine-json.outputs.combined }}' | npx tsx .github/workflows/nightly/process-result.ts | |
| report-failure: | |
| if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'true' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| name: Report failure | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'failure' | |
| message_format: '${{ needs.process-result.outputs.message }}' | |
| notification_title: 'Golden Test Failure' | |
| footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>' | |
| mention_groups: ${{ needs.process-result.outputs.codeowners }} | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| report-success: | |
| if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'false' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| name: Report status | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'success' | |
| message_format: '${{ needs.process-result.outputs.message }}' | |
| notification_title: '✅ Golden Tests: All Passed!' | |
| footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| report-pm-time: | |
| if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| name: Report duration per package manager | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'skipped' | |
| message_format: '${{ needs.process-result.outputs.pm_duration }}' | |
| notification_title: '⌛ Total duration per package manager (ubuntu only)' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| report-proj-time: | |
| if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| name: Report duration per project | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'skipped' | |
| message_format: '${{ needs.process-result.outputs.proj_duration }}' | |
| notification_title: '⌛ E2E Project duration stats' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} |