E2E matrix #1764
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E matrix | |
| on: | |
| # schedule: | |
| # - cron: "0 5 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| type: boolean | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: false | |
| env: | |
| CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress | |
| permissions: { } | |
| jobs: | |
| preinstall: | |
| if: ${{ github.repository_owner == 'nrwl' }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| - windows-latest | |
| node_version: | |
| - 20 | |
| - 22 | |
| # - 23 | |
| exclude: | |
| # run just node v20 on macos and windows | |
| - os: macos-latest | |
| node_version: 22 | |
| # - os: macos-latest | |
| # node_version: 23 | |
| - os: windows-latest | |
| node_version: 22 | |
| # - os: windows-latest | |
| # node_version: 23 | |
| name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| filter: tree:0 | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| version: 9.8.0 | |
| run_install: false | |
| - name: Set node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node_version }} | |
| cache: 'pnpm' | |
| - name: Cache node_modules | |
| id: cache-modules | |
| uses: actions/cache@v4 | |
| with: | |
| lookup-only: true | |
| path: '**/node_modules' | |
| key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ hashFiles('pnpm-lock.yaml') }} | |
| - name: Ensure Python setuptools Installed on Macos | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| id: brew-install-python-setuptools | |
| run: brew install python-setuptools | |
| - name: Install packages | |
| if: steps.cache-modules.outputs.cache-hit != 'true' | |
| run: pnpm install --frozen-lockfile | |
| - name: Homebrew cache directory path | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| id: homebrew-cache-dir-path | |
| run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT | |
| - name: Cache Homebrew | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| uses: actions/cache@v4 | |
| with: | |
| lookup-only: true | |
| path: ${{ steps.homebrew-cache-dir-path.outputs.dir }} | |
| key: brew-${{ matrix.node_version }} | |
| restore-keys: | | |
| brew- | |
| - name: Cache Cypress | |
| id: cache-cypress | |
| uses: actions/cache@v4 | |
| with: | |
| lookup-only: true | |
| path: '${{ github.workspace }}/.cypress' | |
| key: ${{ runner.os }}-cypress | |
| - name: Install Cypress | |
| if: steps.cache-cypress.outputs.cache-hit != 'true' | |
| run: npx cypress install | |
| prepare-matrix: | |
| name: Prepare matrix combinations | |
| if: ${{ github.repository_owner == 'nrwl' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.process-json.outputs.MATRIX }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| filter: tree:0 | |
| - name: Process matrix data | |
| id: process-json | |
| run: | |
| echo "MATRIX=$(npx tsx .github/workflows/nightly/process-matrix.ts | jq -c .)" >> $GITHUB_OUTPUT | |
| e2e: | |
| if: ${{ github.repository_owner == 'nrwl' }} | |
| needs: | |
| - preinstall | |
| - prepare-matrix | |
| permissions: | |
| contents: read | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} # Load matrix from previous job | |
| fail-fast: false | |
| name: ${{ matrix.os_name }}/${{ matrix.package_manager }}/${{ matrix.node_version }} ${{ join(matrix.project) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| filter: tree:0 | |
| - name: Prepare dir for output | |
| run: mkdir -p outputs | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| version: 9.8.0 | |
| run_install: false | |
| - name: Use Node.js ${{ matrix.node_version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node_version }} | |
| cache: 'pnpm' | |
| - name: Install Rust | |
| if: ${{ matrix.os != 'windows-latest' }} | |
| run: | | |
| curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y | |
| source "$HOME/.cargo/env" | |
| rustup toolchain install 1.70.0 | |
| - name: Load Cargo Env | |
| if: ${{ matrix.os != 'windows-latest' }} | |
| run: echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV | |
| - name: Install bun | |
| if: ${{ matrix.os != 'windows-latest' }} | |
| uses: oven-sh/setup-bun@v1 | |
| with: | |
| bun-version: latest | |
| - name: Cache node_modules | |
| id: cache-modules | |
| uses: actions/cache@v4 | |
| with: | |
| path: '**/node_modules' | |
| key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ hashFiles('pnpm-lock.yaml') }} | |
| - name: Install packages | |
| run: pnpm install --frozen-lockfile | |
| - name: Cleanup | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| # Workaround to provide additional free space for testing. | |
| # https://github.com/actions/virtual-environments/issues/2840 | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| sudo apt-get install lsof | |
| echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p | |
| - name: Homebrew cache directory path | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| id: homebrew-cache-dir-path | |
| run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT | |
| - name: Cache Homebrew | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.homebrew-cache-dir-path.outputs.dir }} | |
| key: brew-${{ matrix.node_version }} | |
| restore-keys: | | |
| brew- | |
| - name: Cache Cypress | |
| id: cache-cypress | |
| uses: actions/cache@v4 | |
| with: | |
| path: '${{ github.workspace }}/.cypress' | |
| key: ${{ runner.os }}-cypress | |
| - name: Install Cypress | |
| if: steps.cache-cypress.outputs.cache-hit != 'true' | |
| run: npx cypress install | |
| - name: Configure Detox Environment, Install applesimutils | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| run: | | |
| # Check if applesimutils is already installed | |
| if ! brew list applesimutils &>/dev/null; then | |
| HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null | |
| HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null | |
| else | |
| echo "applesimutils is already installed, skipping installation" | |
| fi | |
| timeout-minutes: 10 | |
| - name: Reset iOS Simulators | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| run: | | |
| xcrun simctl shutdown all && xcrun simctl erase all | |
| timeout-minutes: 5 | |
| - name: Configure git metadata (needed for lerna smoke tests) | |
| run: | | |
| git config --global user.email [email protected] | |
| git config --global user.name "Test Test" | |
| - name: Set starting timestamp | |
| id: before-e2e | |
| shell: bash | |
| run: | | |
| echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT | |
| - name: Setup macOS environment for E2E tests | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| run: | | |
| # 1. Set permissive umask for new files | |
| umask 0000 | |
| # 2. Make /tmp completely permissive | |
| sudo chmod 1777 /tmp | |
| timeout-minutes: 3 | |
| - name: Run e2e tests with pnpm (Linux/Windows) | |
| id: e2e-run-pnpm | |
| if: ${{ matrix.os != 'macos-latest' }} | |
| run: pnpm nx run ${{ matrix.project }}:e2e-local | |
| shell: bash | |
| timeout-minutes: ${{ matrix.os_timeout }} | |
| env: | |
| NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }} | |
| NX_DAEMON: 'true' | |
| NX_PERF_LOGGING: 'false' | |
| NX_E2E_VERBOSE_LOGGING: 'true' | |
| NX_NATIVE_LOGGING: 'false' | |
| NX_E2E_RUN_E2E: 'true' | |
| NX_CLOUD_NO_TIMEOUTS: 'true' | |
| NX_E2E_SKIP_CLEANUP: 'true' | |
| NODE_OPTIONS: --max_old_space_size=8192 | |
| SELECTED_PM: ${{ matrix.package_manager }} | |
| npm_config_registry: http://localhost:4872 | |
| YARN_REGISTRY: http://localhost:4872 | |
| CI: true | |
| - name: Run e2e tests with npm (macOS) | |
| id: e2e-run-npm | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| run: | | |
| # Create a directory to store error logs | |
| mkdir -p /tmp/nx-error-logs | |
| # Set environment variable to tell the test framework to save error logs | |
| export NX_E2E_SAVE_ERROR_LOGS="/tmp/nx-error-logs" | |
| # Add a custom error.log monitor script | |
| cat > /tmp/watch-logs.sh << 'EOF' | |
| #!/bin/bash | |
| while true; do | |
| for ERROR_LOG in /var/folders/**/tmp-*/error.log; do | |
| if [ -f "$ERROR_LOG" ]; then | |
| echo "Found error log at $ERROR_LOG, copying..." | |
| cp "$ERROR_LOG" "/tmp/nx-error-logs/$(basename $(dirname $ERROR_LOG))-error.log" | |
| fi | |
| done | |
| sleep 2 | |
| done | |
| EOF | |
| chmod +x /tmp/watch-logs.sh | |
| # Start watching for error logs in background | |
| /tmp/watch-logs.sh & | |
| WATCHER_PID=$! | |
| # Run the tests | |
| if [[ "${{ matrix.project }}" == "e2e-detox" ]] || [[ "${{ matrix.project }}" == "e2e-react-native" ]] || [[ "${{ matrix.project }}" == "e2e-expo" ]]; then | |
| NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-macos-local || TEST_FAILED=1 | |
| else | |
| NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-local || TEST_FAILED=1 | |
| fi | |
| # Stop the watcher | |
| kill $WATCHER_PID | |
| # Display any captured error logs | |
| echo "=========== CAPTURED ERROR LOGS ===========" | |
| for LOG in /tmp/nx-error-logs/*.log; do | |
| if [ -f "$LOG" ]; then | |
| echo "Contents of $LOG:" | |
| cat "$LOG" | |
| echo "=======================================" | |
| fi | |
| done | |
| # Re-throw the error if the test failed | |
| if [ "$TEST_FAILED" = "1" ]; then | |
| exit 1 | |
| fi | |
| env: | |
| NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }} | |
| NX_PERF_LOGGING: 'false' | |
| NX_CI_EXECUTION_ENV: 'macos' | |
| NX_E2E_VERBOSE_LOGGING: 'true' | |
| NX_NATIVE_LOGGING: 'false' | |
| NX_E2E_RUN_E2E: 'true' | |
| NODE_OPTIONS: --max_old_space_size=8192 | |
| SELECTED_PM: 'npm' | |
| npm_config_registry: http://localhost:4872 | |
| YARN_REGISTRY: http://localhost:4872 | |
| DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer' | |
| CI: true | |
| - name: Save matrix config in file | |
| if: ${{ always() }} | |
| id: save-matrix | |
| shell: bash | |
| run: | | |
| before=${{ steps.before-e2e.outputs.timestamp }} | |
| now=$(date +%s) | |
| delta=$(($now - $before)) | |
| # Determine the outcome based on which step ran | |
| outcome="${{ matrix.os == 'macos-latest' && steps.e2e-run-npm.outcome || steps.e2e-run-pnpm.outcome }}" | |
| matrix=$(( | |
| echo '${{ toJSON(matrix) }}' | |
| ) | jq --argjson delta $delta -c '. + { "status": "'"$outcome"'", "duration": $delta }') | |
| echo "$matrix" > 'outputs/matrix.json' | |
| - name: Upload matrix config | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: ${{ matrix.os_name}}-${{ matrix.node_version}}-${{ matrix.package_manager}}-${{ matrix.project }} | |
| overwrite: true | |
| if-no-files-found: 'ignore' | |
| path: 'outputs/matrix.json' | |
| - name: Setup tmate session | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }} | |
| uses: mxschmitt/[email protected] | |
| timeout-minutes: 15 | |
| with: | |
| sudo: ${{ matrix.os != 'windows-latest' }} # disable sudo for windows debugging | |
| process-result: | |
| if: ${{ always() && github.repository_owner == 'nrwl' }} | |
| runs-on: ubuntu-latest | |
| needs: e2e | |
| outputs: | |
| message: ${{ steps.process-json.outputs.SLACK_MESSAGE }} | |
| proj-duration: ${{ steps.process-json.outputs.SLACK_PROJ_DURATION }} | |
| pm-duration: ${{ steps.process-json.outputs.SLACK_PM_DURATION }} | |
| codeowners: ${{ steps.process-json.outputs.CODEOWNERS }} | |
| steps: | |
| - name: Prepare dir for output | |
| run: mkdir -p outputs | |
| - name: Load outputs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: outputs | |
| - name: Join and stringify matrix configs | |
| id: combine-json | |
| run: | | |
| combined=$((jq -s . outputs/*/matrix.json) | jq tostring) | |
| echo "combined=$combined" >> $GITHUB_OUTPUT | |
| - name: Make slack outputs | |
| id: process-json | |
| uses: actions/github-script@v7 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| script: | | |
| const combined = JSON.parse(${{ steps.combine-json.outputs.combined }}); | |
| const failedProjects = combined.filter(c => c.status === 'failure').sort((a, b) => a.project.localeCompare(b.project)); | |
| // codeowners | |
| const codeowners = new Set(); | |
| failedProjects.forEach(c => { | |
| codeowners.add(c.codeowners); | |
| }); | |
| core.setOutput('CODEOWNERS', Array.from(codeowners).join(',')); | |
| function trimSpace(res) { | |
| return res.split('\n').map((l) => l.trim()).join('\n'); | |
| } | |
| // failed message | |
| let lastProject; | |
| let result = ` | |
| \`\`\` | |
| | Failed project | PM | OS | Node | | |
| |--------------------------------|------|-------|------|`; | |
| failedProjects.forEach(matrix => { | |
| const project = matrix.project !== lastProject ? matrix.project : '...'; | |
| result += `\n| ${project.padEnd(30)} | ${matrix.package_manager.padEnd(4)} | ${matrix.os_name} | v${matrix.node_version} |` | |
| lastProject = matrix.project; | |
| }); | |
| result += `\`\`\``; | |
| core.setOutput('SLACK_MESSAGE', trimSpace(result)); | |
| console.log(trimSpace(result)); | |
| function humanizeDuration(num) { | |
| let res = ''; | |
| const hours = Math.floor(num / 3600); | |
| if (hours) { | |
| res += `${hours}h `; | |
| } | |
| const mins = Math.floor((num % 3600) / 60); | |
| if (mins) { | |
| res += `${mins}m `; | |
| } | |
| const sec = num % 60; | |
| if (sec) { | |
| res += `${sec}s` | |
| } | |
| return res; | |
| } | |
| // duration message | |
| const timeReport = {}; | |
| const pmReport = { | |
| npm: 0, | |
| yarn: 0, | |
| pnpm: 0 | |
| }; | |
| const macosProjects = ['e2e-detox', 'e2e-expo', 'e2e-react-native']; | |
| combined.forEach((matrix) => { | |
| if (matrix.os_name === 'Linux' && matrix.node_version === 20) { | |
| pmReport[matrix.package_manager] += matrix.duration; | |
| } | |
| if (matrix.os_name === 'Linux' || macosProjects.includes(matrix.project)) { | |
| if (timeReport[matrix.project]) { | |
| if (matrix.duration > timeReport[matrix.project].max) { | |
| timeReport[matrix.project].max = matrix.duration; | |
| timeReport[ | |
| matrix.project | |
| ].maxEnv = `${matrix.os_name}, ${matrix.package_manager}`; | |
| } | |
| if (matrix.duration < timeReport[matrix.project].min) { | |
| timeReport[matrix.project].min = matrix.duration; | |
| timeReport[ | |
| matrix.project | |
| ].minEnv = `${matrix.os_name}, ${matrix.package_manager}`; | |
| } | |
| } else { | |
| timeReport[matrix.project] = { | |
| min: matrix.duration, | |
| max: matrix.duration, | |
| minEnv: `${matrix.os_name}, ${matrix.package_manager}`, | |
| maxEnv: `${matrix.os_name}, ${matrix.package_manager}`, | |
| }; | |
| } | |
| } | |
| }); | |
| // project time report | |
| let resultPkg = ` | |
| \`\`\` | |
| | Project | Time | | |
| |--------------------------------|---------------------------|`; | |
| function mapProjectTime(proj, section) { | |
| let res = ''; | |
| res += `${humanizeDuration(timeReport[proj][section])}`; | |
| res += ` (${timeReport[proj][section + 'Env']})` | |
| return res; | |
| } | |
| function durationIcon(proj, section) { | |
| if (timeReport[proj][section] < 12 * 60) { | |
| return `${section} ✅`; | |
| } | |
| if (timeReport[proj][section] < 15 * 60) { | |
| return `${section} ❗`; | |
| } | |
| return `${section} ❌`; | |
| } | |
| Object.keys(timeReport).forEach(proj => { | |
| resultPkg += `\n| ${proj.padEnd(30)} | |`; | |
| resultPkg += `\n| ${durationIcon(proj, 'min').padStart(29)} | ${mapProjectTime(proj, 'min').padEnd(25)} |`; | |
| resultPkg += `\n| ${durationIcon(proj, 'max').padStart(29)} | ${mapProjectTime(proj, 'max').padEnd(25)} |`; | |
| }); | |
| resultPkg += `\`\`\``; | |
| core.setOutput('SLACK_PROJ_DURATION', trimSpace(resultPkg)); | |
| // Print project duration report inline to allow reviewing on manual runs (when no slack message will be sent) | |
| console.log(trimSpace(resultPkg)); | |
| let resultPm = ` | |
| \`\`\` | |
| | PM | Total time | | |
| |------|-------------|`; | |
| Object.keys(pmReport).forEach(pm => { | |
| resultPm += `\n| ${pm.padEnd(4)} | ${humanizeDuration(pmReport[pm]).padEnd(11)} |` | |
| }); | |
| resultPm += `\`\`\``; | |
| core.setOutput('SLACK_PM_DURATION', trimSpace(resultPm)); | |
| // Print package manager duration report inline to allow reviewing on manual runs (when no slack message will be sent) | |
| console.log(trimSpace(resultPm)); | |
| report-failure: | |
| if: ${{ failure() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| name: Report failure | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'failure' | |
| message_format: '{emoji} Workflow has {status_message} ${{ needs.process-result.outputs.message }}' | |
| notification_title: '{workflow}' | |
| footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>' | |
| mention_groups: ${{ needs.process-result.outputs.codeowners }} | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| report-success: | |
| if: ${{ success() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: e2e | |
| runs-on: ubuntu-latest | |
| name: Report status | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: ${{ needs.e2e.result }} | |
| message_format: '{emoji} Workflow has {status_message}' | |
| notification_title: '{workflow}' | |
| footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| report-pm-time: | |
| if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| name: Report duration per package manager | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'skipped' | |
| message_format: '${{ needs.process-result.outputs.pm-duration }}' | |
| notification_title: 'Total duration per package manager (ubuntu only)' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| report-proj-time: | |
| if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} | |
| needs: process-result | |
| runs-on: ubuntu-latest | |
| name: Report duration per package manager | |
| steps: | |
| - name: Send notification | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: 'skipped' | |
| message_format: '${{ needs.process-result.outputs.proj-duration }}' | |
| notification_title: 'E2E Project duration stats' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} |