diff --git a/.github/workflows/component-tests.yml b/.github/workflows/component-tests.yml index fcb6ff55641..60c62050a90 100644 --- a/.github/workflows/component-tests.yml +++ b/.github/workflows/component-tests.yml @@ -22,55 +22,185 @@ jobs: NEXT_PUBLIC_TEST_AUTH_PASSWORD: ${{ secrets.NEXT_PUBLIC_TEST_AUTH_PASSWORD }} NEXT_PUBLIC_SKIP_AUTH_EMAIL: ${{ secrets.NEXT_PUBLIC_SKIP_AUTH_EMAIL }} NEXT_PUBLIC_SKIP_AUTH_PASSWORD: ${{ secrets.NEXT_PUBLIC_SKIP_AUTH_PASSWORD }} - + steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: '20' cache: 'npm' - + - name: Install dependencies run: npm ci - + - name: Run component tests - id: component-tests + id: run-component-tests continue-on-error: true - run: npm run test:component -- --reporter spec - + run: | + mkdir -p cypress/results + npm run test:component -- --reporter junit --reporter-options "mochaFile=cypress/results/component-tests-[hash].xml,toConsole=false" + - name: Generate test summary if: always() + env: + COMPONENT_STEP_OUTCOME: ${{ steps.run-component-tests.outcome }} run: | - echo "## ๐Ÿงช Component Test Results" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - if [ "${{ steps.component-tests.outcome }}" == "success" ]; then - echo "โœ… **Status:** All tests passed" >> $GITHUB_STEP_SUMMARY - else - echo "โŒ **Status:** Some tests failed" >> $GITHUB_STEP_SUMMARY - fi - - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Test Artifacts" >> $GITHUB_STEP_SUMMARY - echo "- ๐Ÿ“ธ Screenshots and videos available in artifacts below" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - if [ -d "cypress/screenshots" ] && [ "$(ls -A cypress/screenshots)" ]; then - echo "### ๐Ÿ“ธ Failed Test Screenshots" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - find cypress/screenshots -type f -name "*.png" | head -10 >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - fi - + node <<'NODE' + const fs = require('fs'); + const path = require('path'); + + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + const outcome = process.env.COMPONENT_STEP_OUTCOME || 'unknown'; + const resultsDir = path.resolve('cypress', 'results'); + const screenshotsDir = path.resolve('cypress', 'screenshots'); + + const counts = { + total: 0, + passed: 0, + failed: 0, + skipped: 0, + durationSec: 0, + }; + + const failedTests = []; + + const decodeXml = (value = '') => + value + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))) + .replace(/&#([0-9]+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10))); + + const getAttr = (tag, name) => { + const match = tag.match(new RegExp(`${name}="([^"]*)"`, 'i')); + return match ? match[1] : ''; + }; + + const toNumber = (value) => { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : 0; + }; + + const xmlFiles = fs.existsSync(resultsDir) + ? fs.readdirSync(resultsDir) + .filter((file) => file.endsWith('.xml')) + .map((file) => path.join(resultsDir, file)) + : []; + + for (const xmlFile of xmlFiles) { + const xml = fs.readFileSync(xmlFile, 'utf8'); + + const suiteTags = xml.match(/]*>/g) || []; + for (const suiteTag of suiteTags) { + counts.total += toNumber(getAttr(suiteTag, 'tests')); + counts.failed += toNumber(getAttr(suiteTag, 'failures')); + counts.skipped += toNumber(getAttr(suiteTag, 'skipped')); + counts.durationSec += toNumber(getAttr(suiteTag, 'time')); + } + + const testcaseRegex = /]*>[\s\S]*?<\/testcase>/g; + let testcaseMatch = testcaseRegex.exec(xml); + while (testcaseMatch) { + const testcaseBlock = testcaseMatch[0]; + + if (testcaseBlock.includes(']*>/); + const failureTagMatch = testcaseBlock.match(/]*>/); + const failureTextMatch = testcaseBlock.match(/]*>([\s\S]*?)<\/failure>/); + + const testcaseTag = testcaseTagMatch ? testcaseTagMatch[0] : ''; + const failureTag = failureTagMatch ? failureTagMatch[0] : ''; + + const rawMessage = + (failureTextMatch && failureTextMatch[1]) || + getAttr(failureTag, 'message') || + 'No failure message captured'; + + const message = decodeXml(rawMessage) + .split('\n') + .map((line) => line.trim()) + .find(Boolean) || 'No failure message captured'; + + failedTests.push({ + suite: decodeXml(getAttr(testcaseTag, 'classname')) || 'unknown-suite', + name: decodeXml(getAttr(testcaseTag, 'name')) || 'unknown test', + file: decodeXml(getAttr(testcaseTag, 'file')) || path.relative(process.cwd(), xmlFile), + message, + }); + } + + testcaseMatch = testcaseRegex.exec(xml); + } + } + + counts.passed = Math.max(counts.total - counts.failed - counts.skipped, 0); + const headline = outcome === 'success' ? 'Component Tests Passed' : 'Component Tests Failed'; + + let md = ''; + md += `## ${headline}\n\n`; + md += '| Metric | Count |\n'; + md += '|---|---:|\n'; + md += `| Tests (total) | ${counts.total} |\n`; + md += `| Tests (passed) | ${counts.passed} |\n`; + md += `| Tests (failed) | ${counts.failed} |\n`; + md += `| Tests (skipped) | ${counts.skipped} |\n`; + md += `| Duration (seconds) | ${counts.durationSec.toFixed(2)} |\n\n`; + + if (xmlFiles.length === 0) { + md += 'JUnit report files were not found in `cypress/results`.\n\n'; + } + + if (failedTests.length > 0) { + md += '
\n'; + md += `Failed tests (${failedTests.length})\n\n`; + for (const test of failedTests.slice(0, 30)) { + md += `- **${test.suite} > ${test.name}** \n`; + md += ` \`${test.file}\` \n`; + md += ` ${test.message}\n`; + } + if (failedTests.length > 30) { + md += `\n- ... and ${failedTests.length - 30} more\n`; + } + md += '\n
\n\n'; + } + + if (fs.existsSync(screenshotsDir)) { + const screenshots = fs + .readdirSync(screenshotsDir, { recursive: true }) + .filter((entry) => typeof entry === 'string' && entry.endsWith('.png')) + .slice(0, 15); + + if (screenshots.length > 0) { + md += '
\n'; + md += `Failed test screenshots (${screenshots.length})\n\n`; + for (const shot of screenshots) { + md += `- \`cypress/screenshots/${shot}\`\n`; + } + md += '\n
\n'; + } + } + + fs.appendFileSync(summaryFile, md); + NODE + - name: Upload test results if: always() uses: actions/upload-artifact@v4 with: name: component-test-results-${{ github.run_id }} path: | + cypress/results cypress/screenshots cypress/videos + if-no-files-found: ignore retention-days: 30 + + - name: Mark job as failed when component tests failed + if: steps.run-component-tests.outcome != 'success' + run: exit 1 diff --git a/CI_CT_DEBUG_INSIGHTS.md b/CI_CT_DEBUG_INSIGHTS.md new file mode 100644 index 00000000000..e6d36e52a30 --- /dev/null +++ b/CI_CT_DEBUG_INSIGHTS.md @@ -0,0 +1,47 @@ +# CI Component Tests Debug Insights +_Last updated: 2026-02-17_ + +## Final Verdict +- Root cause: flaky race in Cypress Component Testing with `justInTimeCompile: true` on GitHub Actions. +- Manifestation: random specs with real tests occasionally end with empty Mocha graph and `cypress:server:project received runnables null`, then `Tests: 0`. +- Confirmation: after switching to `component.justInTimeCompile: false`, focused debug reruns stopped reproducing the issue. + +## Verified Evidence +- Failing specs: compile/network were OK, but right before finish there was `received runnables null` and `0 passing`. +- Passing specs: `received runnables { ... }` was present. +- `normalizeAll(...)` path in Cypress returns empty/undefined when suite has no tests at normalization time. +- This points to runner lifecycle/spec registration timing, not to missing files or transport failure. +- Focused run after fix: `zero-tests = 0`, `received runnables null = 0`, `received runnables { = 10`. +- Second rerun after fix: passed (user-confirmed). + +## Permanent Fix (Mandatory) +- File: `cypress.config.ts` +- Setting: `component.justInTimeCompile: false` +- Why mandatory for this project: + - with JIT enabled, CI had intermittent spec-registration race that produced empty runnables (`Tests: 0`), + - with JIT disabled, spec build/registration became deterministic and the flaky zero-tests symptom disappeared. + +## Hypotheses Log +_Statuses: `confirmed` | `rejected`_ + +- `rejected` H-001: invalid/empty spec files. +- `rejected` H-002: global `uncaught:exception` suppression hides failures. +- `rejected` H-003: `Invalid Host/Origin` reconnect path is primary cause. +- `confirmed` H-004: failure is on registration/runner side after compile, before runnable normalization. +- `confirmed` H-005: issue occurs between `before:spec` and runnable creation. +- `confirmed` H-006: key runtime marker is `received runnables null`. +- `confirmed` H-007: upstream state before `normalizeAll` is empty suite. +- `rejected` H-008: dominant cause is top-level import crash in spec. +- `rejected` H-009: dominant cause is post-registration suite wipe/filtering. +- `confirmed` H-010: socket disconnect/error is not primary trigger in observed failures. +- `confirmed` H-011: part of browser-probe instrumentation was non-deterministic in runMode logs. +- `confirmed` H-012: summary parser could hide `received runnables null` lines. +- `confirmed` H-013: ANSI coloring could break naive marker grep counts. +- `rejected` H-014: hidden `results` payload carried a meaningful top-level test error for zero-tests cases. +- `confirmed` H-015: `justInTimeCompile` race hypothesis; disabling JIT removed flaky zero-tests in reruns. + +## Cleanup After Debug +- Removed temporary CI marker-analysis step and focused `CT_DEBUG_SPEC_LIST` mode. +- Removed temporary server/browser probe logs (`ct-runnables-debug`). +- Removed temporary zero-tests meta dumping from Cypress node events. +- Kept only the permanent `justInTimeCompile: false` fix plus explanatory comment. diff --git a/cypress.config.ts b/cypress.config.ts index 78192b31c98..b5b1c1bfb8d 100644 --- a/cypress.config.ts +++ b/cypress.config.ts @@ -3,6 +3,10 @@ import { defineConfig } from "cypress" export default defineConfig({ projectId: '76trp2', component: { + // Required for CI stability. + // With JIT enabled, Cypress CT can intermittently finish spec evaluation with an empty Mocha suite + // in GitHub Actions (`received runnables null` / `Tests: 0`) because of a spec registration race. + justInTimeCompile: false, devServer: { framework: 'next', bundler: 'webpack',