diff --git a/packages/html-reporter/src/index.tsx b/packages/html-reporter/src/index.tsx index 6f2d3b414f04d..1f018966e1be2 100644 --- a/packages/html-reporter/src/index.tsx +++ b/packages/html-reporter/src/index.tsx @@ -23,6 +23,7 @@ import * as ReactDOM from 'react-dom'; import './colors.css'; import type { LoadedReport } from './loadedReport'; import { ReportView } from './reportView'; +import { mergeReports } from './mergeReports'; // @ts-ignore const zipjs = zipImport as typeof zip; @@ -31,8 +32,12 @@ const ReportLoader: React.FC = () => { React.useEffect(() => { if (report) return; + const shardTotal = window.playwrightShardTotal; const zipReport = new ZipReport(); - zipReport.load().then(() => setReport(zipReport)); + const loadPromise = shardTotal ? + zipReport.loadFromShards(shardTotal) : + zipReport.loadFromBase64(window.playwrightReportBase64!); + loadPromise.then(() => setReport(zipReport)); }, [report]); return ; }; @@ -45,11 +50,27 @@ class ZipReport implements LoadedReport { private _entries = new Map(); private _json!: HTMLReport; - async load() { - const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader((window as any).playwrightReportBase64), { useWebWorkers: false }) as zip.ZipReader; + async loadFromBase64(reportBase64: string) { + const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader(reportBase64), { useWebWorkers: false }) as zip.ZipReader; + this._json = await this._readReportAndTestEntries(zipReader); + } + + async loadFromShards(shardTotal: number) { + const readers = []; + const paddedLen = String(shardTotal).length; + for (let i = 0; i < shardTotal; i++) { + const paddedNumber = String(i + 1).padStart(paddedLen, '0'); + const fileName = `report-${paddedNumber}-of-${shardTotal}.zip`; + const zipReader = new zipjs.ZipReader(new zipjs.HttpReader(fileName), { useWebWorkers: false }) as zip.ZipReader; + readers.push(this._readReportAndTestEntries(zipReader)); + } + this._json = mergeReports(await Promise.all(readers)); + } + + private async _readReportAndTestEntries(zipReader: zip.ZipReader): Promise { for (const entry of await zipReader.getEntries()) this._entries.set(entry.filename, entry); - this._json = await this.entry('report.json') as HTMLReport; + return await this.entry('report.json') as HTMLReport; } json(): HTMLReport { @@ -63,3 +84,4 @@ class ZipReport implements LoadedReport { return JSON.parse(await writer.getData()); } } + diff --git a/packages/html-reporter/src/mergeReports.ts b/packages/html-reporter/src/mergeReports.ts new file mode 100644 index 0000000000000..981fde912f9fc --- /dev/null +++ b/packages/html-reporter/src/mergeReports.ts @@ -0,0 +1,49 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { HTMLReport, Stats } from './types'; + +export function mergeReports(reports: HTMLReport[]): HTMLReport { + const [report, ...rest] = reports; + + for (const currentReport of rest) { + currentReport.files.forEach(file => { + const existingGroup = report.files.find(({ fileId }) => fileId === file.fileId); + + if (existingGroup) { + existingGroup.tests.push(...file.tests); + mergeStats(existingGroup.stats, file.stats); + } else { + report.files.push(file); + } + }); + + mergeStats(report.stats, currentReport.stats); + report.metadata.duration += currentReport.metadata.duration; + } + + return report; +} + +function mergeStats(toStats: Stats, fromStats: Stats) { + toStats.total += fromStats.total; + toStats.expected += fromStats.expected; + toStats.unexpected += fromStats.unexpected; + toStats.flaky += fromStats.flaky; + toStats.skipped += fromStats.skipped; + toStats.duration += fromStats.duration; + toStats.ok = toStats.ok && fromStats.ok; +} diff --git a/packages/html-reporter/src/reportView.tsx b/packages/html-reporter/src/reportView.tsx index 08dd0e11e9693..49eba20eead1d 100644 --- a/packages/html-reporter/src/reportView.tsx +++ b/packages/html-reporter/src/reportView.tsx @@ -31,6 +31,7 @@ import './theme.css'; declare global { interface Window { + playwrightShardTotal?: number; playwrightReportBase64?: string; } } diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index af6007e436f60..282c6d7a7b4f3 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -101,6 +101,7 @@ class HtmlReporter implements Reporter { async onEnd() { const duration = monotonicTime() - this._montonicStartTime; + const shard = this.config.shard; const projectSuites = this.suite.suites; const reports = projectSuites.map(suite => { const rawReporter = new RawReporter(); @@ -109,7 +110,7 @@ class HtmlReporter implements Reporter { }); await removeFolders([this._outputFolder]); const builder = new HtmlBuilder(this._outputFolder); - this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports); + this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports, shard); } async onExit() { @@ -204,7 +205,7 @@ class HtmlBuilder { this._dataZipFile = new yazl.ZipFile(); } - async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[]): Promise<{ ok: boolean, singleTestId: string | undefined }> { + async build(metadata: Metadata & { duration: number }, rawReports: JsonReport[], shard: FullConfigInternal['shard']): Promise<{ ok: boolean, singleTestId: string | undefined }> { const data = new Map(); for (const projectJson of rawReports) { @@ -289,17 +290,11 @@ class HtmlBuilder { } } - // Inline report data. const indexFile = path.join(this._reportFolder, 'index.html'); - fs.appendFileSync(indexFile, ''); + if (shard) + await this._writeShardedReport(indexFile, shard); + else + await this._writeInlineReport(indexFile); let singleTestId: string | undefined; if (htmlReport.stats.total === 1) { @@ -310,6 +305,32 @@ class HtmlBuilder { return { ok, singleTestId }; } + private async _writeShardedReport(indexFile: string, shard: { total: number, current: number }) { + // For each shard write same index.html and store report data in a separate report-num-of-total.zip + // so that they can all be copied in one folder. + await fs.promises.appendFile(indexFile, ``); + const paddedNumber = String(shard.current).padStart(String(shard.total).length, '0'); + const reportZip = path.join(this._reportFolder, `report-${paddedNumber}-of-${shard.total}.zip`); + await new Promise(f => { + this._dataZipFile!.end(undefined, () => { + this._dataZipFile!.outputStream.pipe(fs.createWriteStream(reportZip)).on('close', f); + }); + }); + } + + private async _writeInlineReport(indexFile: string) { + // Inline report data. + await fs.promises.appendFile(indexFile, ''); + } + private _addDataFile(fileName: string, data: any) { this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName); } diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 129e7a377d932..923551430bca3 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -22,11 +22,11 @@ import type { HttpServer } from '../../packages/playwright-core/lib/utils'; import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html'; import { spawnAsync } from 'playwright-core/lib/utils'; -const test = baseTest.extend<{ showReport: () => Promise }>({ +const test = baseTest.extend<{ showReport: (reportFolder?: string) => Promise }>({ showReport: async ({ page }, use, testInfo) => { let server: HttpServer | undefined; - await use(async () => { - const reportFolder = testInfo.outputPath('playwright-report'); + await use(async (reportFolder?: string) => { + reportFolder ??= testInfo.outputPath('playwright-report'); server = startHtmlReportServer(reportFolder); const location = await server.start(); await page.goto(location); @@ -976,3 +976,66 @@ test.describe('report location', () => { }); +test('should shard report', async ({ runInlineTest, showReport, page }, testInfo) => { + const totalShards = 3; + + const testFiles = {}; + for (let i = 0; i < totalShards; i++) { + testFiles[`a-${i}.spec.ts`] = ` + const { test } = pwt; + test('passes', async ({}) => { expect(2).toBe(2); }); + test('fails', async ({}) => { expect(1).toBe(2); }); + test('skipped', async ({}) => { test.skip('Does not work') }); + test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); }); + `; + } + + const allReports = testInfo.outputPath(`aggregated-report`); + await fs.promises.mkdir(allReports, { recursive: true }); + + for (let i = 1; i <= totalShards; i++) { + const result = await runInlineTest(testFiles, + { 'reporter': 'dot,html', 'retries': 1, 'shard': `${i}/${totalShards}` }, + { PW_TEST_HTML_REPORT_OPEN: 'never' }, + { usesCustomReporters: true }); + + + expect(result.exitCode).toBe(1); + const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`)); + expect(new Set(files)).toEqual(new Set([ + 'index.html', + `report-${i}-of-${totalShards}.zip` + ])); + await Promise.all(files.map(name => fs.promises.rename(testInfo.outputPath(`playwright-report/${name}`), `${allReports}/${name}`))); + } + + // Show aggregated report + await showReport(allReports); + + await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('' + (4 * totalShards)); + await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('' + totalShards); + await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('' + totalShards); + await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('' + totalShards); + await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('' + totalShards); + + await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(totalShards); + await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(totalShards); + await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(totalShards); + await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(totalShards); +}); + +test('should pad report numbers with zeros', async ({ runInlineTest, showReport, page }, testInfo) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('passes', async ({}) => {}); + `, + }, { reporter: 'dot,html', shard: '3/100' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }); + + expect(result.exitCode).toBe(0); + const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`)); + expect(new Set(files)).toEqual(new Set([ + 'index.html', + `report-003-of-100.zip` + ])); +});