cherry-pick(#30611): chore: add common env vars for junit and json re… (#30624)

…porters

Author: yury-s

docs/src/test-reporters-js.md

@@ -231,7 +231,7 @@ Blob report supports following configuration options and environment variables:
 |---|---|---|---|
 | `PLAYWRIGHT_BLOB_OUTPUT_DIR` | `outputDir` | Directory to save the output. Existing content is deleted before writing the new report. | `blob-report`
 | `PLAYWRIGHT_BLOB_OUTPUT_NAME` | `fileName` | Report file name. | `report-<project>-<hash>-<shard_number>.zip`
-| `PLAYWRIGHT_BLOB_OUTPUT_FILE` | `outputFile` | Full path for the output. If defined, `outputDir` and `fileName` will be ignored. | `undefined`
+| `PLAYWRIGHT_BLOB_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `outputDir` and `fileName` will be ignored. | `undefined`
 
 ### JSON reporter
 
@@ -267,7 +267,9 @@ JSON report supports following configuration options and environment variables:
 
 | Environment Variable Name | Reporter Config Option| Description | Default
 |---|---|---|---|
-| `PLAYWRIGHT_JUNIT_OUTPUT_NAME` | `outputFile` | Report file path. | JSON report is printed to stdout.
+| `PLAYWRIGHT_JSON_OUTPUT_DIR` | | Directory to save the output file. Ignored if output file is specified. | `cwd` or config directory.
+| `PLAYWRIGHT_JSON_OUTPUT_NAME` | `outputFile` | Base file name for the output, relative to the output dir. | JSON report is printed to the stdout.
+| `PLAYWRIGHT_JSON_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `PLAYWRIGHT_JSON_OUTPUT_DIR` and `PLAYWRIGHT_JSON_OUTPUT_NAME` will be ignored. | JSON report is printed to the stdout.
 
 ### JUnit reporter
 
@@ -303,7 +305,9 @@ JUnit report supports following configuration options and environment variables:
 
 | Environment Variable Name | Reporter Config Option| Description | Default
 |---|---|---|---|
-| `PLAYWRIGHT_JUNIT_OUTPUT_NAME` | `outputFile` | Report file path. | JUnit report is printed to stdout.
+| `PLAYWRIGHT_JUNIT_OUTPUT_DIR` | | Directory to save the output file. Ignored if output file is not specified. | `cwd` or config directory.
+| `PLAYWRIGHT_JUNIT_OUTPUT_NAME` | `outputFile` | Base file name for the output, relative to the output dir. | JUnit report is printed to the stdout.
+| `PLAYWRIGHT_JUNIT_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `PLAYWRIGHT_JUNIT_OUTPUT_DIR` and `PLAYWRIGHT_JUNIT_OUTPUT_NAME` will be ignored. | JUnit report is printed to the stdout.
 |  | `stripANSIControlSequences` | Whether to remove ANSI control sequences from the text before writing it in the report. | By default output text is added as is.
 |  | `includeProjectInTestName` | Whether to include Playwright project name in every test case as a name prefix. | By default not included.
 | `PLAYWRIGHT_JUNIT_SUITE_ID` |  | Value of the `id` attribute on the root `<testsuites/>` report entry. | Empty string.

packages/playwright/src/reporters/base.ts

@@ -19,6 +19,7 @@ import path from 'path';
 import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location } from '../../types/testReporter';
 import { getPackageManagerExecCommand } from 'playwright-core/lib/utils';
 import type { ReporterV2 } from './reporterV2';
+import { resolveReporterOutputPath } from '../util';
 export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
 export const kOutputSymbol = Symbol('output');
 
@@ -547,3 +548,49 @@ function fitToWidth(line: string, width: number, prefix?: string): string {
 function belongsToNodeModules(file: string) {
   return file.includes(`${path.sep}node_modules${path.sep}`);
 }
+
+function resolveFromEnv(name: string): string | undefined {
+  const value = process.env[name];
+  if (value)
+    return path.resolve(process.cwd(), value);
+  return undefined;
+}
+
+// In addition to `outputFile` the function returns `outputDir` which should
+// be cleaned up if present by some reporters contract.
+export function resolveOutputFile(reporterName: string, options: {
+    configDir: string,
+    outputDir?: string,
+    fileName?: string,
+    outputFile?: string,
+    default?: {
+      fileName: string,
+      outputDir: string,
+    }
+  }):  { outputFile: string, outputDir?: string } |undefined {
+  const name = reporterName.toUpperCase();
+  let outputFile;
+  if (options.outputFile)
+    outputFile = path.resolve(options.configDir, options.outputFile);
+  if (!outputFile)
+    outputFile = resolveFromEnv(`PLAYWRIGHT_${name}_OUTPUT_FILE`);
+  // Return early to avoid deleting outputDir.
+  if (outputFile)
+    return { outputFile };
+
+  let outputDir;
+  if (options.outputDir)
+    outputDir = path.resolve(options.configDir, options.outputDir);
+  if (!outputDir)
+    outputDir = resolveFromEnv(`PLAYWRIGHT_${name}_OUTPUT_DIR`);
+  if (!outputDir && options.default)
+    outputDir = resolveReporterOutputPath(options.default.outputDir, options.configDir, undefined);
+
+  if (!outputFile) {
+    const reportName = options.fileName ?? process.env[`PLAYWRIGHT_${name}_OUTPUT_NAME`] ?? options.default?.fileName;
+    if (!reportName)
+      return undefined;
+    outputFile = path.resolve(outputDir ?? process.cwd(), reportName);
+  }
+  return { outputFile, outputDir };
+}

packages/playwright/src/reporters/blob.ts

@@ -24,7 +24,7 @@ import type { FullConfig, FullResult, TestResult } from '../../types/testReporte
 import type { JsonAttachment, JsonEvent } from '../isomorphic/teleReceiver';
 import { TeleReporterEmitter } from './teleEmitter';
 import { yazl } from 'playwright-core/lib/zipBundle';
-import { resolveReporterOutputPath } from '../util';
+import { resolveOutputFile } from './base';
 
 type BlobReporterOptions = {
   configDir: string;
@@ -107,17 +107,15 @@ export class BlobReporter extends TeleReporterEmitter {
   }
 
   private async _prepareOutputFile() {
-    let outputFile = reportOutputFileFromEnv();
-    if (!outputFile && this._options.outputFile)
-      outputFile = path.resolve(this._options.configDir, this._options.outputFile);
-    // Explicit `outputFile` overrides `outputDir` and `fileName` options.
-    if (!outputFile) {
-      const reportName = this._options.fileName || process.env[`PLAYWRIGHT_BLOB_OUTPUT_NAME`] || this._defaultReportName(this._config);
-      const outputDir = resolveReporterOutputPath('blob-report', this._options.configDir, this._options.outputDir ?? reportOutputDirFromEnv());
-      if (!process.env.PWTEST_BLOB_DO_NOT_REMOVE)
-        await removeFolders([outputDir]);
-      outputFile = path.resolve(outputDir, reportName);
-    }
+    const { outputFile, outputDir } = resolveOutputFile('BLOB', {
+      ...this._options,
+      default: {
+        fileName: this._defaultReportName(this._config),
+        outputDir: 'blob-report',
+      }
+    })!;
+    if (!process.env.PWTEST_BLOB_DO_NOT_REMOVE)
+      await removeFolders([outputDir!]);
     await fs.promises.mkdir(path.dirname(outputFile), { recursive: true });
     return outputFile;
   }
@@ -149,15 +147,3 @@ export class BlobReporter extends TeleReporterEmitter {
     });
   }
 }
-
-function reportOutputDirFromEnv(): string | undefined {
-  if (process.env[`PLAYWRIGHT_BLOB_OUTPUT_DIR`])
-    return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_BLOB_OUTPUT_DIR`]);
-  return undefined;
-}
-
-function reportOutputFileFromEnv(): string | undefined {
-  if (process.env[`PLAYWRIGHT_BLOB_OUTPUT_FILE`])
-    return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_BLOB_OUTPUT_FILE`]);
-  return undefined;
-}

packages/playwright/src/reporters/json.ts

@@ -17,24 +17,29 @@
 import fs from 'fs';
 import path from 'path';
 import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep, JSONReportError } from '../../types/testReporter';
-import { formatError, prepareErrorStack } from './base';
-import { MultiMap, assert, toPosixPath } from 'playwright-core/lib/utils';
+import { formatError, prepareErrorStack, resolveOutputFile } from './base';
+import { MultiMap, toPosixPath } from 'playwright-core/lib/utils';
 import { getProjectId } from '../common/config';
 import EmptyReporter from './empty';
 
+type JSONOptions = {
+  outputFile?: string,
+  configDir: string,
+};
+
 class JSONReporter extends EmptyReporter {
   config!: FullConfig;
   suite!: Suite;
   private _errors: TestError[] = [];
-  private _outputFile: string | undefined;
+  private _resolvedOutputFile: string | undefined;
 
-  constructor(options: { outputFile?: string } = {}) {
+  constructor(options: JSONOptions) {
     super();
-    this._outputFile = options.outputFile || reportOutputNameFromEnv();
+    this._resolvedOutputFile = resolveOutputFile('JSON', options)?.outputFile;
   }
 
   override printsToStdio() {
-    return !this._outputFile;
+    return !this._resolvedOutputFile;
   }
 
   override onConfigure(config: FullConfig) {
@@ -50,7 +55,7 @@ class JSONReporter extends EmptyReporter {
   }
 
   override async onEnd(result: FullResult) {
-    await outputReport(this._serializeReport(result), this.config, this._outputFile);
+    await outputReport(this._serializeReport(result), this._resolvedOutputFile);
   }
 
   private _serializeReport(result: FullResult): JSONReport {
@@ -228,13 +233,11 @@ class JSONReporter extends EmptyReporter {
   }
 }
 
-async function outputReport(report: JSONReport, config: FullConfig, outputFile: string | undefined) {
+async function outputReport(report: JSONReport, resolvedOutputFile: string | undefined) {
   const reportString = JSON.stringify(report, undefined, 2);
-  if (outputFile) {
-    assert(config.configFile || path.isAbsolute(outputFile), 'Expected fully resolved path if not using config file.');
-    outputFile = config.configFile ? path.resolve(path.dirname(config.configFile), outputFile) : outputFile;
-    await fs.promises.mkdir(path.dirname(outputFile), { recursive: true });
-    await fs.promises.writeFile(outputFile, reportString);
+  if (resolvedOutputFile) {
+    await fs.promises.mkdir(path.dirname(resolvedOutputFile), { recursive: true });
+    await fs.promises.writeFile(resolvedOutputFile, reportString);
   } else {
     console.log(reportString);
   }
@@ -250,12 +253,6 @@ function removePrivateFields(config: FullConfig): FullConfig {
   return Object.fromEntries(Object.entries(config).filter(([name, value]) => !name.startsWith('_'))) as FullConfig;
 }
 
-function reportOutputNameFromEnv(): string | undefined {
-  if (process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`])
-    return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]);
-  return undefined;
-}
-
 export function serializePatterns(patterns: string | RegExp | (string | RegExp)[]): string[] {
   if (!Array.isArray(patterns))
     patterns = [patterns];

packages/playwright/src/reporters/junit.ts

@@ -17,15 +17,15 @@
 import fs from 'fs';
 import path from 'path';
 import type { FullConfig, FullResult, Suite, TestCase } from '../../types/testReporter';
-import { formatFailure, stripAnsiEscapes } from './base';
+import { formatFailure, resolveOutputFile, stripAnsiEscapes } from './base';
 import EmptyReporter from './empty';
 
 type JUnitOptions = {
   outputFile?: string,
   stripANSIControlSequences?: boolean,
   includeProjectInTestName?: boolean,
 
-  configDir?: string,
+  configDir: string,
 };
 
 class JUnitReporter extends EmptyReporter {
@@ -40,14 +40,12 @@ class JUnitReporter extends EmptyReporter {
   private stripANSIControlSequences = false;
   private includeProjectInTestName = false;
 
-  constructor(options: JUnitOptions = {}) {
+  constructor(options: JUnitOptions) {
     super();
     this.stripANSIControlSequences = options.stripANSIControlSequences || false;
     this.includeProjectInTestName = options.includeProjectInTestName || false;
-    this.configDir = options.configDir || '';
-    const outputFile = options.outputFile || reportOutputNameFromEnv();
-    if (outputFile)
-      this.resolvedOutputFile = path.resolve(this.configDir, outputFile);
+    this.configDir = options.configDir;
+    this.resolvedOutputFile = resolveOutputFile('JUNIT', options)?.outputFile;
   }
 
   override printsToStdio() {
@@ -261,10 +259,4 @@ function escape(text: string, stripANSIControlSequences: boolean, isCharacterDat
   return text;
 }
 
-function reportOutputNameFromEnv(): string | undefined {
-  if (process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`])
-    return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]);
-  return undefined;
-}
-
 export default JUnitReporter;

tests/playwright-test/reporter-blob.spec.ts

@@ -1292,12 +1292,18 @@ test('support PLAYWRIGHT_BLOB_OUTPUT_FILE environment variable', async ({ runInl
       test('math 1 @smoke', async ({}) => {});
     `,
   };
+  const defaultDir = test.info().outputPath('blob-report');
+  fs.mkdirSync(defaultDir, { recursive: true });
+  const file = path.join(defaultDir, 'some.file');
+  fs.writeFileSync(file, 'content');
 
   await runInlineTest(files, { shard: `1/2` }, { PLAYWRIGHT_BLOB_OUTPUT_FILE: 'subdir/report-one.zip' });
   await runInlineTest(files, { shard: `2/2` }, { PLAYWRIGHT_BLOB_OUTPUT_FILE: test.info().outputPath('subdir/report-two.zip') });
   const reportDir = test.info().outputPath('subdir');
   const reportFiles = await fs.promises.readdir(reportDir);
   expect(reportFiles.sort()).toEqual(['report-one.zip', 'report-two.zip']);
+
+  expect(fs.existsSync(file), 'Default directory should not be cleaned up if output file is specified.').toBe(true);
 });
 
 test('keep projects with same name different bot name separate', async ({ runInlineTest, mergeReports, showReport, page }) => {

tests/playwright-test/reporter-json.spec.ts

@@ -288,4 +288,42 @@ test.describe('report location', () => {
     expect(result.passed).toBe(1);
     expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true);
   });
+
+  test('support PLAYWRIGHT_JSON_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => {
+    const result = await runInlineTest({
+      'foo/package.json': `{ "name": "foo" }`,
+      // unused config along "search path"
+      'foo/bar/playwright.config.js': `
+        module.exports = { projects: [ {} ] };
+      `,
+      'foo/bar/baz/tests/a.spec.js': `
+        import { test, expect } from '@playwright/test';
+        const fs = require('fs');
+        test('pass', ({}, testInfo) => {
+        });
+      `
+    }, { 'reporter': 'json' }, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PLAYWRIGHT_JSON_OUTPUT_FILE': '../my-report.json' }, {
+      cwd: 'foo/bar/baz/tests',
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+    expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true);
+  });
+
+  test('support PLAYWRIGHT_JSON_OUTPUT_DIR and PLAYWRIGHT_JSON_OUTPUT_NAME', async ({ runInlineTest }, testInfo) => {
+    const result = await runInlineTest({
+      'playwright.config.js': `
+        module.exports = { projects: [ {} ] };
+      `,
+      'tests/a.spec.js': `
+        import { test, expect } from '@playwright/test';
+        const fs = require('fs');
+        test('pass', ({}, testInfo) => {
+        });
+      `
+    }, { 'reporter': 'json' }, { 'PLAYWRIGHT_JSON_OUTPUT_DIR': 'foo/bar', 'PLAYWRIGHT_JSON_OUTPUT_NAME': 'baz/my-report.json' });
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+    expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true);
+  });
 });

tests/playwright-test/reporter-junit.spec.ts

@@ -504,6 +504,44 @@ for (const useIntermediateMergeReport of [false, true] as const) {
         expect(result.passed).toBe(1);
         expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
       });
+
+      test('support PLAYWRIGHT_JUNIT_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => {
+        const result = await runInlineTest({
+          'foo/package.json': `{ "name": "foo" }`,
+          // unused config along "search path"
+          'foo/bar/playwright.config.js': `
+            module.exports = { projects: [ {} ] };
+          `,
+          'foo/bar/baz/tests/a.spec.js': `
+            import { test, expect } from '@playwright/test';
+            const fs = require('fs');
+            test('pass', ({}, testInfo) => {
+            });
+          `
+        }, { 'reporter': 'junit,line' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_FILE': '../my-report.xml' }, {
+          cwd: 'foo/bar/baz/tests',
+        });
+        expect(result.exitCode).toBe(0);
+        expect(result.passed).toBe(1);
+        expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
+      });
+
+      test('support PLAYWRIGHT_JUNIT_OUTPUT_DIR and PLAYWRIGHT_JUNIT_OUTPUT_NAME', async ({ runInlineTest }, testInfo) => {
+        const result = await runInlineTest({
+          'playwright.config.js': `
+            module.exports = { projects: [ {} ] };
+          `,
+          'tests/a.spec.js': `
+            import { test, expect } from '@playwright/test';
+            const fs = require('fs');
+            test('pass', ({}, testInfo) => {
+            });
+          `
+        }, { 'reporter': 'junit,line' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_DIR': 'foo/bar', 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': 'baz/my-report.xml' });
+        expect(result.exitCode).toBe(0);
+        expect(result.passed).toBe(1);
+        expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
+      });
     });
 
     test('testsuites time is test run wall time', async ({ runInlineTest }) => {