chore: move 'dev-server' extensibility point to plugin (#32448)

Instead of plumbing it through a custom unspecified config field, make
it a part of plugin interface.

Additionally, use task runner for starting/stopping dev server.

packages/playwright-ct-core/index.js

@@ -16,7 +16,7 @@
 
 const { test: baseTest, expect, devices, defineConfig: originalDefineConfig } = require('playwright/test');
 const { fixtures } = require('./lib/mount');
-const { clearCacheCommand, runDevServerCommand, findRelatedTestFilesCommand } = require('./lib/cliOverrides');
+const { clearCacheCommand, findRelatedTestFilesCommand } = require('./lib/cliOverrides');
 const { createPlugin } = require('./lib/vitePlugin');
 
 const defineConfig = (...configs) => {
@@ -31,7 +31,6 @@ const defineConfig = (...configs) => {
       ],
       cli: {
         'clear-cache': clearCacheCommand,
-        'dev-server': runDevServerCommand,
         'find-related-test-files': findRelatedTestFilesCommand,
       },
     }

packages/playwright-ct-core/src/cliOverrides.ts

@@ -18,10 +18,8 @@
 import { affectedTestFiles, cacheDir } from 'playwright/lib/transform/compilationCache';
 import { buildBundle } from './vitePlugin';
 import { resolveDirs } from './viteUtils';
-import { runDevServer } from './devServer';
 import type { FullConfigInternal } from 'playwright/lib/common/config';
 import { removeFolderAndLogToConsole } from 'playwright/lib/runner/testServer';
-import type { FullConfig } from 'playwright/types/test';
 
 export async function clearCacheCommand(config: FullConfigInternal) {
   const dirs = await resolveDirs(config.configDir, config.config);
@@ -34,7 +32,3 @@ export async function findRelatedTestFilesCommand(files: string[],  config: Full
   await buildBundle(config.config, config.configDir);
   return { testFiles: affectedTestFiles(files) };
 }
-
-export async function runDevServerCommand(config: FullConfig) {
-  return await runDevServer(config);
-}

packages/playwright-ct-core/src/vitePlugin.ts

@@ -31,6 +31,7 @@ import type { ImportInfo } from './tsxTransform';
 import type { ComponentRegistry } from './viteUtils';
 import { createConfig, frameworkConfig, hasJSComponents, populateComponentsFromTests, resolveDirs, resolveEndpoint, transformIndexFile } from './viteUtils';
 import { resolveHook } from 'playwright/lib/transform/transform';
+import { runDevServer } from './devServer';
 
 const log = debug('pw:vite');
 
@@ -73,6 +74,10 @@ export function createPlugin(): TestRunnerPlugin {
     populateDependencies: async () => {
       await buildBundle(config, configDir);
     },
+
+    startDevServer: async () => {
+      return await runDevServer(config);
+    },
   };
 }
 

packages/playwright/src/plugins/index.ts

@@ -21,6 +21,7 @@ export interface TestRunnerPlugin {
   name: string;
   setup?(config: FullConfig, configDir: string, reporter: ReporterV2): Promise<void>;
   populateDependencies?(): Promise<void>;
+  startDevServer?(): Promise<() => Promise<void>>;
   begin?(suite: Suite): Promise<void>;
   end?(): Promise<void>;
   teardown?(): Promise<void>;
@@ -29,6 +30,7 @@ export interface TestRunnerPlugin {
 export type TestRunnerPluginRegistration = {
   factory: TestRunnerPlugin | (() => TestRunnerPlugin | Promise<TestRunnerPlugin>);
   instance?: TestRunnerPlugin;
+  devServerCleanup?: any;
 };
 
 export { webServer } from './webServerPlugin';

packages/playwright/src/program.ts

@@ -98,15 +98,10 @@ function addDevServerCommand(program: Command) {
     const config = await loadConfigFromFileRestartIfNeeded(options.config);
     if (!config)
       return;
-    const implementation = (config.config as any)['@playwright/test']?.['cli']?.['dev-server'];
-    if (implementation) {
-      const runner = new Runner(config);
-      await runner.loadAllTests();
-      await implementation(config.config);
-    } else {
-      console.log(`DevServer is not available in the package you are using. Did you mean to use component testing?`);
-      gracefullyProcessExitDoNotHang(1);
-    }
+    const runner = new Runner(config);
+    const { status } = await runner.runDevServer();
+    const exitCode = status === 'interrupted' ? 130 : (status === 'passed' ? 0 : 1);
+    gracefullyProcessExitDoNotHang(exitCode);
   });
 }
 

packages/playwright/src/runner/reporters.ts

@@ -16,7 +16,7 @@
 
 import path from 'path';
 import type { FullConfig, TestError } from '../../types/testReporter';
-import { formatError } from '../reporters/base';
+import { colors, formatError } from '../reporters/base';
 import DotReporter from '../reporters/dot';
 import EmptyReporter from '../reporters/empty';
 import GitHubReporter from '../reporters/github';
@@ -86,6 +86,14 @@ export async function createReporterForTestServer(file: string, messageSink: (me
   }));
 }
 
+export function createConsoleReporter() {
+  return wrapReporterAsV2({
+    onError(error: TestError) {
+      process.stdout.write(formatError(error, colors.enabled).message + '\n');
+    }
+  });
+}
+
 function reporterOptions(config: FullConfigInternal, mode: 'list' | 'test' | 'merge', isTestServer: boolean) {
   return {
     configDir: config.configDir,

packages/playwright/src/runner/runner.ts

@@ -21,8 +21,8 @@ import { monotonicTime } from 'playwright-core/lib/utils';
 import type { FullResult, TestError } from '../../types/testReporter';
 import { webServerPluginsForConfig } from '../plugins/webServerPlugin';
 import { collectFilesForProject, filterProjects } from './projectUtils';
-import { createReporters } from './reporters';
-import { TestRun, createTaskRunner, createTaskRunnerForList } from './tasks';
+import { createConsoleReporter, createReporters } from './reporters';
+import { TestRun, createTaskRunner, createTaskRunnerForDevServer, createTaskRunnerForList } from './tasks';
 import type { FullConfigInternal } from '../common/config';
 import type { Suite } from '../common/test';
 import { wrapReporterAsV2 } from '../reporters/reporterV2';
@@ -143,6 +143,17 @@ export class Runner {
       return await override(resolvedFiles, this._config);
     return { testFiles: affectedTestFiles(resolvedFiles) };
   }
+
+  async runDevServer() {
+    const reporter = new InternalReporter([createConsoleReporter()]);
+    const taskRunner = createTaskRunnerForDevServer(this._config, reporter, 'in-process', true);
+    const testRun = new TestRun(this._config);
+    reporter.onConfigure(this._config.config);
+    const status = await taskRunner.run(testRun, 0);
+    await reporter.onEnd({ status });
+    await reporter.onExit();
+    return { status };
+  }
 }
 
 export type LastRunInfo = {

packages/playwright/src/runner/tasks.ts

@@ -113,6 +113,22 @@ export function createTaskRunnerForListFiles(config: FullConfigInternal, reporte
   return taskRunner;
 }
 
+export function createTaskRunnerForDevServer(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', setupAndWait: boolean): TaskRunner<TestRun> {
+  const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
+  if (setupAndWait) {
+    for (const plugin of config.plugins)
+      taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
+  }
+  taskRunner.addTask('load tests', createLoadTask(mode, { failOnLoadErrors: true, filterOnly: false }));
+  taskRunner.addTask('start dev server', createStartDevServerTask());
+  if (setupAndWait) {
+    taskRunner.addTask('wait until interrupted', {
+      setup: async () => new Promise(() => {}),
+    });
+  }
+  return taskRunner;
+}
+
 function createReportBeginTask(): Task<TestRun> {
   return {
     setup: async (reporter, { rootSuite }) => {
@@ -349,3 +365,25 @@ function createRunTestsTask(): Task<TestRun> {
     },
   };
 }
+
+function createStartDevServerTask(): Task<TestRun> {
+  return {
+    setup: async (reporter, testRun, errors, softErrors) => {
+      if (testRun.config.plugins.some(plugin => !!plugin.devServerCleanup)) {
+        errors.push({ message: `DevServer is already running` });
+        return;
+      }
+      for (const plugin of testRun.config.plugins)
+        plugin.devServerCleanup = await plugin.instance?.startDevServer?.();
+      if (!testRun.config.plugins.some(plugin => !!plugin.devServerCleanup))
+        errors.push({ message: `DevServer is not available in the package you are using. Did you mean to use component testing?` });
+    },
+
+    teardown: async (reporter, testRun) => {
+      for (const plugin of testRun.config.plugins) {
+        await plugin.devServerCleanup?.();
+        plugin.devServerCleanup = undefined;
+      }
+    },
+  };
+}

packages/playwright/src/runner/testServer.ts

@@ -23,7 +23,7 @@ import type * as reporterTypes from '../../types/testReporter';
 import { collectAffectedTestFiles, dependenciesForTestFile } from '../transform/compilationCache';
 import type { ConfigLocation, FullConfigInternal } from '../common/config';
 import { createReporterForTestServer, createReporters } from './reporters';
-import { TestRun, createTaskRunnerForList, createTaskRunnerForTestServer, createTaskRunnerForWatchSetup, createTaskRunnerForListFiles } from './tasks';
+import { TestRun, createTaskRunnerForList, createTaskRunnerForTestServer, createTaskRunnerForWatchSetup, createTaskRunnerForListFiles, createTaskRunnerForDevServer } from './tasks';
 import { open } from 'playwright-core/lib/utilsBundle';
 import ListReporter from '../reporters/list';
 import { SigIntWatcher } from './sigIntWatcher';
@@ -75,12 +75,12 @@ export class TestServerDispatcher implements TestServerInterface {
   readonly transport: Transport;
   private _queue = Promise.resolve();
   private _globalSetup: { cleanup: () => Promise<any>, report: ReportEntry[] } | undefined;
+  private _devServer: { cleanup: () => Promise<any>, report: ReportEntry[] } | undefined;
   readonly _dispatchEvent: TestServerInterfaceEventEmitters['dispatchEvent'];
   private _plugins: TestRunnerPluginRegistration[] | undefined;
   private _serializer = require.resolve('./uiModeReporter');
   private _watchTestDirs = false;
   private _closeOnDisconnect = false;
-  private _devServerHandle: (() => Promise<void>) | undefined;
 
   constructor(configLocation: ConfigLocation) {
     this._configLocation = configLocation;
@@ -174,41 +174,32 @@ export class TestServerDispatcher implements TestServerInterface {
   }
 
   async startDevServer(params: Parameters<TestServerInterface['startDevServer']>[0]): ReturnType<TestServerInterface['startDevServer']> {
-    if (this._devServerHandle)
-      return { status: 'failed', report: [] };
-    const { config, report, reporter, status } = await this._innerListTests({});
+    await this.stopDevServer({});
+
+    const { reporter, report } = await this._collectingInternalReporter();
+    const config = await this._loadConfigOrReportError(reporter);
     if (!config)
-      return { status, report };
-    const devServerCommand = (config.config as any)['@playwright/test']?.['cli']?.['dev-server'];
-    if (!devServerCommand) {
-      reporter.onError({ message: 'No dev-server command found in the configuration' });
-      return { status: 'failed', report };
-    }
-    try {
-      this._devServerHandle = await devServerCommand(config.config);
-      return { status: 'passed', report };
-    } catch (e) {
-      reporter.onError(serializeError(e));
-      return { status: 'failed', report };
+      return { report, status: 'failed' };
+
+    const taskRunner = createTaskRunnerForDevServer(config, reporter, 'out-of-process', false);
+    const testRun = new TestRun(config);
+    reporter.onConfigure(config.config);
+    const { status, cleanup } = await taskRunner.runDeferCleanup(testRun, 0);
+    await reporter.onEnd({ status });
+    await reporter.onExit();
+    if (status !== 'passed') {
+      await cleanup();
+      return { report, status };
     }
+    this._devServer = { cleanup, report };
+    return { report, status };
   }
 
   async stopDevServer(params: Parameters<TestServerInterface['stopDevServer']>[0]): ReturnType<TestServerInterface['stopDevServer']> {
-    if (!this._devServerHandle)
-      return { status: 'failed', report: [] };
-    try {
-      await this._devServerHandle();
-      this._devServerHandle = undefined;
-      return { status: 'passed', report: [] };
-    } catch (e) {
-      const { reporter, report } = await this._collectingInternalReporter();
-      // Produce dummy config when it has an error.
-      reporter.onConfigure(baseFullConfig);
-      reporter.onError(serializeError(e));
-      await reporter.onEnd({ status: 'failed' });
-      await reporter.onExit();
-      return { status: 'failed', report };
-    }
+    const devServer = this._devServer;
+    const status = await devServer?.cleanup();
+    this._devServer = undefined;
+    return { status, report: devServer?.report || [] };
   }
 
   async clearCache(params: Parameters<TestServerInterface['clearCache']>[0]): ReturnType<TestServerInterface['clearCache']> {