feat: support proxy in connect/connectOverCDP

Author: dgozman

docs/src/api/class-browsertype.md

@@ -144,6 +144,16 @@ Some common examples:
 1. `"<loopback>"` to expose localhost network.
 1. `"*.test.internal-domain,*.staging.internal-domain,<loopback>"` to expose test/staging deployments and localhost.
 
+### option: BrowserType.connect.proxy
+* since: v1.52
+- `proxy` <[Object]>
+  - `server` <[string]> Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy.
+  - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
+  - `username` ?<[string]> Optional username to use if HTTP proxy requires authentication.
+  - `password` ?<[string]> Optional password to use if HTTP proxy requires authentication.
+
+Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used by the browser to load web pages.
+
 ## async method: BrowserType.connectOverCDP
 * since: v1.9
 - returns: <[Browser]>
@@ -232,6 +242,16 @@ Logger sink for Playwright logging. Optional.
 Maximum time in milliseconds to wait for the connection to be established. Defaults to
 `30000` (30 seconds). Pass `0` to disable timeout.
 
+### option: BrowserType.connectOverCDP.proxy
+* since: v1.52
+- `proxy` <[Object]>
+  - `server` <[string]> Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy.
+  - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
+  - `username` ?<[string]> Optional username to use if HTTP proxy requires authentication.
+  - `password` ?<[string]> Optional password to use if HTTP proxy requires authentication.
+
+Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used by the browser to load web pages.
+
 ## method: BrowserType.executablePath
 * since: v1.8
 - returns: <[string]>

docs/src/api/params.md

@@ -226,10 +226,8 @@ Dangerous option; use with care. Defaults to `false`.
 ## browser-option-proxy
 - `proxy` <[Object]>
   - `server` <[string]> Proxy to be used for all requests. HTTP and SOCKS proxies are supported, for example
-    `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP
-    proxy.
-  - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org,
-    .domain.com"`.
+    `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy.
+  - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
   - `username` ?<[string]> Optional username to use if HTTP proxy requires authentication.
   - `password` ?<[string]> Optional password to use if HTTP proxy requires authentication.
 

packages/playwright-client/types/types.d.ts

@@ -21793,6 +21793,34 @@ export interface ConnectOverCDPOptions {
    */
   logger?: Logger;
 
+  /**
+   * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used
+   * by the browser to load web pages.
+   */
+  proxy?: {
+    /**
+     * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example
+     * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP
+     * proxy.
+     */
+    server: string;
+
+    /**
+     * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
+     */
+    bypass?: string;
+
+    /**
+     * Optional username to use if HTTP proxy requires authentication.
+     */
+    username?: string;
+
+    /**
+     * Optional password to use if HTTP proxy requires authentication.
+     */
+    password?: string;
+  };
+
   /**
    * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going
    * on. Defaults to 0.
@@ -21834,6 +21862,34 @@ export interface ConnectOptions {
    */
   logger?: Logger;
 
+  /**
+   * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used
+   * by the browser to load web pages.
+   */
+  proxy?: {
+    /**
+     * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example
+     * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP
+     * proxy.
+     */
+    server: string;
+
+    /**
+     * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
+     */
+    bypass?: string;
+
+    /**
+     * Optional username to use if HTTP proxy requires authentication.
+     */
+    username?: string;
+
+    /**
+     * Optional password to use if HTTP proxy requires authentication.
+     */
+    password?: string;
+  };
+
   /**
    * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going
    * on. Defaults to 0.

packages/playwright-core/src/client/browserType.ts

@@ -129,6 +129,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel> imple
         exposeNetwork: params.exposeNetwork ?? params._exposeNetwork,
         slowMo: params.slowMo,
         timeout: params.timeout,
+        proxy: params.proxy,
       };
       if ((params as any).__testHookRedirectPortForwarding)
         connectParams.socksProxyRedirectPortForTest = (params as any).__testHookRedirectPortForwarding;
@@ -188,7 +189,8 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel> imple
       endpointURL,
       headers,
       slowMo: params.slowMo,
-      timeout: params.timeout
+      timeout: params.timeout,
+      proxy: params.proxy,
     });
     const browser = Browser.from(result.browser);
     this._didLaunchBrowser(browser, {}, params.logger);

packages/playwright-core/src/client/types.ts

@@ -103,6 +103,12 @@ export type ConnectOptions = {
   slowMo?: number,
   timeout?: number,
   logger?: Logger,
+  proxy?: {
+    server: string,
+    bypass?: string,
+    username?: string,
+    password?: string
+  },
 };
 export type LaunchServerOptions = {
   channel?: channels.BrowserTypeLaunchOptions['channel'],

packages/playwright-core/src/protocol/validator.ts

@@ -327,6 +327,12 @@ scheme.LocalUtilsConnectParams = tObject({
   exposeNetwork: tOptional(tString),
   slowMo: tOptional(tNumber),
   timeout: tOptional(tNumber),
+  proxy: tOptional(tObject({
+    server: tString,
+    bypass: tOptional(tString),
+    username: tOptional(tString),
+    password: tOptional(tString),
+  })),
   socksProxyRedirectPortForTest: tOptional(tNumber),
 });
 scheme.LocalUtilsConnectResult = tObject({
@@ -650,6 +656,12 @@ scheme.BrowserTypeConnectOverCDPParams = tObject({
   headers: tOptional(tArray(tType('NameValue'))),
   slowMo: tOptional(tNumber),
   timeout: tOptional(tNumber),
+  proxy: tOptional(tObject({
+    server: tString,
+    bypass: tOptional(tString),
+    username: tOptional(tString),
+    password: tOptional(tString),
+  })),
 });
 scheme.BrowserTypeConnectOverCDPResult = tObject({
   browser: tChannel(['Browser']),

packages/playwright-core/src/server/browserType.ts

@@ -292,7 +292,7 @@ export abstract class BrowserType extends SdkObject {
       await fs.promises.mkdir(options.tracesDir, { recursive: true });
   }
 
-  async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number }, timeout?: number): Promise<Browser> {
+  async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number, proxy?: types.ProxySettings, timeout?: number, headers?: types.HeadersArray }): Promise<Browser> {
     throw new Error('CDP connections are only supported by Chromium');
   }
 

packages/playwright-core/src/server/chromium/chromium.ts

@@ -64,15 +64,15 @@ export class Chromium extends BrowserType {
       this._devtools = this._createDevTools();
   }
 
-  override async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number, headers?: types.HeadersArray }, timeout?: number) {
+  override async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number, proxy?: types.ProxySettings, timeout?: number, headers?: types.HeadersArray }) {
     const controller = new ProgressController(metadata, this);
     controller.setLogName('browser');
     return controller.run(async progress => {
       return await this._connectOverCDPInternal(progress, endpointURL, options);
-    }, TimeoutSettings.timeout({ timeout }));
+    }, TimeoutSettings.timeout(options));
   }
 
-  async _connectOverCDPInternal(progress: Progress, endpointURL: string, options: types.LaunchOptions & { headers?: types.HeadersArray }, onClose?: () => Promise<void>) {
+  async _connectOverCDPInternal(progress: Progress, endpointURL: string, options: types.LaunchOptions & { headers?: types.HeadersArray, proxy?: types.ProxySettings }, onClose?: () => Promise<void>) {
     let headersMap: { [key: string]: string; } | undefined;
     if (options.headers)
       headersMap = headersArrayToObject(options.headers, false);
@@ -84,10 +84,10 @@ export class Chromium extends BrowserType {
 
     const artifactsDir = await fs.promises.mkdtemp(ARTIFACTS_FOLDER);
 
-    const wsEndpoint = await urlToWSEndpoint(progress, endpointURL, headersMap);
+    const wsEndpoint = await urlToWSEndpoint(progress, endpointURL, headersMap, options.proxy);
     progress.throwIfAborted();
 
-    const chromeTransport = await WebSocketTransport.connect(progress, wsEndpoint, headersMap);
+    const chromeTransport = await WebSocketTransport.connect(progress, wsEndpoint, { headers: headersMap, proxy: options.proxy });
     const cleanedUp = new ManualPromise<void>();
     const doCleanup = async () => {
       await removeFolders([artifactsDir]);
@@ -365,18 +365,30 @@ class ChromiumReadyState extends BrowserReadyState {
   }
 }
 
-async function urlToWSEndpoint(progress: Progress, endpointURL: string, headers: { [key: string]: string; }) {
+async function urlToWSEndpoint(progress: Progress, endpointURL: string, headers: { [key: string]: string; }, proxy?: types.ProxySettings) {
   if (endpointURL.startsWith('ws'))
     return endpointURL;
   progress.log(`<ws preparing> retrieving websocket url from ${endpointURL}`);
   const httpURL = endpointURL.endsWith('/') ? `${endpointURL}json/version/` : `${endpointURL}/json/version/`;
+  // Chromium insists on localhost "Host" header for security reasons, and in the case of proxy
+  // we end up with the remote host instead of localhost.
+  const extraHeaders = proxy ? { Host: `localhost:9222` } : {};
   const json = await fetchData({
     url: httpURL,
-    headers,
+    headers: { ...headers, ...extraHeaders },
+    proxy,
   }, async (_, resp) => new Error(`Unexpected status ${resp.statusCode} when connecting to ${httpURL}.\n` +
     `This does not look like a DevTools server, try connecting via ws://.`)
   );
-  return JSON.parse(json).webSocketDebuggerUrl;
+  const wsUrl = JSON.parse(json).webSocketDebuggerUrl;
+  if (proxy) {
+    // webSocketDebuggerUrl will be a localhost URL, accessible from the browser's computer.
+    // When using a proxy, assume we need to connect through the original endpointURL.
+    const url = new URL(endpointURL);
+    url.pathname += (url.pathname.endsWith('/') ? '' : '/') + new URL(wsUrl).pathname.substring(1);
+    return url.toString();
+  }
+  return wsUrl;
 }
 
 async function seleniumErrorHandler(params: HTTPRequestParams, response: http.IncomingMessage) {

packages/playwright-core/src/server/dispatchers/browserTypeDispatcher.ts

@@ -43,7 +43,7 @@ export class BrowserTypeDispatcher extends Dispatcher<BrowserType, channels.Brow
   }
 
   async connectOverCDP(params: channels.BrowserTypeConnectOverCDPParams, metadata: CallMetadata): Promise<channels.BrowserTypeConnectOverCDPResult> {
-    const browser = await this._object.connectOverCDP(metadata, params.endpointURL, params, params.timeout);
+    const browser = await this._object.connectOverCDP(metadata, params.endpointURL, params);
     const browserDispatcher = new BrowserDispatcher(this, browser);
     return {
       browser: browserDispatcher,

packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts

@@ -33,6 +33,7 @@ import type { RootDispatcher } from './dispatcher';
 import type * as channels from '@protocol/channels';
 import type * as http from 'http';
 import type { HTTPRequestParams } from '../utils/network';
+import type { ProxySettings } from '../types';
 
 export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.LocalUtilsChannel, RootDispatcher> implements channels.LocalUtilsChannel {
   _type_LocalUtils: boolean;
@@ -90,9 +91,9 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.
         'x-playwright-proxy': params.exposeNetwork ?? '',
         ...params.headers,
       };
-      const wsEndpoint = await urlToWSEndpoint(progress, params.wsEndpoint);
+      const wsEndpoint = await urlToWSEndpoint(progress, params.wsEndpoint, params.proxy);
 
-      const transport = await WebSocketTransport.connect(progress, wsEndpoint, wsHeaders, true, 'x-playwright-debug-log');
+      const transport = await WebSocketTransport.connect(progress, wsEndpoint, { headers: wsHeaders, followRedirects: true, debugLogHeader: 'x-playwright-debug-log', proxy: params.proxy });
       const socksInterceptor = new SocksInterceptor(transport, params.exposeNetwork, params.socksProxyRedirectPortForTest);
       const pipe = new JsonPipeDispatcher(this);
       transport.onmessage = json => {
@@ -128,7 +129,7 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.
   }
 }
 
-async function urlToWSEndpoint(progress: Progress | undefined, endpointURL: string): Promise<string> {
+async function urlToWSEndpoint(progress: Progress | undefined, endpointURL: string, proxy?: ProxySettings): Promise<string> {
   if (endpointURL.startsWith('ws'))
     return endpointURL;
 
@@ -142,6 +143,7 @@ async function urlToWSEndpoint(progress: Progress | undefined, endpointURL: stri
     method: 'GET',
     timeout: progress?.timeUntilDeadline() ?? 30_000,
     headers: { 'User-Agent': getUserAgent() },
+    proxy,
   }, async (params: HTTPRequestParams, response: http.IncomingMessage) => {
     return new Error(`Unexpected status ${response.statusCode} when connecting to ${fetchUrl.toString()}.\n` +
         `This does not look like a Playwright server, try connecting via ws://.`);