diff --git a/docs/src/api/class-browsertype.md b/docs/src/api/class-browsertype.md index 1480d567c5d3e..f970d4aef3d39 100644 --- a/docs/src/api/class-browsertype.md +++ b/docs/src/api/class-browsertype.md @@ -144,16 +144,6 @@ Some common examples: 1. `""` to expose localhost network. 1. `"*.test.internal-domain,*.staging.internal-domain,"` to expose test/staging deployments and localhost. -### option: BrowserType.connect.proxy -* since: v1.52 -- `proxy` <[Object]> - - `server` <[string]> Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy. - - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. - - `username` ?<[string]> Optional username to use if HTTP proxy requires authentication. - - `password` ?<[string]> Optional password to use if HTTP proxy requires authentication. - -Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used by the browser to load web pages. - ## async method: BrowserType.connectOverCDP * since: v1.9 - returns: <[Browser]> @@ -242,16 +232,6 @@ Logger sink for Playwright logging. Optional. Maximum time in milliseconds to wait for the connection to be established. Defaults to `30000` (30 seconds). Pass `0` to disable timeout. -### option: BrowserType.connectOverCDP.proxy -* since: v1.52 -- `proxy` <[Object]> - - `server` <[string]> Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy. - - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. - - `username` ?<[string]> Optional username to use if HTTP proxy requires authentication. - - `password` ?<[string]> Optional password to use if HTTP proxy requires authentication. - -Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used by the browser to load web pages. - ## method: BrowserType.executablePath * since: v1.8 - returns: <[string]> diff --git a/docs/src/api/params.md b/docs/src/api/params.md index aeb8f7fde383d..4c49370044b2e 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -226,8 +226,10 @@ Dangerous option; use with care. Defaults to `false`. ## browser-option-proxy - `proxy` <[Object]> - `server` <[string]> Proxy to be used for all requests. HTTP and SOCKS proxies are supported, for example - `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy. - - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. + `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP + proxy. + - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, + .domain.com"`. - `username` ?<[string]> Optional username to use if HTTP proxy requires authentication. - `password` ?<[string]> Optional password to use if HTTP proxy requires authentication. diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 8ad9c6e34882d..aff26e89421f2 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -21793,34 +21793,6 @@ export interface ConnectOverCDPOptions { */ logger?: Logger; - /** - * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used - * by the browser to load web pages. - */ - proxy?: { - /** - * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example - * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP - * proxy. - */ - server: string; - - /** - * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. - */ - bypass?: string; - - /** - * Optional username to use if HTTP proxy requires authentication. - */ - username?: string; - - /** - * Optional password to use if HTTP proxy requires authentication. - */ - password?: string; - }; - /** * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going * on. Defaults to 0. @@ -21862,34 +21834,6 @@ export interface ConnectOptions { */ logger?: Logger; - /** - * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used - * by the browser to load web pages. - */ - proxy?: { - /** - * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example - * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP - * proxy. - */ - server: string; - - /** - * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. - */ - bypass?: string; - - /** - * Optional username to use if HTTP proxy requires authentication. - */ - username?: string; - - /** - * Optional password to use if HTTP proxy requires authentication. - */ - password?: string; - }; - /** * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going * on. Defaults to 0. diff --git a/packages/playwright-core/src/client/browserType.ts b/packages/playwright-core/src/client/browserType.ts index 47e572233581a..f67c97ced866c 100644 --- a/packages/playwright-core/src/client/browserType.ts +++ b/packages/playwright-core/src/client/browserType.ts @@ -129,7 +129,6 @@ export class BrowserType extends ChannelOwner imple exposeNetwork: params.exposeNetwork ?? params._exposeNetwork, slowMo: params.slowMo, timeout: params.timeout, - proxy: params.proxy, }; if ((params as any).__testHookRedirectPortForwarding) connectParams.socksProxyRedirectPortForTest = (params as any).__testHookRedirectPortForwarding; @@ -189,8 +188,7 @@ export class BrowserType extends ChannelOwner imple endpointURL, headers, slowMo: params.slowMo, - timeout: params.timeout, - proxy: params.proxy, + timeout: params.timeout }); const browser = Browser.from(result.browser); this._didLaunchBrowser(browser, {}, params.logger); diff --git a/packages/playwright-core/src/client/types.ts b/packages/playwright-core/src/client/types.ts index 0a7d93c8a491b..5235fef8b4379 100644 --- a/packages/playwright-core/src/client/types.ts +++ b/packages/playwright-core/src/client/types.ts @@ -103,12 +103,6 @@ export type ConnectOptions = { slowMo?: number, timeout?: number, logger?: Logger, - proxy?: { - server: string, - bypass?: string, - username?: string, - password?: string - }, }; export type LaunchServerOptions = { channel?: channels.BrowserTypeLaunchOptions['channel'], diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 96b31310b631d..0484fd1c11d6b 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -327,12 +327,6 @@ scheme.LocalUtilsConnectParams = tObject({ exposeNetwork: tOptional(tString), slowMo: tOptional(tNumber), timeout: tOptional(tNumber), - proxy: tOptional(tObject({ - server: tString, - bypass: tOptional(tString), - username: tOptional(tString), - password: tOptional(tString), - })), socksProxyRedirectPortForTest: tOptional(tNumber), }); scheme.LocalUtilsConnectResult = tObject({ @@ -656,12 +650,6 @@ scheme.BrowserTypeConnectOverCDPParams = tObject({ headers: tOptional(tArray(tType('NameValue'))), slowMo: tOptional(tNumber), timeout: tOptional(tNumber), - proxy: tOptional(tObject({ - server: tString, - bypass: tOptional(tString), - username: tOptional(tString), - password: tOptional(tString), - })), }); scheme.BrowserTypeConnectOverCDPResult = tObject({ browser: tChannel(['Browser']), diff --git a/packages/playwright-core/src/server/browserType.ts b/packages/playwright-core/src/server/browserType.ts index f0f3c53a583da..c4c44d83dec30 100644 --- a/packages/playwright-core/src/server/browserType.ts +++ b/packages/playwright-core/src/server/browserType.ts @@ -292,7 +292,7 @@ export abstract class BrowserType extends SdkObject { await fs.promises.mkdir(options.tracesDir, { recursive: true }); } - async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number, proxy?: types.ProxySettings, timeout?: number, headers?: types.HeadersArray }): Promise { + async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number }, timeout?: number): Promise { throw new Error('CDP connections are only supported by Chromium'); } diff --git a/packages/playwright-core/src/server/chromium/chromium.ts b/packages/playwright-core/src/server/chromium/chromium.ts index 0f478c9d50e74..35acc621e29ec 100644 --- a/packages/playwright-core/src/server/chromium/chromium.ts +++ b/packages/playwright-core/src/server/chromium/chromium.ts @@ -64,15 +64,15 @@ export class Chromium extends BrowserType { this._devtools = this._createDevTools(); } - override async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number, proxy?: types.ProxySettings, timeout?: number, headers?: types.HeadersArray }) { + override async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number, headers?: types.HeadersArray }, timeout?: number) { const controller = new ProgressController(metadata, this); controller.setLogName('browser'); return controller.run(async progress => { return await this._connectOverCDPInternal(progress, endpointURL, options); - }, TimeoutSettings.timeout(options)); + }, TimeoutSettings.timeout({ timeout })); } - async _connectOverCDPInternal(progress: Progress, endpointURL: string, options: types.LaunchOptions & { headers?: types.HeadersArray, proxy?: types.ProxySettings }, onClose?: () => Promise) { + async _connectOverCDPInternal(progress: Progress, endpointURL: string, options: types.LaunchOptions & { headers?: types.HeadersArray }, onClose?: () => Promise) { let headersMap: { [key: string]: string; } | undefined; if (options.headers) headersMap = headersArrayToObject(options.headers, false); @@ -84,10 +84,10 @@ export class Chromium extends BrowserType { const artifactsDir = await fs.promises.mkdtemp(ARTIFACTS_FOLDER); - const wsEndpoint = await urlToWSEndpoint(progress, endpointURL, headersMap, options.proxy); + const wsEndpoint = await urlToWSEndpoint(progress, endpointURL, headersMap); progress.throwIfAborted(); - const chromeTransport = await WebSocketTransport.connect(progress, wsEndpoint, { headers: headersMap, proxy: options.proxy }); + const chromeTransport = await WebSocketTransport.connect(progress, wsEndpoint, headersMap); const cleanedUp = new ManualPromise(); const doCleanup = async () => { await removeFolders([artifactsDir]); @@ -365,35 +365,18 @@ class ChromiumReadyState extends BrowserReadyState { } } -async function urlToWSEndpoint(progress: Progress, endpointURL: string, headers: { [key: string]: string; }, proxy?: types.ProxySettings) { +async function urlToWSEndpoint(progress: Progress, endpointURL: string, headers: { [key: string]: string; }) { if (endpointURL.startsWith('ws')) return endpointURL; progress.log(` retrieving websocket url from ${endpointURL}`); const httpURL = endpointURL.endsWith('/') ? `${endpointURL}json/version/` : `${endpointURL}/json/version/`; - // Chromium insists on localhost "Host" header for security reasons, and in the case of proxy - // we end up with the remote host instead of localhost. - const extraHeaders = proxy ? { Host: `localhost:9222` } : {}; const json = await fetchData({ url: httpURL, - headers: { ...headers, ...extraHeaders }, - proxy, + headers, }, async (_, resp) => new Error(`Unexpected status ${resp.statusCode} when connecting to ${httpURL}.\n` + `This does not look like a DevTools server, try connecting via ws://.`) ); - const wsUrl = JSON.parse(json).webSocketDebuggerUrl; - if (proxy) { - // webSocketDebuggerUrl will be a localhost URL, accessible from the browser's computer. - // When using a proxy, assume we need to connect through the original endpointURL. - // Example: - // connecting to http://example.com/ - // making request to http://example.com/json/version/ - // webSocketDebuggerUrl ends up as ws://localhost:9222/devtools/page/ - // we construct ws://example.com/devtools/page/ by appending the pathname to the original URL - const url = new URL(endpointURL); - url.pathname += (url.pathname.endsWith('/') ? '' : '/') + new URL(wsUrl).pathname.substring(1); - return url.toString(); - } - return wsUrl; + return JSON.parse(json).webSocketDebuggerUrl; } async function seleniumErrorHandler(params: HTTPRequestParams, response: http.IncomingMessage) { diff --git a/packages/playwright-core/src/server/dispatchers/browserTypeDispatcher.ts b/packages/playwright-core/src/server/dispatchers/browserTypeDispatcher.ts index 40bf3eb9d80d7..bf21e726bcc01 100644 --- a/packages/playwright-core/src/server/dispatchers/browserTypeDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/browserTypeDispatcher.ts @@ -43,7 +43,7 @@ export class BrowserTypeDispatcher extends Dispatcher { - const browser = await this._object.connectOverCDP(metadata, params.endpointURL, params); + const browser = await this._object.connectOverCDP(metadata, params.endpointURL, params, params.timeout); const browserDispatcher = new BrowserDispatcher(this, browser); return { browser: browserDispatcher, diff --git a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts index 2a66d0a788022..a681b349d325f 100644 --- a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts @@ -33,7 +33,6 @@ import type { RootDispatcher } from './dispatcher'; import type * as channels from '@protocol/channels'; import type * as http from 'http'; import type { HTTPRequestParams } from '../utils/network'; -import type { ProxySettings } from '../types'; export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.LocalUtilsChannel, RootDispatcher> implements channels.LocalUtilsChannel { _type_LocalUtils: boolean; @@ -91,9 +90,9 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels. 'x-playwright-proxy': params.exposeNetwork ?? '', ...params.headers, }; - const wsEndpoint = await urlToWSEndpoint(progress, params.wsEndpoint, params.proxy); + const wsEndpoint = await urlToWSEndpoint(progress, params.wsEndpoint); - const transport = await WebSocketTransport.connect(progress, wsEndpoint, { headers: wsHeaders, followRedirects: true, debugLogHeader: 'x-playwright-debug-log', proxy: params.proxy }); + const transport = await WebSocketTransport.connect(progress, wsEndpoint, wsHeaders, true, 'x-playwright-debug-log'); const socksInterceptor = new SocksInterceptor(transport, params.exposeNetwork, params.socksProxyRedirectPortForTest); const pipe = new JsonPipeDispatcher(this); transport.onmessage = json => { @@ -129,7 +128,7 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels. } } -async function urlToWSEndpoint(progress: Progress | undefined, endpointURL: string, proxy?: ProxySettings): Promise { +async function urlToWSEndpoint(progress: Progress | undefined, endpointURL: string): Promise { if (endpointURL.startsWith('ws')) return endpointURL; @@ -143,7 +142,6 @@ async function urlToWSEndpoint(progress: Progress | undefined, endpointURL: stri method: 'GET', timeout: progress?.timeUntilDeadline() ?? 30_000, headers: { 'User-Agent': getUserAgent() }, - proxy, }, async (params: HTTPRequestParams, response: http.IncomingMessage) => { return new Error(`Unexpected status ${response.statusCode} when connecting to ${fetchUrl.toString()}.\n` + `This does not look like a Playwright server, try connecting via ws://.`); diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts index 71f9de0833e0b..9760ea89a1aa5 100644 --- a/packages/playwright-core/src/server/fetch.ts +++ b/packages/playwright-core/src/server/fetch.ts @@ -18,12 +18,14 @@ import http from 'http'; import https from 'https'; import { Transform, pipeline } from 'stream'; import { TLSSocket } from 'tls'; +import url from 'url'; import * as zlib from 'zlib'; import { TimeoutSettings } from './timeoutSettings'; -import { assert, constructURLBasedOnBaseURL, createProxyAgent, eventsHelper, monotonicTime } from '../utils'; +import { assert, constructURLBasedOnBaseURL, eventsHelper, monotonicTime } from '../utils'; import { createGuid } from './utils/crypto'; import { getUserAgent } from './utils/userAgent'; +import { HttpsProxyAgent, SocksProxyAgent } from '../utilsBundle'; import { BrowserContext, verifyClientCertificates } from './browserContext'; import { CookieStore, domainMatches, parseRawCookie } from './cookieStore'; import { MultipartFormData } from './formData'; @@ -181,8 +183,8 @@ export abstract class APIRequestContext extends SdkObject { let agent; // We skip 'per-context' in order to not break existing users. 'per-context' was previously used to // workaround an upstream Chromium bug. Can be removed in the future. - if (proxy?.server !== 'per-context') - agent = createProxyAgent(proxy, requestUrl); + if (proxy && proxy.server !== 'per-context' && !shouldBypassProxy(requestUrl, proxy.bypass)) + agent = createProxyAgent(proxy); let maxRedirects = params.maxRedirects ?? (defaults.maxRedirects ?? 20); maxRedirects = maxRedirects === 0 ? -1 : maxRedirects; @@ -645,6 +647,13 @@ export class GlobalAPIRequestContext extends APIRequestContext { const timeoutSettings = new TimeoutSettings(); if (options.timeout !== undefined) timeoutSettings.setDefaultTimeout(options.timeout); + const proxy = options.proxy; + if (proxy?.server) { + let url = proxy?.server.trim(); + if (!/^\w+:\/\//.test(url)) + url = 'http://' + url; + proxy.server = url; + } if (options.storageState) { this._origins = options.storageState.origins?.map(origin => ({ indexedDB: [], ...origin })); this._cookieStore.addCookies(options.storageState.cookies || []); @@ -659,7 +668,7 @@ export class GlobalAPIRequestContext extends APIRequestContext { maxRedirects: options.maxRedirects, httpCredentials: options.httpCredentials, clientCertificates: options.clientCertificates, - proxy: options.proxy, + proxy, timeoutSettings, }; this._tracing = new Tracing(this, options.tracesDir); @@ -696,6 +705,20 @@ export class GlobalAPIRequestContext extends APIRequestContext { } } +export function createProxyAgent(proxy: types.ProxySettings) { + const proxyOpts = url.parse(proxy.server); + if (proxyOpts.protocol?.startsWith('socks')) { + return new SocksProxyAgent({ + host: proxyOpts.hostname, + port: proxyOpts.port || undefined, + }); + } + if (proxy.username) + proxyOpts.auth = `${proxy.username}:${proxy.password || ''}`; + // TODO: We should use HttpProxyAgent conditional on proxyOpts.protocol instead of always using CONNECT method. + return new HttpsProxyAgent(proxyOpts); +} + function toHeadersArray(rawHeaders: string[]): types.HeadersArray { const result: types.HeadersArray = []; for (let i = 0; i < rawHeaders.length; i += 2) @@ -768,6 +791,19 @@ function removeHeader(headers: { [name: string]: string }, name: string) { delete headers[name]; } +function shouldBypassProxy(url: URL, bypass?: string): boolean { + if (!bypass) + return false; + const domains = bypass.split(',').map(s => { + s = s.trim(); + if (!s.startsWith('.')) + s = '.' + s; + return s; + }); + const domain = '.' + url.hostname; + return domains.some(d => domain.endsWith(d)); +} + function setBasicAuthorizationHeader(headers: { [name: string]: string }, credentials: HTTPCredentials) { const { username, password } = credentials; const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64'); diff --git a/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts b/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts index 2cb588378208b..a68cb88abb779 100644 --- a/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts +++ b/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts @@ -23,7 +23,7 @@ import tls from 'tls'; import { SocksProxy } from './utils/socksProxy'; import { ManualPromise, escapeHTML, generateSelfSignedCertificate, rewriteErrorMessage } from '../utils'; import { verifyClientCertificates } from './browserContext'; -import { createProxyAgent } from './utils/network'; +import { createProxyAgent } from './fetch'; import { debugLogger } from './utils/debugLogger'; import { createSocket, createTLSSocket } from './utils/happyEyeballs'; @@ -242,7 +242,7 @@ export class ClientCertificatesProxy { ignoreHTTPSErrors: boolean | undefined; secureContextMap: Map = new Map(); alpnCache: ALPNCache; - proxyAgentFromOptions: ReturnType; + proxyAgentFromOptions: ReturnType | undefined; constructor( contextOptions: Pick @@ -250,7 +250,7 @@ export class ClientCertificatesProxy { verifyClientCertificates(contextOptions.clientCertificates); this.alpnCache = new ALPNCache(); this.ignoreHTTPSErrors = contextOptions.ignoreHTTPSErrors; - this.proxyAgentFromOptions = createProxyAgent(contextOptions.proxy); + this.proxyAgentFromOptions = contextOptions.proxy ? createProxyAgent(contextOptions.proxy) : undefined; this._initSecureContexts(contextOptions.clientCertificates); this._socksProxy = new SocksProxy(); this._socksProxy.setPattern('*'); diff --git a/packages/playwright-core/src/server/transport.ts b/packages/playwright-core/src/server/transport.ts index a4e092ef6eb5b..ec2caec0c550e 100644 --- a/packages/playwright-core/src/server/transport.ts +++ b/packages/playwright-core/src/server/transport.ts @@ -15,13 +15,13 @@ * limitations under the License. */ -import { createProxyAgent, makeWaitForNextTask } from '../utils'; +import { makeWaitForNextTask } from '../utils'; import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './utils/happyEyeballs'; import { ws } from '../utilsBundle'; import type { WebSocket } from '../utilsBundle'; import type { Progress } from './progress'; -import type { HeadersArray, ProxySettings } from './types'; +import type { HeadersArray } from './types'; import type { ClientRequest, IncomingMessage } from 'http'; export const perMessageDeflate = { @@ -60,13 +60,6 @@ export interface ConnectionTransport { onclose?: (reason?: string) => void, } -type WebSocketTransportOptions = { - headers?: { [key: string]: string; }; - followRedirects?: boolean; - debugLogHeader?: string; - proxy?: ProxySettings; -}; - export class WebSocketTransport implements ConnectionTransport { private _ws: WebSocket; private _progress?: Progress; @@ -77,14 +70,14 @@ export class WebSocketTransport implements ConnectionTransport { readonly wsEndpoint: string; readonly headers: HeadersArray = []; - static async connect(progress: (Progress|undefined), url: string, options: WebSocketTransportOptions = {}): Promise { - return await WebSocketTransport._connect(progress, url, options, false /* hadRedirects */); + static async connect(progress: (Progress|undefined), url: string, headers?: { [key: string]: string; }, followRedirects?: boolean, debugLogHeader?: string): Promise { + return await WebSocketTransport._connect(progress, url, headers || {}, { follow: !!followRedirects, hadRedirects: false }, debugLogHeader); } - static async _connect(progress: (Progress|undefined), url: string, options: WebSocketTransportOptions, hadRedirects: boolean): Promise { + static async _connect(progress: (Progress|undefined), url: string, headers: { [key: string]: string; }, redirect: { follow: boolean, hadRedirects: boolean }, debugLogHeader?: string): Promise { const logUrl = stripQueryParams(url); progress?.log(` ${logUrl}`); - const transport = new WebSocketTransport(progress, url, logUrl, { ...options, followRedirects: !!options.followRedirects && hadRedirects }); + const transport = new WebSocketTransport(progress, url, logUrl, headers, redirect.follow && redirect.hadRedirects, debugLogHeader); let success = false; progress?.cleanupWhenAborted(async () => { if (!success) @@ -101,13 +94,13 @@ export class WebSocketTransport implements ConnectionTransport { transport._ws.close(); }); transport._ws.on('unexpected-response', (request: ClientRequest, response: IncomingMessage) => { - if (options.followRedirects && !hadRedirects && (response.statusCode === 301 || response.statusCode === 302 || response.statusCode === 307 || response.statusCode === 308)) { + if (redirect.follow && !redirect.hadRedirects && (response.statusCode === 301 || response.statusCode === 302 || response.statusCode === 307 || response.statusCode === 308)) { fulfill({ redirect: response }); transport._ws.close(); return; } for (let i = 0; i < response.rawHeaders.length; i += 2) { - if (options.debugLogHeader && response.rawHeaders[i] === options.debugLogHeader) + if (debugLogHeader && response.rawHeaders[i] === debugLogHeader) progress?.log(response.rawHeaders[i + 1]); } const chunks: Buffer[] = []; @@ -124,34 +117,32 @@ export class WebSocketTransport implements ConnectionTransport { if (result.redirect) { // Strip authorization headers from the redirected request. - const newHeaders = Object.fromEntries(Object.entries(options.headers || {}).filter(([name]) => { + const newHeaders = Object.fromEntries(Object.entries(headers || {}).filter(([name]) => { return !name.includes('access-key') && name.toLowerCase() !== 'authorization'; })); - return WebSocketTransport._connect(progress, result.redirect.headers.location!, { ...options, headers: newHeaders }, true /* hadRedirects */); + return WebSocketTransport._connect(progress, result.redirect.headers.location!, newHeaders, { follow: true, hadRedirects: true }, debugLogHeader); } success = true; return transport; } - constructor(progress: Progress|undefined, url: string, logUrl: string, options: WebSocketTransportOptions) { + constructor(progress: Progress|undefined, url: string, logUrl: string, headers?: { [key: string]: string; }, followRedirects?: boolean, debugLogHeader?: string) { this.wsEndpoint = url; this._logUrl = logUrl; - const proxyAgent = createProxyAgent(options.proxy, new URL(url)); - const happyEyeballsAgent = (/^(https|wss):\/\//.test(url)) ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent; this._ws = new ws(url, [], { maxPayload: 256 * 1024 * 1024, // 256Mb, // Prevent internal http client error when passing negative timeout. handshakeTimeout: Math.max(progress?.timeUntilDeadline() ?? 30_000, 1), - headers: options.headers, - followRedirects: options.followRedirects, - agent: proxyAgent || happyEyeballsAgent, + headers, + followRedirects, + agent: (/^(https|wss):\/\//.test(url)) ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent, perMessageDeflate, }); this._ws.on('upgrade', response => { for (let i = 0; i < response.rawHeaders.length; i += 2) { this.headers.push({ name: response.rawHeaders[i], value: response.rawHeaders[i + 1] }); - if (options.debugLogHeader && response.rawHeaders[i] === options.debugLogHeader) + if (debugLogHeader && response.rawHeaders[i] === debugLogHeader) progress?.log(response.rawHeaders[i + 1]); } }); diff --git a/packages/playwright-core/src/server/utils/network.ts b/packages/playwright-core/src/server/utils/network.ts index d954a69bdfaa7..8e0357f958122 100644 --- a/packages/playwright-core/src/server/utils/network.ts +++ b/packages/playwright-core/src/server/utils/network.ts @@ -20,11 +20,10 @@ import http2 from 'http2'; import https from 'https'; import url from 'url'; -import { HttpsProxyAgent, SocksProxyAgent, getProxyForUrl } from '../../utilsBundle'; +import { HttpsProxyAgent, getProxyForUrl } from '../../utilsBundle'; import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './happyEyeballs'; import type net from 'net'; -import type { ProxySettings } from '../types'; export type HTTPRequestParams = { url: string, @@ -33,7 +32,6 @@ export type HTTPRequestParams = { data?: string | Buffer, timeout?: number, rejectUnauthorized?: boolean, - proxy?: ProxySettings, }; export const NET_DEFAULT_TIMEOUT = 30_000; @@ -51,26 +49,22 @@ export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.Inco const timeout = params.timeout ?? NET_DEFAULT_TIMEOUT; - if (params.proxy) { - options.agent = createProxyAgent(params.proxy, new URL(params.url)); - } else { - const proxyURL = getProxyForUrl(params.url); - if (proxyURL) { - const parsedProxyURL = url.parse(proxyURL); - if (params.url.startsWith('http:')) { - options = { - path: parsedUrl.href, - host: parsedProxyURL.hostname, - port: parsedProxyURL.port, - headers: options.headers, - method: options.method - }; - } else { - (parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:'; - - options.agent = new HttpsProxyAgent(parsedProxyURL); - options.rejectUnauthorized = false; - } + const proxyURL = getProxyForUrl(params.url); + if (proxyURL) { + const parsedProxyURL = url.parse(proxyURL); + if (params.url.startsWith('http:')) { + options = { + path: parsedUrl.href, + host: parsedProxyURL.hostname, + port: parsedProxyURL.port, + headers: options.headers, + method: options.method + }; + } else { + (parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:'; + + options.agent = new HttpsProxyAgent(parsedProxyURL); + options.rejectUnauthorized = false; } } @@ -115,47 +109,6 @@ export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequ }); } -function shouldBypassProxy(url: URL, bypass?: string): boolean { - if (!bypass) - return false; - const domains = bypass.split(',').map(s => { - s = s.trim(); - if (!s.startsWith('.')) - s = '.' + s; - return s; - }); - const domain = '.' + url.hostname; - return domains.some(d => domain.endsWith(d)); -} - -export function createProxyAgent(proxy?: ProxySettings, forUrl?: URL) { - if (!proxy) - return; - if (forUrl && proxy.bypass && shouldBypassProxy(forUrl, proxy.bypass)) - return; - - // Browsers allow to specify proxy without a protocol, defaulting to http. - let proxyServer = proxy.server.trim(); - if (!/^\w+:\/\//.test(proxyServer)) - proxyServer = 'http://' + proxyServer; - - const proxyOpts = url.parse(proxyServer); - if (proxyOpts.protocol?.startsWith('socks')) { - return new SocksProxyAgent({ - host: proxyOpts.hostname, - port: proxyOpts.port || undefined, - }); - } - if (proxy.username) - proxyOpts.auth = `${proxy.username}:${proxy.password || ''}`; - if (forUrl && ['ws:', 'wss:'].includes(forUrl.protocol)) { - // Force CONNECT method for WebSockets. - return new HttpsProxyAgent(proxyOpts); - } - // TODO: We should use HttpProxyAgent conditional on proxyOpts.protocol instead of always using CONNECT method. - return new HttpsProxyAgent(proxyOpts); -} - export function createHttpServer(requestListener?: (req: http.IncomingMessage, res: http.ServerResponse) => void): http.Server; export function createHttpServer(options: http.ServerOptions, requestListener?: (req: http.IncomingMessage, res: http.ServerResponse) => void): http.Server; export function createHttpServer(...args: any[]): http.Server { diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 8ad9c6e34882d..aff26e89421f2 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -21793,34 +21793,6 @@ export interface ConnectOverCDPOptions { */ logger?: Logger; - /** - * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used - * by the browser to load web pages. - */ - proxy?: { - /** - * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example - * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP - * proxy. - */ - server: string; - - /** - * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. - */ - bypass?: string; - - /** - * Optional username to use if HTTP proxy requires authentication. - */ - username?: string; - - /** - * Optional password to use if HTTP proxy requires authentication. - */ - password?: string; - }; - /** * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going * on. Defaults to 0. @@ -21862,34 +21834,6 @@ export interface ConnectOptions { */ logger?: Logger; - /** - * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used - * by the browser to load web pages. - */ - proxy?: { - /** - * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example - * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP - * proxy. - */ - server: string; - - /** - * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. - */ - bypass?: string; - - /** - * Optional username to use if HTTP proxy requires authentication. - */ - username?: string; - - /** - * Optional password to use if HTTP proxy requires authentication. - */ - password?: string; - }; - /** * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going * on. Defaults to 0. diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index 5a9c5687b6dd8..2e1c447b88648 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -540,12 +540,6 @@ export type LocalUtilsConnectParams = { exposeNetwork?: string, slowMo?: number, timeout?: number, - proxy?: { - server: string, - bypass?: string, - username?: string, - password?: string, - }, socksProxyRedirectPortForTest?: number, }; export type LocalUtilsConnectOptions = { @@ -553,12 +547,6 @@ export type LocalUtilsConnectOptions = { exposeNetwork?: string, slowMo?: number, timeout?: number, - proxy?: { - server: string, - bypass?: string, - username?: string, - password?: string, - }, socksProxyRedirectPortForTest?: number, }; export type LocalUtilsConnectResult = { @@ -1177,23 +1165,11 @@ export type BrowserTypeConnectOverCDPParams = { headers?: NameValue[], slowMo?: number, timeout?: number, - proxy?: { - server: string, - bypass?: string, - username?: string, - password?: string, - }, }; export type BrowserTypeConnectOverCDPOptions = { headers?: NameValue[], slowMo?: number, timeout?: number, - proxy?: { - server: string, - bypass?: string, - username?: string, - password?: string, - }, }; export type BrowserTypeConnectOverCDPResult = { browser: BrowserChannel, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index a169f1a54cb3b..c146e995da8b9 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -702,13 +702,6 @@ LocalUtils: exposeNetwork: string? slowMo: number? timeout: number? - proxy: - type: object? - properties: - server: string - bypass: string? - username: string? - password: string? socksProxyRedirectPortForTest: number? returns: pipe: JsonPipe @@ -1017,13 +1010,6 @@ BrowserType: items: NameValue slowMo: number? timeout: number? - proxy: - type: object? - properties: - server: string - bypass: string? - username: string? - password: string? returns: browser: Browser defaultContext: BrowserContext? diff --git a/tests/library/browsertype-connect.spec.ts b/tests/library/browsertype-connect.spec.ts index df182c32aeb3d..1298e38b5e6a4 100644 --- a/tests/library/browsertype-connect.spec.ts +++ b/tests/library/browsertype-connect.spec.ts @@ -765,24 +765,6 @@ for (const kind of ['launchServer', 'run-server'] as const) { await browser.close(); }); - test('should connect over http proxy', { - annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/33894' }, - }, async ({ connect, startRemoteServer, proxyServer }) => { - const remoteServer = await startRemoteServer(kind); - const url = new URL(remoteServer.wsEndpoint()); - proxyServer.forwardTo(+url.port, { allowConnectRequests: true }); - const browser = await connect(`http://some.random.host.does.not.exist:1337`, { - proxy: { server: `localhost:${proxyServer.PORT}` }, - }); - const page = await browser.newPage(); - expect(await page.evaluate('11 * 11')).toBe(121); - await browser.close(); - // We should "CONNECT" twice: - // - to convert http url into ws url - // - actually connect to the ws endpoint - expect(proxyServer.connectHosts).toEqual(['some.random.host.does.not.exist:1337', 'some.random.host.does.not.exist:1337']); - }); - test.describe('socks proxy', () => { test.skip(({ mode }) => mode !== 'default'); test.skip(kind === 'launchServer', 'not supported yet'); diff --git a/tests/library/chromium/connect-over-cdp.spec.ts b/tests/library/chromium/connect-over-cdp.spec.ts index 71ceefbe45495..3191403be8cbb 100644 --- a/tests/library/chromium/connect-over-cdp.spec.ts +++ b/tests/library/chromium/connect-over-cdp.spec.ts @@ -446,32 +446,6 @@ test('should be able to connect via localhost', async ({ browserType }, testInfo } }); -test('should be able to connect over http proxy', { - annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/35206' }, -}, async ({ browserType, proxyServer }, testInfo) => { - const port = 9339 + testInfo.workerIndex; - const browserServer = await browserType.launch({ - args: ['--remote-debugging-port=' + port] - }); - proxyServer.forwardTo(port, { allowConnectRequests: true }); - try { - const cdpBrowser = await browserType.connectOverCDP(`http://some.random.host.does.not.exist:1337`, { - proxy: { server: `localhost:${proxyServer.PORT}` }, - }); - const contexts = cdpBrowser.contexts(); - expect(contexts.length).toBe(1); - const page = await contexts[0].newPage(); - expect(await page.evaluate('11 * 11')).toBe(121); - await cdpBrowser.close(); - // We should "CONNECT" twice: - // - to convert http url into ws url - // - actually connect to the ws endpoint - expect(proxyServer.connectHosts).toEqual(['some.random.host.does.not.exist:1337', 'some.random.host.does.not.exist:1337']); - } finally { - await browserServer.close(); - } -}); - test('emulate media should not be affected by second connectOverCDP', async ({ browserType }, testInfo) => { test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/24109' }); test.fixme(); diff --git a/tests/library/proxy.spec.ts b/tests/library/proxy.spec.ts index 71233106b5045..b7df2b66f7a7f 100644 --- a/tests/library/proxy.spec.ts +++ b/tests/library/proxy.spec.ts @@ -323,7 +323,7 @@ it('should use SOCKS proxy for websocket requests', async ({ browserType, server await closeProxyServer(); }); -it('should use http proxy for websocket requests', async ({ isWindows, browserName, browserType, server, proxyServer }) => { +it('should use http proxy for websocket requests', async ({ browserName, browserType, server, proxyServer }) => { proxyServer.forwardTo(server.PORT, { allowConnectRequests: true }); const browser = await browserType.launch({ proxy: { server: `localhost:${proxyServer.PORT}` } @@ -350,7 +350,7 @@ it('should use http proxy for websocket requests', async ({ isWindows, browserNa // WebKit does not use CONNECT for websockets, but other browsers do. if (browserName === 'webkit') - expect(proxyServer.wsUrls).toContain(isWindows ? '/ws' : 'ws://fake-localhost-127-0-0-1.nip.io:1337/ws'); + expect(proxyServer.wsUrls).toContain('ws://fake-localhost-127-0-0-1.nip.io:1337/ws'); else expect(proxyServer.connectHosts).toContain('fake-localhost-127-0-0-1.nip.io:1337');