Skip to content

Commit

Permalink
feat: Support proxy option (#614)
Browse files Browse the repository at this point in the history
* feat: Support `proxy` option

* refactor: Do not override user-provided `agent`

* docs: correction

* feat: Support `no_proxy` complete URL matching

* test: Add proxy tests

* docs: corrections
  • Loading branch information
danielbankhead authored Apr 8, 2024
1 parent 8b20c8b commit 2d14b3f
Show file tree
Hide file tree
Showing 4 changed files with 400 additions and 172 deletions.
37 changes: 35 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ over other authentication methods, i.e., application default credentials.

## Request Options

```js
{
```ts
interface GaxiosOptions = {
// The url to which the request should be sent. Required.
url: string,

Expand Down Expand Up @@ -155,6 +155,39 @@ over other authentication methods, i.e., application default credentials.
// See https://github.com/bitinn/node-fetch#request-cancellation-with-abortsignal
signal?: AbortSignal

/**
* A collection of parts to send as a `Content-Type: multipart/related` request.
*/
multipart?: GaxiosMultipartOptions;

/**
* An optional proxy to use for requests.
* Available via `process.env.HTTP_PROXY` and `process.env.HTTPS_PROXY` as well - with a preference for the this config option when multiple are available.
* The `agent` option overrides this.
*
* @see {@link GaxiosOptions.noProxy}
* @see {@link GaxiosOptions.agent}
*/
proxy?: string | URL;
/**
* A list for excluding traffic for proxies.
* Available via `process.env.NO_PROXY` as well as a common-separated list of strings - merged with any local `noProxy` rules.
*
* - When provided a string, it is matched by
* - Wildcard `*.` and `.` matching are available. (e.g. `.example.com` or `*.example.com`)
* - When provided a URL, it is matched by the `.origin` property.
* - For example, requesting `https://example.com` with the following `noProxy`s would result in a no proxy use:
* - new URL('https://example.com')
* - new URL('https://example.com:443')
* - The following would be used with a proxy:
* - new URL('http://example.com:80')
* - new URL('https://example.com:8443')
* - When provided a regular expression it is used to match the stringified URL
*
* @see {@link GaxiosOptions.proxy}
*/
noProxy?: (string | URL | RegExp)[];

/**
* An experimental, customizable error redactor.
*
Expand Down
32 changes: 32 additions & 0 deletions src/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,9 @@ export interface GaxiosOptions {
*/
maxRedirects?: number;
follow?: number;
/**
* A collection of parts to send as a `Content-Type: multipart/related` request.
*/
multipart?: GaxiosMultipartOptions[];
params?: any;

Check warning on line 188 in src/common.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type
paramsSerializer?: (params: {[index: string]: string | number}) => string;
Expand Down Expand Up @@ -213,6 +216,35 @@ export interface GaxiosOptions {
// Configure client to use mTLS:
cert?: string;
key?: string;

/**
* An optional proxy to use for requests.
* Available via `process.env.HTTP_PROXY` and `process.env.HTTPS_PROXY` as well - with a preference for the this config option when multiple are available.
* The {@link GaxiosOptions.agent `agent`} option overrides this.
*
* @see {@link GaxiosOptions.noProxy}
* @see {@link GaxiosOptions.agent}
*/
proxy?: string | URL;
/**
* A list for excluding traffic for proxies.
* Available via `process.env.NO_PROXY` as well as a common-separated list of strings - merged with any local `noProxy` rules.
*
* - When provided a string, it is matched by
* - Wildcard `*.` and `.` matching are available. (e.g. `.example.com` or `*.example.com`)
* - When provided a URL, it is matched by the `.origin` property.
* - For example, requesting `https://example.com` with the following `noProxy`s would result in a no proxy use:
* - new URL('https://example.com')
* - new URL('https://example.com:443')
* - The following would be used with a proxy:
* - new URL('http://example.com:80')
* - new URL('https://example.com:8443')
* - When provided a regular expression it is used to match the stringified URL
*
* @see {@link GaxiosOptions.proxy}
*/
noProxy?: (string | URL | RegExp)[];

/**
* An experimental error redactor.
*
Expand Down
168 changes: 97 additions & 71 deletions src/gaxios.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import {
} from './common';
import {getRetryConfig} from './retry';
import {PassThrough, Stream, pipeline} from 'stream';
import {HttpsProxyAgent as httpsProxyAgent} from 'https-proxy-agent';
import {v4} from 'uuid';

/* eslint-disable @typescript-eslint/no-explicit-any */
Expand Down Expand Up @@ -64,54 +63,11 @@ function getHeader(options: GaxiosOptions, header: string): string | undefined {
return undefined;
}

let HttpsProxyAgent: any;

function loadProxy() {
const proxy =
process?.env?.HTTPS_PROXY ||
process?.env?.https_proxy ||
process?.env?.HTTP_PROXY ||
process?.env?.http_proxy;
if (proxy) {
HttpsProxyAgent = httpsProxyAgent;
}

return proxy;
}

loadProxy();

function skipProxy(url: string | URL) {
const noProxyEnv = process.env.NO_PROXY ?? process.env.no_proxy;
if (!noProxyEnv) {
return false;
}
const noProxyUrls = noProxyEnv.split(',');
const parsedURL = url instanceof URL ? url : new URL(url);
return !!noProxyUrls.find(url => {
if (url.startsWith('*.') || url.startsWith('.')) {
url = url.replace(/^\*\./, '.');
return parsedURL.hostname.endsWith(url);
} else {
return url === parsedURL.origin || url === parsedURL.hostname;
}
});
}

// Figure out if we should be using a proxy. Only if it's required, load
// the https-proxy-agent module as it adds startup cost.
function getProxy(url: string | URL) {
// If there is a match between the no_proxy env variables and the url, then do not proxy
if (skipProxy(url)) {
return undefined;
// If there is not a match between the no_proxy env variables and the url, check to see if there should be a proxy
} else {
return loadProxy();
}
}

export class Gaxios {
protected agentCache = new Map<string, Agent | ((parsedUrl: URL) => Agent)>();
protected agentCache = new Map<
string | URL,
Agent | ((parsedUrl: URL) => Agent)
>();

/**
* Default HTTP options that will be used for every HTTP request.
Expand All @@ -131,15 +87,15 @@ export class Gaxios {
* @param opts Set of HTTP options that will be used for this HTTP request.
*/
async request<T = any>(opts: GaxiosOptions = {}): GaxiosPromise<T> {
opts = this.validateOpts(opts);
opts = await this.#prepareRequest(opts);
return this._request(opts);
}

private async _defaultAdapter<T>(
opts: GaxiosOptions
): Promise<GaxiosResponse<T>> {
const fetchImpl = opts.fetchImplementation || fetch;
const res = (await fetchImpl(opts.url!, opts)) as FetchResponse;
const res = (await fetchImpl(opts.url, opts)) as FetchResponse;
const data = await this.getResponseData(opts, res);
return this.translateResponse<T>(opts, res, data);
}
Expand Down Expand Up @@ -228,11 +184,59 @@ export class Gaxios {
}
}

#urlMayUseProxy(
url: string | URL,
noProxy: GaxiosOptions['noProxy'] = []
): boolean {
const candidate = new URL(url);
const noProxyList = [...noProxy];
const noProxyEnvList =
(process.env.NO_PROXY ?? process.env.no_proxy)?.split(',') || [];

for (const rule of noProxyEnvList) {
noProxyList.push(rule.trim());
}

for (const rule of noProxyList) {
// Match regex
if (rule instanceof RegExp) {
if (rule.test(candidate.toString())) {
return false;
}
}
// Match URL
else if (rule instanceof URL) {
if (rule.origin === candidate.origin) {
return false;
}
}
// Match string regex
else if (rule.startsWith('*.') || rule.startsWith('.')) {
const cleanedRule = rule.replace(/^\*\./, '.');
if (candidate.hostname.endsWith(cleanedRule)) {
return false;
}
}
// Basic string match
else if (
rule === candidate.origin ||
rule === candidate.hostname ||
rule === candidate.href
) {
return false;
}
}

return true;
}

/**
* Validates the options, and merges them with defaults.
* @param opts The original options passed from the client.
* Validates the options, merges them with defaults, and prepare request.
*
* @param options The original options passed from the client.
* @returns Prepared options, ready to make a request
*/
private validateOpts(options: GaxiosOptions): GaxiosOptions {
async #prepareRequest(options: GaxiosOptions): Promise<GaxiosOptions> {
const opts = extend(true, {}, this.defaults, options);
if (!opts.url) {
throw new Error('URL is required.');
Expand Down Expand Up @@ -318,36 +322,39 @@ export class Gaxios {
}
opts.method = opts.method || 'GET';

const proxy = getProxy(opts.url);
if (proxy) {
const proxy =
opts.proxy ||
process?.env?.HTTPS_PROXY ||
process?.env?.https_proxy ||
process?.env?.HTTP_PROXY ||
process?.env?.http_proxy;
const urlMayUseProxy = this.#urlMayUseProxy(opts.url, opts.noProxy);

if (opts.agent) {
// don't do any of the following options - use the user-provided agent.
} else if (proxy && urlMayUseProxy) {
const HttpsProxyAgent = await Gaxios.#getProxyAgent();

if (this.agentCache.has(proxy)) {
opts.agent = this.agentCache.get(proxy);
} else {
// Proxy is being used in conjunction with mTLS.
if (opts.cert && opts.key) {
const parsedURL = new URL(proxy);
opts.agent = new HttpsProxyAgent({
port: parsedURL.port,
host: parsedURL.host,
protocol: parsedURL.protocol,
cert: opts.cert,
key: opts.key,
});
} else {
opts.agent = new HttpsProxyAgent(proxy);
}
this.agentCache.set(proxy, opts.agent!);
opts.agent = new HttpsProxyAgent(proxy, {
cert: opts.cert,
key: opts.key,
});

this.agentCache.set(proxy, opts.agent);
}
} else if (opts.cert && opts.key) {
// Configure client for mTLS:
// Configure client for mTLS
if (this.agentCache.has(opts.key)) {
opts.agent = this.agentCache.get(opts.key);
} else {
opts.agent = new HTTPSAgent({
cert: opts.cert,
key: opts.key,
});
this.agentCache.set(opts.key, opts.agent!);
this.agentCache.set(opts.key, opts.agent);
}
}

Expand Down Expand Up @@ -459,4 +466,23 @@ export class Gaxios {
}
yield finale;
}

/**
* A cache for the lazily-loaded proxy agent.
*
* Should use {@link Gaxios[#getProxyAgent]} to retrieve.
*/
// using `import` to dynamically import the types here
static #proxyAgent?: typeof import('https-proxy-agent').HttpsProxyAgent;

/**
* Imports, caches, and returns a proxy agent - if not already imported
*
* @returns A proxy agent
*/
static async #getProxyAgent() {
this.#proxyAgent ||= (await import('https-proxy-agent')).HttpsProxyAgent;

return this.#proxyAgent;
}
}
Loading

0 comments on commit 2d14b3f

Please sign in to comment.