feat(http): Force default rate limits for some known hosts (#30207)

Co-authored-by: HonkingGoose <[email protected]>
Co-authored-by: Michael Kriese <[email protected]>

lib/modules/datasource/rubygems/index.ts

@@ -2,14 +2,13 @@ import { Marshal } from '@qnighy/marshal';
 import type { ZodError } from 'zod';
 import { logger } from '../../../logger';
 import { cache } from '../../../util/cache/package/decorator';
-import { HttpError } from '../../../util/http';
+import { Http, HttpError } from '../../../util/http';
 import { AsyncResult, Result } from '../../../util/result';
 import { getQueryString, joinUrlParts, parseUrl } from '../../../util/url';
 import * as rubyVersioning from '../../versioning/ruby';
 import { Datasource } from '../datasource';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 import { getV1Releases } from './common';
-import { RubygemsHttp } from './http';
 import { MetadataCache } from './metadata-cache';
 import { GemInfo, MarshalledVersionInfo } from './schema';
 import { VersionsEndpointCache } from './versions-endpoint-cache';
@@ -34,7 +33,7 @@ export class RubyGemsDatasource extends Datasource {
 
   constructor() {
     super(RubyGemsDatasource.id);
-    this.http = new RubygemsHttp(RubyGemsDatasource.id);
+    this.http = new Http(RubyGemsDatasource.id);
     this.versionsEndpointCache = new VersionsEndpointCache(this.http);
     this.metadataCache = new MetadataCache(this.http);
   }

lib/util/host-rules.ts

@@ -77,7 +77,7 @@ export interface HostRuleSearch {
   readOnly?: boolean;
 }
 
-function matchesHost(url: string, matchHost: string): boolean {
+export function matchesHost(url: string, matchHost: string): boolean {
   if (isHttpUrl(url) && isHttpUrl(matchHost)) {
     return url.startsWith(matchHost);
   }

lib/util/http/host-rules.ts

@@ -217,17 +217,3 @@ export function applyHostRule<GotOptions extends HostRulesGotOptions>(
 
   return options;
 }
-
-export function getConcurrentRequestsLimit(url: string): number | null {
-  const { concurrentRequestLimit } = hostRules.find({ url });
-  return is.number(concurrentRequestLimit) && concurrentRequestLimit > 0
-    ? concurrentRequestLimit
-    : null;
-}
-
-export function getThrottleIntervalMs(url: string): number | null {
-  const { maxRequestsPerSecond } = hostRules.find({ url });
-  return is.number(maxRequestsPerSecond) && maxRequestsPerSecond > 0
-    ? Math.ceil(1000 / maxRequestsPerSecond)
-    : null;
-}

lib/util/http/index.ts

@@ -18,7 +18,7 @@ import { hooks } from './hooks';
 import { applyHostRule, findMatchingRule } from './host-rules';
 import { getQueue } from './queue';
 import { getRetryAfter, wrapWithRetry } from './retry-after';
-import { Throttle, getThrottle } from './throttle';
+import { getThrottle } from './throttle';
 import type {
   GotJSONOptions,
   GotOptions,
@@ -134,10 +134,6 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
     );
   }
 
-  protected getThrottle(url: string): Throttle | null {
-    return getThrottle(url);
-  }
-
   protected async request<T>(
     requestUrl: string | URL,
     httpOptions: InternalHttpOptions,
@@ -212,7 +208,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
         return gotTask(url, options, { queueMs });
       };
 
-      const throttle = this.getThrottle(url);
+      const throttle = getThrottle(url);
       const throttledTask: GotTask<T> = throttle
         ? () => throttle.add<HttpResponse<T>>(httpTask)
         : httpTask;

lib/util/http/queue.ts

@@ -1,7 +1,7 @@
 import PQueue from 'p-queue';
 import { logger } from '../../logger';
 import { parseUrl } from '../url';
-import { getConcurrentRequestsLimit } from './host-rules';
+import { getConcurrentRequestsLimit } from './rate-limits';
 
 const hostQueues = new Map<string, PQueue | null>();
 

lib/util/http/rate-limit.spec.ts

@@ -0,0 +1,83 @@
+import * as hostRules from '../host-rules';
+import {
+  getConcurrentRequestsLimit,
+  getThrottleIntervalMs,
+  setHttpRateLimits,
+} from './rate-limits';
+
+describe('util/http/rate-limit', () => {
+  beforeEach(() => {
+    hostRules.clear();
+    setHttpRateLimits([]);
+  });
+
+  describe('getConcurrentRequestsLimit', () => {
+    it('returns null if no limits are set', () => {
+      expect(getConcurrentRequestsLimit('https://example.com')).toBeNull();
+    });
+
+    it('returns null if host does not match', () => {
+      setHttpRateLimits([
+        { matchHost: 'https://crates.io/api/', throttleMs: 1000 },
+      ]);
+      expect(getConcurrentRequestsLimit('https://index.crates.io')).toBeNull();
+    });
+
+    it('gets the limit from the host rules', () => {
+      hostRules.add({ matchHost: 'example.com', concurrentRequestLimit: 123 });
+      expect(getConcurrentRequestsLimit('https://example.com')).toBe(123);
+    });
+
+    it('selects default value if host rule is greater', () => {
+      setHttpRateLimits([{ matchHost: 'example.com', concurrency: 123 }]);
+      hostRules.add({ matchHost: 'example.com', concurrentRequestLimit: 456 });
+      expect(getConcurrentRequestsLimit('https://example.com')).toBe(123);
+    });
+
+    it('selects host rule value if default is greater', () => {
+      setHttpRateLimits([{ matchHost: 'example.com', concurrency: 456 }]);
+      hostRules.add({ matchHost: 'example.com', concurrentRequestLimit: 123 });
+      expect(getConcurrentRequestsLimit('https://example.com')).toBe(123);
+    });
+
+    it('matches wildcard host', () => {
+      setHttpRateLimits([{ matchHost: '*', concurrency: 123 }]);
+      expect(getConcurrentRequestsLimit('https://example.com')).toBe(123);
+    });
+  });
+
+  describe('getThrottleIntervalMs', () => {
+    it('returns null if no limits are set', () => {
+      expect(getThrottleIntervalMs('https://example.com')).toBeNull();
+    });
+
+    it('returns null if host does not match', () => {
+      setHttpRateLimits([
+        { matchHost: 'https://crates.io/api/', concurrency: 123 },
+      ]);
+      expect(getThrottleIntervalMs('https://index.crates.io')).toBeNull();
+    });
+
+    it('gets the limit from the host rules', () => {
+      hostRules.add({ matchHost: 'example.com', maxRequestsPerSecond: 8 });
+      expect(getThrottleIntervalMs('https://example.com')).toBe(125);
+    });
+
+    it('selects maximum throttle when default is greater', () => {
+      setHttpRateLimits([{ matchHost: 'example.com', throttleMs: 500 }]);
+      hostRules.add({ matchHost: 'example.com', maxRequestsPerSecond: 8 });
+      expect(getThrottleIntervalMs('https://example.com')).toBe(500);
+    });
+
+    it('selects maximum throttle when host rule is greater', () => {
+      setHttpRateLimits([{ matchHost: 'example.com', throttleMs: 125 }]);
+      hostRules.add({ matchHost: 'example.com', maxRequestsPerSecond: 2 });
+      expect(getThrottleIntervalMs('https://example.com')).toBe(500);
+    });
+
+    it('matches wildcard host', () => {
+      setHttpRateLimits([{ matchHost: '*', throttleMs: 123 }]);
+      expect(getThrottleIntervalMs('https://example.com')).toBe(123);
+    });
+  });
+});

lib/util/http/rate-limits.ts

@@ -0,0 +1,87 @@
+import is from '@sindresorhus/is';
+import { matchesHost } from '../host-rules';
+import * as hostRules from '../host-rules';
+import type { RateLimitRule } from './types';
+
+const defaults: RateLimitRule[] = [
+  {
+    // https://guides.rubygems.org/rubygems-org-rate-limits/
+    matchHost: 'rubygems.org',
+    throttleMs: 125,
+  },
+  {
+    // https://crates.io/data-access#api
+    matchHost: 'https://crates.io/api/',
+    throttleMs: 1000,
+  },
+  {
+    matchHost: '*',
+    concurrency: 16,
+  },
+];
+
+let limits: RateLimitRule[] = [];
+
+export function setHttpRateLimits(rules?: RateLimitRule[]): void {
+  limits = rules ?? defaults;
+}
+
+function matches(url: string, host: string): boolean {
+  if (host === '*') {
+    return true;
+  }
+
+  return matchesHost(url, host);
+}
+
+export function getConcurrentRequestsLimit(url: string): number | null {
+  let result: number | null = null;
+
+  const { concurrentRequestLimit: hostRuleLimit } = hostRules.find({ url });
+  if (
+    is.number(hostRuleLimit) &&
+    hostRuleLimit > 0 &&
+    hostRuleLimit < Number.MAX_SAFE_INTEGER
+  ) {
+    result = hostRuleLimit;
+  }
+
+  for (const { matchHost, concurrency: limit } of limits) {
+    if (!matches(url, matchHost) || !is.number(limit)) {
+      continue;
+    }
+
+    if (result && result <= limit) {
+      continue;
+    }
+
+    result = limit;
+    break;
+  }
+
+  return result;
+}
+
+export function getThrottleIntervalMs(url: string): number | null {
+  let result: number | null = null;
+
+  const { maxRequestsPerSecond } = hostRules.find({ url });
+  if (is.number(maxRequestsPerSecond) && maxRequestsPerSecond > 0) {
+    result = Math.ceil(1000 / maxRequestsPerSecond);
+  }
+
+  for (const { matchHost, throttleMs: limit } of limits) {
+    if (!matches(url, matchHost) || !is.number(limit)) {
+      continue;
+    }
+
+    if (result && result >= limit) {
+      continue;
+    }
+
+    result = limit;
+    break;
+  }
+
+  return result;
+}

lib/util/http/throttle.ts

@@ -1,7 +1,7 @@
 import pThrottle from 'p-throttle';
 import { logger } from '../../logger';
 import { parseUrl } from '../url';
-import { getThrottleIntervalMs } from './host-rules';
+import { getThrottleIntervalMs } from './rate-limits';
 
 const hostThrottles = new Map<string, Throttle | null>();
 
@@ -33,11 +33,10 @@ export function getThrottle(url: string): Throttle | null {
   let throttle = hostThrottles.get(host);
   if (throttle === undefined) {
     throttle = null; // null represents "no throttle", as opposed to undefined
-    const throttleOptions = getThrottleIntervalMs(url);
-    if (throttleOptions) {
-      const intervalMs = throttleOptions;
-      logger.debug(`Using throttle ${intervalMs} intervalMs for host ${host}`);
-      throttle = new Throttle(intervalMs);
+    const throttleMs = getThrottleIntervalMs(url);
+    if (throttleMs) {
+      logger.debug(`Using throttle ${throttleMs} intervalMs for host ${host}`);
+      throttle = new Throttle(throttleMs);
     } else {
       logger.trace({ host }, 'No throttle');
     }

lib/util/http/types.ts

@@ -91,3 +91,9 @@ export interface HttpResponse<T = string> {
 
 export type Task<T> = () => Promise<T>;
 export type GotTask<T> = Task<HttpResponse<T>>;
+
+export interface RateLimitRule {
+  matchHost: string;
+  throttleMs?: number;
+  concurrency?: number;
+}

lib/workers/global/initialize.ts

@@ -10,6 +10,7 @@ import * as packageCache from '../../util/cache/package';
 import { setEmojiConfig } from '../../util/emoji';
 import { validateGitVersion } from '../../util/git';
 import * as hostRules from '../../util/host-rules';
+import { setHttpRateLimits } from '../../util/http/rate-limits';
 import { initMergeConfidence } from '../../util/merge-confidence';
 import { setMaxLimit } from './limits';
 
@@ -79,6 +80,7 @@ export async function globalInitialize(
   config_: AllConfig,
 ): Promise<RenovateConfig> {
   let config = config_;
+  setHttpRateLimits();
   await checkVersions();
   setGlobalHostRules(config);
   config = await initPlatform(config);