feat(http): Force default rate limits for some known hosts

lib/modules/datasource/rubygems/index.ts

@@ -2,14 +2,13 @@ import { Marshal } from '@qnighy/marshal';
 import type { ZodError } from 'zod';
 import { logger } from '../../../logger';
 import { cache } from '../../../util/cache/package/decorator';
-import { HttpError } from '../../../util/http';
+import { Http, HttpError } from '../../../util/http';
 import { AsyncResult, Result } from '../../../util/result';
 import { getQueryString, joinUrlParts, parseUrl } from '../../../util/url';
 import * as rubyVersioning from '../../versioning/ruby';
 import { Datasource } from '../datasource';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 import { getV1Releases } from './common';
-import { RubygemsHttp } from './http';
 import { MetadataCache } from './metadata-cache';
 import { GemInfo, MarshalledVersionInfo } from './schema';
 import { VersionsEndpointCache } from './versions-endpoint-cache';
@@ -34,7 +33,7 @@ export class RubyGemsDatasource extends Datasource {
 
   constructor() {
     super(RubyGemsDatasource.id);
-    this.http = new RubygemsHttp(RubyGemsDatasource.id);
+    this.http = new Http(RubyGemsDatasource.id);
     this.versionsEndpointCache = new VersionsEndpointCache(this.http);
     this.metadataCache = new MetadataCache(this.http);
   }

lib/util/host-rules.ts

@@ -77,7 +77,7 @@ export interface HostRuleSearch {
   readOnly?: boolean;
 }
 
-function matchesHost(url: string, matchHost: string): boolean {
+export function matchesHost(url: string, matchHost: string): boolean {
   if (isHttpUrl(url) && isHttpUrl(matchHost)) {
     return url.startsWith(matchHost);
   }

lib/util/http/default-limits.ts

@@ -0,0 +1,40 @@
+import is from '@sindresorhus/is';
+import { matchesHost } from '../host-rules';
+import type { RateLimitRule } from './types';
+
+const defaultLimits: RateLimitRule[] = [
+  {
+    matchHost: 'rubygems.org',
+    throttleIntervalMs: 125,
+  },
+  {
+    matchHost: 'https://crates.io/api/',
+    throttleIntervalMs: 1000,
+  },
+];
+
+export function getDefaultThrottleIntervalMs(url: string): number | null {
+  for (const rule of defaultLimits) {
+    if (
+      matchesHost(url, rule.matchHost) &&
+      is.number(rule.throttleIntervalMs)
+    ) {
+      return rule.throttleIntervalMs;
+    }
+  }
+
+  return null;
+}
+
+export function getDefaultConcurrentRequestsLimit(url: string): number | null {
+  for (const rule of defaultLimits) {
+    if (
+      matchesHost(url, rule.matchHost) &&
+      is.number(rule.maxConcurrentRequests)
+    ) {
+      return rule.maxConcurrentRequests;
+    }
+  }
+
+  return null;
+}

lib/util/http/index.ts

@@ -18,7 +18,7 @@ import { hooks } from './hooks';
 import { applyHostRule, findMatchingRule } from './host-rules';
 import { getQueue } from './queue';
 import { getRetryAfter, wrapWithRetry } from './retry-after';
-import { Throttle, getThrottle } from './throttle';
+import { getThrottle } from './throttle';
 import type {
   GotJSONOptions,
   GotOptions,
@@ -134,10 +134,6 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
     );
   }
 
-  protected getThrottle(url: string): Throttle | null {
-    return getThrottle(url);
-  }
-
   protected async request<T>(
     requestUrl: string | URL,
     httpOptions: InternalHttpOptions,
@@ -212,7 +208,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
         return gotTask(url, options, { queueMs });
       };
 
-      const throttle = this.getThrottle(url);
+      const throttle = getThrottle(url);
       const throttledTask: GotTask<T> = throttle
         ? () => throttle.add<HttpResponse<T>>(httpTask)
         : httpTask;

lib/util/http/queue.ts

@@ -1,6 +1,7 @@
 import PQueue from 'p-queue';
 import { logger } from '../../logger';
 import { parseUrl } from '../url';
+import { getDefaultConcurrentRequestsLimit } from './default-limits';
 import { getConcurrentRequestsLimit } from './host-rules';
 
 const hostQueues = new Map<string, PQueue | null>();
@@ -16,7 +17,11 @@
   let queue = hostQueues.get(host);
   if (queue === undefined) {
     queue = null; // null represents "no queue", as opposed to undefined
-    const concurrency = getConcurrentRequestsLimit(url);
+    const hostConcurrency = getConcurrentRequestsLimit(url);
+    const defaultConcurrency = getDefaultConcurrentRequestsLimit(url);
+    const concurrency = Math.min(
+      ...[hostConcurrency, defaultConcurrency].filter((x) => x !== null),
+    );
     if (concurrency) {
       logger.debug(`Using queue: host=${host}, concurrency=${concurrency}`);
       queue = new PQueue({ concurrency });

lib/util/http/throttle.ts

@@ -1,6 +1,7 @@
 import pThrottle from 'p-throttle';
 import { logger } from '../../logger';
 import { parseUrl } from '../url';
+import { getDefaultThrottleIntervalMs } from './default-limits';
 import { getThrottleIntervalMs } from './host-rules';
 
 const hostThrottles = new Map<string, Throttle | null>();
@@ -9,7 +10,7 @@
   private throttle: ReturnType<typeof pThrottle>;
 
   constructor(interval: number) {
     this.throttle = pThrottle({
       strict: true,
       limit: 1,
       interval,
@@ -33,9 +34,13 @@
   let throttle = hostThrottles.get(host);
   if (throttle === undefined) {
     throttle = null; // null represents "no throttle", as opposed to undefined
-    const throttleOptions = getThrottleIntervalMs(url);
-    if (throttleOptions) {
-      const intervalMs = throttleOptions;
+    const hostThrottleMs = getThrottleIntervalMs(url);
+    const defaultThrottleMs = getDefaultThrottleIntervalMs(url);
+    const throttleMs = Math.max(
+      ...[hostThrottleMs, defaultThrottleMs].filter((x) => x !== null),
+    );
+    if (throttleMs) {
+      const intervalMs = throttleMs;
       logger.debug(`Using throttle ${intervalMs} intervalMs for host ${host}`);
       throttle = new Throttle(intervalMs);
     } else {

lib/util/http/types.ts

@@ -91,3 +91,9 @@ export interface HttpResponse<T = string> {
 
 export type Task<T> = () => Promise<T>;
 export type GotTask<T> = Task<HttpResponse<T>>;
+
+export interface RateLimitRule {
+  matchHost: string;
+  throttleIntervalMs?: number;
+  maxConcurrentRequests?: number;
+}