Refactor

lib/util/http/host-rules.ts

@@ -217,17 +217,3 @@ export function applyHostRule<GotOptions extends HostRulesGotOptions>(
 
   return options;
 }
-
-export function getConcurrentRequestsLimit(url: string): number | null {
-  const { concurrentRequestLimit } = hostRules.find({ url });
-  return is.number(concurrentRequestLimit) && concurrentRequestLimit > 0
-    ? concurrentRequestLimit
-    : null;
-}
-
-export function getThrottleIntervalMs(url: string): number | null {
-  const { maxRequestsPerSecond } = hostRules.find({ url });
-  return is.number(maxRequestsPerSecond) && maxRequestsPerSecond > 0
-    ? Math.ceil(1000 / maxRequestsPerSecond)
-    : null;
-}

lib/util/http/queue.ts

@@ -1,8 +1,7 @@
 import PQueue from 'p-queue';
 import { logger } from '../../logger';
 import { parseUrl } from '../url';
-import { getDefaultConcurrentRequestsLimit } from './default-limits';
-import { getConcurrentRequestsLimit } from './host-rules';
+import { getConcurrentRequestsLimit } from './rate-limits';
 
 const hostQueues = new Map<string, PQueue | null>();
 
@@ -17,11 +16,7 @@ export function getQueue(url: string): PQueue | null {
   let queue = hostQueues.get(host);
   if (queue === undefined) {
     queue = null; // null represents "no queue", as opposed to undefined
-    const hostConcurrency = getConcurrentRequestsLimit(url);
-    const defaultConcurrency = getDefaultConcurrentRequestsLimit(url);
-    const concurrency = Math.min(
-      ...[hostConcurrency, defaultConcurrency].filter((x) => x !== null),
-    );
+    const concurrency = getConcurrentRequestsLimit(url);
     if (concurrency) {
       logger.debug(`Using queue: host=${host}, concurrency=${concurrency}`);
       queue = new PQueue({ concurrency });

lib/util/http/rate-limits.ts

@@ -0,0 +1,73 @@
+import is from '@sindresorhus/is';
+import { matchesHost } from '../host-rules';
+import * as hostRules from '../host-rules';
+import type { RateLimitRule } from './types';
+
+const defaults: RateLimitRule[] = [
+  {
+    matchHost: 'rubygems.org',
+    throttleMs: 125,
+  },
+  {
+    matchHost: 'https://crates.io/api/',
+    throttleMs: 1000,
+  },
+];
+
+export function getConcurrentRequestsLimit(url: string): number | null {
+  let result: number | null = null;
+
+  const { concurrentRequestLimit: hostRuleLimit } = hostRules.find({ url });
+  if (
+    is.number(hostRuleLimit) &&
+    hostRuleLimit > 0 &&
+    hostRuleLimit < Number.MAX_SAFE_INTEGER
+  ) {
+    result = hostRuleLimit;
+  }
+
+  for (const { matchHost, concurrency: limit } of defaults) {
+    if (!matchesHost(url, matchHost)) {
+      continue;
+    }
+
+    if (!is.number(limit)) {
+      continue;
+    }
+
+    if (result && result <= limit) {
+      continue;
+    }
+
+    result = limit;
+  }
+
+  return result;
+}
+
+export function getThrottleIntervalMs(url: string): number | null {
+  let result: number | null = null;
+
+  const { maxRequestsPerSecond } = hostRules.find({ url });
+  if (is.number(maxRequestsPerSecond) && maxRequestsPerSecond > 0) {
+    result = Math.ceil(1000 / maxRequestsPerSecond);
+  }
+
+  for (const { matchHost, throttleMs: limit } of defaults) {
+    if (!matchesHost(url, matchHost)) {
+      continue;
+    }
+
+    if (!is.number(limit)) {
+      continue;
+    }
+
+    if (result && result >= limit) {
+      continue;
+    }
+
+    result = limit;
+  }
+
+  return result;
+}

lib/util/http/throttle.ts

@@ -1,8 +1,7 @@
 import pThrottle from 'p-throttle';
 import { logger } from '../../logger';
 import { parseUrl } from '../url';
-import { getDefaultThrottleIntervalMs } from './default-limits';
-import { getThrottleIntervalMs } from './host-rules';
+import { getThrottleIntervalMs } from './rate-limits';
 
 const hostThrottles = new Map<string, Throttle | null>();
 
@@ -34,11 +33,7 @@ export function getThrottle(url: string): Throttle | null {
   let throttle = hostThrottles.get(host);
   if (throttle === undefined) {
     throttle = null; // null represents "no throttle", as opposed to undefined
-    const hostThrottleMs = getThrottleIntervalMs(url);
-    const defaultThrottleMs = getDefaultThrottleIntervalMs(url);
-    const throttleMs = Math.max(
-      ...[hostThrottleMs, defaultThrottleMs].filter((x) => x !== null),
-    );
+    const throttleMs = getThrottleIntervalMs(url);
     if (throttleMs) {
       const intervalMs = throttleMs;
       logger.debug(`Using throttle ${intervalMs} intervalMs for host ${host}`);

lib/util/http/types.ts

@@ -94,6 +94,6 @@ export type GotTask<T> = Task<HttpResponse<T>>;
 
 export interface RateLimitRule {
   matchHost: string;
-  throttleIntervalMs?: number;
-  maxConcurrentRequests?: number;
+  throttleMs?: number;
+  concurrency?: number;
 }