Skip to content

Commit

Permalink
feat(requests): add user-agent and httpsAgent (#646)
Browse files Browse the repository at this point in the history
  • Loading branch information
Samuel Bodin authored Jul 6, 2021
1 parent fbe2e97 commit 5a48ad3
Show file tree
Hide file tree
Showing 5 changed files with 154 additions and 127 deletions.
7 changes: 4 additions & 3 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
"doctoc": "doctoc --notitle --maxlevel 3 README.md",
"lint": "eslint --ext=jsx,ts,tsx,js .",
"prepare": "husky install",
"start": "UV_THREADPOOL_SIZE=12 node -r dotenv/config --async-stack-traces --max-old-space-size=920 dist/index.js",
"start_new": "indexName=npm-search-new bootstrapIndexName=npm-search-new.tmp UV_THREADPOOL_SIZE=22 node --async-stack-traces --max-old-space-size=920 dist/index.js",
"start": "UV_THREADPOOL_SIZE=64 node -r dotenv/config --async-stack-traces --max-old-space-size=920 dist/index.js",
"start_new": "indexName=npm-search-new bootstrapIndexName=npm-search-new.tmp UV_THREADPOOL_SIZE=64 node --async-stack-traces --max-old-space-size=920 dist/index.js",
"test:watch": "jest --watchAll --no-watchman",
"test": "jest --forceExit"
},
Expand All @@ -27,8 +27,9 @@
},
"license": "MIT",
"dependencies": {
"@algolia/requester-node-http": "4.10.3",
"@sentry/node": "6.5.1",
"algoliasearch": "4.9.3",
"algoliasearch": "4.10.3",
"async": "3.2.0",
"bunyan": "1.8.15",
"bunyan-debug-stream": "2.0.0",
Expand Down
13 changes: 12 additions & 1 deletion src/algolia/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
import { createNodeHttpRequester } from '@algolia/requester-node-http';
import type { SearchClient, SearchIndex } from 'algoliasearch';
import algoliasearch from 'algoliasearch';

import type { Config } from '../config';
import { httpAgent, httpsAgent, USER_AGENT } from '../utils/request';

const requester = createNodeHttpRequester({
agent: httpsAgent,
httpAgent,
httpsAgent,
});

function createClient({
appId,
Expand All @@ -12,7 +20,10 @@ function createClient({
apiKey: string;
indexName: string;
}): { index: SearchIndex; client: SearchClient } {
const client = algoliasearch(appId, apiKey);
const client = algoliasearch(appId, apiKey, {
requester,
});
client.addAlgoliaAgent(USER_AGENT);
return {
index: client.initIndex(indexName),
client,
Expand Down
8 changes: 7 additions & 1 deletion src/npm/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,18 @@ import type { RawPkg } from '../@types/pkg';
import { config } from '../config';
import { datadog } from '../utils/datadog';
import { log } from '../utils/log';
import { request } from '../utils/request';
import { httpsAgent, request, USER_AGENT } from '../utils/request';

import type { GetInfo, GetPackage, PackageDownload } from './types';

const registry = nano({
url: config.npmRegistryEndpoint,
requestDefaults: {
agent: httpsAgent,
headers: {
'user-agent': USER_AGENT,
},
},
});
const db = registry.use<GetPackage>(config.npmRegistryDBName);

Expand Down
9 changes: 9 additions & 0 deletions src/utils/request.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ import https from 'https';
import type { OptionsOfJSONResponseBody } from 'got';
import got from 'got';

// eslint-disable-next-line @typescript-eslint/no-var-requires, import/no-commonjs
const { version } = require('../../package.json');

export const USER_AGENT = `Algolia npm-search/${version} (https://github.com/algolia/npm-search)`;

const options = {
keepAlive: true,
timeout: 60000,
Expand All @@ -22,6 +27,10 @@ export async function request<TRes>(
) {
return await got<TRes>(url, {
...opts,
headers: {
...(opts.headers || {}),
'user-agent': USER_AGENT,
},
agent: {
http: httpAgent,
https: httpsAgent,
Expand Down
244 changes: 122 additions & 122 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,109 +2,109 @@
# yarn lockfile v1


"@algolia/cache-browser-local-storage@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.9.3.tgz#44e6306a548a76c410b5f64a8a0a1b65f63183c9"
integrity sha512-t9yKMfPNxxEUk/PPbZtXj0GCttDk1pk0wV2eA5udIOgf+Wqb/77yH75zz1u8EmCBGPe+FWXjSVT/wS1tlQz7SA==
dependencies:
"@algolia/cache-common" "4.9.3"

"@algolia/cache-common@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/cache-common/-/cache-common-4.9.3.tgz#0b3ca07c9af108433b4d3423a03511c3d053fed5"
integrity sha512-4dvzz28ESs7lRHmpBIjlmRloD9oGeD90E2C0QWNQYuAYosSdXGwW7vw4vdGRdPoL32t6u6S+47Bk6Dhcbw2ftA==

"@algolia/cache-in-memory@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/cache-in-memory/-/cache-in-memory-4.9.3.tgz#0bae2ad1de6537ca28efaf5280051265903bfca5"
integrity sha512-e1eRpP/Ht9qmLw5Sp674N6Y0c59K0L2LBI71EBOlq1j+kVc+JxVO03he5g+nQ7JOwLijyJPrkbm3RvXb5CX0sA==
dependencies:
"@algolia/cache-common" "4.9.3"

"@algolia/client-account@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/client-account/-/client-account-4.9.3.tgz#af9bf3612d05e87aa36372da50a2f0a9265de201"
integrity sha512-mSF0jiAo/tWKf/Z7mqhz6ETltrl+L+Zt2xuM3W5y1UOZvj47fn2ZcMRce8MQ+dd54t9iA8qIa+0XGlCSQf9lxA==
dependencies:
"@algolia/client-common" "4.9.3"
"@algolia/client-search" "4.9.3"
"@algolia/transporter" "4.9.3"

"@algolia/client-analytics@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/client-analytics/-/client-analytics-4.9.3.tgz#cddb4a97d796291d91bd15735de008b2d80a0b1b"
integrity sha512-Z3EjegxsdTMRmOLoDBnCZJjdL3ZM4J/G7TMe2PIArdCJFWM4iDnO7/MvYasqpK0PPOCHRh0wS4yKG9rZOz6Vsw==
dependencies:
"@algolia/client-common" "4.9.3"
"@algolia/client-search" "4.9.3"
"@algolia/requester-common" "4.9.3"
"@algolia/transporter" "4.9.3"

"@algolia/client-common@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/client-common/-/client-common-4.9.3.tgz#167a6863b55ffe7788ab9ac98b1b6fd0078f79df"
integrity sha512-6GAfuNqMrBN3094H0DzvQyxJoKUkyQpEr5OiFhH8I3lihI1rTtjEUrNDTsVp6e9VsR2OCRpnL9EEDv2HcGe8cw==
dependencies:
"@algolia/requester-common" "4.9.3"
"@algolia/transporter" "4.9.3"

"@algolia/client-recommendation@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/client-recommendation/-/client-recommendation-4.9.3.tgz#e2031237b2904c9d9b946fc846c87a21ad67bc5a"
integrity sha512-r+MNluwnUTr1tgHWQ5BPRw0A0YJZp9sXjSVxPCY3a+N6BgLaX4E02+FA8VrqVs8uR7mMQSLaJHoeCKnmNPrk9w==
dependencies:
"@algolia/client-common" "4.9.3"
"@algolia/requester-common" "4.9.3"
"@algolia/transporter" "4.9.3"

"@algolia/client-search@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/client-search/-/client-search-4.9.3.tgz#6259645ad5a7e7388727343806bcf2d0cf3e343a"
integrity sha512-8C6woYf6bY4Fh9H9nKY5IDDeBPwQ3nZn9QMQdgUj9ffDU8UzPqSivtLER1A+I81p1j9h+aBADRifwzIYtSXOkA==
dependencies:
"@algolia/client-common" "4.9.3"
"@algolia/requester-common" "4.9.3"
"@algolia/transporter" "4.9.3"

"@algolia/logger-common@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/logger-common/-/logger-common-4.9.3.tgz#d9b976524313b11f11c6ec546e2ed451c7964a11"
integrity sha512-8hGQ5HQvjx2kr7GWOmpON1tcRX2+VHqVg4p+qJqCBsPFlXbAshUyRJkxuen20eem2EAA5Cmmo1fPy/jlqdMMHA==

"@algolia/logger-console@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/logger-console/-/logger-console-4.9.3.tgz#f690bf8d5262a82425da26b42a519d7ec6c3784a"
integrity sha512-7FGulrAjS/oCVRShKJw5qFuyHOZk/44jolEtNtXvO/tZRR8hPPiow16Vrd3ByRSIhghkC5zj6at4nQhoPK+KqA==
dependencies:
"@algolia/logger-common" "4.9.3"

"@algolia/requester-browser-xhr@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.9.3.tgz#1c0fd594e253b41786b3408ade9f63862fe0c577"
integrity sha512-hP4YgxcY1kol0d+joXpO4BJuXjgF+vy3eBPk8WCXvZucv8hl5Vqb4BLccDMck+sTqP4Tqglwh/KwVTQrpmi/wA==
dependencies:
"@algolia/requester-common" "4.9.3"

"@algolia/requester-common@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/requester-common/-/requester-common-4.9.3.tgz#58fb72ca3f4f7714d75287ff568000ee6c2afac6"
integrity sha512-AgUw1iA/JkanZC+dhkSLyeiVgBhaaM3bI20f3cokuuDdz4X6F+hzi0vEpUZrEuNfnMLbUg8gxq3Vcg1/L9+9MA==

"@algolia/requester-node-http@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/requester-node-http/-/requester-node-http-4.9.3.tgz#a2bf0e7048fe3b81c6b6b170f8f68a5789dadafe"
integrity sha512-+nz7rRnI9qNcdZjHpyAyvcDLAO9mGobqsAi0aicxMka/szU1HVUX6+pvSOiiOsD8ST3R13rJuufgHfWdDUysQg==
dependencies:
"@algolia/requester-common" "4.9.3"

"@algolia/transporter@4.9.3":
version "4.9.3"
resolved "https://registry.yarnpkg.com/@algolia/transporter/-/transporter-4.9.3.tgz#5a0933d4e59acdf88712156b2ab6f1b46c0a7f88"
integrity sha512-oJ68VSSpmUyB9EByqoyx25wgcrO9fgXtjH+pOtKoKmCW+RfxHW5agltJoQ808N8uq/AvP5ugMkRLGL3xf4WdzQ==
dependencies:
"@algolia/cache-common" "4.9.3"
"@algolia/logger-common" "4.9.3"
"@algolia/requester-common" "4.9.3"
"@algolia/cache-browser-local-storage@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.10.3.tgz#3bf81e0f66a4a1079a75914a987eb1ef432c7c68"
integrity sha512-TD1N7zg5lb56/PLjjD4bBl2eccEvVHhC7yfgFu2r9k5tf+gvbGxEZ3NhRZVKu2MObUIcEy2VR4LVLxOQu45Hlg==
dependencies:
"@algolia/cache-common" "4.10.3"

"@algolia/cache-common@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/cache-common/-/cache-common-4.10.3.tgz#311b2b5ae06d55300f4230944c99bc39ad15847d"
integrity sha512-q13cPPUmtf8a2suBC4kySSr97EyulSXuxUkn7l1tZUCX/k1y5KNheMp8npBy8Kc8gPPmHpacxddRSfOncjiKFw==

"@algolia/cache-in-memory@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/cache-in-memory/-/cache-in-memory-4.10.3.tgz#697e4994538426272ea29ccf2b32b46ea4c48862"
integrity sha512-JhPajhOXAjUP+TZrZTh6KJpF5VKTKyWK2aR1cD8NtrcVHwfGS7fTyfXfVm5BqBqkD9U0gVvufUt/mVyI80aZww==
dependencies:
"@algolia/cache-common" "4.10.3"

"@algolia/client-account@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/client-account/-/client-account-4.10.3.tgz#f2cbefb1abce74c341115607d6af199df1b056ae"
integrity sha512-S/IsJB4s+e1xYctdpW3nAbwrR2y3pjSo9X21fJGoiGeIpTRdvQG7nydgsLkhnhcgAdLnmqBapYyAqMGmlcyOkg==
dependencies:
"@algolia/client-common" "4.10.3"
"@algolia/client-search" "4.10.3"
"@algolia/transporter" "4.10.3"

"@algolia/client-analytics@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/client-analytics/-/client-analytics-4.10.3.tgz#43d934ef8df0cf551c78e6b2e9f2452e7fb27d93"
integrity sha512-vlHTbBqJktRgclh3v7bPQLfZvFIqY4erNFIZA5C7nisCj9oLeTgzefoUrr+R90+I+XjfoLxnmoeigS1Z1yg1vw==
dependencies:
"@algolia/client-common" "4.10.3"
"@algolia/client-search" "4.10.3"
"@algolia/requester-common" "4.10.3"
"@algolia/transporter" "4.10.3"

"@algolia/client-common@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/client-common/-/client-common-4.10.3.tgz#c4257dd5c57c5c8ec4bd48a7b1897573e372d403"
integrity sha512-uFyP2Z14jG2hsFRbAoavna6oJf4NTXaSDAZgouZUZlHlBp5elM38sjNeA5HR9/D9J/GjwaB1SgB7iUiIWYBB4w==
dependencies:
"@algolia/requester-common" "4.10.3"
"@algolia/transporter" "4.10.3"

"@algolia/client-personalization@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/client-personalization/-/client-personalization-4.10.3.tgz#58c800f90ab8ab4aa29abdf29a97e89e6bda419e"
integrity sha512-NS7Nx8EJ/nduGXT8CFo5z7kLF0jnFehTP3eC+z+GOEESH3rrs7uR12IZHxv5QhQswZa9vl925zCOZDcDVoENCg==
dependencies:
"@algolia/client-common" "4.10.3"
"@algolia/requester-common" "4.10.3"
"@algolia/transporter" "4.10.3"

"@algolia/client-search@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/client-search/-/client-search-4.10.3.tgz#aa6b02c2d528cb264830f276739b7f68b58988ef"
integrity sha512-Zwnp2G94IrNFKWCG/k7epI5UswRkPvL9FCt7/slXe2bkjP2y/HA37gzRn+9tXoLVRwd7gBzrtOA4jFKIyjrtVw==
dependencies:
"@algolia/client-common" "4.10.3"
"@algolia/requester-common" "4.10.3"
"@algolia/transporter" "4.10.3"

"@algolia/logger-common@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/logger-common/-/logger-common-4.10.3.tgz#6773d2e38581bf9ac57e2dda02f0c4f1bc72ce94"
integrity sha512-M6xi+qov2bkgg1H9e1Qtvq/E/eKsGcgz8RBbXNzqPIYoDGZNkv+b3b8YMo3dxd4Wd6M24HU1iqF3kmr1LaXndg==

"@algolia/logger-console@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/logger-console/-/logger-console-4.10.3.tgz#bd8bdc1f9dba89db37be25d673ac1f2e68de7913"
integrity sha512-vVgRI7b4PHjgBdRkv/cRz490twvkLoGdpC4VYzIouSrKj8SIVLRhey3qgXk7oQXi3xoxVAv6NrklHfpO8Bpx0w==
dependencies:
"@algolia/logger-common" "4.10.3"

"@algolia/requester-browser-xhr@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.10.3.tgz#81ae8f6caf562a28f96102f03da7f4b19bba568c"
integrity sha512-4WIk1zreFbc1EF6+gsfBTQvwSNjWc20zJAAExRWql/Jq5yfVHmwOqi/CajA53/cXKFBqo80DAMRvOiwP+hOLYw==
dependencies:
"@algolia/requester-common" "4.10.3"

"@algolia/requester-common@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/requester-common/-/requester-common-4.10.3.tgz#c3112393cff97be79863bc28de76f9c69b2f5a95"
integrity sha512-PNfLHmg0Hujugs3rx55uz/ifv7b9HVdSFQDb2hj0O5xZaBEuQCNOXC6COrXR8+9VEfqp2swpg7zwgtqFxh+BtQ==

"@algolia/requester-node-http@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/requester-node-http/-/requester-node-http-4.10.3.tgz#75ea7805ac0ba25a1124989d8632ef39c31441c1"
integrity sha512-A9ZcGfEvgqf0luJApdNcIhsRh6MShn2zn2tbjwjGG1joF81w+HUY+BWuLZn56vGwAA9ZB9n00IoJJpxibbfofg==
dependencies:
"@algolia/requester-common" "4.10.3"

"@algolia/transporter@4.10.3":
version "4.10.3"
resolved "https://registry.yarnpkg.com/@algolia/transporter/-/transporter-4.10.3.tgz#0aeee752923957cffe63e4cf1c7a22ca48d96dde"
integrity sha512-n1lRyKDbrckbMEgm7QXtj3nEWUuzA3aKLzVQ43/F/RCFib15j4IwtmYhXR6OIBRSc7+T0Hm48S0J6F+HeYCQkw==
dependencies:
"@algolia/cache-common" "4.10.3"
"@algolia/logger-common" "4.10.3"
"@algolia/requester-common" "4.10.3"

"@babel/[email protected]":
version "7.12.11"
Expand Down Expand Up @@ -1317,25 +1317,25 @@ ajv@^8.0.1:
require-from-string "^2.0.2"
uri-js "^4.2.2"

algoliasearch@4.9.3:
version "4.9.3"
resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-4.9.3.tgz#b22ef0ae0450304cdf5264369a29cefa71ea2b30"
integrity sha512-VLl9pYXhVB397xTW369sy13qw3m1hHzCfj9zSdeDDYVwTxHiiok/QvhPKAMIzjqyUoY07O8j+941UxYZjugsMQ==
dependencies:
"@algolia/cache-browser-local-storage" "4.9.3"
"@algolia/cache-common" "4.9.3"
"@algolia/cache-in-memory" "4.9.3"
"@algolia/client-account" "4.9.3"
"@algolia/client-analytics" "4.9.3"
"@algolia/client-common" "4.9.3"
"@algolia/client-recommendation" "4.9.3"
"@algolia/client-search" "4.9.3"
"@algolia/logger-common" "4.9.3"
"@algolia/logger-console" "4.9.3"
"@algolia/requester-browser-xhr" "4.9.3"
"@algolia/requester-common" "4.9.3"
"@algolia/requester-node-http" "4.9.3"
"@algolia/transporter" "4.9.3"
algoliasearch@4.10.3:
version "4.10.3"
resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-4.10.3.tgz#22df4bb02fbf13a765b18b85df8745ee9c04f00a"
integrity sha512-OLY0AWlPKGLbSaw14ivMB7BT5fPdp8VdzY4L8FtzZnqmLKsyes24cltGlf7/X96ACkYEcT390SReCDt/9SUIRg==
dependencies:
"@algolia/cache-browser-local-storage" "4.10.3"
"@algolia/cache-common" "4.10.3"
"@algolia/cache-in-memory" "4.10.3"
"@algolia/client-account" "4.10.3"
"@algolia/client-analytics" "4.10.3"
"@algolia/client-common" "4.10.3"
"@algolia/client-personalization" "4.10.3"
"@algolia/client-search" "4.10.3"
"@algolia/logger-common" "4.10.3"
"@algolia/logger-console" "4.10.3"
"@algolia/requester-browser-xhr" "4.10.3"
"@algolia/requester-common" "4.10.3"
"@algolia/requester-node-http" "4.10.3"
"@algolia/transporter" "4.10.3"

anchor-markdown-header@~0.5.7:
version "0.5.7"
Expand Down

0 comments on commit 5a48ad3

Please sign in to comment.