fix: add 429 rate limit for unknown user-agents (#1050)

There are a lot of API requests now that Package Phobia is so popular.

We need to ensure that clients are setting a proper `user-agent` so we
know where these requests are coming from and can block bad traffic.

Users who wish to call the API should add the expected user agent to
[API.md](https://github.com/styfle/packagephobia/blob/main/API.md)
document.

API.md

@@ -1,5 +1,9 @@
 # API
 
+If you intend to use this API, please add your website to the list below and set your client's `user-agent` request header to match.
+
+If you forget to set a `user-agent`, you will likely be blocked.
+
 ## Users
 
 Current websites using this API:
@@ -10,8 +14,6 @@ Current websites using this API:
 - https://bestofjs.org
 - https://socket.dev
 
-If you intend to use this API, please add your website to the list and set user-agent header to match.
-
 ## Endpoints
 
 - v1: `GET /api.json`

src/server.ts

@@ -26,10 +26,28 @@ console.log('TMPDIR: ', TMPDIR);
 console.log('HOME: ', process.env.HOME);
 console.log('AWS_SECRET_ACCESS_KEY: ', process.env.AWS_SECRET_ACCESS_KEY);
 
+let botCount = 0;
+
 export async function handler(req: IncomingMessage, res: ServerResponse) {
     let { method, url, headers } = req;
+    const userAgent = headers['user-agent'] || '';
     console.log(`${method} ${headers.host}${url}`);
-    console.log(`user-agent: ${headers['user-agent']}`);
+    console.log(`user-agent: ${userAgent}`);
+    if (
+        !userAgent ||
+        userAgent === 'node' ||
+        userAgent.startsWith('axios') ||
+        userAgent.startsWith('got')
+    ) {
+        botCount++;
+        if (botCount % 100 === 0) {
+            res.statusCode = 429;
+            res.end(
+                'Too many requests from unknown user-agent. See https://github.com/styfle/packagephobia/blob/main/API.md',
+            );
+            return;
+        }
+    }
     let { pathname = '/', query = {} } = parse(url || '', true);
     if (!pathname || pathname === '/') {
         pathname = pages.index;