Refactor har generation utilities for reuse

packages/toolpad-app/src/toolpadDataSources/function/execFunction.spec.ts

@@ -1,10 +1,9 @@
-import * as http from 'http';
-import getPort from 'get-port';
-import { Readable } from 'stream';
 import formidable from 'formidable';
 import { execa, ExecaChildProcess } from 'execa';
 import path from 'path';
 import execFunction from './execFunction';
+import { streamToString } from '../../utils/streams';
+import { startServer } from '../../utils/tests';
 
 function redirectOutput(cp: ExecaChildProcess): ExecaChildProcess {
   if (cp.stdin) {
@@ -19,44 +18,6 @@ function redirectOutput(cp: ExecaChildProcess): ExecaChildProcess {
   return cp;
 }
 
-function streamToString(stream: Readable) {
-  const chunks: Buffer[] = [];
-  return new Promise((resolve, reject) => {
-    stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
-    stream.on('error', (err) => reject(err));
-    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
-  });
-}
-
-async function startServer(handler?: http.RequestListener) {
-  const server = http.createServer(handler);
-  const port = await getPort({ port: 3000 });
-  let app: http.Server | undefined;
-  await new Promise((resolve, reject) => {
-    app = server.listen(port);
-    app.once('listening', resolve);
-    app.once('error', reject);
-  });
-  return {
-    port,
-    async stopServer() {
-      await new Promise<void>((resolve, reject) => {
-        if (app) {
-          app.close((err) => {
-            if (err) {
-              reject(err);
-            } else {
-              resolve();
-            }
-          });
-        } else {
-          resolve();
-        }
-      });
-    },
-  };
-}
-
 async function buildRuntime() {
   await redirectOutput(
     execa('yarn', ['run', 'build:function-runtime'], {

packages/toolpad-app/src/toolpadDataSources/function/execFunction.ts

@@ -1,14 +1,15 @@
 import ivm from 'isolated-vm';
 import * as esbuild from 'esbuild';
 import type * as harFormat from 'har-format';
-import { withHar, createHarLog } from 'node-fetch-har';
+import { createHarLog } from 'node-fetch-har';
 import * as fs from 'fs/promises';
 import fetch from 'node-fetch';
 import * as path from 'path';
 import { FunctionResult } from './types';
 import { LogEntry } from '../../components/Console';
 import { FetchOptions } from './runtime/types';
 import projectRoot from '../../server/projectRoot';
+import { withHarInstrumentation } from '../../utils/har';
 
 async function fetchRuntimeModule() {
   const filePath = path.resolve(projectRoot, './src/toolpadDataSources/function/dist/index.js');
@@ -42,16 +43,10 @@ export default async function execFunction(
   const logs: LogEntry[] = [];
   const har: harFormat.Har = createHarLog();
 
-  const instrumentedFetch: typeof fetch = withHar(fetch, { har });
+  const instrumentedFetch = withHarInstrumentation(fetch, { har });
 
   const fetchStub = new ivm.Reference((url: string, rawOptions: FetchOptions) => {
-    const options = {
-      ...rawOptions,
-      // node-fetch-har doesn't deal well with arrays here
-      headers: Object.fromEntries(rawOptions.headers || []),
-    };
-
-    return instrumentedFetch(url, options).then(
+    return instrumentedFetch(url, rawOptions).then(
       (res) => {
         const resHeadersInit = Array.from(res.headers.entries());
 

packages/toolpad-app/src/toolpadDataSources/function/runtime/types.ts

@@ -3,7 +3,7 @@ import type * as ivm from 'isolated-vm';
 export interface FetchOptions {
   headers: [string, string][];
   method: string;
-  mode?: string;
+  mode?: RequestMode;
   body?: string;
 }
 

packages/toolpad-app/src/utils/har.spec.ts

@@ -0,0 +1,46 @@
+import fetch from 'node-fetch';
+import { createHarLog, withHarInstrumentation } from './har';
+import { streamToString } from './streams';
+import { startServer } from './tests';
+
+describe('har', () => {
+  test('headers in array form', async () => {
+    const { port, stopServer } = await startServer(async (req, res) => {
+      res.write(
+        JSON.stringify({
+          body: await streamToString(req),
+          method: req.method,
+          headers: req.headers,
+        }),
+      );
+      res.end();
+    });
+
+    try {
+      const har = createHarLog();
+      const instruentedfetch = withHarInstrumentation(fetch, { har });
+
+      const res = await instruentedfetch(`http://localhost:${port}`, {
+        headers: [['foo', 'bar']],
+        method: 'POST',
+        body: 'baz',
+      });
+
+      expect(res.ok).toBeTruthy();
+
+      const body = await res.json();
+
+      expect(body).toEqual(
+        expect.objectContaining({
+          body: 'baz',
+          method: 'POST',
+          headers: expect.objectContaining({
+            foo: 'bar',
+          }),
+        }),
+      );
+    } finally {
+      await stopServer();
+    }
+  });
+});

packages/toolpad-app/src/utils/har.ts

@@ -0,0 +1,30 @@
+import { withHar as withHarOriginal } from 'node-fetch-har';
+import fetch, { Request } from 'node-fetch';
+
+const withHarInstrumentation: typeof withHarOriginal = function withHar(fetchFn, options) {
+  const withHarFetch = withHarOriginal(fetchFn, options);
+
+  const patchedfetch = (...args: Parameters<typeof fetch>): ReturnType<typeof fetch> => {
+    // node-fetch-har doesn't deal well with certain ways of passing parameters e.g. passing headers as [string, string][]
+    // We're normalizing them here to a format that we know it can handle correctly:
+    const req = new Request(...args);
+    const input = req.url;
+    return withHarFetch(input, {
+      agent: req.agent,
+      body: req.body,
+      compress: req.compress,
+      follow: req.follow,
+      headers: req.headers,
+      method: req.method,
+      redirect: req.redirect,
+    });
+  };
+
+  patchedfetch.isRedirect = fetch.isRedirect;
+
+  return patchedfetch;
+};
+
+export { createHarLog } from 'node-fetch-har';
+export type { WithHarOptions } from 'node-fetch-har';
+export { withHarInstrumentation };

packages/toolpad-app/src/utils/streams.ts

@@ -0,0 +1,10 @@
+import { Readable } from 'stream';
+
+export function streamToString(stream: Readable) {
+  const chunks: Buffer[] = [];
+  return new Promise((resolve, reject) => {
+    stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
+    stream.on('error', (err) => reject(err));
+    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+  });
+}

packages/toolpad-app/src/utils/tests.ts

@@ -0,0 +1,31 @@
+import * as http from 'http';
+import getPort from 'get-port';
+
+export async function startServer(handler?: http.RequestListener) {
+  const server = http.createServer(handler);
+  const port = await getPort({ port: 3000 });
+  let app: http.Server | undefined;
+  await new Promise((resolve, reject) => {
+    app = server.listen(port);
+    app.once('listening', resolve);
+    app.once('error', reject);
+  });
+  return {
+    port,
+    async stopServer() {
+      await new Promise<void>((resolve, reject) => {
+        if (app) {
+          app.close((err) => {
+            if (err) {
+              reject(err);
+            } else {
+              resolve();
+            }
+          });
+        } else {
+          resolve();
+        }
+      });
+    },
+  };
+}

packages/toolpad-app/typings/node-fetch-har.d.ts

@@ -1,10 +1,11 @@
 declare module 'node-fetch-har' {
   import { Har } from 'har-format';
+  import fetch from 'node-fetch';
 
   export interface WithHarOptions {
     har?: Har;
   }
 
-  export function withHar(fetch: typeof fetch, options?: WithHarOptions): typeof fetch;
+  export function withHar(fetchFn: typeof fetch, options?: WithHarOptions): typeof fetch;
   export function createHarLog(): Har;
 }