inspector: add protocol methods retrieving sent/received data

Add protocol method `Network.dataSent` to buffer request data. And
expose protocol methods `Network.getRequestPostData` and
`Network.getResponseBody` allowing devtool to retrieve buffered data.

Author: legendecas

doc/api/inspector.md

@@ -524,6 +524,20 @@ This feature is only available with the `--experimental-network-inspection` flag
 Broadcasts the `Network.dataReceived` event to connected frontends, or buffers the data if
 `Network.streamResourceContent` command was not invoked for the given request yet.
 
+Also enables `Network.getResponseBody` command to retrieve the response data.
+
+### `inspector.Network.dataSent([params])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `params` {Object}
+
+This feature is only available with the `--experimental-network-inspection` flag enabled.
+
+Enables `Network.getRequestPostData` command to retrieve the request data.
+
 ### `inspector.Network.requestWillBeSent([params])`
 
 <!-- YAML

lib/inspector.js

@@ -214,6 +214,7 @@ const Network = {
   responseReceived: (params) => broadcastToFrontend('Network.responseReceived', params),
   loadingFinished: (params) => broadcastToFrontend('Network.loadingFinished', params),
   loadingFailed: (params) => broadcastToFrontend('Network.loadingFailed', params),
+  dataSent: (params) => broadcastToFrontend('Network.dataSent', params),
   dataReceived: (params) => broadcastToFrontend('Network.dataReceived', params),
 };
 

lib/internal/inspector/network.js

@@ -6,6 +6,7 @@ const {
 } = primordials;
 
 const { now } = require('internal/perf/utils');
+const { MIMEType } = require('internal/mime');
 const kInspectorRequestId = Symbol('kInspectorRequestId');
 
 // https://chromedevtools.github.io/devtools-protocol/1-3/Network/#type-ResourceType
@@ -46,9 +47,29 @@ function getNextRequestId() {
   return `node-network-event-${++requestId}`;
 };
 
+function sniffMimeType(contentType) {
+  let mimeType;
+  let charset;
+  try {
+    const mimeTypeObj = new MIMEType(contentType);
+    mimeType = mimeTypeObj.essence || '';
+    charset = mimeTypeObj.params.get('charset') || '';
+  } catch {
+    mimeType = '';
+    charset = '';
+  }
+
+  return {
+    __proto__: null,
+    mimeType,
+    charset,
+  };
+}
+
 module.exports = {
   kInspectorRequestId,
   kResourceType,
   getMonotonicTime,
   getNextRequestId,
+  sniffMimeType,
 };

lib/internal/inspector/network_http.js

@@ -13,35 +13,43 @@ const {
   kResourceType,
   getMonotonicTime,
   getNextRequestId,
+  sniffMimeType,
 } = require('internal/inspector/network');
 const dc = require('diagnostics_channel');
 const { Network } = require('inspector');
-const { MIMEType } = require('internal/mime');
 
 const kRequestUrl = Symbol('kRequestUrl');
 
 // Convert a Headers object (Map<string, number | string | string[]>) to a plain object (Map<string, string>)
 const convertHeaderObject = (headers = {}) => {
   // The 'host' header that contains the host and port of the URL.
   let host;
+  let charset;
+  let mimeType;
   const dict = {};
   for (const { 0: key, 1: value } of ObjectEntries(headers)) {
-    if (key.toLowerCase() === 'host') {
+    const lowerCasedKey = key.toLowerCase();
+    if (lowerCasedKey === 'host') {
       host = value;
     }
+    if (lowerCasedKey === 'content-type') {
+      const result = sniffMimeType(value);
+      charset = result.charset;
+      mimeType = result.mimeType;
+    }
     if (typeof value === 'string') {
       dict[key] = value;
     } else if (ArrayIsArray(value)) {
-      if (key.toLowerCase() === 'cookie') dict[key] = value.join('; ');
+      if (lowerCasedKey === 'cookie') dict[key] = value.join('; ');
       // ChromeDevTools frontend treats 'set-cookie' as a special case
       // https://github.com/ChromeDevTools/devtools-frontend/blob/4275917f84266ef40613db3c1784a25f902ea74e/front_end/core/sdk/NetworkRequest.ts#L1368
-      else if (key.toLowerCase() === 'set-cookie') dict[key] = value.join('\n');
+      else if (lowerCasedKey === 'set-cookie') dict[key] = value.join('\n');
       else dict[key] = value.join(', ');
     } else {
       dict[key] = String(value);
     }
   }
-  return [host, dict];
+  return [dict, host, charset, mimeType];
 };
 
 /**
@@ -52,14 +60,15 @@ const convertHeaderObject = (headers = {}) => {
 function onClientRequestCreated({ request }) {
   request[kInspectorRequestId] = getNextRequestId();
 
-  const { 0: host, 1: headers } = convertHeaderObject(request.getHeaders());
+  const { 0: headers, 1: host, 2: charset } = convertHeaderObject(request.getHeaders());
   const url = `${request.protocol}//${host}${request.path}`;
   request[kRequestUrl] = url;
 
   Network.requestWillBeSent({
     requestId: request[kInspectorRequestId],
     timestamp: getMonotonicTime(),
     wallTime: DateNow(),
+    charset,
     request: {
       url,
       method: request.method,
@@ -95,16 +104,7 @@ function onClientResponseFinish({ request, response }) {
     return;
   }
 
-  let mimeType;
-  let charset;
-  try {
-    const mimeTypeObj = new MIMEType(response.headers['content-type']);
-    mimeType = mimeTypeObj.essence || '';
-    charset = mimeTypeObj.params.get('charset') || '';
-  } catch {
-    mimeType = '';
-    charset = '';
-  }
+  const { 0: headers, 2: charset, 3: mimeType } = convertHeaderObject(response.headers);
 
   Network.responseReceived({
     requestId: request[kInspectorRequestId],
@@ -114,7 +114,7 @@ function onClientResponseFinish({ request, response }) {
       url: request[kRequestUrl],
       status: response.statusCode,
       statusText: response.statusMessage ?? '',
-      headers: convertHeaderObject(response.headers)[1],
+      headers,
       mimeType,
       charset,
     },

lib/internal/inspector/network_undici.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const {
-  ArrayPrototypeFindIndex,
   DateNow,
 } = primordials;
 
@@ -10,10 +9,10 @@ const {
   kResourceType,
   getMonotonicTime,
   getNextRequestId,
+  sniffMimeType,
 } = require('internal/inspector/network');
 const dc = require('diagnostics_channel');
 const { Network } = require('inspector');
-const { MIMEType } = require('internal/mime');
 
 // Convert an undici request headers array to a plain object (Map<string, string>)
 function requestHeadersArrayToDictionary(headers) {
@@ -29,21 +28,30 @@ function requestHeadersArrayToDictionary(headers) {
 // Convert an undici response headers array to a plain object (Map<string, string>)
 function responseHeadersArrayToDictionary(headers) {
   const dict = {};
+  let charset;
+  let mimeType;
   for (let idx = 0; idx < headers.length; idx += 2) {
     const key = `${headers[idx]}`;
+    const lowerCasedKey = key.toLowerCase();
     const value = `${headers[idx + 1]}`;
     const prevValue = dict[key];
 
+    if (lowerCasedKey === 'content-type') {
+      const result = sniffMimeType(value);
+      charset = result.charset;
+      mimeType = result.mimeType;
+    }
+
     if (typeof prevValue === 'string') {
       // ChromeDevTools frontend treats 'set-cookie' as a special case
       // https://github.com/ChromeDevTools/devtools-frontend/blob/4275917f84266ef40613db3c1784a25f902ea74e/front_end/core/sdk/NetworkRequest.ts#L1368
-      if (key.toLowerCase() === 'set-cookie') dict[key] = `${prevValue}\n${value}`;
+      if (lowerCasedKey === 'set-cookie') dict[key] = `${prevValue}\n${value}`;
       else dict[key] = `${prevValue}, ${value}`;
     } else {
       dict[key] = value;
     }
   }
-  return dict;
+  return [dict, charset, mimeType];
 };
 
 /**
@@ -54,10 +62,15 @@ function responseHeadersArrayToDictionary(headers) {
 function onClientRequestStart({ request }) {
   const url = `${request.origin}${request.path}`;
   request[kInspectorRequestId] = getNextRequestId();
+
+  const headers = requestHeadersArrayToDictionary(request.headers);
+  const { charset } = sniffMimeType(headers);
+
   Network.requestWillBeSent({
     requestId: request[kInspectorRequestId],
     timestamp: getMonotonicTime(),
     wallTime: DateNow(),
+    charset,
     request: {
       url,
       method: request.method,
@@ -94,19 +107,7 @@ function onClientResponseHeaders({ request, response }) {
     return;
   }
 
-  let mimeType;
-  let charset;
-  try {
-    const contentTypeKeyIndex =
-      ArrayPrototypeFindIndex(response.headers, (header) => header.toString().toLowerCase() === 'content-type');
-    const contentType = contentTypeKeyIndex !== -1 ? response.headers[contentTypeKeyIndex + 1].toString() : '';
-    const mimeTypeObj = new MIMEType(contentType);
-    mimeType = mimeTypeObj.essence || '';
-    charset = mimeTypeObj.params.get('charset') || '';
-  } catch {
-    mimeType = '';
-    charset = '';
-  }
+  const { 0: headers, 1: charset, 2: mimeType } = responseHeadersArrayToDictionary(response.headers);
 
   const url = `${request.origin}${request.path}`;
   Network.responseReceived({
@@ -118,7 +119,7 @@ function onClientResponseHeaders({ request, response }) {
       url,
       status: response.statusCode,
       statusText: response.statusText,
-      headers: responseHeadersArrayToDictionary(response.headers),
+      headers,
       mimeType,
       charset,
     },