fix: decode page in native automation mode (#8099)

<!--
Thank you for your contribution.

Before making a PR, please read our contributing guidelines at

https://github.com/DevExpress/testcafe/blob/master/CONTRIBUTING.md#code-contribution

We recommend creating a *draft* PR, so that you can mark it as 'ready
for review' when you are done.
-->

## Purpose
Incorrect decoding page in native automation mode.

## Approach
The methods for decoding a page to a string have been modified.

[PR-hammerhead](DevExpress/testcafe-hammerhead#2984)

## References
_Provide a link to the existing issue(s), if any._

## Pre-Merge TODO
- [x] Write tests for your proposed changes
- [x] Make sure that existing tests do not fail

Author: PavelMor25

package-lock.json

@@ -142,7 +142,7 @@
     "source-map-support": "^0.5.16",
     "strip-bom": "^2.0.0",
     "testcafe-browser-tools": "2.0.26",
-    "testcafe-hammerhead": "31.7.0",
+    "testcafe-hammerhead": "31.7.1",
     "testcafe-legacy-api": "5.1.6",
     "testcafe-reporter-json": "^2.1.0",
     "testcafe-reporter-list": "^2.2.0",

package.json

@@ -12,7 +12,8 @@ import CertificateErrorEvent = Protocol.Security.CertificateErrorEvent;
 import HeaderEntry = Protocol.Fetch.HeaderEntry;
 import NativeAutomationRequestHookEventProvider from '../request-hooks/event-provider';
 import ResourceInjector, { ResourceInjectorOptions } from '../resource-injector';
-import { convertToHeaderEntries } from '../utils/headers';
+import { convertToHeaderEntries, getHeaderEntry } from '../utils/headers';
+import httpHeaders from '../../utils/http-headers';
 
 import {
     createRequestPausedEventForResponse,
@@ -157,12 +158,13 @@ export default class NativeAutomationRequestPipeline extends NativeAutomationApi
             await this._resourceInjector.processNonProxiedContent(fulfillInfo, this._client, sessionId);
         else {
             const userScripts = await this._getUserScripts(event);
+            const contentType = getHeaderEntry(event.responseHeaders, httpHeaders.contentType)?.value;
 
             await this._resourceInjector.processHTMLPageContent(fulfillInfo, {
                 isIframe:       false,
                 contextStorage: this.contextStorage,
                 userScripts,
-            }, this._client, sessionId);
+            }, this._client, sessionId, contentType);
         }
 
         requestPipelineMockLogger(`sent mocked response for the ${event.requestId}`);
@@ -201,7 +203,8 @@ export default class NativeAutomationRequestPipeline extends NativeAutomationApi
             return;
         }
 
-        const resourceInfo = await this._resourceInjector.getDocumentResourceInfo(event, this._client);
+        const contentType = getHeaderEntry(event.responseHeaders, httpHeaders.contentType)?.value;
+        const resourceInfo = await this._resourceInjector.getDocumentResourceInfo(event, this._client, contentType);
 
         if (resourceInfo.error) {
             if (this._shouldRedirectToErrorPage(event)) {
@@ -242,7 +245,7 @@ export default class NativeAutomationRequestPipeline extends NativeAutomationApi
                     contextStorage:    this.contextStorage,
                     userScripts,
                 },
-                this._client, sessionId);
+                this._client, sessionId, contentType);
 
             this._contextInfo.dispose(getRequestId(event));
 
@@ -258,9 +261,7 @@ export default class NativeAutomationRequestPipeline extends NativeAutomationApi
     }
 
     private static _isPage (responseHeaders: HeaderEntry[] | undefined): boolean {
-        const contentType = responseHeaders
-            ?.find(header => header.name.toLowerCase() === 'content-type')
-            ?.value;
+        const contentType = getHeaderEntry(responseHeaders, httpHeaders.contentType)?.value;
 
         if (contentType)
             return contentTypeUtils.isPage(contentType);
@@ -502,7 +503,7 @@ export default class NativeAutomationRequestPipeline extends NativeAutomationApi
         const headers = this._formatHeadersForContinueResponse(event.request.headers);
 
         for (const changedHeader of reqOpts._changedHeaders) {
-            const targetHeader = headers.find(header => header.name.toLowerCase() === changedHeader.name) as HeaderEntry;
+            const targetHeader = getHeaderEntry(headers, changedHeader.name);
 
             if (targetHeader)
                 targetHeader.value = changedHeader.value;

src/native-automation/request-pipeline/index.ts

@@ -137,13 +137,13 @@ export default class ResourceInjector {
         return stringifyHeaderValues(headers);
     }
 
-    private async _fulfillRequest (client: ProtocolApi, fulfillRequestInfo: FulfillRequestRequest, body: string, sessionId: SessionId): Promise<void> {
+    private async _fulfillRequest (client: ProtocolApi, fulfillRequestInfo: FulfillRequestRequest, body: string, sessionId: SessionId, contentType?: string): Promise<void> {
         await safeFulfillRequest(client, {
             requestId:       fulfillRequestInfo.requestId,
             responseCode:    fulfillRequestInfo.responseCode || StatusCodes.OK,
             responsePhrase:  fulfillRequestInfo.responsePhrase,
             responseHeaders: this._processResponseHeaders(fulfillRequestInfo.responseHeaders),
-            body:            toBase64String(body),
+            body:            toBase64String(body, contentType),
         }, sessionId);
     }
 
@@ -158,7 +158,7 @@ export default class ResourceInjector {
         await navigateTo(client, this._options.specialServiceRoutes.errorPage1);
     }
 
-    public async getDocumentResourceInfo (event: RequestPausedEvent, client: ProtocolApi): Promise<DocumentResourceInfo> {
+    public async getDocumentResourceInfo (event: RequestPausedEvent, client: ProtocolApi, contentType?: string): Promise<DocumentResourceInfo> {
         const {
             requestId,
             request,
@@ -184,7 +184,7 @@ export default class ResourceInjector {
             }
 
             const responseObj = await client.Fetch.getResponseBody({ requestId });
-            const responseStr = getResponseAsString(responseObj);
+            const responseStr = getResponseAsString(responseObj, contentType);
 
             return {
                 error: null,
@@ -213,7 +213,7 @@ export default class ResourceInjector {
         });
     }
 
-    public async processHTMLPageContent (fulfillRequestInfo: FulfillRequestRequest, injectableResourcesOptions: InjectableResourcesOptions, client: ProtocolApi, sessionId: SessionId): Promise<void> {
+    public async processHTMLPageContent (fulfillRequestInfo: FulfillRequestRequest, injectableResourcesOptions: InjectableResourcesOptions, client: ProtocolApi, sessionId: SessionId, contentType?: string): Promise<void> {
         const injectableResources = await this._prepareInjectableResources(injectableResourcesOptions);
 
         // NOTE: an unhandled exception interrupts the test execution,
@@ -227,7 +227,7 @@ export default class ResourceInjector {
                 this._getPageInjectableResourcesOptions(injectableResourcesOptions),
             );
 
-            await this._fulfillRequest(client, fulfillRequestInfo, updatedResponseStr, sessionId);
+            await this._fulfillRequest(client, fulfillRequestInfo, updatedResponseStr, sessionId, contentType);
         }
     }
 

src/native-automation/resource-injector.ts

@@ -25,3 +25,7 @@ export function convertToOutgoingHttpHeaders (headers: HeaderEntry[] | undefined
     }, {});
 }
 
+export function getHeaderEntry (headers: HeaderEntry[] | undefined, headerName: string): HeaderEntry | undefined {
+    return headers?.find(header => header.name.toLowerCase() === headerName);
+}
+

src/native-automation/utils/headers.ts

@@ -1,11 +1,15 @@
 import Protocol from 'devtools-protocol';
 import GetResponseBodyResponse = Protocol.Network.GetResponseBodyResponse;
 import HeaderEntry = Protocol.Fetch.HeaderEntry;
+import { decodeBufferToString, encodeStringToBuffer } from 'testcafe-hammerhead';
 
-export function getResponseAsString (response: GetResponseBodyResponse): string {
-    return response.base64Encoded
-        ? Buffer.from(response.body, 'base64').toString()
-        : response.body;
+export function getResponseAsString (response: GetResponseBodyResponse, contentType?: string): string {
+    if (!contentType)
+        return response.base64Encoded ? Buffer.from(response.body, 'base64').toString() : response.body;
+
+    const bufferBody = getResponseAsBuffer(response);
+
+    return decodeBufferToString(bufferBody, contentType);
 }
 
 export function getResponseAsBuffer (response: GetResponseBodyResponse): Buffer {
@@ -14,8 +18,10 @@ export function getResponseAsBuffer (response: GetResponseBodyResponse): Buffer
         : Buffer.from(response.body);
 }
 
-export function toBase64String (str: string): string {
-    return Buffer.from(str).toString('base64');
+export function toBase64String (str: string, contentType?: string): string {
+    const bufferBody = contentType ? encodeStringToBuffer(str, contentType) : Buffer.from(str);
+
+    return bufferBody.toString('base64');
 }
 
 export function fromBase64String (str: string): Buffer {

src/native-automation/utils/string.ts

@@ -0,0 +1,7 @@
+const { onlyDescribeInNativeAutomation } = require('../../../utils/skip-in');
+
+onlyDescribeInNativeAutomation('[Regression](GH-7529)', function () {
+    it('In Native Automation mode, the page should be decoded as in proxy mode', function () {
+        return runTests('./testcafe-fixtures/index.js');
+    });
+});

test/functional/fixtures/regression/gh-7529/test.js

@@ -0,0 +1,10 @@
+import { Selector } from 'testcafe';
+
+fixture `Regression GH-7529`
+    .page `http://localhost:3000/fixtures/regression/gh-7529/`;
+
+test('Decode page in native automation mode', async t => {
+    const title = await Selector('h1').textContent;
+
+    await t.expect(title).eql('codage réussi');
+});

test/functional/fixtures/regression/gh-7529/testcafe-fixtures/index.js

@@ -149,6 +149,26 @@ Server.prototype._setupRoutes = function (apiRouter) {
         `);
     });
 
+    this.app.get('/fixtures/regression/gh-7529/', function (req, res) {
+        const html = `
+            <!DOCTYPE html>
+            <html lang="fr">
+            <head>
+                <meta charset="ISO-8859-15">
+                <title>GH-7529</title>
+            </head>
+            <body>
+            <h1>codage réussi</h1>
+            </body>
+            </html>
+        `;
+
+        const content = Buffer.from(html, 'latin1');
+
+        res.setHeader('content-type', 'text/html; charset=iso-8859-15');
+        res.send(content);
+    });
+
     this.app.get('*', function (req, res) {
         const reqPath      = req.params[0] || '';
         const resourcePath = path.join(server.basePath, reqPath);