diff --git a/src/lib/connectors/debugging-protocol-common/debugging-protocol-connector.ts b/src/lib/connectors/debugging-protocol-common/debugging-protocol-connector.ts index 7bb3c74605b..2264625ca11 100644 --- a/src/lib/connectors/debugging-protocol-common/debugging-protocol-connector.ts +++ b/src/lib/connectors/debugging-protocol-common/debugging-protocol-connector.ts @@ -298,13 +298,19 @@ export class Connector implements IConnector { } } - private async getResponseBody(cdpResponse) { + private async getResponseBody(cdpResponse): Promise<{ content: string, rawContent: Buffer, rawResponse(): Promise }> { let content: string = ''; let rawContent: Buffer = null; - let rawResponse: Buffer = null; + const rawResponse = (): Promise => { + return Promise.resolve(null); + }; + const fetchContent = this.fetchContent; + + const defaultBody = { content, rawContent, rawResponse }; if (cdpResponse.response.status !== 200) { - return { content, rawContent, rawResponse }; + // TODO: is this right? no-friendly-error-pages won't have a problem? + return defaultBody; } try { @@ -314,22 +320,51 @@ export class Connector implements IConnector { content = body; rawContent = Buffer.from(body, encoding); - if (rawContent.length.toString() === cdpResponse.response.headers['Content-Length']) { - // Response wasn't compressed so both buffers are the same - rawResponse = rawContent; - } else { - rawResponse = null; // TODO: Find a way to get this data - } + const returnValue = { + content, + rawContent, + rawResponse: () => { + const self = (this as any); + const cached = self._rawResponse; + + if (cached) { + return Promise.resolve(cached); + } + + if (rawContent.length.toString() === cdpResponse.response.headers['Content-Length']) { + // Response wasn't compressed so both buffers are the same + return Promise.resolve(rawContent); + } + + const { url: responseUrl, requestHeaders: headers } = cdpResponse.response; + + return fetchContent(responseUrl, headers) + .then((result) => { + const { response: { body: { rawResponse: rr } } } = result; + + return rr(); + }) + .then((value) => { + self._rawResponse = value; + + return value; + }); + } + }; + + debug(`Content for ${cutString(cdpResponse.response.url)} downloaded`); + + return returnValue; } catch (e) { debug(`Body requested after connection closed for request ${cdpResponse.requestId}`); rawContent = Buffer.alloc(0); } debug(`Content for ${cutString(cdpResponse.response.url)} downloaded`); - return { content, rawContent, rawResponse }; + return defaultBody; } - /** Returns a Response for the given request */ + /** Returns a Response for the given request. */ private async createResponse(cdpResponse): Promise { const resourceUrl: string = cdpResponse.response.url; const hops: Array = this._redirects.calculate(resourceUrl); @@ -802,7 +837,8 @@ export class Connector implements IConnector { public async fetchContent(target: URL | string, customHeaders?: object): Promise { // TODO: This should create a new tab, navigate to the // resource and control what is received somehow via an event. - const headers = Object.assign({}, this._headers, customHeaders); + const assigns = _.compact([this && this._headers, customHeaders]); + const headers = Object.assign({}, ...assigns); const href: string = typeof target === 'string' ? target : target.href; const request: Requester = new Requester({ headers }); const response: INetworkData = await request.get(href); diff --git a/src/lib/connectors/jsdom/jsdom.ts b/src/lib/connectors/jsdom/jsdom.ts index 6965d8ba411..8f3d714df12 100644 --- a/src/lib/connectors/jsdom/jsdom.ts +++ b/src/lib/connectors/jsdom/jsdom.ts @@ -111,7 +111,9 @@ class JSDOMConnector implements IConnector { content: body, contentEncoding: null, rawContent: null, - rawResponse: null + rawResponse() { + return Promise.resolve(null); + } }, headers: null, hops: [], diff --git a/src/lib/connectors/utils/requester.ts b/src/lib/connectors/utils/requester.ts index 8d4916af3d4..87f470ef236 100644 --- a/src/lib/connectors/utils/requester.ts +++ b/src/lib/connectors/utils/requester.ts @@ -118,7 +118,9 @@ export class Requester { content: body, contentEncoding: charset, rawContent: rawBody, - rawResponse: rawBodyResponse + rawResponse: () => { + return Promise.resolve(rawBodyResponse); + } }, headers: response.headers, hops, diff --git a/src/lib/types/network.ts b/src/lib/types/network.ts index a5426642591..6bc18f8f353 100644 --- a/src/lib/types/network.ts +++ b/src/lib/types/network.ts @@ -14,7 +14,7 @@ export interface IResponseBody { /** The uncompressed bytes of the response's body. */ rawContent: Buffer; /** The original bytes of the body. They could be compressed or not. */ - rawResponse: Buffer; + rawResponse(): Promise; } /** Response data from fetching an item using a connector. */ diff --git a/tests/lib/connectors/events.ts b/tests/lib/connectors/events.ts index c80048c5df9..a162e21a084 100644 --- a/tests/lib/connectors/events.ts +++ b/tests/lib/connectors/events.ts @@ -290,7 +290,7 @@ const testConnectorEvents = (connectorInfo) => { } // List of events that only have to be called once per execution - const singles = ['fetch::error', 'scan::start', 'scan::end', 'manifestfetch::missing']; + const singles = ['fetch::error', 'scan::start', 'scan::end', 'manifestfetch::missing', 'targetfetch::start', 'targetfetch::end']; const groupedEvents = _.groupBy(invokes, (invoke) => { return invoke[0]; }); diff --git a/tests/lib/connectors/fetchContent.ts b/tests/lib/connectors/fetchContent.ts index 1074454dfbd..bdb1cc96760 100644 --- a/tests/lib/connectors/fetchContent.ts +++ b/tests/lib/connectors/fetchContent.ts @@ -31,7 +31,7 @@ test.afterEach.always(async (t) => { await t.context.connector.close(); }); -const testConnectorEvaluate = (connectorInfo) => { +const testConnectorFetchContent = (connectorInfo) => { const connectorBuilder: IConnectorBuilder = connectorInfo.builder; const name: string = connectorInfo.name; @@ -46,15 +46,16 @@ const testConnectorEvaluate = (connectorInfo) => { server.configure({ '/edge.png': { content: file } }); const result: INetworkData = await connector.fetchContent(url.parse(`http://localhost:${server.port}/edge.png`)); + const rawResponse = await result.response.body.rawResponse(); t.is(result.response.statusCode, 200); t.true(file.equals(result.response.body.rawContent), 'rawContent is the same'); - // Because it is an image, the rawResponse should be the same - t.true(file.equals(result.response.body.rawResponse), 'rawResponse is the same'); + // Because it is an image and it is not send compressed, the rawResponse should be the same + t.true(file.equals(rawResponse), 'rawResponse is the same'); }); }; builders.forEach((connector) => { - testConnectorEvaluate(connector); + testConnectorFetchContent(connector); }); diff --git a/tests/lib/connectors/requestResponse.ts b/tests/lib/connectors/requestResponse.ts new file mode 100644 index 00000000000..f8b3cf21d4e --- /dev/null +++ b/tests/lib/connectors/requestResponse.ts @@ -0,0 +1,162 @@ +/** + * @fileoverview Minimum event functionality a connector must implement + * in order to be valid. + */ + +/* eslint-disable no-sync */ + +import * as fs from 'fs'; +import * as path from 'path'; +import * as url from 'url'; +import * as zlib from 'zlib'; + +import * as _ from 'lodash'; +import * as sinon from 'sinon'; +import test from 'ava'; + +import { builders } from '../../helpers/connectors'; +import { createServer } from '../../helpers/test-server'; +import { IConnector, IConnectorBuilder } from '../../../src/lib/types'; + +const sourceHtml = fs.readFileSync(path.join(__dirname, './fixtures/common/index.html'), 'utf8'); + +/** + * Updates all references to localhost to use the right port for the current instance. + * + * This does a deep search in all the object properties. + */ +const updateLocalhost = (content, port) => { + if (typeof content === 'string') { + return content.replace(/localhost\//g, `localhost:${port}/`); + } + + if (typeof content === 'number' || !content) { + return content; + } + + if (Array.isArray(content)) { + const transformed = _.map(content, (value) => { + return updateLocalhost(value, port); + }); + + return transformed; + } + + const transformed = _.reduce(content, (obj, value, key) => { + obj[key] = updateLocalhost(value, port); + + return obj; + }, {}); + + return transformed; +}; + + +test.beforeEach(async (t) => { + const sonar = { + emit() { }, + emitAsync() { } + }; + + sinon.spy(sonar, 'emitAsync'); + sinon.spy(sonar, 'emit'); + + const server = createServer(); + + await server.start(); + + const html = updateLocalhost(sourceHtml, server.port); + const gzipHtml = zlib.gzipSync(Buffer.from(html)); + + t.context = { + gzipHtml, + html, + server, + sonar + }; +}); + +test.afterEach.always(async (t) => { + t.context.sonar.emitAsync.restore(); + t.context.sonar.emit.restore(); + t.context.server.stop(); + await t.context.connector.close(); +}); + +const findEvent = (func, eventName) => { + for (let i = 0; i < func.callCount; i++) { + const args = func.getCall(i).args; + + if (args[0] === eventName) { + return args[1]; + } + } + + return null; +}; + +const testRequestResponse = (connectorInfo) => { + const connectorBuilder: IConnectorBuilder = connectorInfo.builder; + const name: string = connectorInfo.name; + + test(`[${name}] requestResponse`, async (t) => { + const { sonar } = t.context; + const { emit, emitAsync } = sonar; + const connector: IConnector = await (connectorBuilder)(sonar, {}); + const server = t.context.server; + + t.context.connector = connector; + + server.configure({ + '/': { + content: t.context.gzipHtml, + headers: { + 'content-encoding': 'gzip', + 'content-type': 'text/html' + } + } + }); + + await connector.collect(url.parse(`http://localhost:${server.port}/`)); + + const invokedTargetFetchEnd = findEvent(emitAsync, 'targetfetch::end') || findEvent(emit, 'targetfetch::end'); + /* eslint-disable sort-keys */ + const expectedTargetFetchEnd = { + resource: `http://localhost:${server.port}/`, + request: { url: `http://localhost:${server.port}/` }, + response: { + body: { + content: t.context.html, + contentEncoding: 'utf-8', + rawContent: Buffer.from(t.context.html), + rawResponse() { + return Promise.resolve(t.context.gzipHtml); + } + }, + hops: [], + statusCode: 200, + url: 'http://localhost/' + } + }; + /* eslint-enable sort-keys */ + + if (!invokedTargetFetchEnd) { + t.fail(`targetfetch::end' event not found`); + + return; + } + + const { body: invokedBody } = invokedTargetFetchEnd.response; + const { body: expectedBody } = expectedTargetFetchEnd.response; + const [invokedRawResponse, expectedRawResponse] = await Promise.all([invokedBody.rawResponse(), expectedBody.rawResponse()]); + + t.true(expectedRawResponse.equals(invokedRawResponse), 'rawResponses are different'); + t.true(expectedBody.content === invokedBody.content, 'content is different'); + t.true(expectedBody.contentEncoding === invokedBody.contentEncoding, 'content-encoding is different'); + t.true(expectedBody.rawContent.equals(invokedBody.rawContent), 'rawContent is different'); + }); +}; + +builders.forEach((connector) => { + testRequestResponse(connector); +}); diff --git a/tests/lib/connectors/utils/requester.ts b/tests/lib/connectors/utils/requester.ts index ed5e8eb67cf..2f315096baf 100644 --- a/tests/lib/connectors/utils/requester.ts +++ b/tests/lib/connectors/utils/requester.ts @@ -65,7 +65,7 @@ const testTextDecoding = async (t, encoding: string, contentType: string, useCom const { requester, server } = t.context; const originalBytes = iconv.encode(text, encoding); const transformedText = iconv.decode(originalBytes, encoding); - const content = useCompression ? await compress(originalBytes) : originalBytes; + const content: Buffer = useCompression ? await compress(originalBytes) : originalBytes; server.configure({ '/': { @@ -78,15 +78,16 @@ const testTextDecoding = async (t, encoding: string, contentType: string, useCom }); const { response: { body } } = await requester.get(`http://localhost:${server.port}`); + const rawResponse = await body.rawResponse(); // body is a `string` t.is(body.content, transformedText); // rawBody is a `Buffer` with the uncompressed bytes of the response - t.deepEqual(body.rawContent, originalBytes); + t.true(originalBytes.equals(body.rawContent), 'rawContent is not the same'); // rawBodyResponse is a `Buffer` with the original bytes of the response - t.deepEqual(body.rawResponse, content); + t.true(content.equals(rawResponse)); }; supportedEncodings.forEach((encoding) => {