diff --git a/.sonarwhalrc b/.sonarwhalrc index 2aa92ed50af..48a4d92c099 100644 --- a/.sonarwhalrc +++ b/.sonarwhalrc @@ -17,6 +17,7 @@ "highest-available-document-mode": "warning", "html-checker": "warning", "http-cache": "warning", + "http-compression": "warning", "image-optimization-cloudinary": "warning", "manifest-app-name": "error", "manifest-exists": "warning", diff --git a/docs/user-guide/rules/http-compression.md b/docs/user-guide/rules/http-compression.md new file mode 100644 index 00000000000..218b68e4d86 --- /dev/null +++ b/docs/user-guide/rules/http-compression.md @@ -0,0 +1,568 @@ +# Require resources to be served compressed (`http-compression`) + +`http-compression` warns against not serving resources compressed when +requested as such using the most appropriate encoding. + +## Why is this important? + +One of the fastest and easiest ways one can improve the web site's/app's +performance is to reduce the amount of data that needs to get delivered +to the client by using HTTP compression. This not only [reduces the data +used by the user][wdmsc], but can also significallty cut down on the +server costs. + +However, there are a few things that need to be done right in order for +get the most out of compression: + +* Only compress resources for which the result of the compression + will be smaller than original size. + + In general text-based resources (HTML, CSS, JavaScript, SVGs, etc.) + compresss very well especially if the file is not very small. + The same goes for some other file formats (e.g.: ICO files, web fonts + such as EOT, OTF, and TTF, etc.) + + However, compressing resources that are already compressed (e.g.: + images, audio files, PDFs, etc.) not only waste CPU resources, but + usually result in little to no reduction, or in some cases even + a bigger file size. + + The same goes for resources that are very small because of the + overhead of compression file formats. + +* Use most efficien compression method. + + `gzip` is the most used encoding nowadays as it strikes a good + balance between compression ratio (as [high as 70%][gzip ratio] + especially for larger files) and encoding time, and is supported + pretty much everywhere. + + Better savings can be achieved using [`Zopfli`][zopfli] which + can reduce the size on average [3–8% more than `gzip`][zopfli + blog post]. Since `Zopfli` output (for the `gzip` option) is valid + `gzip` content, `Zopfli` works eveywere `gzip` works. The only + downsize is that encoding takes more time than with `gzip`, thus, + making `Zopfli` more suitable for static content (i.e. encoding + resources as part of build script, not on the fly). + + But, things can be improved even futher using [Brotli][brotli]. + This encoding allows to get [20–26% higher compression ratios][brotli + blog post] even over `Zopfli`. However, this encoding is not compatible + with `gzip`, limiting the support to modern browsers and its usage to + [only over HTTPS (as proxies misinterpreting unknown encodings)][brotli + over https]. + + So, in general, for best performance and interoperability resources + should be served compress with `Zopfli`, and `Brotli` over HTTPS with + a fallback to `Zopfli` if not supported HTTPS. + +* Avoid using deprecated or not widlly supported compression formats, + and `Content-Type` values. + + Avoid using deprecated `Content-Type` values such as `x-gzip`. Some + user agents may alias them to the correct, current equivalent value + (e.g.: alias `x-gzip` to `gzip`), but that is not always true. + + Also avoid using encoding that are not widely supported (e.g.: + `compress`, `bzip2`, [`SDCH`][unship sdch], etc.), and/or may not + be as efficient, or can create problems (e.g.: [`deflate`][deflate + issues]). In general these should be avoided, and one should just + stick to the encoding specified in the previous point. + +* Avoid potential caching related issues. + + When resources are served compressed, they should be served with + the `Vary` header containing the `Accept-Encoding` value (or with + something such as `Cache-Control: private` that prevents caching + in proxy caches and such altogether). + + This needs to be done in order to avoid problems such as an + intermediate proxy caching the compress version of the resource and + then sending it to all user agents regardless if they support that + particular encoding or not, or if they even want the compressed + version or not. + +* Resources should be served compressed only when requested as such, + appropriately encoded, and without relying on user agent sniffing. + + The `Accept-Encoding` request header specified should be respected. + Sending a content encoded with a different encoding than one of the + ones accepted can lead to problems. + +* Dealing with special cases. + + One such special case are `SVGZ` files that are just `SVG` files + compressed with `gzip`. Since they are already compressed, they + shouldn't be compressed again. However sending them without the + `Content-Encoding: gzip` header will create problems as user agents + will not know they need to decompress them before displaying them, + and thus, try to display them directly. + +## What does the rule check? + +The rule checks for the use cases previously specified, namely, it +checks that: + +* Only resources for which the result of the compression is smaller + than original size are served compressed. + +* The most efficient encodigs are used (by default the rule check if + `Zopfli` is used over HTTP and Brotli over `HTTPS`, however that can + be changed, see: [`Can the rule be configured?` + section](#can-the-rule-be-configured)). + +* Deprecated or not widely supported encodigs, and `Content-Type` + values are not used. + +* Potential caching related issues are avoided. + +* Resources are served compressed only when requested as such, are + appropriately encoded, and no user sniffing is done. + +* Special cases (such as `SVGZ`) are handled correctly. + +### Examples that **trigger** the rule + +Resource that should be compressed is not served compressed. + +e.g.: When the request for `https://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate, br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Type: text/javascript + + +``` + +Resource that should not be compressed is served compressed. + +e.g.: When the request for `https://example.com/example.png` contains + +```text +... +Accept-Encoding: gzip, deflate, br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: br +Content-Type: image/png +Vary: Accept-Encoding + + +``` + +Resource that compressed results in a bigger or equal size to the +uncompressed size is still served compressed. + +e.g.: For `http://example.com/example.js` containing only `const x = 5;`, +using the defaults, the sizes may be as follows. + +```text +origina size: 13 bytes + +gzip size: 38 bytes +zopfli size: 33 bytes +brotli size: 17 bytes +``` + +When the request for `http://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: gzip +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +Resource that should be compressed is served compressed with deprecated +or disallowed compression method or `Content-Encoding` value. + +e.g.: When the request for `http://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate +``` + +response contains deprecated `x-gzip` value for `Content-Encoding` + +```text +HTTP/... 200 OK + +... +Content-Encoding: x-gzip +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +response is compressed with disallowed `compress` compression method + +```text +HTTP/... 200 OK + +... +Content-Encoding: compress +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +or response tries to use deprecated SDCH + +```text +HTTP/... 200 OK + +... +Content-Encoding: gzip, +Content-Type: text/javascript +Get-Dictionary: /dictionaries/search_dict, /dictionaries/help_dict +Vary: Accept-Encoding + + +``` + +Resource that should be compressed is not served compressed using +`Zopfli` over HTTP. + +e.g.: When the request for `http://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: gzip +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +Resource that should be compressed is served compressed using `Brotli` +over HTTP. + +e.g.: When the request for `http://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate, br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: br +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +Resource that should be compressed is not served compressed using +`Brotli` over HTTPS. + +e.g.: When the request for `https://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate, br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: gzip +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +Resource that is served compressed doesn't account for caching +(e.g: is not served with the `Vary` header with the `Accept-Encoding` +value included, or something such as `Cache-Control: private`). + +e.g.: When the request for `https://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate, br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: br +Content-Type: text/javascript + + +``` + +Resource is blindly served compressed using `gzip` no matter what the +user agent advertises as supporting. + +E.g.: When the request for `https://example.com/example.js` contains + +```text +... +Accept-Encoding: br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: gzip +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +Resource is served compressed only for certain user agents. + +E.g.: When the request for `https://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate, br +User-Agent: Mozilla/5.0 Gecko +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Type: text/javascript + + +``` + +however when requested with + +```text +... +Accept-Encoding: gzip, deflate, br +User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:57.0) Gecko/20100101 Firefox/57.0 +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: br +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +`SVGZ` resource is not served with `Content-Encoding: gzip` header: + +E.g.: When the request for `https://example.com/example.svgz` contains + +```text +... +Accept-Encoding: gzip, deflate, br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Type: image/svg+xml + + +``` + +### Examples that **pass** the rule + +Resource that should be compressed is served compressed using `Zopfli` +over HTTP and with the `Vary: Accept-Encoding` header. + +e.g.: When the request for `http://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: gzip +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +Resource that should be compressed is served compressed using `Brotli` +over HTTPS and with the `Vary: Accept-Encoding` header. + +e.g.: When the request for `https://example.com/example.js` contains + +```text +... +Accept-Encoding: gzip, deflate, br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: br +Content-Type: text/javascript +Vary: Accept-Encoding + + +``` + +Resource that should not be compressed is not served compressed. + +e.g.: When the request for `https://example.com/example.png` contains + +```text +... +Accept-Encoding: gzip, deflate, br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Type: image/png + + +``` + +`SVGZ` resource is served with `Content-Encoding: gzip` header: + +e.g.: When the request for `https://example.com/example.svgz` contains + +```text +... +Accept-Encoding: gzip, deflate, br +``` + +response is + +```text +HTTP/... 200 OK + +... +Content-Encoding: gzip +Content-Type: image/svg+xml + + +``` + +## Can the rule be configured? + +You can override the defaults by specifying what type of compression +you don't want the rule to check for. This can be done for the `target` +(main page) and/or the `resources` the rule determines should be served +compressed, using the following format: + +```json +"http-compression": [ "warning", { + "resource": { + "": , + ... + }, + "target": { + "": , + ... + } +} +``` + +Where `` can be one of: `brotli`, `gzip`, or +`zopfli`. + +E.g. If you want the rule to check if only the page resources are +served compressed using `Brotli`, and not the page itself, you can +use the following configuration: + +```json +"http-compression": [ "warning", { + "target": { + "brotli": false + } +}] +``` + +Note: You can also use the [`ignoredUrls`](../index.md#rule-configuration) +property from the `.sonarwhalrc` file to exclude domains you don’t control +(e.g.: CDNs) from these checks. + + + +[brotli blog post]: https://opensource.googleblog.com/2015/09/introducing-brotli-new-compression.html +[brotli over https]: https://medium.com/@yoavweiss/well-the-technical-reason-for-brotli-being-https-only-is-that-otherwise-there-s-a-very-high-508f15f0ad95 +[brotli]: https://github.com/google/brotli +[deflate issues]: https://stackoverflow.com/questions/9170338/why-are-major-web-sites-using-GZIP/9186091#9186091 +[gzip is not enough]: https://www.youtube.com/watch?v=whGwm0Lky2s +[gzip ratio]: https://www.youtube.com/watch?v=Mjab_aZsdxw&t=24s +[unship sdch]: https://groups.google.com/a/chromium.org/forum/#!topic/blink-dev/nQl0ORHy7sw +[wdmsc]: https://whatdoesmysitecost.com/ +[zopfli blog post]: https://developers.googleblog.com/2013/02/compress-data-more-densely-with-zopfli.html +[zopfli]: https://github.com/google/zopfli diff --git a/docs/user-guide/rules/index.md b/docs/user-guide/rules/index.md index 4ecf5a791a1..6b168d8c084 100644 --- a/docs/user-guide/rules/index.md +++ b/docs/user-guide/rules/index.md @@ -23,6 +23,7 @@ * [`amp-validator`](amp-validator.md) * [`http-cache`](http-cache.md) +* [`http-compression`](http-compression.md) * [`image-optimization-cloudinary`](image-optimization-cloudinary.md) * [`no-html-only-headers`](no-html-only-headers.md) * [`no-http-redirects`](no-http-redirects.md) diff --git a/package-lock.json b/package-lock.json index 35c564122a2..e01fe3deeb0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2118,6 +2118,11 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "base64-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.2.1.tgz", + "integrity": "sha512-dwVUVIXsBZXwTuwnXI9RK8sBmgq09NDHzyR9SAph9eqk76gKK2JSQmZARC2zRC81JC2QTtxD0ARU5qTS25gIGw==" + }, "bcrypt-pbkdf": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", @@ -2267,6 +2272,14 @@ "repeat-element": "1.1.2" } }, + "brotli": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.2.tgz", + "integrity": "sha1-UlqcrU/LqWR119OI9q7LE+7VL0Y=", + "requires": { + "base64-js": "1.2.1" + } + }, "browserslist": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-2.11.0.tgz", diff --git a/package.json b/package.json index 693256fc866..ccc4fadb823 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "dependencies": { "amphtml-validator": "^1.0.21", "axe-core": "^2.6.1", + "brotli": "^1.3.2", "browserslist": "^2.11.0", "caniuse-api": "^2.0.0", "canvas-prebuilt": "^1.6.0", diff --git a/src/bin/sonarwhal.ts b/src/bin/sonarwhal.ts index 0fdb563889c..944c57d0a2b 100644 --- a/src/bin/sonarwhal.ts +++ b/src/bin/sonarwhal.ts @@ -42,11 +42,13 @@ process.once('uncaughtException', (err) => { }); process.once('unhandledRejection', (reason) => { + const source = reason.error ? reason.error : reason; + console.error(`Unhandled rejection promise: - uri: ${reason.uri} - message: ${reason.error.message} + uri: ${source.uri} + message: ${source.message} stack: -${reason.error.stack}`); +${source.stack}`); process.exit(1); }); diff --git a/src/lib/connectors/debugging-protocol-common/debugging-protocol-connector.ts b/src/lib/connectors/debugging-protocol-common/debugging-protocol-connector.ts index d84df75b995..9d2e556fac3 100644 --- a/src/lib/connectors/debugging-protocol-common/debugging-protocol-connector.ts +++ b/src/lib/connectors/debugging-protocol-common/debugging-protocol-connector.ts @@ -380,7 +380,7 @@ export class Connector implements IConnector { * */ - const validHeaders = Object.entries(headers).reduce((final, [key, value]) => { + const validHeaders = Object.entries(headers || {}).reduce((final, [key, value]) => { if (key.startsWith(':')) { return final; } diff --git a/src/lib/connectors/utils/requester.ts b/src/lib/connectors/utils/requester.ts index 27c3ff7c2f3..38e3ff687a1 100644 --- a/src/lib/connectors/utils/requester.ts +++ b/src/lib/connectors/utils/requester.ts @@ -48,6 +48,7 @@ export class Requester { public constructor(customOptions?: request.CoreOptions) { if (customOptions) { customOptions.followRedirect = false; + customOptions.rejectUnauthorized = false; this._maxRedirects = customOptions.maxRedirects || this._maxRedirects; } const options: request.CoreOptions = Object.assign({}, defaults, customOptions); diff --git a/src/lib/rules/http-compression/compression-check-options.ts b/src/lib/rules/http-compression/compression-check-options.ts new file mode 100644 index 00000000000..fbdebd206b0 --- /dev/null +++ b/src/lib/rules/http-compression/compression-check-options.ts @@ -0,0 +1,5 @@ +export type CompressionCheckOptions = { + brotli: boolean; + gzip: boolean; + zopfli: boolean; +}; diff --git a/src/lib/rules/http-compression/http-compression.ts b/src/lib/rules/http-compression/http-compression.ts new file mode 100644 index 00000000000..c55888fb9a9 --- /dev/null +++ b/src/lib/rules/http-compression/http-compression.ts @@ -0,0 +1,682 @@ +/** + * @fileoverview Check if resources are served compressed when requested + * as such using the most appropriate encoding. + */ + +/* + * ------------------------------------------------------------------------------ + * Requirements + * ------------------------------------------------------------------------------ + */ + +import * as decompressBrotli from 'brotli/decompress'; +import * as mimeDB from 'mime-db'; + +import { Category } from '../../enums/category'; +import { getFileExtension, isTextMediaType } from '../../utils/content-type'; +import { getHeaderValueNormalized, isRegularProtocol, isHTTP, normalizeString } from '../../utils/misc'; +import { IAsyncHTMLElement, IResponse, IRule, IRuleBuilder, IFetchEnd } from '../../types'; +import { RuleContext } from '../../rule-context'; +import { CompressionCheckOptions } from './compression-check-options'; + +const uaString = 'Mozilla/5.0 Gecko'; + +/* + * ------------------------------------------------------------------------------ + * Public + * ------------------------------------------------------------------------------ + */ + +const rule: IRuleBuilder = { + create(context: RuleContext): IRule { + + const getRuleOptions = (property: string): CompressionCheckOptions => { + return Object.assign( + {}, + { + brotli: true, + gzip: true, + zopfli: true + }, + (context.ruleOptions && context.ruleOptions[property]) + ); + }; + + const resourceOptions: CompressionCheckOptions = getRuleOptions('resource'); + const targetOptions: CompressionCheckOptions = getRuleOptions('target'); + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + const checkIfBytesMatch = (rawResponse: Buffer, magicNumbers) => { + return rawResponse && magicNumbers.every((b, i) => { + return rawResponse[i] === b; + }); + }; + + const getHeaderValues = (headers, headerName) => { + return (getHeaderValueNormalized(headers, headerName) || '').split(','); + }; + + const checkVaryHeader = async (resource, element, headers) => { + const varyHeaderValues = getHeaderValues(headers, 'vary'); + const cacheControlValues = getHeaderValues(headers, 'cache-control'); + + if (!cacheControlValues.includes('private') && + !varyHeaderValues.includes('accept-encoding')) { + await context.report(resource, element, `Should be served with the 'Vary' header containing 'Accept-Encoding' value.`); + } + }; + + const generateDisallowedCompressionMessage = (encoding: string) => { + return `Disallowed compression method: '${encoding}'.`; + }; + + const generateContentEncodingMessage = (encoding: string, notRequired?: boolean, suffix?: string) => { + return `Should${notRequired ? ' not' : ''} be served with the 'content-encoding${encoding ? `: ${encoding}` : ''}' header${suffix ? ` ${suffix}` : ''}.`; + }; + + const generateCompressionMessage = (encoding: string, notRequired?: boolean, suffix?: string) => { + return `Should${notRequired ? ' not' : ''} be served compressed${encoding ? ` with ${encoding}` : ''}${suffix ? ` ${suffix}` : ''}.`; + }; + + const generateSizeMessage = async (resource: string, element: IAsyncHTMLElement, encoding: string, sizeDifference) => { + await context.report(resource, element, `Should not be served compressed with ${encoding} as the compressed size is ${sizeDifference > 0 ? 'bigger than' : 'the same size as'} the uncompressed one.`); + }; + + const getNetworkData = async (resource: string, requestHeaders) => { + const networkData = await context.fetchContent(resource, requestHeaders); + + return { + contentEncodingHeaderValue: getHeaderValueNormalized(networkData.response.headers, 'content-encoding'), + rawContent: networkData.response.body.rawContent, + rawResponse: await networkData.response.body.rawResponse(), + response: networkData.response + }; + }; + + const isCompressedWithBrotli = (rawResponse: Buffer): boolean => { + + /* + * Brotli doesn't currently contain any magic numbers. + * https://github.com/google/brotli/issues/298#issuecomment-172549140 + */ + + try { + decompressBrotli(rawResponse); + } catch (e) { + return false; + } + + return true; + }; + + const isCompressedWithGzip = (rawContent: Buffer): boolean => { + // See: https://tools.ietf.org/html/rfc1952#page-5. + return checkIfBytesMatch(rawContent, [0x1f, 0x8b]); + }; + + const isNotCompressedWithZopfli = (rawResponse: Buffer): boolean => { + + /* + * Since the Zopfli output (for the gzip option) is valid + * gzip content, there doesn't seem to be a straightforward + * and foolproof way to identify files compressed with Zopfli. + * + * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * + * From an email discussion with @lvandeve: + * + * " There is no way to tell for sure. Adding information + * to the output to indicate zopfli, would actually + * add bits to the output so such thing is not done :) + * Any compressor can set the FLG, MTIME, and so on + * to anything it wants, and users of zopfli can also + * change the MTIME bytes that zopfli had output to an + * actual time. + * + * One heuristic to tell that it was compressed with + * zopfli or another dense deflate compressor is to + * compress it with regular gzip -9 (which is fast), + * and compare that the size of the file to test is + * for example more than 3% smaller. " + * + * Using the above mentioned for every resource `sonarwhal` + * encounters can be expensive, plus, for the online scanner, + * it might also cause some security (?) problems. + * + * So, since this is not a foolproof way to identify files + * compressed with Zopfli, the following still not foolproof, + * but faster way is used. + * + * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * + * 1. gzip + * + * A gzip member header has the following structure + * + * +---+---+---+---+---+---+---+---+---+---+ + * |ID1|ID2|CM |FLG| MTIME |XFL|OS | (more-->) + * +---+---+---+---+---+---+---+---+---+---+ + * + * where: + * + * ID1 = 1f and ID2 = 8b - these are the magic + * numbers that uniquely identify the content + * as being gzip. + * + * CM = 8 - this is a value customarily used by gzip + * + * FLG and MTIME are usually non-zero values. + * + * XFL will be either 0, 2, or 4: + * + * 0 - default, compressor used intermediate levels + * of compression (when any of the -2 ... -8 + * options are used). + * + * 2 - the compressor used maximum compression, + * slowest algorithm (when the -9 or --best + * option is used). + * + * 4 - the compressor used fastest algorithm (when + * the -1 or --fast option is used). + * + * 2. Zopfli + * + * One thing that Zopfli does is that it sets FLG and + * MTIME to 0, XFL to 2, and OS to 3 [1], so basically + * files compressed with Zopfli will most likely start + * with `1f8b 0800 0000 0000 0203`, unless things are + * changed by the user (which in general doesn't seem + * very likely to happen). + * + * Now, regular gzip output might also start with that, + * even thought the chance of doing so is smaller: + * + * * Most web servers (e.g.: Apache², NGINX³), by default, + * will not opt users into the best compression level, + * therefore, the output shouldn't have XFL set to 2. + * + * * Most utilities that output regular gzip will have + * non-zero values for MTIME and FLG. + * + * So, if a file does not start with: + * + * `1f8b 0800 0000 0000 0203` + * + * it's a good (not perfect) indication that Zopfli wasn't + * used, but it's a fast check compared to compressing + * files and comparing file sizes. However, if a file does + * start with that, it can be either Zopfli or gzip, and + * we cannot really make assumptions here. + * + * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * + * Ref: + * + * ¹ https://github.com/google/zopfli/blob/6818a0859063b946094fb6f94732836404a0d89a/src/zopfli/gzip_container.c#L90-L101) + * ² https://httpd.apache.org/docs/current/mod/mod_deflate.html#DeflateCompressionLevel + * ³ https://nginx.org/en/docs/http/ngx_http_gzip_module.html#gzip_comp_level + */ + + return !checkIfBytesMatch(rawResponse, [0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x03]); + }; + + const checkBrotli = async (resource, element) => { + const { contentEncodingHeaderValue, rawResponse, response } = await getNetworkData(resource, { 'Accept-Encoding': 'br' }); + + const compressedWithBrotli = isCompressedWithBrotli(rawResponse); + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // Check if compressed with Brotli over HTTP. + + if (isHTTP(resource)) { + if (compressedWithBrotli) { + await context.report(resource, element, generateCompressionMessage('Brotli', true, 'over HTTP')); + } + + return; + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // Check compressed vs. uncompressed sizes. + + /* + * TODO: Remove the following once connectors + * support Brotli compression. + */ + const rawContent = compressedWithBrotli ? decompressBrotli(rawResponse) : response.body.rawContent; + + const itShouldNotBeCompressed = contentEncodingHeaderValue === 'br' && + rawContent.byteLength <= rawResponse.byteLength; + + if (compressedWithBrotli && itShouldNotBeCompressed) { + generateSizeMessage(resource, element, 'Brotli', rawResponse.byteLength - rawContent.byteLength); + + return; + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // Check if compressed. + + if (!compressedWithBrotli) { + await context.report(resource, element, generateCompressionMessage('Brotli', false, 'over HTTPS')); + + return; + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // Check related headers. + + await checkVaryHeader(resource, element, response.headers); + + if (contentEncodingHeaderValue !== 'br') { + await context.report(resource, element, generateContentEncodingMessage('br')); + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // Check for user agent sniffing. + + const { rawResponse: uaRawResponse } = await getNetworkData(resource, { + 'Accept-Encoding': 'br', + 'User-Agent': uaString + }); + + if (!isCompressedWithBrotli(uaRawResponse)) { + await context.report(resource, element, generateCompressionMessage('Brotli', false, `over HTTPS regardless of the user agent`)); + } + }; + + const checkGzipZopfli = async (resource: string, element: IAsyncHTMLElement, shouldCheckIfCompressedWith: CompressionCheckOptions) => { + const { contentEncodingHeaderValue, rawContent, rawResponse, response } = await getNetworkData(resource, { 'Accept-Encoding': 'gzip' }); + + const compressedWithGzip = isCompressedWithGzip(rawResponse); + const notCompressedWithZopfli = isNotCompressedWithZopfli(rawResponse); + const itShouldNotBeCompressed = contentEncodingHeaderValue === 'gzip' && + rawContent.byteLength <= rawResponse.byteLength; + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // Check compressed vs. uncompressed sizes. + + if (compressedWithGzip && itShouldNotBeCompressed) { + generateSizeMessage(resource, element, notCompressedWithZopfli ? 'gzip' : 'Zopfli', rawResponse.byteLength - rawContent.byteLength); + + return; + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // Check if compressed. + + if (!compressedWithGzip && shouldCheckIfCompressedWith.gzip) { + await context.report(resource, element, generateCompressionMessage('gzip')); + + return; + } + + if (notCompressedWithZopfli && shouldCheckIfCompressedWith.zopfli) { + await context.report(resource, element, generateCompressionMessage('Zopfli')); + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // Check related headers. + + if (shouldCheckIfCompressedWith.gzip || + shouldCheckIfCompressedWith.zopfli) { + await checkVaryHeader(resource, element, response.headers); + + if (contentEncodingHeaderValue !== 'gzip') { + await context.report(resource, element, generateContentEncodingMessage('gzip')); + } + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // Check for user agent sniffing. + + const { rawResponse: uaRawResponse } = await getNetworkData(resource, { + 'Accept-Encoding': 'gzip', + 'User-Agent': uaString + }); + + if (!isCompressedWithGzip(uaRawResponse) && + shouldCheckIfCompressedWith.gzip) { + await context.report(resource, element, generateCompressionMessage('gzip', false, `regardless of the user agent`)); + + return; + } + + if (isNotCompressedWithZopfli(uaRawResponse) && + !notCompressedWithZopfli && + shouldCheckIfCompressedWith.zopfli) { + await context.report(resource, element, generateCompressionMessage('Zopfli', false, `regardless of the user agent`)); + } + + }; + + const responseIsCompressed = (rawResponse: Buffer, contentEncodingHeaderValue: string): boolean => { + return isCompressedWithGzip(rawResponse) || + isCompressedWithBrotli(rawResponse) || + + /* + * Other compression methods may be used, but there is + * no way to check for all possible cases. So, if this + * point is reached, just consider 'content-encoding' + * header as a possible indication of the response being + * compressed. + */ + + (contentEncodingHeaderValue && + + /* + * Although `identity` should not be sent as + * a value for `content-encoding`, if sent, for + * the * scope of this function, just ignore it + * and consider no encoding was specified. + * + * From (now kinda obsolete) + * https://tools.ietf.org/html/rfc2616#page-24: + * + * " identity + * + * The default (identity) encoding; the use of no + * transformation whatsoever. This content-coding + * is used only in the Accept-Encoding header, and + * SHOULD NOT be used in the Content-Encoding header. " + * + * See also: http://httpwg.org/specs/rfc7231.html#content.coding.registration + */ + + (contentEncodingHeaderValue !== 'identity')); + }; + + const checkForDisallowedCompressionMethods = async (resource: string, element: IAsyncHTMLElement, response: IResponse) => { + + // See: https://www.iana.org/assignments/http-parameters/http-parameters.xml. + + const contentEncodingHeaderValue = getHeaderValueNormalized(response.headers, 'content-encoding'); + + if (!contentEncodingHeaderValue) { + return; + } + + const encodings = contentEncodingHeaderValue.split(','); + + for (const encoding of encodings) { + if (!['gzip', 'br'].includes(encoding)) { + + /* + * `x-gzip` is deprecated but usually user agents + * alias it to `gzip`, so if the content is actual + * `gzip`, don't trigger an error here as the gzip + * related check will show an error for the response + * not being served with `content-encoding: gzip`. + */ + + if (encoding === 'x-gzip' && isCompressedWithGzip(await response.body.rawResponse())) { + return; + } + + // For anything else just flag it as disallowed. + await context.report(resource, element, generateDisallowedCompressionMessage(encoding)); + } + } + + /* + * Special cases: + * + * * SDCH (Shared Dictionary Compression over HTTP) + * https://lists.w3.org/Archives/Public/ietf-http-wg/2008JulSep/att-0441/Shared_Dictionary_Compression_over_HTTP.pdf + * Theoretically this should only happen if the user + * agent advertises support for SDCH, but yet again, + * server might be misconfigured. + * + * For SDCH, the first response will not contain anything + * special regarding the `content-encoding` header, however, + * it will contain the `get-dictionary` header. + */ + + if (normalizeString(response.headers['get-dictionary'])) { + await context.report(resource, element, generateDisallowedCompressionMessage('sdch')); + } + }; + + const checkUncompressed = async (resource: string, element: IAsyncHTMLElement) => { + + /* + * From: http://httpwg.org/specs/rfc7231.html#header.accept-encoding + * + * " An "identity" token is used as a synonym for + * "no encoding" in order to communicate when no + * encoding is preferred. + * + * ... + * + * If no Accept-Encoding field is in the request, + * any content-coding is considered acceptable by + * the user agent. " + */ + + const { contentEncodingHeaderValue, rawResponse } = await getNetworkData(resource, { 'Accept-Encoding': 'identity' }); + + if (responseIsCompressed(rawResponse, contentEncodingHeaderValue)) { + await context.report(resource, element, generateCompressionMessage('', true, `for requests made with 'Accept-Encoding: identity'`)); + } + + if (contentEncodingHeaderValue) { + await context.report(resource, element, generateContentEncodingMessage('', true, `for requests made with 'Accept-Encoding: identity'`)); + } + }; + + const isCompressibleAccordingToMediaType = (mediaType: string): boolean => { + const COMMON_MEDIA_TYPES_THAT_SHOULD_BE_COMPRESSED = [ + 'image/x-icon', + 'image/bmp' + ]; + + const COMMON_MEDIA_TYPES_THAT_SHOULD_NOT_BE_COMPRESSED = [ + 'image/jpeg', + 'image/png', + 'image/gif', + 'font/woff2', + 'font/woff', + 'font/otf', + 'font/ttf' + ]; + + if (!mediaType) { + return false; + } + + /* + * The reason for doing the following is because + * `mime-db` is quite big, so querying it for + * eveything is expensive. + */ + + /* + * Check if the media type is one of the common + * ones for which it is known the response should + * be compressed. + */ + + if (isTextMediaType(mediaType) || + COMMON_MEDIA_TYPES_THAT_SHOULD_BE_COMPRESSED.includes(mediaType)) { + return true; + } + + /* + * Check if the media type is one of the common + * ones for which it is known the response should + * not be compressed. + */ + + if (COMMON_MEDIA_TYPES_THAT_SHOULD_NOT_BE_COMPRESSED.includes(mediaType)) { + return false; + } + + /* + * If the media type is not included in any of the + * above, check `mime-db`. + */ + + const typeInfo = mimeDB[mediaType]; + + return typeInfo && typeInfo.compressible; + + }; + + const isSpecialCase = async (resource: string, element: IAsyncHTMLElement, response: IResponse): Promise => { + + /* + * Check for special cases: + * + * * Files that are by default compressed with gzip. + * + * SVGZ files are by default compressed with gzip, so + * by not sending them with the `Content-Encoding: gzip` + * header, browsers will not be able to display them + * correctly. + */ + + if ((response.mediaType === 'image/svg+xml' || getFileExtension(resource) === 'svgz') && + isCompressedWithGzip(await response.body.rawResponse())) { + + if (getHeaderValueNormalized(response.headers, 'content-encoding') !== 'gzip') { + await context.report(resource, element, generateContentEncodingMessage('gzip')); + } + + return true; + } + + return false; + }; + + const validate = (shouldCheckIfCompressedWith: CompressionCheckOptions) => { + return async (fetchEnd: IFetchEnd) => { + const { element, resource, response }: { element: IAsyncHTMLElement, resource: string, response: IResponse } = fetchEnd; + + /* + * We shouldn't validate error responses, and 204 (response with no body). + * Also some sites return body with 204 status code and that breaks `request`: + * https://github.com/request/request/issues/2669 + */ + if (response.statusCode !== 200) { + return; + } + + // It doesn't make sense for things that are not served over http(s) + if (!isRegularProtocol(resource)) { + return; + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + /* + * Check if this is a special case, and if it is, do the + * specific checks, but ignore all the checks that follow. + */ + + if (await isSpecialCase(resource, element, response)) { + return; + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + // If the resource should not be compressed: + if (!isCompressibleAccordingToMediaType(response.mediaType)) { + + const rawResponse = await response.body.rawResponse(); + const contentEncodingHeaderValue = getHeaderValueNormalized(response.headers, 'content-encoding'); + + // * Check if the resource is actually compressed. + if (responseIsCompressed(rawResponse, contentEncodingHeaderValue)) { + await context.report(resource, element, generateCompressionMessage('', true)); + } + + // * Check if resource is sent with the `Content-Encoding` header. + if (contentEncodingHeaderValue) { + await context.report(resource, element, `Should not be served with the 'content-encoding' header.`); + } + + return; + } + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + /* + * If the resource should be compressed: + * + * * Check if the resource is sent compressed with an + * deprecated or not recommended compression method. + */ + + await checkForDisallowedCompressionMethods(resource, element, response); + + // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + /* + * * Check if it's actually compressed and served in + * the correct/required compressed format. + * + * Note: Checking if servers respect the qvalue + * is beyond of the scope for the time being, + * so the followings won't check that. + */ + + await checkUncompressed(resource, element); + + if (shouldCheckIfCompressedWith.gzip || + shouldCheckIfCompressedWith.zopfli) { + await checkGzipZopfli(resource, element, shouldCheckIfCompressedWith); + } + + if (shouldCheckIfCompressedWith.brotli) { + await checkBrotli(resource, element); + } + }; + }; + + return { + 'fetch::end': validate(resourceOptions), + 'manifestfetch::end': validate(resourceOptions), + 'targetfetch::end': validate(targetOptions) + }; + }, + meta: { + docs: { + category: Category.performance, + description: 'Require resources to be served compressed' + }, + recommended: true, + schema: [{ + additionalProperties: false, + definitions: { + options: { + additionalProperties: false, + minProperties: 1, + properties: { + brotli: { type: 'boolean' }, + gzip: { type: 'boolean' }, + zopfli: { type: 'boolean' } + } + } + }, + properties: { + resource: { $ref: '#/definitions/options' }, + target: { $ref: '#/definitions/options' } + }, + type: 'object' + }], + worksWithLocalFiles: false + } +}; + +export default rule; diff --git a/src/lib/utils/misc.ts b/src/lib/utils/misc.ts index c23f9eb3837..39112767ace 100644 --- a/src/lib/utils/misc.ts +++ b/src/lib/utils/misc.ts @@ -47,6 +47,15 @@ const getFileExtension = (resource: string): string => { .pop(); }; +/** + * Remove whitespace from both ends of a header value and lowercase it. + * If `defaultValue` is provided, it will be return instead of the actual + * return value if that value is `null`. + */ +const getHeaderValueNormalized = (headers: object, headerName: string, defaultValue?: string) => { + return normalizeString(headers && headers[normalizeString(headerName)], defaultValue); // eslint-disable-line no-use-before-define, typescript/no-use-before-define +}; + /** Convenience function to check if a resource uses a specific protocol. */ const hasProtocol = (resource: string, protocol: string): boolean => { return url.parse(resource).protocol === protocol; @@ -85,6 +94,11 @@ const isHTMLDocument = (targetURL: string, responseHeaders: object): boolean => return mediaType === 'text/html'; }; +/** Convenience function to check if a resource is served over HTTP. */ +const isHTTP = (resource: string): boolean => { + return hasProtocol(resource, 'http:'); +}; + /** Convenience function to check if a resource is served over HTTPS. */ const isHTTPS = (resource: string): boolean => { return hasProtocol(resource, 'https:'); @@ -263,12 +277,14 @@ export { findNodeModulesRoot, findPackageRoot, getFileExtension, + getHeaderValueNormalized, hasAttributeWithValue, hasProtocol, isDataURI, isDirectory, isFile, isHTMLDocument, + isHTTP, isHTTPS, isLocalFile, isRegularProtocol, diff --git a/tests/helpers/connectors.ts b/tests/helpers/connectors.ts index d68d79e0357..3d7395d65a9 100644 --- a/tests/helpers/connectors.ts +++ b/tests/helpers/connectors.ts @@ -1,11 +1,17 @@ -// We need to import this interface to generate the definition of this file +/* + * This interface needs to be imported in order + * to generate the definition of this file. + */ import { IConnectorBuilder } from '../../src/lib/types'; // eslint-disable-line no-unused-vars import chromeBuilder from '../../src/lib/connectors/chrome/chrome'; import jsdomBuilder from '../../src/lib/connectors/jsdom/jsdom'; -/** The ids of the available connectors to test. */ -export const ids = ['jsdom', 'chrome']; +/** The IDs of the available connectors to test. */ +export const ids = [ + 'chrome', + 'jsdom' +]; /** The builders of the available connectors to test. */ export const builders: Array<{builder: IConnectorBuilder, name: string}> = [ diff --git a/tests/helpers/test-server.ts b/tests/helpers/test-server.ts index 02c242729a0..d57ffad6563 100644 --- a/tests/helpers/test-server.ts +++ b/tests/helpers/test-server.ts @@ -10,6 +10,8 @@ import * as _ from 'lodash'; import * as express from 'express'; import * as onHeaders from 'on-headers'; +import { getHeaderValueNormalized, normalizeString } from '../../src/lib/utils/misc'; + export type ServerConfiguration = string | object; //eslint-disable-line const maxPort = 65535; @@ -35,11 +37,11 @@ export class Server { * Because we don't know the port until we start the server, we need to update * the references to http://localhost in the HTML to http://localhost:finalport. */ - private updateLocalhost(html: string): string { + private updateLocalhost (html: string): string { return html.replace(/\/\/localhost\//g, `//localhost:${this._port}/`); } - private handleHeaders = (res, headers) => { + private handleHeaders (res, headers) { onHeaders(res, () => { Object.entries(headers).forEach(([header, value]) => { if (value !== null) { @@ -49,7 +51,173 @@ export class Server { } }); }); - }; + } + + private getContent (value): string { + if (typeof value === 'string') { + return this.updateLocalhost(value); + } else if (value && typeof value.content !== 'undefined') { + return typeof value.content === 'string' ? this.updateLocalhost(value.content) : value.content; + } + + return ''; + } + + private getNumberOfMatches (req, requestConditions) { + const headers = requestConditions.request && requestConditions.request.headers; + + /* + * Matching is done only based on headers, as for the time + * beeing there is no need to match based on other things. + */ + + if (!headers) { + return 0; + } + + let numberOfMatches = 0; + + for (const [header, value] of Object.entries(headers)) { + const headerValue = getHeaderValueNormalized(req.headers, header); + + if ((headerValue !== normalizeString(value)) || (!headerValue && (value === null))) { + return 0; + } + + numberOfMatches++; + } + + return numberOfMatches; + } + + private getValue (req, config) { + let bestNumberOfMatches = 1; + let bestMatch = null; + + for (const [key, value] of Object.entries(config)) { + let requestConditions; + + try { + requestConditions = JSON.parse(key); + + const newNumberOfMatches = this.getNumberOfMatches(req, requestConditions); + + if (newNumberOfMatches >= bestNumberOfMatches) { + bestMatch = value; + bestNumberOfMatches = newNumberOfMatches; + } + + } catch (e) { + // Ignore invalid keys. + } + } + + return bestMatch; + } + + private isConditionalConfig (configuration): boolean { + /* + * The following is done to quickly determine the type of + * configuration. Possible options are: + * + * 1) Simple config + * (server response is always the same) + * + * { + * name: ..., + * reports: [{ message: ... }], + * serverConfig: { + * '/': { + * content: ..., + * headers: ... + * }, + * ... + * } + * } + * + * + * 2) Conditional config + * (server response depends on the request) + * + * { + * name: ... + * reports: [{ message: ... }], + * serverConfig: { + * [JSON.stringify({ + * headers: { + * ... + * } + * })]: { + * '/': { + * content: ..., + * headers: ... + * }, + * }, + * ... + * } + * } + */ + + try { + return typeof JSON.parse(Object.entries(configuration)[0][0]) === 'object'; + } catch (e) { + // Ignore. + } + + return false; + } + + private normalizeConfig (configuration) { + const config = {}; + + /* + * This function convers something such as: + * + * { + * '{"request":{"headers":{"Accept-Encoding":"gzip"}}}': { + * '/': { + * content: ... + * headers: ... + * }, + * ... + * } + * '{"request":{"headers":{"Accept-Encoding":"br"}}}': { + * '/': { + * content: ... + * headers: ... + * }, + * ... + * } + * ... + * } + * + * to + * + * { + * '/': { + * '{"request":{"headers":{"Accept-Encoding":"gzip"}}}': { + * content: ... + * headers: ... + * }, + * '{"request":{"headers":{"Accept-Encoding":"br"}}}': { + * content: ... + * headers: ... + * }, + * ... + * } + * ... + * } + * + */ + + for (const [k, v] of Object.entries(configuration)) { + for (const [key, value] of Object.entries(v)) { + config[key] = Object.assign({}, config[key], { [k]: value}); + } + } + + return config; + } /** Applies the configuration for routes to the server. */ public configure(configuration: ServerConfiguration) { @@ -63,20 +231,17 @@ export class Server { return; } - _.forEach(configuration, (value, key) => { - customFavicon = customFavicon || key === '/favicon.ico'; - let content; + const conditionalConfig = this.isConditionalConfig(configuration); + const config = conditionalConfig ? this.normalizeConfig(configuration) : configuration; - if (typeof value === 'string') { - content = this.updateLocalhost(value); - } else if (value && typeof value.content !== 'undefined') { - content = typeof value.content === 'string' ? this.updateLocalhost(value.content) : value.content; - } else { - content = ''; - } + _.forEach(config, (val, key) => { + customFavicon = customFavicon || key === '/favicon.ico'; this._app.get(key, (req, res) => { + const value = conditionalConfig ? this.getValue(req, val): val; + const content = this.getContent(value); + /* * Hacky way to make `request` fail, but required * for testing cases such as the internet connection diff --git a/tests/lib/rules/http-compression/_tests.ts b/tests/lib/rules/http-compression/_tests.ts new file mode 100644 index 00000000000..f27e34d6184 --- /dev/null +++ b/tests/lib/rules/http-compression/_tests.ts @@ -0,0 +1,707 @@ +import * as fs from 'fs'; + +import { IRuleTest } from '../../../helpers/rule-test-type'; + +const uaString = 'Mozilla/5.0 Gecko'; + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// Error messages. + +const generateCompressionMessage = (encoding?: string, notRequired?: boolean, suffix?: string) => { + return `Should${notRequired ? ' not' : ''} be served compressed${encoding ? ` with ${encoding}` : ''}${suffix ? ` ${suffix}` : ''}.`; +}; + +const generateContentEncodingMessage = (encoding?: string, notRequired?: boolean, suffix?: string) => { + return `Should${notRequired ? ' not' : ''} be served with the 'content-encoding${encoding ? `: ${encoding}`: ''}' header${suffix ? ` ${suffix}` : ''}.`; +}; + +const generateDisallowedCompressionMessage = (encoding: string) => { + return `Disallowed compression method: '${encoding}'.`; +}; + +const generateSizeMessage = (encoding: string, differentSize: boolean) => { + return `Should not be served compressed with ${encoding} as the compressed size is ${differentSize ? 'bigger than' : 'the same size as'} the uncompressed one.`; +}; + +const generateUnneededContentEncodingMessage = (encoding?: string) => { + return `Should not be served with the 'content-encoding' header${encoding ? ` for requests made with 'Accept-Encoding: ${encoding}'` : ''}.`; +}; + +const varyMessage = `Should be served with the 'Vary' header containing 'Accept-Encoding' value.`; + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// Files. + +/* eslint-disable no-sync */ +const faviconFile = { + brotli: fs.readFileSync(`${__dirname}/fixtures/favicon.br`), + gzip: fs.readFileSync(`${__dirname}/fixtures/favicon.gz`), + original: fs.readFileSync(`${__dirname}/fixtures/favicon.ico`), + zopfli: fs.readFileSync(`${__dirname}/fixtures/favicon.zopfli.gz`) +}; + +const htmlFile = { + brotli: fs.readFileSync(`${__dirname}/fixtures/page.br`), + gzip: fs.readFileSync(`${__dirname}/fixtures/page.gz`), + original: fs.readFileSync(`${__dirname}/fixtures/page.html`), + zopfli: fs.readFileSync(`${__dirname}/fixtures/page.zopfli.gz`) +}; + +const imageFile = { + brotli: fs.readFileSync(`${__dirname}/fixtures/image.br`), + gzip: fs.readFileSync(`${__dirname}/fixtures/image.gz`), + original: fs.readFileSync(`${__dirname}/fixtures/image.png`), + zopfli: fs.readFileSync(`${__dirname}/fixtures/image.zopfli.gz`) +}; + +const scriptFile = { + brotli: fs.readFileSync(`${__dirname}/fixtures/script.br`), + gzip: fs.readFileSync(`${__dirname}/fixtures/script.gz`), + original: fs.readFileSync(`${__dirname}/fixtures/script.js`), + zopfli: fs.readFileSync(`${__dirname}/fixtures/script.zopfli.gz`) +}; + +const scriptSmallFile = { + brotli: fs.readFileSync(`${__dirname}/fixtures/script-small.br`), + gzip: fs.readFileSync(`${__dirname}/fixtures/script-small.gz`), + original: fs.readFileSync(`${__dirname}/fixtures/script-small.js`), + zopfli: fs.readFileSync(`${__dirname}/fixtures/script-small.zopfli.gz`) +}; + +const svgzFile = fs.readFileSync(`${__dirname}/fixtures/image.svgz`); +/* eslint-enable no-sync */ + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// Server configs. + +const createConfig = ({ + faviconFileContent = faviconFile.zopfli, + faviconFileHeaders = { + 'Content-Encoding': 'gzip', + Vary: 'Accept-Encoding' + }, + htmlFileContent = htmlFile.zopfli, + htmlFileHeaders = { + 'Content-Encoding': 'gzip', + Vary: 'Accept-Encoding' + }, + imageFileContent = imageFile.original, + imageFileHeaders = {}, + request = { headers: { 'Accept-Encoding': 'gzip, deflate, br' }}, + scriptFileContent = scriptFile.zopfli, + scriptFileHeaders = { + 'Content-Encoding': 'gzip', + Vary: 'Accept-Encoding' + }, + svgzFileContent = svgzFile, + svgzFileHeaders = { 'Content-Encoding': 'gzip' } +} = {}) => { + return { + [JSON.stringify({ request })]: { + '/': { + content: htmlFileContent, + headers: Object.assign({ 'Content-Type': 'text/html; charset=utf-8' }, htmlFileHeaders) + }, + '/favicon.ico': { + content: faviconFileContent, + headers: Object.assign({ 'Content-Type': 'image/x-icon' }, faviconFileHeaders) + }, + '/image.png': { + content: imageFileContent, + headers: Object.assign({ 'Content-Type': 'image/png' }, imageFileHeaders) + }, + '/image.svgz': { + content: svgzFileContent, + headers: Object.assign({ 'Content-Type': 'image/svg+xml' }, svgzFileHeaders) + }, + '/script.js': { + content: scriptFileContent, + headers: Object.assign({ 'Content-Type': 'text/javascript; charset=utf-8' }, scriptFileHeaders) + } + } + }; +}; + +const brotliConfigs = { + faviconFileContent: faviconFile.brotli, + faviconFileHeaders: { + 'Content-Encoding': 'br', + Vary: 'Accept-Encoding' + }, + htmlFileContent: htmlFile.brotli, + htmlFileHeaders: { + 'Content-Encoding': 'br', + Vary: 'Accept-Encoding' + }, + request: { headers: { 'Accept-Encoding': 'br' } }, + scriptFileContent: scriptFile.brotli, + scriptFileHeaders: { + 'Content-Encoding': 'br', + Vary: 'Accept-Encoding' + } +}; + +const noCompressionConfigs = { + faviconFileContent: faviconFile.original, + faviconFileHeaders: { + 'Content-Encoding': null, + Vary: null + }, + htmlFileContent: htmlFile.original, + htmlFileHeaders: { + 'Content-Encoding': null, + Vary: null + }, + request: { headers: { 'Accept-Encoding': 'identity' } }, + scriptFileContent: scriptFile.original, + scriptFileHeaders: { + 'Content-Encoding': null, + Vary: null + } +}; + +const createGzipZopfliConfigs = (configs = {}) => { + return Object.assign( + // Accept-Encoding: gzip + createConfig(Object.assign({ request: { headers: { 'Accept-Encoding': 'gzip' } } }, configs)), + + // Accept-Encoding: gzip, deflate (jsdom) + createConfig(Object.assign({ request: { headers: { 'Accept-Encoding': 'gzip, deflate' } } }, configs)), + + // Accept-Encoding: gzip, deflate, br (chrome) + createConfig(configs) + ); +}; + +const createServerConfig = (configs = {}, https: boolean = false) => { + return Object.assign( + + // Accept-Encoding: identity + createConfig(noCompressionConfigs), + + /* + * Accept-Encoding: gzip + * Accept-Encoding: gzip, deflate (jsdom) + * Accept-Encoding: gzip, deflate, br (chrome) + */ + createGzipZopfliConfigs(configs), + + // Accept-Encoding: br + createConfig(Object.assign( + { request: { headers: { 'Accept-Encoding': 'br' } } }, + https ? Object.assign({}, brotliConfigs, configs) : configs + )) + ); +}; + +const createGzipZopfliServerConfig = (configs, https: boolean = false) => { + return Object.assign( + createServerConfig({}, https), + createGzipZopfliConfigs(configs) + ); +}; + +const createBrotliServerConfig = (configs) => { + return createGzipZopfliServerConfig( + Object.assign( + {}, + brotliConfigs, + configs + ), + true + ); +}; + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +const testsForBrotli: Array = [ + { + name: `Resource is not served compressed with Brotli when Brotli compression is requested`, + reports: [{ message: generateCompressionMessage('Brotli', false, 'over HTTPS') }], + serverConfig: createBrotliServerConfig({ + scriptFileContent: scriptFile.original, + scriptFileHeaders: { 'Content-Encoding': null } + }) + }, + { + name: `Resource is served compressed with Brotli and without the 'Content-Encoding' header when Brotli compression is requested`, + reports: [{ message: generateContentEncodingMessage('br') }], + serverConfig: createBrotliServerConfig({ + scriptFileContent: scriptFile.brotli, + scriptFileHeaders: { + 'Content-Encoding': null, + vary: 'Accept-Encoding' + } + }) + } +]; + +const testsForBrotliOverHTTP: Array = [ + { + name: `Resource is served compressed with Brotli over HTTP`, + reports: [{ message: 'Should not be served compressed with Brotli over HTTP.' }], + serverConfig: createGzipZopfliServerConfig( + Object.assign( + { request: { headers: { 'Accept-Encoding': 'br' } } }, + { + scriptFileContent: scriptFile.brotli, + scriptFileHeaders: { + 'Content-Encoding': 'br', + vary: 'accept-encoding' + } + } + ) + ) + } +]; + +const testsForBrotliSmallSize: Array = [ + { + name: `Resource is served compressed with Brotli when Brotli compression is requested but uncompressed size is smaller the compressed size`, + reports: [{ message: generateSizeMessage('Brotli', true) }], + serverConfig: createBrotliServerConfig({ scriptFileContent: scriptSmallFile.brotli}) + } +]; + +const testsForBrotliUASniffing = (): Array => { + const headersConfig = { + request: { + headers: { + 'Accept-Encoding': 'br', + 'User-Agent': uaString + } + } + }; + + return [ + { + name: `Resource is not served compressed with Brotli when Brotli compression is requested with uncommon user agent string`, + reports: [{ message: generateCompressionMessage('Brotli', false, `over HTTPS regardless of the user agent`) }], + serverConfig: createBrotliServerConfig( + Object.assign( + headersConfig, + { + scriptFileContent: scriptFile.original, + scriptFileHeaders: { 'Content-Encoding': null} + } + ) + ) + } + ]; +}; + +const testsForDefaults = (https: boolean = false): Array => { + return [ + { + name: `Only resources that should be served compressed are served compressed`, + serverConfig: createServerConfig({}, https) + }, + + // Compression is applied to resources that should not be compressed.. + + { + name: `Resource that should not be served compressed is served compressed.`, + reports: [{ message: generateCompressionMessage('', true) }], + serverConfig: createGzipZopfliServerConfig( + { imageFileContent: imageFile.gzip }, + https + ) + }, + /* + * TODO: This breaks connectors. + * + * { + * name: `Resource that should not be served compressed is served with the 'Content-Encoding' header`, + * reports: [{ message: generateCompressionMessage('', true) }], + * serverConfig: createGzipZopfliServerConfig( + * { imageFileHeaders: { 'content-encoding': 'gzip' }, + * https + * ) + * }, + */ + { + name: `Resource that should not be served compressed is served compressed and with the 'Content-Encoding' header`, + reports: [ + { message: generateCompressionMessage('', true) }, + { message: generateContentEncodingMessage('', true) } + ], + serverConfig: createGzipZopfliServerConfig( + { + imageFileContent: imageFile.gzip, + imageFileHeaders: { 'Content-Encoding': 'gzip'} + }, + https + ) + } + ]; +}; + +const testsForDisallowedCompressionMethods = (https: boolean = false): Array => { + return [ + { + name: `Compressed resource is served with disallowed 'Content-Encoding: x-gzip' header`, + reports: [{ message: generateContentEncodingMessage('gzip') }], + serverConfig: createGzipZopfliServerConfig( + { + scriptFileHeaders: { + 'Content-Encoding': 'x-gzip', + vary: 'Accept-Encoding' + } + }, + https + ) + }, + { + name: `Compressed resource is served with disallowed 'Content-Encoding: x-compress' header`, + reports: [ + { message: generateDisallowedCompressionMessage('x-compress') }, + { message: generateCompressionMessage('gzip') } + ], + serverConfig: createGzipZopfliServerConfig( + { + scriptFileContent: scriptFile.original, + scriptFileHeaders: { + 'Content-Encoding': 'x-compress', + vary: 'Accept-Encoding' + } + }, + https + ) + }, + { + name: `Compressed resource is served with 'Get-Dictionary' header`, + reports: [{ message: generateDisallowedCompressionMessage('sdch') }], + serverConfig: createGzipZopfliServerConfig( + { + scriptFileHeaders: { + 'Content-Encoding': 'gzip', + 'geT-dictionary': '/dictionaries/search_dict, /dictionaries/help_dict', + vary: 'Accept-Encoding' + } + }, + https + ) + } + ]; +}; + +const testsForGzipZopfli = (https: boolean = false): Array => { + return [ + { + name: `Resource is not served compressed with gzip when gzip compression is requested`, + reports: [{ message: generateCompressionMessage('gzip') }], + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileContent: scriptFile.original, + scriptFileHeaders: { 'Content-Encoding': null} + }, + https + ) + }, + { + name: `Resource is served compressed with gzip and without the 'Content-Encoding' header when gzip compression is requested`, + reports: [ + { message: generateCompressionMessage('Zopfli') }, + { message: generateContentEncodingMessage('gzip') } + ], + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileContent: scriptFile.gzip, + scriptFileHeaders: { + 'Content-Encoding': null, + vary: 'Accept-Encoding' + } + }, + https + ) + }, + { + name: `Resource is not served compressed with Zopfli when gzip compression is requested`, + reports: [{ message: generateCompressionMessage('Zopfli') }], + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileContent: scriptFile.gzip, + scriptFileHeaders: { + 'Content-Encoding': 'gzip', + vary: 'Accept-Encoding' + } + }, + https + ) + }, + { + name: `Resource is served compressed with Zopfli and without the 'Content-Encoding' header when gzip compression is requested`, + reports: [{ message: generateContentEncodingMessage('gzip') }], + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileContent: scriptFile.zopfli, + scriptFileHeaders: { + 'Content-Encoding': null, + vary: 'Accept-Encoding' + } + }, + https + ) + } + ]; +}; + +const testsForGzipZopfliCaching = (https: boolean = false): Array => { + return [ + { + name: `Resource is served compressed with Zopfli and without the 'Vary' or 'Cache-Control' header when gzip compression is requested`, + reports: [{ message: varyMessage }], + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileHeaders: { + 'Cache-control': null, + 'Content-Encoding': 'gzip', + vary: null + } + }, + https + ) + }, + { + name: `Resource is served compressed with Zopfli and with 'Cache-Control: private, max-age=0' header when gzip compression is requested`, + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileHeaders: { + 'Cache-Control': 'private, max-age=0', + 'Content-Encoding': 'gzip', + vary: null + } + }, + https + ) + }, + { + name: `Resource is served compressed with Zopfli and with 'Vary: user-agent' header when gzip compression is requested`, + reports: [{ message: varyMessage }], + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileHeaders: { + 'Cache-Control': null, + 'Content-Encoding': 'gzip', + vary: 'user-agent' + } + }, + https + ) + }, + { + name: `Resource is served compressed with Zopfli and with 'Vary: user-agent, Accept-encoding' header when gzip compression is requested`, + reports: [{ message: varyMessage }], + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileHeaders: { + 'Content-Encoding': 'gzip', + vary: 'user-agent, Accept-encoding' + } + }, + https + ) + } + ]; +}; + +const testsForGzipZopfliSmallSize = (https: boolean = false): Array => { + return [ + { + name: `Resource is served compressed with gzip when gzip compression is requested but uncompressed size is smaller the compressed size`, + reports: [{ message: generateSizeMessage('gzip', true) }], + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileContent: scriptSmallFile.gzip + }, + https + ) + }, + { + name: `Resource is served compressed with Zopfli when gzip compression is requested but uncompressed size is smaller the compressed size`, + reports: [{ message: generateSizeMessage('Zopfli', true) }], + serverConfig: createGzipZopfliServerConfig( + { + request: { headers: { 'Accept-Encoding': 'gzip' }}, + scriptFileContent: scriptSmallFile.zopfli + }, + https + ) + } + ]; +}; + +const testsForGzipZopfliUASniffing = (https: boolean = false): Array => { + const headersConfig = { + request: { + headers: { + 'Accept-Encoding': 'gzip', + 'User-Agent': uaString + } + } + }; + + return [ + { + name: `Resource is not served compressed with gzip when gzip compression is requested with uncommon user agent string`, + reports: [{ message: generateCompressionMessage('gzip', false, `regardless of the user agent`) }], + serverConfig: createGzipZopfliServerConfig( + Object.assign( + headersConfig, + { + scriptFileContent: scriptFile.original, + scriptFileHeaders: { 'Content-Encoding': null} + } + ), + https + ) + }, + { + name: `Resource is not served compressed with Zopfli when Zopfli compression is requested with uncommon user agent string`, + reports: [{ message: generateCompressionMessage('Zopfli', false, `regardless of the user agent`) }], + serverConfig: createGzipZopfliServerConfig( + Object.assign( + headersConfig, + { + scriptFileContent: scriptFile.gzip, + scriptFileHeaders: { 'Content-Encoding': 'gzip'} + } + ), + https + ) + } + ]; +}; + +const testsForNoCompression = (https: boolean = false): Array => { + return [ + { + name: `Resource is served compressed when requested uncompressed`, + reports: [ + { message: `Should not be served compressed for requests made with 'Accept-Encoding: identity'.` }, + { message: `Should not be served with the 'content-encoding' header for requests made with 'Accept-Encoding: identity'.` } + ], + serverConfig: createGzipZopfliServerConfig( + { + faviconFileContent: faviconFile.original, + faviconFileHeaders: { 'Content-Encoding': null }, + htmlFileContent: htmlFile.original, + htmlFileHeaders: { 'Content-Encoding': null }, + request: { headers: { 'Accept-Encoding': 'identity' }} + }, + https + ) + }, + { + name: `Resource is served uncompressed and with the 'Content-Encoding: identity' header when requested uncompressed`, + + reports: [{ message: generateUnneededContentEncodingMessage('identity') }], + serverConfig: createGzipZopfliServerConfig( + { + faviconFileContent: faviconFile.original, + faviconFileHeaders: { 'Content-Encoding': null }, + htmlFileContent: htmlFile.original, + htmlFileHeaders: { 'Content-Encoding': null }, + request: { headers: { 'Accept-Encoding': 'identity' }}, + scriptFileContent: scriptFile.original, + scriptFileHeaders: { 'Content-Encoding': 'identity'} + }, + https + ) + }, + { + name: `Resources are served uncompressed when requested uncompressed`, + serverConfig: createGzipZopfliServerConfig( + { + faviconFileContent: faviconFile.original, + faviconFileHeaders: { 'Content-Encoding': null }, + htmlFileContent: htmlFile.original, + htmlFileHeaders: { 'Content-Encoding': null }, + request: { headers: { 'Accept-Encoding': 'identity' }}, + scriptFileContent: scriptFile.original, + scriptFileHeaders: { 'Content-Encoding': null } + }, + https + ) + } + ]; +}; + +const testsForSpecialCases = (https: boolean = false): Array => { + return [ + + // SVGZ. + + { + name: `SVGZ image is served without the 'Content-Encoding: gzip' header`, + reports: [{ message: generateContentEncodingMessage('gzip') }], + serverConfig: createServerConfig({ svgzFileHeaders: { 'Content-Encoding': null } }, https) + }, + { + name: `SVGZ image is served with the wrong 'Content-Encoding: br' header`, + reports: [{ message: generateContentEncodingMessage('gzip') }], + serverConfig: createServerConfig({ svgzFileHeaders: { 'Content-Encoding': 'x-gzip' } }, https) + } + ]; +}; + +const testsForUserConfigs = (encoding, isTarget: boolean = true, https: boolean = false): Array => { + const isBrotli = encoding === 'Brotli'; + const isGzip = encoding === 'gzip'; + + const configs = { request: { headers: { 'Accept-Encoding': isBrotli ? 'br' : 'gzip' }}}; + + if (!isBrotli) { + Object.assign(configs, { request: { headers: { vary: 'Accept-encoding' }}}); + } + + Object.assign( + configs, + isTarget ? + { + htmlFileContent: isGzip ? htmlFile.zopfli: htmlFile.gzip, + htmlFileHeaders: { 'Content-Encoding': null } + } : + { + scriptFileContent: isGzip ? scriptFile.zopfli: scriptFile.gzip, + scriptFileHeaders: { 'Content-Encoding': null } + } + ); + + return [ + { + name: `${isTarget ? 'Target' : 'Resource'} is not served compressed with ${encoding} when ${isBrotli ? 'Brotli': 'gzip'} compression is requested but the user configuration allows it`, + serverConfig: isBrotli && https ? createBrotliServerConfig(configs): createGzipZopfliServerConfig(configs, https) + } + ]; +}; + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +export { + testsForBrotli, + testsForBrotliOverHTTP, + testsForBrotliSmallSize, + testsForBrotliUASniffing, + testsForDefaults, + testsForDisallowedCompressionMethods, + testsForGzipZopfli, + testsForGzipZopfliCaching, + testsForGzipZopfliSmallSize, + testsForGzipZopfliUASniffing, + testsForNoCompression, + testsForSpecialCases, + testsForUserConfigs +}; diff --git a/tests/lib/rules/http-compression/fixtures/favicon.br b/tests/lib/rules/http-compression/fixtures/favicon.br new file mode 100644 index 00000000000..dde54cd6e1c Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/favicon.br differ diff --git a/tests/lib/rules/http-compression/fixtures/favicon.gz b/tests/lib/rules/http-compression/fixtures/favicon.gz new file mode 100644 index 00000000000..f6200f7a382 Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/favicon.gz differ diff --git a/tests/lib/rules/http-compression/fixtures/favicon.ico b/tests/lib/rules/http-compression/fixtures/favicon.ico new file mode 100644 index 00000000000..9bf95ef2348 Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/favicon.ico differ diff --git a/tests/lib/rules/http-compression/fixtures/favicon.zopfli.gz b/tests/lib/rules/http-compression/fixtures/favicon.zopfli.gz new file mode 100644 index 00000000000..9b079fe7c9f Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/favicon.zopfli.gz differ diff --git a/tests/lib/rules/http-compression/fixtures/image.br b/tests/lib/rules/http-compression/fixtures/image.br new file mode 100644 index 00000000000..c36571a641b Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/image.br differ diff --git a/tests/lib/rules/http-compression/fixtures/image.gz b/tests/lib/rules/http-compression/fixtures/image.gz new file mode 100644 index 00000000000..eab261afc81 Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/image.gz differ diff --git a/tests/lib/rules/http-compression/fixtures/image.png b/tests/lib/rules/http-compression/fixtures/image.png new file mode 100644 index 00000000000..8065fad7965 Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/image.png differ diff --git a/tests/lib/rules/http-compression/fixtures/image.svgz b/tests/lib/rules/http-compression/fixtures/image.svgz new file mode 100644 index 00000000000..70e3b21322f Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/image.svgz differ diff --git a/tests/lib/rules/http-compression/fixtures/image.zopfli.gz b/tests/lib/rules/http-compression/fixtures/image.zopfli.gz new file mode 100644 index 00000000000..e4cf7a48b2e Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/image.zopfli.gz differ diff --git a/tests/lib/rules/http-compression/fixtures/page.br b/tests/lib/rules/http-compression/fixtures/page.br new file mode 100644 index 00000000000..39421825eb6 Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/page.br differ diff --git a/tests/lib/rules/http-compression/fixtures/page.gz b/tests/lib/rules/http-compression/fixtures/page.gz new file mode 100644 index 00000000000..b0edcf0bac4 Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/page.gz differ diff --git a/tests/lib/rules/http-compression/fixtures/page.html b/tests/lib/rules/http-compression/fixtures/page.html new file mode 100644 index 00000000000..e32d505e998 --- /dev/null +++ b/tests/lib/rules/http-compression/fixtures/page.html @@ -0,0 +1,55 @@ + + + + + test + + + + + Lorem ipsum dolor sit amet, consectetur adipiscing + elit. Phasellus dictum dolor ac sodales gravida. Sed in libero arcu. + Vestibulum tincidunt massa quis orci faucibus, in luctus odio bibendum. + Sed egestas ugue diam, eu dictum elit interdum in. In ac lectus hendrerit, + egestas diam at, aliquet massa. In vitae porttitor quam. Maecenas quis + risus quis leo molestie vestibulum. Fusce vestibulum mattis porttitor. + + Maecenas eu justo mi. Nam nec hendrerit elit, at semper urna. Aliquam + erat volutpat. Nam placerat in massa tristique aliquam. Proin laoreet + dolor vitae arcu luctus, ut blandit felis pellentesque. Curabitur eleif + end at eros sit amet ornare. Etiam fringilla blandit rhoncus. + + Phasellus varius, dui sed porta viverra, mauris ligula rutrum orci, + vel tempor lorem neque et tellus. Phasellus lobortis lectus ac ligula + blandit hendrerit. Nam faucibus dolor sit amet urna varius, et volutpat + dui interdum. Etiam at aliquam mauris. Proin a porta ligula. Sed at arcu + tristique, sollicitudin sem non, bibendum odio. Interdum et malesuada + fames ac ante ipsum primis in faucibus. Morbi interdum est eget molestie + mattis. In luctus sollicitudin viverra. Pellentesque scelerisque a lacus + a lacinia. Nam mattis venenatis ligula, eu elementum nulla tincidunt sed. + Curabitur porttitor viverra purus, quis vulputate urna finibus at. + Curabitur in auctor libero, vitae condimentum mauris. Cras tincidunt + justo quis dictum tristique. + + Quisque quis libero mattis, imperdiet turpis non, posuere turpis. + Ut cursus ex sit amet ultrices viverra. Sed mollis mi enim, id lacinia + magna vulputate eu. Sed molestie dolor velit, luctus blandit diam euismod + at. Donec sit amet nibh at nisl pretium varius quis non lectus. Duis + pharetra rutrum nisi quis varius. Fusce interdum risus lacus, nec + facilisis orci varius in. Duis nec purus turpis. Ut bibendum vitae elit + sed accumsan. Integer blandit risus id metus cursus hendrerit. Aliquam + faucibus bibendum pellentesque. Pellentesque habitant morbi tristique + senectus et netus et malesuada fames ac turpis egestas. + + Vestibulum varius erat quis sapien blandit iaculis eu eu orci. Aenean + eleifend sed lectus sit amet ultrices. Curabitur justo turpis, consequat + eleifend convallis sit amet, ullamcorper ac risus. Fusce non ligula nec + ligula viverra vulputate id sit amet justo. Aliquam quis neque dapibus, + interdum nibh ut, viverra risus. In vulputate diam lacinia condimentum + suscipit. Praesent pharetra fermentum orci sit amet consectetur. + + + + + + diff --git a/tests/lib/rules/http-compression/fixtures/page.zopfli.gz b/tests/lib/rules/http-compression/fixtures/page.zopfli.gz new file mode 100644 index 00000000000..d5aa51d5bfe Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/page.zopfli.gz differ diff --git a/tests/lib/rules/http-compression/fixtures/script-small.br b/tests/lib/rules/http-compression/fixtures/script-small.br new file mode 100644 index 00000000000..bfb9762cc6a --- /dev/null +++ b/tests/lib/rules/http-compression/fixtures/script-small.br @@ -0,0 +1,3 @@ + €/* eslint-disable no-unused-vars */ +const x = 5; + \ No newline at end of file diff --git a/tests/lib/rules/http-compression/fixtures/script-small.gz b/tests/lib/rules/http-compression/fixtures/script-small.gz new file mode 100644 index 00000000000..e8ef38e6da9 Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/script-small.gz differ diff --git a/tests/lib/rules/http-compression/fixtures/script-small.js b/tests/lib/rules/http-compression/fixtures/script-small.js new file mode 100644 index 00000000000..0cc61fa9304 --- /dev/null +++ b/tests/lib/rules/http-compression/fixtures/script-small.js @@ -0,0 +1,2 @@ +/* eslint-disable no-unused-vars */ +const x = 5; diff --git a/tests/lib/rules/http-compression/fixtures/script-small.zopfli.gz b/tests/lib/rules/http-compression/fixtures/script-small.zopfli.gz new file mode 100644 index 00000000000..d6ebb5eec57 Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/script-small.zopfli.gz differ diff --git a/tests/lib/rules/http-compression/fixtures/script.br b/tests/lib/rules/http-compression/fixtures/script.br new file mode 100644 index 00000000000..b5a83558e6e Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/script.br differ diff --git a/tests/lib/rules/http-compression/fixtures/script.gz b/tests/lib/rules/http-compression/fixtures/script.gz new file mode 100644 index 00000000000..a2d395d825c Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/script.gz differ diff --git a/tests/lib/rules/http-compression/fixtures/script.js b/tests/lib/rules/http-compression/fixtures/script.js new file mode 100644 index 00000000000..bee419b1fb1 --- /dev/null +++ b/tests/lib/rules/http-compression/fixtures/script.js @@ -0,0 +1,41 @@ +/* eslint-disable no-unused-vars */ +const test = `Lorem ipsum dolor sit amet, consectetur adipiscing +elit. Phasellus dictum dolor ac sodales gravida. Sed in libero arcu. +Vestibulum tincidunt massa quis orci faucibus, in luctus odio bibendum. +Sed egestas ugue diam, eu dictum elit interdum in. In ac lectus hendrerit, +egestas diam at, aliquet massa. In vitae porttitor quam. Maecenas quis +risus quis leo molestie vestibulum. Fusce vestibulum mattis porttitor. + +Maecenas eu justo mi. Nam nec hendrerit elit, at semper urna. Aliquam +erat volutpat. Nam placerat in massa tristique aliquam. Proin laoreet +dolor vitae arcu luctus, ut blandit felis pellentesque. Curabitur eleif +end at eros sit amet ornare. Etiam fringilla blandit rhoncus. + +Phasellus varius, dui sed porta viverra, mauris ligula rutrum orci, +vel tempor lorem neque et tellus. Phasellus lobortis lectus ac ligula +blandit hendrerit. Nam faucibus dolor sit amet urna varius, et volutpat +dui interdum. Etiam at aliquam mauris. Proin a porta ligula. Sed at arcu +tristique, sollicitudin sem non, bibendum odio. Interdum et malesuada +fames ac ante ipsum primis in faucibus. Morbi interdum est eget molestie +mattis. In luctus sollicitudin viverra. Pellentesque scelerisque a lacus +a lacinia. Nam mattis venenatis ligula, eu elementum nulla tincidunt sed. +Curabitur porttitor viverra purus, quis vulputate urna finibus at. +Curabitur in auctor libero, vitae condimentum mauris. Cras tincidunt +justo quis dictum tristique. + +Quisque quis libero mattis, imperdiet turpis non, posuere turpis. +Ut cursus ex sit amet ultrices viverra. Sed mollis mi enim, id lacinia +magna vulputate eu. Sed molestie dolor velit, luctus blandit diam euismod +at. Donec sit amet nibh at nisl pretium varius quis non lectus. Duis +pharetra rutrum nisi quis varius. Fusce interdum risus lacus, nec +facilisis orci varius in. Duis nec purus turpis. Ut bibendum vitae elit +sed accumsan. Integer blandit risus id metus cursus hendrerit. Aliquam +faucibus bibendum pellentesque. Pellentesque habitant morbi tristique +senectus et netus et malesuada fames ac turpis egestas. + +Vestibulum varius erat quis sapien blandit iaculis eu eu orci. Aenean +eleifend sed lectus sit amet ultrices. Curabitur justo turpis, consequat +eleifend convallis sit amet, ullamcorper ac risus. Fusce non ligula nec +ligula viverra vulputate id sit amet justo. Aliquam quis neque dapibus, +interdum nibh ut, viverra risus. In vulputate diam lacinia condimentum +suscipit. Praesent pharetra fermentum orci sit amet consectetur. `; diff --git a/tests/lib/rules/http-compression/fixtures/script.zopfli.gz b/tests/lib/rules/http-compression/fixtures/script.zopfli.gz new file mode 100644 index 00000000000..5f5e6c2afaa Binary files /dev/null and b/tests/lib/rules/http-compression/fixtures/script.zopfli.gz differ diff --git a/tests/lib/rules/http-compression/tests-http.ts b/tests/lib/rules/http-compression/tests-http.ts new file mode 100644 index 00000000000..f3ed4ac7a48 --- /dev/null +++ b/tests/lib/rules/http-compression/tests-http.ts @@ -0,0 +1,50 @@ +import { getRuleName } from '../../../../src/lib/utils/rule-helpers'; +import * as ruleRunner from '../../../helpers/rule-runner'; + +import { + testsForBrotliOverHTTP, + testsForDefaults, + testsForDisallowedCompressionMethods, + testsForGzipZopfli, + testsForGzipZopfliCaching, + testsForGzipZopfliSmallSize, + testsForGzipZopfliUASniffing, + testsForNoCompression, + testsForSpecialCases, + testsForUserConfigs +} from './_tests'; + +const ruleName = getRuleName(__dirname); + +/* + * TODO: Remove `ignoredConnectors` part once headless + * Chrome on Travis CI doesn't fail miserably. :( + */ +const testConfigs = { ignoredConnectors: ['chrome'] }; + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +ruleRunner.testRule(ruleName, testsForDefaults(), testConfigs); +ruleRunner.testRule(ruleName, testsForSpecialCases(), testConfigs); +ruleRunner.testRule(ruleName, testsForDisallowedCompressionMethods(), testConfigs); +ruleRunner.testRule(ruleName, testsForNoCompression(), testConfigs); +ruleRunner.testRule(ruleName, testsForGzipZopfli(), testConfigs); +ruleRunner.testRule(ruleName, testsForGzipZopfliCaching(), testConfigs); +ruleRunner.testRule(ruleName, testsForGzipZopfliSmallSize(), testConfigs); +ruleRunner.testRule(ruleName, testsForGzipZopfliUASniffing(), testConfigs); +ruleRunner.testRule(ruleName, testsForBrotliOverHTTP, testConfigs); + +// Tests for the user options. +[true, false].forEach((isTarget) => { + ['gzip', 'zopfli', 'brotli'].forEach((encoding) => { + ruleRunner.testRule( + ruleName, + testsForUserConfigs(`${encoding}`, isTarget), + Object.assign( + {}, + testConfigs, + { ruleOptions: { [isTarget ? 'target' : 'resource']: { [encoding]: false } } } + ) + ); + }); +}); diff --git a/tests/lib/rules/http-compression/tests-https.ts b/tests/lib/rules/http-compression/tests-https.ts new file mode 100644 index 00000000000..826be792182 --- /dev/null +++ b/tests/lib/rules/http-compression/tests-https.ts @@ -0,0 +1,58 @@ +import { getRuleName } from '../../../../src/lib/utils/rule-helpers'; +import * as ruleRunner from '../../../helpers/rule-runner'; + +import { + testsForBrotli, + testsForBrotliSmallSize, + testsForBrotliUASniffing, + testsForDefaults, + testsForDisallowedCompressionMethods, + testsForGzipZopfli, + testsForGzipZopfliCaching, + testsForGzipZopfliSmallSize, + testsForGzipZopfliUASniffing, + testsForNoCompression, + testsForSpecialCases, + testsForUserConfigs +} from './_tests'; + +const ruleName = getRuleName(__dirname); + +/* + * TODO: Remove `ignoredConnectors` part once headless + * Chrome on Travis CI doesn't fail miserably. :( + */ +const testConfigs = { + https: true, + ignoredConnectors: ['chrome'] +}; + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +ruleRunner.testRule(ruleName, testsForDefaults(true), testConfigs); +ruleRunner.testRule(ruleName, testsForSpecialCases(true), testConfigs); +ruleRunner.testRule(ruleName, testsForDisallowedCompressionMethods(true), testConfigs); +ruleRunner.testRule(ruleName, testsForNoCompression(true), testConfigs); +ruleRunner.testRule(ruleName, testsForGzipZopfli(true), testConfigs); +ruleRunner.testRule(ruleName, testsForGzipZopfliCaching(true), testConfigs); +ruleRunner.testRule(ruleName, testsForGzipZopfliSmallSize(true), testConfigs); +ruleRunner.testRule(ruleName, testsForGzipZopfliUASniffing(true), testConfigs); + +ruleRunner.testRule(ruleName, testsForBrotli, testConfigs); +ruleRunner.testRule(ruleName, testsForBrotliSmallSize, testConfigs); +ruleRunner.testRule(ruleName, testsForBrotliUASniffing(), testConfigs); + +// Tests for the user options. +[true, false].forEach((isTarget) => { + ['gzip', 'zopfli', 'brotli'].forEach((encoding) => { + ruleRunner.testRule( + ruleName, + testsForUserConfigs(`${encoding}`, isTarget, true), + Object.assign( + {}, + testConfigs, + { ruleOptions: { [isTarget ? 'target' : 'resource']: { [encoding]: false } } } + ) + ); + }); +});