Ability to take multiple measures on the same page

Author: aomarks

CHANGELOG.md

@@ -7,6 +7,11 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+- Add ability to specify multiple measurements from the same page load by
+  setting the `measurement` property in the JSON config file to an array. For
+  example, you can now use the performance API to define two intervals on the
+  same page, and compare them to each other or to other pages.
+
 - Add ability to pull measurements from the browser performance measurement API,
   e.g.:
 

README.md

@@ -86,6 +86,10 @@ confidence in them.
 Tachometer supports four kinds of time interval measurements, controlled with
 the `measurement` config file property, or the `--measure` flag.
 
+If `measurement` is an array, then all of the given measurements will be
+retrieved from each page load. Each measurement from a page is treated as its
+own benchmark.
+
 #### Performance API
 
 Retrieve a measure, mark, or paint timing from the

config.schema.json

@@ -113,6 +113,22 @@
                         {
                             "$ref": "#/definitions/ExpressionMeasurement"
                         },
+                        {
+                            "items": {
+                                "anyOf": [
+                                    {
+                                        "$ref": "#/definitions/CallbackMeasurement"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/PerformanceEntryMeasurement"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/ExpressionMeasurement"
+                                    }
+                                ]
+                            },
+                            "type": "array"
+                        },
                         {
                             "enum": [
                                 "callback",

package.json

@@ -92,4 +92,4 @@
     "typescript": "^3.6.4",
     "typescript-json-schema": "^0.42.0"
   }
-}
+}

src/automatic.ts

@@ -17,10 +17,10 @@ import ansi = require('ansi-escape-sequences');
 
 import {jsonOutput, legacyJsonOutput} from './json-output';
 import {browserSignature, makeDriver, openAndSwitchToNewTab} from './browser';
-import {measure} from './measure';
+import {measure, measurementName} from './measure';
 import {BenchmarkResult, BenchmarkSpec} from './types';
 import {formatCsvStats, formatCsvRaw} from './csv';
-import {ResultStats, ResultStatsWithDifferences, horizonsResolved, summaryStats, computeDifferences} from './stats';
+import {ResultStatsWithDifferences, horizonsResolved, summaryStats, computeDifferences} from './stats';
 import {verticalTermResultTable, horizontalTermResultTable, verticalHtmlResultTable, horizontalHtmlResultTable, automaticResultTable, spinner, benchmarkOneLiner} from './format';
 import {Config} from './config';
 import * as github from './github';
@@ -41,7 +41,7 @@ export class AutomaticMode {
   private browsers = new Map<string, Browser>();
   private bar: ProgressBar;
   private completeGithubCheck?: (markdown: string) => void;
-  private specResults = new Map<BenchmarkSpec, BenchmarkResult[]>();
+  private results = new Map<BenchmarkSpec, BenchmarkResult[]>();
   private hitTimeout = false;
 
   constructor(config: Config, servers: Map<BenchmarkSpec, Server>) {
@@ -62,9 +62,6 @@ export class AutomaticMode {
     }
     console.log('Running benchmarks\n');
     await this.warmup();
-    for (const spec of this.specs) {
-      this.specResults.set(spec, []);
-    }
     await this.takeMinimumSamples();
     await this.takeAdditionalSamples();
     await this.closeBrowsers();
@@ -111,22 +108,39 @@ export class AutomaticMode {
       bar.tick(0, {
         status: `warmup ${i + 1}/${specs.length} ${benchmarkOneLiner(spec)}`,
       });
-      await this.takeSample(spec);
+      await this.takeSamples(spec);
       bar.tick(1);
     }
   }
 
+  private recordSamples(spec: BenchmarkSpec, newResults: BenchmarkResult[]) {
+    let specResults = this.results.get(spec);
+    if (specResults === undefined) {
+      specResults = [];
+      this.results.set(spec, specResults);
+    }
+
+    for (const newResult of newResults) {
+      const primary = specResults[newResult.measurementIdx];
+      if (primary === undefined) {
+        specResults[newResult.measurementIdx] = newResult;
+      } else {
+        primary.millis.push(...newResult.millis);
+      }
+    }
+  }
+
   private async takeMinimumSamples() {
     // Always collect our minimum number of samples.
-    const {config, specs, bar, specResults} = this;
+    const {config, specs, bar} = this;
     const numRuns = specs.length * config.sampleSize;
     let run = 0;
     for (let sample = 0; sample < config.sampleSize; sample++) {
       for (const spec of specs) {
         bar.tick(0, {
           status: `${++run}/${numRuns} ${benchmarkOneLiner(spec)}`,
         });
-        specResults.get(spec)!.push(await this.takeSample(spec));
+        this.recordSamples(spec, await this.takeSamples(spec));
         if (bar.curr === bar.total - 1) {
           // Note if we tick with 0 after we've completed, the status is
           // rendered on the next line for some reason.
@@ -139,7 +153,7 @@ export class AutomaticMode {
   }
 
   private async takeAdditionalSamples() {
-    const {config, specs, specResults} = this;
+    const {config, specs} = this;
     if (config.timeout > 0) {
       console.log();
       const timeoutMs = config.timeout * 60 * 1000;  // minutes -> millis
@@ -170,14 +184,14 @@ export class AutomaticMode {
             process.stdout.write(
                 `\r${spinner[run % spinner.length]} Auto-sample ${sample} ` +
                 `(timeout in ${mins}m${secs}s)` + ansi.erase.inLine(0));
-            specResults.get(spec)!.push(await this.takeSample(spec));
+            this.recordSamples(spec, await this.takeSamples(spec));
           }
         }
       }
     }
   }
 
-  private async takeSample(spec: BenchmarkSpec): Promise<BenchmarkResult> {
+  private async takeSamples(spec: BenchmarkSpec): Promise<BenchmarkResult[]> {
     const {servers, config, browsers} = this;
 
     let server;
@@ -192,19 +206,31 @@ export class AutomaticMode {
     const {driver, initialTabHandle} =
         browsers.get(browserSignature(spec.browser))!;
 
-    let millis: number|undefined;
     let bytesSent = 0;
     let userAgent = '';
     // TODO(aomarks) Make maxAttempts and timeouts configurable.
     const maxAttempts = 3;
+    const measurements = spec.measurement;
+    let millis: number[];
+    let numPending: number;
     for (let attempt = 1;; attempt++) {
+      millis = [];
+      numPending = measurements.length;
       await openAndSwitchToNewTab(driver, spec.browser);
       await driver.get(url);
-      for (let waited = 0; millis === undefined && waited <= 10000;
-           waited += 50) {
+      for (let waited = 0; numPending > 0 && waited <= 10000; waited += 50) {
         // TODO(aomarks) You don't have to wait in callback mode!
         await wait(50);
-        millis = await measure(driver, spec.measurement, server);
+        for (let i = 0; i < measurements.length; i++) {
+          if (millis[i] !== undefined) {
+            continue;
+          }
+          const result = await measure(driver, measurements[i], server);
+          if (result !== undefined) {
+            millis[i] = result;
+            numPending--;
+          }
+        }
       }
 
       // Close the active tab (but not the whole browser, since the
@@ -218,7 +244,7 @@ export class AutomaticMode {
         userAgent = session.userAgent;
       }
 
-      if (millis !== undefined || attempt >= maxAttempts) {
+      if (numPending === 0 || attempt >= maxAttempts) {
         break;
       }
 
@@ -228,44 +254,40 @@ export class AutomaticMode {
           `in ${spec.browser.name} from ${url}. Retrying.`);
     }
 
-    if (millis === undefined) {
+    if (numPending > 0) {
       console.log();
       throw new Error(
           `\n\nFailed ${maxAttempts}/${maxAttempts} times ` +
           `to get a measurement ` +
           `in ${spec.browser.name} from ${url}. Retrying.`);
     }
 
-    return {
-      name: spec.name,
-      queryString: spec.url.kind === 'local' ? spec.url.queryString : '',
-      version: spec.url.kind === 'local' && spec.url.version !== undefined ?
-          spec.url.version.label :
-          '',
-      millis: [millis],
-      bytesSent,
-      browser: spec.browser,
-      userAgent,
-    };
+    return measurements.map(
+        (measurement, measurementIdx) => ({
+          name: measurements.length === 1 ?
+              spec.name :
+              `${spec.name} [${measurementName(measurement)}]`,
+          measurementIdx,
+          queryString: spec.url.kind === 'local' ? spec.url.queryString : '',
+          version: spec.url.kind === 'local' && spec.url.version !== undefined ?
+              spec.url.version.label :
+              '',
+          millis: [millis[measurementIdx]],
+          bytesSent,
+          browser: spec.browser,
+          userAgent,
+        }));
   }
 
   makeResults() {
-    const results: BenchmarkResult[] = [];
-    for (const sr of this.specResults.values()) {
-      const combined: BenchmarkResult = {
-        ...sr[0],
-        millis: [],
-      };
-      for (const result of sr) {
-        combined.millis.push(...result.millis);
+    const resultStats = [];
+    for (const results of this.results.values()) {
+      for (let r = 0; r < results.length; r++) {
+        const result = results[r];
+        resultStats.push({result, stats: summaryStats(result.millis)});
       }
-      results.push(combined);
     }
-    const withStats = results.map((result): ResultStats => ({
-                                    result,
-                                    stats: summaryStats(result.millis),
-                                  }));
-    return computeDifferences(withStats);
+    return computeDifferences(resultStats);
   }
 
   private async outputResults(withDifferences: ResultStatsWithDifferences[]) {

src/config.ts

@@ -123,12 +123,14 @@ export async function makeConfig(opts: Opts): Promise<Config> {
   }
 
   for (const spec of config.benchmarks) {
-    if (spec.measurement.kind === 'performance' &&
-        spec.measurement.entryName === 'first-contentful-paint' &&
-        !fcpBrowsers.has(spec.browser.name)) {
-      throw new Error(
-          `Browser ${spec.browser.name} does not support the ` +
-          `first contentful paint (FCP) measurement`);
+    for (const measurement of spec.measurement) {
+      if (measurement.kind === 'performance' &&
+          measurement.entryName === 'first-contentful-paint' &&
+          !fcpBrowsers.has(spec.browser.name)) {
+        throw new Error(
+            `Browser ${spec.browser.name} does not support the ` +
+            `first contentful paint (FCP) measurement`);
+      }
     }
   }
 

src/configfile.ts

@@ -136,7 +136,8 @@ interface ConfigFileBenchmark {
   expand?: ConfigFileBenchmark[];
 }
 
-type ConfigFileMeasurement = 'callback'|'fcp'|'global'|Measurement;
+type ConfigFileMeasurement =
+    'callback'|'fcp'|'global'|Measurement|Array<Measurement>;
 
 type BrowserConfigs =
     ChromeConfig|FirefoxConfig|SafariConfig|EdgeConfig|IEConfig;
@@ -327,22 +328,24 @@ async function parseBenchmark(benchmark: ConfigFileBenchmark, root: string):
   }
 
   if (benchmark.measurement === 'callback') {
-    spec.measurement = {
+    spec.measurement = [{
       kind: 'callback',
-    };
+    }];
   } else if (benchmark.measurement === 'fcp') {
-    spec.measurement = {
+    spec.measurement = [{
       kind: 'performance',
       entryName: 'first-contentful-paint',
-    };
+    }];
   } else if (benchmark.measurement === 'global') {
-    spec.measurement = {
+    spec.measurement = [{
       kind: 'expression',
       expression:
           benchmark.measurementExpression || defaults.measurementExpression,
-    };
-  } else {
+    }];
+  } else if (Array.isArray(benchmark.measurement)) {
     spec.measurement = benchmark.measurement;
+  } else if (benchmark.measurement !== undefined) {
+    spec.measurement = [benchmark.measurement];
   }
 
   const url = benchmark.url;
@@ -457,7 +460,7 @@ function applyDefaults(partialSpec: Partial<BenchmarkSpec>): BenchmarkSpec {
     };
   }
   if (measurement === undefined) {
-    measurement = defaults.measurement(url);
+    measurement = [defaults.measurement(url)];
   }
   return {name, url, browser, measurement};
 }

src/measure.ts

@@ -108,4 +108,25 @@ async function queryForExpression(
     }
     return result;
   }
+}
+
+/**
+ * Return a good-enough label for the given measurement, to disambiguate cases
+ * where there are multiple measurements on the same page.
+ */
+export function measurementName(measurement: Measurement): string {
+  switch (measurement.kind) {
+    case 'callback':
+      return 'callback';
+    case 'expression':
+      return measurement.expression;
+    case 'performance':
+      return measurement.entryName === 'first-contentful-paint' ?
+          'fcp' :
+          measurement.entryName;
+  }
+  throwUnreachable(
+      measurement,
+      `Internal error: unknown measurement type ` +
+          JSON.stringify(measurement));
 }

src/specs.ts

@@ -106,8 +106,9 @@ export async function specsFromOpts(opts: Opts): Promise<BenchmarkSpec[]> {
         const spec: BenchmarkSpec = {
           name: arg.alias || arg.url,
           browser,
-          measurement: measurement === undefined ? defaults.measurement(url) :
-                                                   measurement,
+          measurement:
+              [measurement === undefined ? defaults.measurement(url) :
+                                           measurement],
           url,
         };
         specs.push(spec);
@@ -132,8 +133,9 @@ export async function specsFromOpts(opts: Opts): Promise<BenchmarkSpec[]> {
           const spec: BenchmarkSpec = {
             name,
             browser,
-            measurement: measurement === undefined ? defaults.measurement(url) :
-                                                     measurement,
+            measurement:
+                [measurement === undefined ? defaults.measurement(url) :
+                                             measurement],
             url,
           };
           specs.push(spec);