Merge pull request #174 from Polymer/multi

Ability to take multiple measures on the same page

Author: aomarks

CHANGELOG.md

@@ -7,14 +7,19 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+- Add ability to specify multiple measurements from the same page load by
+  setting the `measurement` property in the JSON config file to an array. For
+  example, you can now use the performance API to define two intervals on the
+  same page, and compare them to each other or to other pages.
+
 - Add ability to pull measurements from the browser performance measurement API,
   e.g.:
 
   ```
   "benchmarks": [
     {
       "measurement": {
-        "kind": "performance",
+        "mode": "performance",
         "entryName": "foo"
       }
     }
@@ -27,12 +32,12 @@ project adheres to [Semantic Versioning](http://semver.org/).
   "benchmarks": [
     {
       "measurement": {
-        "kind": "callback"
+        "mode": "callback"
       }
     },
     {
       "measurement": {
-        "kind": "expression",
+        "mode": "expression",
         "expression": "window.tachometerResult"
       }
     }

README.md

@@ -83,9 +83,13 @@ confidence in them.
 
 ## Measurement modes
 
-Tachometer supports four kinds of time interval measurements, controlled with
+Tachometer supports four modes of time interval measurements, controlled with
 the `measurement` config file property, or the `--measure` flag.
 
+If `measurement` is an array, then all of the given measurements will be
+retrieved from each page load. Each measurement from a page is treated as its
+own benchmark.
+
 #### Performance API
 
 Retrieve a measure, mark, or paint timing from the
@@ -107,7 +111,7 @@ And in your config file:
 "benchmarks": [
   {
     "measurement": {
-      "kind": "performance",
+      "mode": "performance",
       "entryName": "foo"
     }
   }

config.schema.json

@@ -5,15 +5,15 @@
         "CallbackMeasurement": {
             "additionalProperties": false,
             "properties": {
-                "kind": {
+                "mode": {
                     "enum": [
                         "callback"
                     ],
                     "type": "string"
                 }
             },
             "required": [
-                "kind"
+                "mode"
             ],
             "type": "object"
         },
@@ -113,6 +113,22 @@
                         {
                             "$ref": "#/definitions/ExpressionMeasurement"
                         },
+                        {
+                            "items": {
+                                "anyOf": [
+                                    {
+                                        "$ref": "#/definitions/CallbackMeasurement"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/PerformanceEntryMeasurement"
+                                    },
+                                    {
+                                        "$ref": "#/definitions/ExpressionMeasurement"
+                                    }
+                                ]
+                            },
+                            "type": "array"
+                        },
                         {
                             "enum": [
                                 "callback",
@@ -191,7 +207,7 @@
                 "expression": {
                     "type": "string"
                 },
-                "kind": {
+                "mode": {
                     "enum": [
                         "expression"
                     ],
@@ -200,7 +216,7 @@
             },
             "required": [
                 "expression",
-                "kind"
+                "mode"
             ],
             "type": "object"
         },
@@ -291,7 +307,7 @@
                 "entryName": {
                     "type": "string"
                 },
-                "kind": {
+                "mode": {
                     "enum": [
                         "performance"
                     ],
@@ -300,7 +316,7 @@
             },
             "required": [
                 "entryName",
-                "kind"
+                "mode"
             ],
             "type": "object"
         },

package.json

@@ -92,4 +92,4 @@
     "typescript": "^3.6.4",
     "typescript-json-schema": "^0.42.0"
   }
-}
+}

src/config.ts

@@ -123,12 +123,14 @@ export async function makeConfig(opts: Opts): Promise<Config> {
   }
 
   for (const spec of config.benchmarks) {
-    if (spec.measurement.kind === 'performance' &&
-        spec.measurement.entryName === 'first-contentful-paint' &&
-        !fcpBrowsers.has(spec.browser.name)) {
-      throw new Error(
-          `Browser ${spec.browser.name} does not support the ` +
-          `first contentful paint (FCP) measurement`);
+    for (const measurement of spec.measurement) {
+      if (measurement.mode === 'performance' &&
+          measurement.entryName === 'first-contentful-paint' &&
+          !fcpBrowsers.has(spec.browser.name)) {
+        throw new Error(
+            `Browser ${spec.browser.name} does not support the ` +
+            `first contentful paint (FCP) measurement`);
+      }
     }
   }
 

src/configfile.ts

@@ -136,7 +136,8 @@ interface ConfigFileBenchmark {
   expand?: ConfigFileBenchmark[];
 }
 
-type ConfigFileMeasurement = 'callback'|'fcp'|'global'|Measurement;
+type ConfigFileMeasurement =
+    'callback'|'fcp'|'global'|Measurement|Array<Measurement>;
 
 type BrowserConfigs =
     ChromeConfig|FirefoxConfig|SafariConfig|EdgeConfig|IEConfig;
@@ -341,22 +342,24 @@ async function parseBenchmark(benchmark: ConfigFileBenchmark, root: string):
   }
 
   if (benchmark.measurement === 'callback') {
-    spec.measurement = {
-      kind: 'callback',
-    };
+    spec.measurement = [{
+      mode: 'callback',
+    }];
   } else if (benchmark.measurement === 'fcp') {
-    spec.measurement = {
-      kind: 'performance',
+    spec.measurement = [{
+      mode: 'performance',
       entryName: 'first-contentful-paint',
-    };
+    }];
   } else if (benchmark.measurement === 'global') {
-    spec.measurement = {
-      kind: 'expression',
+    spec.measurement = [{
+      mode: 'expression',
       expression:
           benchmark.measurementExpression || defaults.measurementExpression,
-    };
-  } else {
+    }];
+  } else if (Array.isArray(benchmark.measurement)) {
     spec.measurement = benchmark.measurement;
+  } else if (benchmark.measurement !== undefined) {
+    spec.measurement = [benchmark.measurement];
   }
 
   const url = benchmark.url;
@@ -471,7 +474,7 @@ function applyDefaults(partialSpec: Partial<BenchmarkSpec>): BenchmarkSpec {
     };
   }
   if (measurement === undefined) {
-    measurement = defaults.measurement(url);
+    measurement = [defaults.measurement(url)];
   }
   return {name, url, browser, measurement};
 }

src/defaults.ts

@@ -28,9 +28,9 @@ export const measurementExpression = 'window.tachometerResult';
 export function measurement(url: LocalUrl|RemoteUrl): Measurement {
   if (url.kind === 'remote') {
     return {
-      kind: 'performance',
+      mode: 'performance',
       entryName: 'first-contentful-paint',
     };
   }
-  return {kind: 'callback'};
+  return {mode: 'callback'};
 }

src/measure.ts

@@ -24,7 +24,7 @@ export async function measure(
     driver: webdriver.WebDriver,
     measurement: Measurement,
     server: Server|undefined): Promise<number|undefined> {
-  switch (measurement.kind) {
+  switch (measurement.mode) {
     case 'callback':
       if (server === undefined) {
         throw new Error('Internal error: no server for spec');
@@ -119,3 +119,24 @@ function escapeStringLiteral(unescaped: string): string {
       .replace(/`/g, '\\`')
       .replace(/\$/g, '\\$');
 }
+
+/**
+ * Return a good-enough label for the given measurement, to disambiguate cases
+ * where there are multiple measurements on the same page.
+ */
+export function measurementName(measurement: Measurement): string {
+  switch (measurement.mode) {
+    case 'callback':
+      return 'callback';
+    case 'expression':
+      return measurement.expression;
+    case 'performance':
+      return measurement.entryName === 'first-contentful-paint' ?
+          'fcp' :
+          measurement.entryName;
+  }
+  throwUnreachable(
+      measurement,
+      `Internal error: unknown measurement type ` +
+          JSON.stringify(measurement));
+}