benchmark: improve reproducibility (#2039)

.eslintrc.yml

@@ -358,6 +358,7 @@ overrides:
     parserOptions:
       sourceType: script
     rules:
+      no-await-in-loop: off
       no-restricted-syntax: off
       no-console: off
       no-sync: off

package.json

@@ -28,7 +28,7 @@
     "testonly": "mocha --full-trace src/**/__tests__/**/*-test.js",
     "testonly:cover": "nyc npm run testonly",
     "lint": "eslint --cache --report-unused-disable-directives src resources",
-    "benchmark": "node ./resources/benchmark.js",
+    "benchmark": "node --predictable ./resources/benchmark.js",
     "prettier": "prettier --ignore-path .gitignore --write --list-different '**/*.{js,md,json,yml}'",
     "prettier:check": "prettier --ignore-path .gitignore --check '**/*.{js,md,json,yml}'",
     "check": "flow check",

resources/benchmark-fork.js

@@ -0,0 +1,50 @@
+// @noflow
+
+'use strict';
+
+const assert = require('assert');
+const cp = require('child_process');
+
+// Clocks the time taken to execute a test per cycle (secs).
+function clock(count, fn) {
+  const start = process.hrtime.bigint();
+  for (let i = 0; i < count; ++i) {
+    fn();
+  }
+  return Number(process.hrtime.bigint() - start) / count;
+}
+
+if (require.main === module) {
+  const modulePath = process.env.BENCHMARK_MODULE_PATH;
+  assert(typeof modulePath === 'string');
+  assert(process.send);
+  const module = require(modulePath);
+
+  clock(7, module.measure); // warm up
+  process.nextTick(() => {
+    process.send({
+      name: module.name,
+      clocked: clock(module.count, module.measure),
+    });
+  });
+}
+
+function sampleModule(modulePath) {
+  return new Promise((resolve, reject) => {
+    const env = { BENCHMARK_MODULE_PATH: modulePath };
+    const child = cp.fork(__filename, { env });
+    let message;
+    let error;
+
+    child.on('message', msg => (message = msg));
+    child.on('error', e => (error = e));
+    child.on('close', () => {
+      if (message) {
+        return resolve(message);
+      }
+      reject(error || new Error('Forked process closed without error'));
+    });
+  });
+}
+
+module.exports = { sampleModule };

resources/benchmark.js

@@ -16,17 +16,15 @@ const {
   mkdirRecursive,
   readdirRecursive,
 } = require('./utils');
+const { sampleModule } = require('./benchmark-fork');
 
 const NS_PER_SEC = 1e9;
 const LOCAL = 'local';
 
-const minTime = 0.05 * NS_PER_SEC;
-// The maximum time a benchmark is allowed to run before finishing.
-const maxTime = 5 * NS_PER_SEC;
+// The maximum time in secounds a benchmark is allowed to run before finishing.
+const maxTime = 5;
 // The minimum sample size required to perform statistical analysis.
-const minSamples = 15;
-// The default number of times to execute a test on a benchmark's first cycle.
-const initCount = 10;
+const minSamples = 5;
 
 function LOCAL_DIR(...paths) {
   return path.join(__dirname, '..', ...paths);
@@ -97,44 +95,19 @@ function findFiles(cwd, pattern) {
   return out.split('\n').filter(Boolean);
 }
 
-function collectSamples(fn) {
-  clock(initCount, fn); // initial warm up
-
-  // Cycles a benchmark until a run `count` can be established.
-  // Resolve time span required to achieve a percent uncertainty of at most 1%.
-  // For more information see http://spiff.rit.edu/classes/phys273/uncert/uncert.html.
-  let count = initCount;
-  let clocked = 0;
-  while ((clocked = clock(count, fn)) < minTime) {
-    // Calculate how many more iterations it will take to achieve the `minTime`.
-    count += Math.ceil(((minTime - clocked) * count) / clocked);
-  }
-
-  let elapsed = 0;
+async function collectSamples(modulePath) {
   const samples = [];
 
   // If time permits, increase sample size to reduce the margin of error.
-  while (samples.length < minSamples || elapsed < maxTime) {
-    clocked = clock(count, fn);
+  const start = Date.now();
+  while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) {
+    const { clocked } = await sampleModule(modulePath);
     assert(clocked > 0);
-
-    elapsed += clocked;
-    // Compute the seconds per operation.
-    samples.push(clocked / count);
+    samples.push(clocked);
   }
-
   return samples;
 }
 
-// Clocks the time taken to execute a test per cycle (secs).
-function clock(count, fn) {
-  const start = process.hrtime.bigint();
-  for (let i = 0; i < count; ++i) {
-    fn();
-  }
-  return Number(process.hrtime.bigint() - start);
-}
-
 // T-Distribution two-tailed critical values for 95% confidence.
 // See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm.
 const tTable = /* prettier-ignore */ {
@@ -238,7 +211,7 @@ function maxBy(array, fn) {
 }
 
 // Prepare all revisions and run benchmarks matching a pattern against them.
-function prepareAndRunBenchmarks(benchmarkPatterns, revisions) {
+async function prepareAndRunBenchmarks(benchmarkPatterns, revisions) {
   const environments = revisions.map(revision => ({
     revision,
     distPath: prepareRevision(revision),
@@ -248,22 +221,24 @@ function prepareAndRunBenchmarks(benchmarkPatterns, revisions) {
     const results = [];
     for (let i = 0; i < environments.length; ++i) {
       const environment = environments[i];
-      const module = require(path.join(environment.distPath, benchmark));
+      const modulePath = path.join(environment.distPath, benchmark);
 
-      if (i) {
-        console.log('⏱️   ' + module.name);
+      if (i === 0) {
+        const { name } = await sampleModule(modulePath);
+        console.log('⏱️   ' + name);
       }
 
       try {
-        const samples = collectSamples(module.measure);
+        const samples = await collectSamples(modulePath);
+
         results.push({
           name: environment.revision,
           samples,
           ...computeStats(samples),
         });
         process.stdout.write('  ' + cyan(i + 1) + ' tests completed.\u000D');
       } catch (error) {
-        console.log('  ' + module.name + ': ' + red(String(error)));
+        console.log('  ' + environment.revision + ': ' + red(String(error)));
       }
     }
     console.log('\n');

src/language/__tests__/parser-benchmark.js

@@ -4,6 +4,7 @@ import { kitchenSinkQuery } from '../../__fixtures__';
 import { parse } from '../parser';
 
 export const name = 'Parse kitchen sink';
+export const count = 1000;
 export function measure() {
   parse(kitchenSinkQuery);
 }

src/utilities/__tests__/buildASTSchema-benchmark.js

@@ -8,6 +8,7 @@ import { buildASTSchema } from '../buildASTSchema';
 const schemaAST = parse(bigSchemaSDL);
 
 export const name = 'Build Schema from AST';
+export const count = 10;
 export function measure() {
   buildASTSchema(schemaAST, { assumeValid: true });
 }

src/utilities/__tests__/buildClientSchema-benchmark.js

@@ -5,6 +5,7 @@ import { bigSchemaIntrospectionResult } from '../../__fixtures__';
 import { buildClientSchema } from '../buildClientSchema';
 
 export const name = 'Build Schema from Introspection';
+export const count = 10;
 export function measure() {
   buildClientSchema(bigSchemaIntrospectionResult.data, { assumeValid: true });
 }

src/utilities/__tests__/introspectionFromSchema-benchmark.js

@@ -10,6 +10,7 @@ const queryAST = parse(getIntrospectionQuery());
 const schema = buildSchema(bigSchemaSDL, { assumeValid: true });
 
 export const name = 'Execute Introspection Query';
+export const count = 10;
 export function measure() {
   execute(schema, queryAST);
 }

src/validation/__tests__/validateGQL-benchmark.js

@@ -9,6 +9,7 @@ const schema = buildSchema(bigSchemaSDL, { assumeValid: true });
 const queryAST = parse(getIntrospectionQuery());
 
 export const name = 'Validate Introspection Query';
+export const count = 50;
 export function measure() {
   validate(schema, queryAST);
 }

src/validation/__tests__/validateSDL-benchmark.js

@@ -8,6 +8,7 @@ import { validateSDL } from '../validate';
 const sdlAST = parse(bigSchemaSDL);
 
 export const name = 'Validate SDL Document';
+export const count = 10;
 export function measure() {
   validateSDL(sdlAST);
 }