Stress test/random tree

apps/stress-test/README.md

@@ -26,16 +26,16 @@ $ yarn workspace @fluentui/stress-test stress-test build --build-deps
 $ yarn workspace @fluentui/stress-test stress-test build
 ```
 
-> NOTE: `build:local` is much slower, but is required the first time you're building the application or if you've pulled in lots of changes. Use `build:app` if you don't need to build dependencies like `@fluentui/react` as it's much faster.
+> NOTE: `build --build-deps` is much slower, but is required the first time you're building the application or if you've pulled in lots of changes. Use `build` if you don't need to build dependencies like `@fluentui/react` as it's much faster.
 
 ### Examples
 
 ```shell
 # Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against Firefox with small sizes and low sample size
-$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --sample-size 2 --test-cases mount prop-update --browsers firefox --sizes xs s
+$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --sample-size 2 --test-cases mount prop-update --browsers firefox --sizes xs s --targets v8/stress-tree?r=button v9/stress-tree?r=button
 
 # Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against the default browsers at the default sizes and sample size
-$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --test-cases mount prop-update
+$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --test-cases mount prop-update --targets v8/stress-tree?r=button v9/stress-tree?r=button
 ```
 
 > NOTE: Tests should be run against production builds. While tests can be run against development builds, and this is useful for gathering quick results and debugging, the performance characteristics of development and production builds can differ quite a bit.
@@ -65,6 +65,6 @@ The `benchmarks` folder houses Tachometer configurations and test results; and h
 
 The `scripts` folder house the Stress Test CLI app that is used to run tests.
 
-### Adding test cases
+### Adding test pages
 
-Add tests cases to the appropriate `src/pages` sub-folder. For example to create a new test, "my test" for Fluent v9 add it to `src/pages/v9/my-test`. Use an existing page as a guide for the files you need to add. Pages are automatically picked up by Webpack when the dev server is started.
+Add tests pages to the appropriate `src/pages` sub-folder. For example to create a new test, "my test" for Fluent v9 add it to `src/pages/v9/my-test`. Use an existing page as a guide for the files you need to add. Pages are automatically picked up by Webpack when the dev server is started.

apps/stress-test/package.json

@@ -15,6 +15,7 @@
     "@fluentui/web-components": "^2.5.5",
     "@microsoft/fast-element": "^1.10.4",
     "react": "17.0.2",
-    "react-dom": "17.0.2"
+    "react-dom": "17.0.2",
+    "random-seedable": "1.0.8"
   }
 }

apps/stress-test/scripts/commands/buildTestConfig.js

@@ -2,8 +2,6 @@ const fs = require('fs-extra');
 const path = require('path');
 const { getConfigDir, getResultsDir, ensureClean } = require('../utils/paths');
 const configureYargs = require('../utils/configureYargs');
-const getScenarioConfig = require('../utils/getScenarioConfig');
-const processOptions = require('../utils/processOptions');
 const querystring = require('querystring');
 
 /**
@@ -15,6 +13,8 @@ const querystring = require('querystring');
  * @property {number} sampleSize
  * @property {string[]} targets
  * @property {number} port
+ * @property {TestOptions} testOptions
+ * @property {string[]} renderers
  */
 
 /**
@@ -35,8 +35,7 @@ const command = 'build-test-config';
  * @returns {ConfigResult[]} Paths to generated config files
  */
 const buildTestConfig = options => {
-  const { scenario, browsers, testCases, sampleSize, targets, sizes, port } = options;
-  const config = getScenarioConfig(scenario);
+  const { scenario, browsers, testCases, sampleSize, targets, sizes, port, testOptions, renderers } = options;
   const configDir = getConfigDir(scenario);
   ensureClean(configDir);
 
@@ -45,7 +44,17 @@ const buildTestConfig = options => {
   for (const browser of browsers) {
     for (const testCase of testCases) {
       for (const size of sizes) {
-        const json = makeConfigJson(scenario, browser, testCase, sampleSize, targets, size, config.sizes[size], port);
+        const json = makeConfigJson(
+          scenario,
+          browser,
+          testCase,
+          sampleSize,
+          targets,
+          size,
+          testOptions,
+          port,
+          renderers,
+        );
         const configName = [browser, testCase, size].join('.') + '.json';
         const configPath = path.join(configDir, configName);
         fs.writeFileSync(configPath, json, { encoding: 'utf8' });
@@ -76,9 +85,10 @@ const buildTestConfig = options => {
  * @param {string} size
  * @param {TestOptions} testOptions
  * @param {number} port
+ * @param {string} renderer
  * @returns {string} Stringified JSON
  */
-const makeConfigJson = (scenario, browser, testCase, sampleSize, targets, size, testOptions, port) => {
+const makeConfigJson = (scenario, browser, testCase, sampleSize, targets, size, testOptions, port, renderers) => {
   const baseUrl = `http://localhost:${port}`;
   const json = {
     $schema: 'https://raw.githubusercontent.com/Polymer/tachometer/master/config.schema.json',
@@ -96,13 +106,34 @@ const makeConfigJson = (scenario, browser, testCase, sampleSize, targets, size,
           },
         ],
 
-        expand: targets.map(target => {
-          const params = querystring.stringify({ test: testCase, ...testOptions });
+        expand: targets.flatMap(target => {
+          const targetParams = target.includes('?') ? querystring.parse(target.substring(target.indexOf('?') + 1)) : {};
+          const params = querystring.stringify({
+            ...targetParams,
+            test: testCase,
+            fixtureName: `${size}_1`,
+            ...testOptions,
+          });
+
+          const targetWithoutParams = target.substring(0, target.indexOf('?'));
 
           return {
             name: `${target} - ${testCase} - ${size}`,
-            url: `${baseUrl}/${target}/${scenario}/?${params}`,
+            url: `${baseUrl}/${targetWithoutParams}/?${params}`,
           };
+          // return renderers.map(renderer => {
+          //   const params = querystring.stringify({
+          //     test: testCase,
+          //     fixtureName: `${size}_1`,
+          //     rendererName: renderer,
+          //     ...testOptions,
+          //   });
+
+          //   return {
+          //     name: `${renderer}\n${target} - ${testCase} - ${size}`,
+          //     url: `${baseUrl}/${target}/?${params}`,
+          //   };
+          // });
         }),
       },
     ],
@@ -122,9 +153,7 @@ const api = {
    * @param {CLIBuildTestConfigOptions} argv
    */
   handler: argv => {
-    const options = processOptions(argv);
-    // @ts-ignore
-    buildTestConfig(options);
+    buildTestConfig(argv);
   },
 };
 

apps/stress-test/scripts/commands/processResults.js

@@ -19,24 +19,34 @@ const handler = argv => {
   const resultsDir = getResultsDir(scenario);
   const files = readDirJson(resultsDir);
 
-  const browserData = {};
+  const browserData = {
+    scenario,
+    testCases: {},
+  };
   for (const file of files) {
     const contents = fs.readFileSync(path.join(resultsDir, file), {
       encoding: 'utf8',
     });
     const json = JSON.parse(contents);
     const benchmark = json.benchmarks[0];
     const browser = benchmark.browser.name;
-    const testCase = benchmark.name.split('-')[1].trim();
 
-    browserData[browser] = browserData[browser] || {};
+    const [, testCase, size] = benchmark.name.split(' - ');
+
+    browserData.testCases[testCase] = browserData.testCases[testCase] || { sizes: {} };
+    browserData.testCases[testCase].sizes[size] = browserData.testCases[testCase].sizes[size] || { browsers: {} };
+
+    browserData.testCases[testCase].sizes[size].browsers[browser] = json.benchmarks.map(bm => {
+      const [bmTarget, bmTestCase, bmSize] = bm.name.split(' - ');
 
-    browserData[browser][testCase] = json.benchmarks.map(test => {
       return {
-        name: test.name,
-        mean: test.mean,
-        differences: test.differences,
-        samples: test.samples,
+        name: bm.name,
+        mean: bm.mean,
+        differences: bm.differences,
+        samples: bm.samples,
+        target: bmTarget,
+        testCase: bmTestCase,
+        size: bmSize,
       };
     });
   }

apps/stress-test/scripts/commands/run.js

@@ -1,4 +1,3 @@
-const processOptions = require('../utils/processOptions');
 const configureYargs = require('../utils/configureYargs');
 const { startServer, stopServer } = require('../utils/server');
 const runTachometer = require('../utils/tachometer');
@@ -35,14 +34,12 @@ const run = async (testConfigs, options) => {
  * @param {CLIRunOptions} argv
  */
 const handler = argv => {
-  const options = processOptions(argv);
-
-  const testConfigs = buildTestConfig(options);
-  run(testConfigs, options).finally(() => {
+  const testConfigs = buildTestConfig(argv);
+  run(testConfigs, argv).finally(() => {
     stopServer();
 
-    if (options.processResults) {
-      processResults(options);
+    if (argv.processResults) {
+      processResults(argv);
     }
   });
 };

apps/stress-test/scripts/utils/configureYargs.js

@@ -25,12 +25,8 @@ const cliOptions = {
     default: 25,
   },
   targets: {
-    describe: 'Libraries to target.',
-    default: ['v8', 'v9', 'wc'],
-  },
-  'use-config': {
-    describe: 'Use options from config, overriding any config values with command line arguments.',
-    default: true,
+    describe: 'Tests to target',
+    demand: true,
   },
   'process-results': {
     describe: 'Process the test results for display.',
@@ -48,6 +44,24 @@ const cliOptions = {
     describe: 'Optimization mode for Griffel.',
     default: 'buildtime',
   },
+  renderers: {
+    describe: 'Renderers to use for testing. This determines what is actually tested.',
+  },
+  'test-options': {
+    describe: 'Options to apply to each test. E.g., option1=value1 option2=value2',
+    coerce: arg => {
+      return arg.reduce((map, current) => {
+        const [key, value] = current.split('=');
+        if (!key || !value) {
+          throw new Error(`Invalid test option. Got ${current}. Expected the form "key=value".`);
+        }
+
+        map[key] = value;
+        return map;
+      }, {});
+    },
+    default: [],
+  },
   mode: {
     describe: 'Build mode.',
     default: 'production',
@@ -78,6 +92,8 @@ const configure = (yargs, options) => {
       case 'test-cases':
       case 'browsers':
       case 'targets':
+      case 'test-options':
+      case 'renderers':
         y = y.array(option);
         break;
 
@@ -123,8 +139,9 @@ const configureYargs = (command, yargs) => {
         browsers,
         'sample-size': sampleSize,
         targets,
-        'use-config': useConfig,
+        'test-options': testOptions,
         port,
+        renderers,
       } = cliOptions;
       configure(yargs, {
         scenario,
@@ -133,8 +150,9 @@ const configureYargs = (command, yargs) => {
         browsers,
         'sample-size': sampleSize,
         targets,
-        'use-config': useConfig,
+        'test-options': testOptions,
         port,
+        renderers,
       });
       break;
     }
@@ -147,10 +165,11 @@ const configureYargs = (command, yargs) => {
         browsers,
         'sample-size': sampleSize,
         targets,
-        'use-config': useConfig,
+        'test-options': testOptions,
         'process-results': processResults,
         port,
         root,
+        renderers,
       } = cliOptions;
       configure(yargs, {
         scenario,
@@ -159,10 +178,11 @@ const configureYargs = (command, yargs) => {
         browsers,
         'sample-size': sampleSize,
         targets,
-        'use-config': useConfig,
+        'test-options': testOptions,
         'process-results': processResults,
         port,
         root,
+        renderers,
       });
       break;
     }

apps/stress-test/scripts/utils/getBrowsers.js

@@ -2,10 +2,12 @@ const os = require('os');
 
 module.exports.getBrowsers = () => {
   // https://github.com/Polymer/tachometer#webdriver-plugins
-  const browsers = ['chrome', 'firefox' /*'edge'*/];
+  const browsers = ['chrome', 'firefox'];
 
   if (os.type() === 'Darwin') {
     browsers.push('safari');
+  } else if (os.type() === 'Windows_NT') {
+    browsers.push('edge');
   }
 
   return browsers;