fix: Comparing results (#16)

.circleci/config.yml

@@ -68,7 +68,7 @@ jobs:
 
       - run:
           name: Comparing perf examples
-          command: cd examples/simple_benchmark/ && best --compareStats d46ad97 320b079 --externalStorage=@best/store-aws --gitIntegration
+          command: cd examples/simple_benchmark/ && best --compareStats ${BASE_COMMIT} ${TARGET_COMMIT} --externalStorage=@best/store-aws --gitIntegration
 
 workflows:
   version: 2

package.json

@@ -14,7 +14,7 @@
         "build": "node scripts/build.js",
         "watch": "yarn build && node ./scripts/watch.js",
         "test": "echo 'No tests yet'",
-        "perf:example": "cd examples/simple_benchmark && yarn perf",
+        "perf:example": "cd examples/simple_benchmark && best --externalStorage=@best/store-aws",
         "release": "yarn prepare && lerna publish --exact --force-publish=* --registry='https://npm.lwcjs.org'",
         "release:internal": "yarn prepare && lerna publish --exact --force-publish=* --skip-git --repo-version `node -pe \"require('./lerna.json').version\"` --yes --registry='https://nexus.soma.salesforce.com/nexus/content/repositories/npmjs-internal/'"
     },

packages/best-cli/src/cli/output.js

@@ -72,5 +72,6 @@ export function generateReportTables(results, stream) {
 
 
 export function generateComparisonTable(comparison, stream) {
+    console.log('WIP: OUTPUT Comparison table here!');
     console.log(comparison);
 }

packages/best-cli/src/run_compare.js

@@ -18,7 +18,7 @@ export async function runCompare(globalConfig, configs, outputStream) {
     }
 
     const projectConfig = configs[0];
-    const [baseCommit, compareCommit] = commits.map(c => c.slice(0, 7));
+    const [baseCommit, compareCommit] = commits;
     const { projectName } = projectConfig;
     let storageProvider;
     try {
@@ -31,7 +31,7 @@ export async function runCompare(globalConfig, configs, outputStream) {
     const compareResults = await compareBenchmarkStats(baseCommit, compareCommit, projectName, storageProvider);
 
     if (gitIntegration) {
-        await pushBenchmarkComparison(compareResults);
+        await pushBenchmarkComparison(baseCommit, compareCommit, compareResults);
     }
 
     return compareResults;

packages/best-compare/src/index.js

@@ -1,5 +1,6 @@
 import { compareSamples } from "@best/analyzer";
 import { preRunMessager } from "@best/messager";
+import { basename } from "path";
 
 function compareEnvironment(baseEnv, targetEnv) {
     // TODO
@@ -57,6 +58,8 @@ export async function compareBenchmarkStats(baseCommit, targetCommit, projectNam
         comparison: []
     };
 
+    //console.log('>> ', baseBenchmarks, targetBenchmarks);
+
     baseBenchmarks.forEach((baseBenchmarkBundle) => {
         const { benchmarkName } = baseBenchmarkBundle;
         const targetBenchmarkBundle = targetBenchmarks.find(b => b.benchmarkName === benchmarkName);

packages/best-config/src/index.js

@@ -7,6 +7,8 @@ import { addGitInformation } from "./git";
 
 import DEFAULT_CONFIG from './defaults';
 
+const TARGET_COMMIT = process.env.TARGET_COMMIT;
+const BASE_COMMIT = process.env.BASE_COMMIT;
 const specialArgs = ['_', '$0', 'h', 'help', 'config'];
 const isFile = filePath => fs.existsSync(filePath) && !fs.lstatSync(filePath).isDirectory();
 
@@ -85,15 +87,15 @@ function setFromArgs(initialOptions, argsCLI) {
                     options.benchmarkIterations = argsCLI[key];
                     break;
                 case 'compareStats':
-                    options[key] = argsCLI[key][0] !== undefined ? argsCLI[key].filter(Boolean) : undefined;
+                    options[key] = argsCLI[key].filter(Boolean);
                     break;
                 default:
                     options[key] = argsCLI[key];
                     break;
             }
             return options;
         }, {});
-
+    console.log(argvToOptions);
     return Object.assign({}, initialOptions, argvToOptions);
 }
 
@@ -153,6 +155,12 @@ function normalizePlugins(plugins, globalOptions) {
     }, {});
 }
 
+function normalizeCommits([base, target]) {
+    base = base || BASE_COMMIT || '';
+    target = target || TARGET_COMMIT || '';
+    return [base.slice(0, 7), target.slice(0, 7)];
+}
+
 function normalize(options, argsCLI) {
     options = normalizeRootDir(setFromArgs(options, argsCLI));
     const newOptions = Object.assign({}, DEFAULT_CONFIG);
@@ -162,6 +170,9 @@ function normalize(options, argsCLI) {
             case 'plugins':
                 value = normalizePlugins(options[key], options);
                 break;
+            case 'compareStats':
+                value = options[key].length ? normalizeCommits(options[key], options) : undefined;
+                break;
             default: value = options[key];
         }
         newOptions[key] = value;

packages/best-github-integration/README.md

@@ -0,0 +1,3 @@
+benchmark | base4f8f259 | targeta6b76f4 | trend
+--- | --- | --- | ---
+simple-item.benchmark:create_and_render | 32.60 (± 1.42 ms) | 32.86 (± 1.42 ms) | 👌

packages/best-github-integration/package.json

@@ -13,6 +13,7 @@
     "@best/messager": "0.0.3",
     "github": "^13.0.1",
     "jsonwebtoken": "8.1.0",
-    "expand-tilde":"~2.0.2"
+    "expand-tilde":"~2.0.2",
+    "json2md": "~1.5.11"
   }
 }

packages/best-github-integration/src/comment.js

@@ -1,10 +1,45 @@
-function template({ targetCommit, baseCommit }) {
-    return `## Benchmark comparison
-Base commit: \`${targetCommit}\` | Target commit: \`${baseCommit}\`
+import json2md from "json2md";
 
-WIP`;
+function template({ targetCommit, baseCommit, table }) {
+    return json2md([
+        { h2 : 'Benchmark comparison '},
+        { p : `Base commit: \`${targetCommit}\` | Target commit: \`${baseCommit}\`` },
+        table
+    ]);
+}
+
+function generateRows(stats, name = '') {
+    return stats.comparison.map((node) => {
+        if (node.comparison) {
+            return generateRows(node, `${node.benchmarkName || node.name}:`).reduce((a, b) => a.concat(b));
+        }
+
+        const durationMetric = node.metrics.duration;
+        const { baseStats, targetStats, samplesComparison } = durationMetric;
+
+        return [
+            name + node.name,
+            `${baseStats.median.toFixed(2)} (± ${targetStats.medianAbsoluteDeviation.toFixed(2)} ms)`,
+            `${targetStats.median.toFixed(2)} (± ${targetStats.medianAbsoluteDeviation.toFixed(2)} ms)`,
+            samplesComparison === 0 ? '👌' : samplesComparison === 1 ? '👎' : '👍'
+        ];
+    });
+}
+
+function generateTable(baseCommit, targetCommit, stats) {
+    return {
+        table: {
+            headers: ['benchmark', `base(\`${baseCommit}\`)`, `target(\`${targetCommit}\`)`, 'trend'],
+            rows: generateRows(stats)
+        }
+    };
 }
 
 export function generateComparisonComment(baseCommit, targetCommit, stats) {
-    return template({ baseCommit, targetCommit });
+    const table = generateTable(baseCommit, targetCommit, stats);
+    return template({
+        baseCommit,
+        targetCommit,
+        table
+    });
 }

packages/best-github-integration/src/index.js

@@ -2,14 +2,15 @@ import { isCI } from "@best/utils";
 import { createGithubApp } from "./git-app";
 import { generateComparisonComment } from "./comment";
 
-const TARGET_COMMIT = process.env.TARGET_COMMIT;
-const BASE_COMMIT = process.env.BASE_COMMIT;
 const PULL_REQUEST_URL = process.env.PULL_REQUEST;
 const REPO_NAME = process.env.REPO_NAME;
 
-export async function pushBenchmarkComparison(compareStats) {
+export async function pushBenchmarkComparison(baseCommit, targetCommit, compareStats) {
     if (!isCI) {
-        throw new Error('GitIntegration is only supposed to run on a CI environment');
+        // throw new Error('GitIntegration is only supposed to run on a CI environment');
+        console.log('[NOT A CI] - The output will not be pushed.\n');
+        console.log(generateComparisonComment(baseCommit, targetCommit, compareStats), '\n');
+        return;
     }
 
     const APP = createGithubApp();
@@ -20,12 +21,12 @@ export async function pushBenchmarkComparison(compareStats) {
     const owner = repoInstallation.account.login;
     const gitHubInstallation = await APP.authAsInstallation(installationId);
     const pullRequestId = parseInt(PULL_REQUEST_URL.split('/').pop(), 10);
-    const comparisonComment = generateComparisonComment(BASE_COMMIT, TARGET_COMMIT, compareStats);
+    const body = generateComparisonComment(baseCommit, targetCommit, compareStats);
 
     await gitHubInstallation.issues.createComment({
         owner,
+        body,
         repo: REPO_NAME,
         number: pullRequestId,
-        body: comparisonComment
     });
 }

packages/best-store-aws/src/index.js

@@ -37,9 +37,10 @@ export async function storeBenchmarkResults(fileMap, { benchmarkName, benchmarkS
 export async function getBenchmarkStats(projectName, commit) {
     const s3 = getS3Instance();
     const benchmarks = await s3.getBenchmarkUrlsForCommit(projectName, commit);
-    console.log(AWS_TEXT + ` Fetching benchmark results for commit ${commit}...`);
+    console.log(AWS_TEXT + ` Fetching benchmarks for commit ${commit}...`);
     return Promise.all(benchmarks.map(async (url) => {
-        const fullUrl = benchmarks[0] + '/stats.json';
+        const fullUrl = url + '/stats.json';
+        console.log(AWS_TEXT + ` Fetching benchmark ${fullUrl}`);
         const response = await fetch(fullUrl);
         return response.json();
     }));

packages/best-utils/src/index.js

@@ -3,7 +3,7 @@ import clearLine from './clear-line';
 import cacheDirectory from "./cache-directory";
 export {
     cacheDirectory,
+    clearLine,
     isInteractive,
     isCI,
-    clearLine
 };