fix: almost done with types for analyzer

packages/@best/analyzer/package.json

@@ -6,5 +6,6 @@
     "Best",
     "performance"
   ],
-  "main": "build/index.js"
+  "main": "build/index.js",
+  "types": "build/index.d.ts"
 }

packages/@best/analyzer/src/index.ts

@@ -1,12 +1,13 @@
+import { BenchmarkResultsSnapshot, BenchmarkResultNode, BenchmarkMetricNames, BenchmarkStats } from "@best/types";
 import { VERSION } from './constants';
 import { quantile, mean, median, variance, medianAbsoluteDeviation, compare as compareSamples } from './stats';
 
-function computeSampleStats(arr: number[], config: any) {
-    const { samplesQuantileThreshold } = config;
+function computeSampleStats(arr: number[], samplesQuantileThreshold: number): BenchmarkStats {
     if (samplesQuantileThreshold < 1) {
         const q = quantile(arr, samplesQuantileThreshold);
         arr = arr.filter(v => v <= q);
     }
+
     return {
         samples: arr,
         sampleSize: arr.length,
@@ -18,62 +19,89 @@ function computeSampleStats(arr: number[], config: any) {
     };
 }
 
-function collectResults({ name, duration, runDuration, benchmarks }: any, collector: any) {
-    let cNode = collector[name];
-    if (typeof cNode !== 'object') {
-        cNode = collector[name] = { duration: [], runDuration: [] };
+type BenchmarkMetricsAggregate = { [key in BenchmarkMetricNames]?: number[] }
+
+// This type will hold as keys all benchmark names, and then an array with all results
+interface BenchmarkMetricsMap {
+    [key: string]: BenchmarkMetricsAggregate
+}
+
+// Given an iteration benchmark (whith nested benchmarks), collect its metrics
+function collectResults({ name, metrics, nodes, aggregate }: BenchmarkResultNode, collector: BenchmarkMetricsMap) {
+    let collectorNode = collector[name];
+    if (!collectorNode) {
+        collectorNode = collector[name] = { script: [], aggregate: [] };
     }
 
-    if (duration > 0) {
-        cNode.duration.push(duration);
+    if (aggregate > 0 && collectorNode.aggregate) {
+        collectorNode.aggregate.push(aggregate);
     }
 
-    if (runDuration !== undefined) {
-        cNode.runDuration.push(runDuration);
-    } else {
-        benchmarks.forEach((node: any) => collectResults(node, collector));
+    if (metrics) {
+        Object.keys(metrics).reduce((collector: BenchmarkMetricsAggregate, key: string) => {
+            const bucket = collector[key as BenchmarkMetricNames];
+            const value = metrics[key as BenchmarkMetricNames];
+            if (bucket && value) {
+                bucket.push(value);
+            }
+
+            return collector;
+        }, collectorNode);
     }
+
+    if (nodes) {
+        nodes.forEach((node: BenchmarkResultNode) => collectResults(node, collector));
+    }
+
     return collector;
 }
 
-function createStructure({ benchmarks, name, runDuration }: any, collector: any) {
-    if (runDuration !== undefined) {
+function createStructure({ nodes, name, type }: BenchmarkResultNode, collector: any): any {
+    if (type === "benchmark") {
         const newNode = collector[name];
         newNode.name = name;
         return newNode;
     }
 
-    return {
-        name,
-        benchmarks: benchmarks.map((childNode: any) => createStructure(childNode, collector)),
-    };
+    if (nodes) {
+        return {
+            name,
+            nodes: nodes.map((childNode: any) => createStructure(childNode, collector)),
+        };
+    }
+
+    return collector[name];
 }
 
-export async function analyzeBenchmarks(benchmarkResults: any) {
+export async function analyzeBenchmarks(benchmarkResults: BenchmarkResultsSnapshot[]) {
     return Promise.all(
-        benchmarkResults.map(async (benchmarkResult: any) => {
-            const { results, environment, benchmarkName, projectConfig } = benchmarkResult;
+        // For each benchmark file runned...
+        benchmarkResults.map(async (benchmarkResult: BenchmarkResultsSnapshot) => {
+            const { results, environment, benchmarkInfo: { benchmarkName }, projectConfig } = benchmarkResult;
             const structure = results[0];
-            const collector = results.reduce((c: any, result: any) => collectResults(result, c), {});
-
-            const benchmarkStats = Object.keys(collector).reduce((stats: any, bName) => {
-                const benchmarkMetrics = collector[bName];
-                stats[bName] = Object.keys(benchmarkMetrics).reduce((mc: any, metric) => {
-                    const list = benchmarkMetrics[metric];
-                    if (Array.isArray(list)) {
-                        if (list.length) {
-                            mc[metric] = computeSampleStats(benchmarkMetrics[metric], projectConfig);
-                        }
-                    } else {
-                        mc[metric] = benchmarkMetrics[metric];
+
+            // Collect the metrics for the nested benchmarks within
+            const collector: BenchmarkMetricsMap = results.reduce((reducer: BenchmarkMetricsMap, node: BenchmarkResultNode) => collectResults(node, reducer), {});
+
+            // For each metric
+            const benchmarkStats = Object.keys(collector).reduce((stats: any, benchmarkName: string) => {
+                const benchmarkMetrics = collector[benchmarkName];
+
+                stats[benchmarkName] = Object.keys(benchmarkMetrics).reduce((metricStats: any, metric: string) => {
+                    const metricResults = benchmarkMetrics[metric as BenchmarkMetricNames];
+                    if (Array.isArray(metricResults) && metricResults.length > 0) {
+                        metricStats[metric] = computeSampleStats(metricResults, projectConfig.samplesQuantileThreshold);
                     }
-                    return mc;
+                    return metricStats;
                 }, {});
+
                 return stats;
             }, {});
 
             const benchmarkStructure = createStructure(structure, benchmarkStats);
 
+            console.log(benchmarkStructure);
+
             benchmarkResult.stats = {
                 version: VERSION,
                 benchmarkName,

packages/@best/analyzer/src/stats.ts

@@ -20,7 +20,7 @@ export function variance(arr: number[]) {
 
 export function median(arr: number[]) {
     if (!arr.length) {
-        return null;
+        return 0;
     }
 
     const sorted = sort(arr);
@@ -33,6 +33,8 @@ export function medianAbsoluteDeviation(arr: number[]) {
     const med = median(arr);
     if (med) {
         return median(arr.map(x => Math.abs(x - med)));
+    } else {
+        return 0;
     }
 }
 

packages/@best/analyzer/tsconfig.json

@@ -3,5 +3,8 @@
   "compilerOptions": {
     "rootDir": "src",
     "outDir": "build",
-  }
+  },
+  "references": [
+    { "path": "../types" }
+  ]
 }

packages/@best/config/src/index.ts

@@ -46,7 +46,8 @@ function generateProjectConfigs(options: NormalizedConfig, isRoot: boolean, gitI
         benchmarkOutput: normalizeRootDirPattern(options.benchmarkOutput, options.rootDir),
         benchmarkCustomAssets: normalizeRootDirPattern(options.benchmarkCustomAssets, options.rootDir),
         testMatch: options.testMatch,
-        testPathIgnorePatterns: options.testPathIgnorePatterns
+        testPathIgnorePatterns: options.testPathIgnorePatterns,
+        samplesQuantileThreshold: options.samplesQuantileThreshold
     });
 
     return { globalConfig, projectConfig };

packages/@best/config/src/utils/defaults.ts

@@ -34,7 +34,7 @@ const defaultOptions = {
     ],
 
     // Calculate statistics on entire distributions (including possible outliers).
-    // samplesQuantileThreshold: 1,
+    samplesQuantileThreshold: 0.8,
 
     // Don't try to normalize distributions.
     // normalize: false,

packages/@best/runner-headless/src/index.ts

@@ -22,7 +22,7 @@ export default class Runner extends AbstractRunner {
             const { results } = await this.runIterations(browser, state, runtimeOptions, runnerLogStream);
             const version = await browser.version();
             const environment = await this.getEnvironment({ version }, projectConfig, globalConfig);
-            return { results, environment, benchmarkInfo };
+            return { results, environment, benchmarkInfo, projectConfig };
 
         } catch (e) {
             runnerLogStream.onBenchmarkError(benchmarkEntry);

packages/@best/types/src/benchmark.ts

@@ -1,4 +1,4 @@
-import { EnvironmentConfig } from "./config";
+import { EnvironmentConfig, FrozenProjectConfig } from "./config";
 
 export interface BenchmarkInfo {
     benchmarkName: string;
@@ -14,9 +14,11 @@ export interface BenchmarkRuntimeConfig {
     iterateOnClient: boolean;
 }
 
-export interface BenchmarkMetrics {
-    [key: string]: number;
-    script: number;
+
+export type BenchmarkMetricNames = "script" | "aggregate" | "paint" | "layout" | "system" | "idle";
+
+export type BenchmarkMetrics = {
+    [key in BenchmarkMetricNames]?: number;
 }
 
 export type ResultNodeTypes = "group" | "benchmark";
@@ -51,10 +53,21 @@ export interface BenchmarkResultsSnapshot {
     results: BenchmarkResultNode[];
     environment: EnvironmentConfig;
     benchmarkInfo: BenchmarkInfo;
+    projectConfig: FrozenProjectConfig;
+    stats?: any;
 }
 
 export interface BenchmarkResultsState {
     executedTime: number,
     executedIterations: number,
     results: BenchmarkResultNode[];
 }
+export interface BenchmarkStats {
+    samples: number[],
+    sampleSize: number,
+    samplesQuantileThreshold: number,
+    mean: number,
+    median: number,
+    variance: number,
+    medianAbsoluteDeviation: number,
+}

packages/@best/types/src/config.ts

@@ -75,6 +75,7 @@ export interface NormalizedConfig {
     benchmarkCustomAssets: string,
     testMatch: string[],
     testPathIgnorePatterns: string[],
+    samplesQuantileThreshold: number;
     rootDir: string
 }
 
@@ -106,6 +107,7 @@ export interface ProjectConfig {
     rootDir: string;
     testMatch: string[];
     testPathIgnorePatterns: string[];
+    samplesQuantileThreshold: number;
 }
 
 export type FrozenGlobalConfig = Readonly<GlobalConfig>;