From 91ecf4ffd9a1c1dde8e61590a34522cd24a877cc Mon Sep 17 00:00:00 2001 From: Diego Ferreiro Val Date: Tue, 18 Jun 2019 23:05:06 -0700 Subject: [PATCH] fix: almost done with types for analyzer --- packages/@best/analyzer/package.json | 3 +- packages/@best/analyzer/src/index.ts | 96 +++++++++++++-------- packages/@best/analyzer/src/stats.ts | 4 +- packages/@best/analyzer/tsconfig.json | 5 +- packages/@best/config/src/index.ts | 3 +- packages/@best/config/src/utils/defaults.ts | 2 +- packages/@best/runner-headless/src/index.ts | 2 +- packages/@best/types/src/benchmark.ts | 21 ++++- packages/@best/types/src/config.ts | 2 + 9 files changed, 94 insertions(+), 44 deletions(-) diff --git a/packages/@best/analyzer/package.json b/packages/@best/analyzer/package.json index 7370ebd6..ccc161d6 100644 --- a/packages/@best/analyzer/package.json +++ b/packages/@best/analyzer/package.json @@ -6,5 +6,6 @@ "Best", "performance" ], - "main": "build/index.js" + "main": "build/index.js", + "types": "build/index.d.ts" } diff --git a/packages/@best/analyzer/src/index.ts b/packages/@best/analyzer/src/index.ts index 936cde95..7a49ce0d 100644 --- a/packages/@best/analyzer/src/index.ts +++ b/packages/@best/analyzer/src/index.ts @@ -1,12 +1,13 @@ +import { BenchmarkResultsSnapshot, BenchmarkResultNode, BenchmarkMetricNames, BenchmarkStats } from "@best/types"; import { VERSION } from './constants'; import { quantile, mean, median, variance, medianAbsoluteDeviation, compare as compareSamples } from './stats'; -function computeSampleStats(arr: number[], config: any) { - const { samplesQuantileThreshold } = config; +function computeSampleStats(arr: number[], samplesQuantileThreshold: number): BenchmarkStats { if (samplesQuantileThreshold < 1) { const q = quantile(arr, samplesQuantileThreshold); arr = arr.filter(v => v <= q); } + return { samples: arr, sampleSize: arr.length, @@ -18,62 +19,89 @@ function computeSampleStats(arr: number[], config: any) { }; } -function collectResults({ name, duration, runDuration, benchmarks }: any, collector: any) { - let cNode = collector[name]; - if (typeof cNode !== 'object') { - cNode = collector[name] = { duration: [], runDuration: [] }; +type BenchmarkMetricsAggregate = { [key in BenchmarkMetricNames]?: number[] } + +// This type will hold as keys all benchmark names, and then an array with all results +interface BenchmarkMetricsMap { + [key: string]: BenchmarkMetricsAggregate +} + +// Given an iteration benchmark (whith nested benchmarks), collect its metrics +function collectResults({ name, metrics, nodes, aggregate }: BenchmarkResultNode, collector: BenchmarkMetricsMap) { + let collectorNode = collector[name]; + if (!collectorNode) { + collectorNode = collector[name] = { script: [], aggregate: [] }; } - if (duration > 0) { - cNode.duration.push(duration); + if (aggregate > 0 && collectorNode.aggregate) { + collectorNode.aggregate.push(aggregate); } - if (runDuration !== undefined) { - cNode.runDuration.push(runDuration); - } else { - benchmarks.forEach((node: any) => collectResults(node, collector)); + if (metrics) { + Object.keys(metrics).reduce((collector: BenchmarkMetricsAggregate, key: string) => { + const bucket = collector[key as BenchmarkMetricNames]; + const value = metrics[key as BenchmarkMetricNames]; + if (bucket && value) { + bucket.push(value); + } + + return collector; + }, collectorNode); } + + if (nodes) { + nodes.forEach((node: BenchmarkResultNode) => collectResults(node, collector)); + } + return collector; } -function createStructure({ benchmarks, name, runDuration }: any, collector: any) { - if (runDuration !== undefined) { +function createStructure({ nodes, name, type }: BenchmarkResultNode, collector: any): any { + if (type === "benchmark") { const newNode = collector[name]; newNode.name = name; return newNode; } - return { - name, - benchmarks: benchmarks.map((childNode: any) => createStructure(childNode, collector)), - }; + if (nodes) { + return { + name, + nodes: nodes.map((childNode: any) => createStructure(childNode, collector)), + }; + } + + return collector[name]; } -export async function analyzeBenchmarks(benchmarkResults: any) { +export async function analyzeBenchmarks(benchmarkResults: BenchmarkResultsSnapshot[]) { return Promise.all( - benchmarkResults.map(async (benchmarkResult: any) => { - const { results, environment, benchmarkName, projectConfig } = benchmarkResult; + // For each benchmark file runned... + benchmarkResults.map(async (benchmarkResult: BenchmarkResultsSnapshot) => { + const { results, environment, benchmarkInfo: { benchmarkName }, projectConfig } = benchmarkResult; const structure = results[0]; - const collector = results.reduce((c: any, result: any) => collectResults(result, c), {}); - - const benchmarkStats = Object.keys(collector).reduce((stats: any, bName) => { - const benchmarkMetrics = collector[bName]; - stats[bName] = Object.keys(benchmarkMetrics).reduce((mc: any, metric) => { - const list = benchmarkMetrics[metric]; - if (Array.isArray(list)) { - if (list.length) { - mc[metric] = computeSampleStats(benchmarkMetrics[metric], projectConfig); - } - } else { - mc[metric] = benchmarkMetrics[metric]; + + // Collect the metrics for the nested benchmarks within + const collector: BenchmarkMetricsMap = results.reduce((reducer: BenchmarkMetricsMap, node: BenchmarkResultNode) => collectResults(node, reducer), {}); + + // For each metric + const benchmarkStats = Object.keys(collector).reduce((stats: any, benchmarkName: string) => { + const benchmarkMetrics = collector[benchmarkName]; + + stats[benchmarkName] = Object.keys(benchmarkMetrics).reduce((metricStats: any, metric: string) => { + const metricResults = benchmarkMetrics[metric as BenchmarkMetricNames]; + if (Array.isArray(metricResults) && metricResults.length > 0) { + metricStats[metric] = computeSampleStats(metricResults, projectConfig.samplesQuantileThreshold); } - return mc; + return metricStats; }, {}); + return stats; }, {}); const benchmarkStructure = createStructure(structure, benchmarkStats); + console.log(benchmarkStructure); + benchmarkResult.stats = { version: VERSION, benchmarkName, diff --git a/packages/@best/analyzer/src/stats.ts b/packages/@best/analyzer/src/stats.ts index ec4f24c4..68a7bf7d 100644 --- a/packages/@best/analyzer/src/stats.ts +++ b/packages/@best/analyzer/src/stats.ts @@ -20,7 +20,7 @@ export function variance(arr: number[]) { export function median(arr: number[]) { if (!arr.length) { - return null; + return 0; } const sorted = sort(arr); @@ -33,6 +33,8 @@ export function medianAbsoluteDeviation(arr: number[]) { const med = median(arr); if (med) { return median(arr.map(x => Math.abs(x - med))); + } else { + return 0; } } diff --git a/packages/@best/analyzer/tsconfig.json b/packages/@best/analyzer/tsconfig.json index b8635ca4..fdd98b7e 100644 --- a/packages/@best/analyzer/tsconfig.json +++ b/packages/@best/analyzer/tsconfig.json @@ -3,5 +3,8 @@ "compilerOptions": { "rootDir": "src", "outDir": "build", - } + }, + "references": [ + { "path": "../types" } + ] } diff --git a/packages/@best/config/src/index.ts b/packages/@best/config/src/index.ts index 52580477..260d2a5e 100644 --- a/packages/@best/config/src/index.ts +++ b/packages/@best/config/src/index.ts @@ -46,7 +46,8 @@ function generateProjectConfigs(options: NormalizedConfig, isRoot: boolean, gitI benchmarkOutput: normalizeRootDirPattern(options.benchmarkOutput, options.rootDir), benchmarkCustomAssets: normalizeRootDirPattern(options.benchmarkCustomAssets, options.rootDir), testMatch: options.testMatch, - testPathIgnorePatterns: options.testPathIgnorePatterns + testPathIgnorePatterns: options.testPathIgnorePatterns, + samplesQuantileThreshold: options.samplesQuantileThreshold }); return { globalConfig, projectConfig }; diff --git a/packages/@best/config/src/utils/defaults.ts b/packages/@best/config/src/utils/defaults.ts index 3b3752f6..4503ac83 100644 --- a/packages/@best/config/src/utils/defaults.ts +++ b/packages/@best/config/src/utils/defaults.ts @@ -34,7 +34,7 @@ const defaultOptions = { ], // Calculate statistics on entire distributions (including possible outliers). - // samplesQuantileThreshold: 1, + samplesQuantileThreshold: 0.8, // Don't try to normalize distributions. // normalize: false, diff --git a/packages/@best/runner-headless/src/index.ts b/packages/@best/runner-headless/src/index.ts index 65f8dc62..11a50d55 100644 --- a/packages/@best/runner-headless/src/index.ts +++ b/packages/@best/runner-headless/src/index.ts @@ -22,7 +22,7 @@ export default class Runner extends AbstractRunner { const { results } = await this.runIterations(browser, state, runtimeOptions, runnerLogStream); const version = await browser.version(); const environment = await this.getEnvironment({ version }, projectConfig, globalConfig); - return { results, environment, benchmarkInfo }; + return { results, environment, benchmarkInfo, projectConfig }; } catch (e) { runnerLogStream.onBenchmarkError(benchmarkEntry); diff --git a/packages/@best/types/src/benchmark.ts b/packages/@best/types/src/benchmark.ts index d79da39b..d10b7026 100644 --- a/packages/@best/types/src/benchmark.ts +++ b/packages/@best/types/src/benchmark.ts @@ -1,4 +1,4 @@ -import { EnvironmentConfig } from "./config"; +import { EnvironmentConfig, FrozenProjectConfig } from "./config"; export interface BenchmarkInfo { benchmarkName: string; @@ -14,9 +14,11 @@ export interface BenchmarkRuntimeConfig { iterateOnClient: boolean; } -export interface BenchmarkMetrics { - [key: string]: number; - script: number; + +export type BenchmarkMetricNames = "script" | "aggregate" | "paint" | "layout" | "system" | "idle"; + +export type BenchmarkMetrics = { + [key in BenchmarkMetricNames]?: number; } export type ResultNodeTypes = "group" | "benchmark"; @@ -51,6 +53,8 @@ export interface BenchmarkResultsSnapshot { results: BenchmarkResultNode[]; environment: EnvironmentConfig; benchmarkInfo: BenchmarkInfo; + projectConfig: FrozenProjectConfig; + stats?: any; } export interface BenchmarkResultsState { @@ -58,3 +62,12 @@ export interface BenchmarkResultsState { executedIterations: number, results: BenchmarkResultNode[]; } +export interface BenchmarkStats { + samples: number[], + sampleSize: number, + samplesQuantileThreshold: number, + mean: number, + median: number, + variance: number, + medianAbsoluteDeviation: number, +} diff --git a/packages/@best/types/src/config.ts b/packages/@best/types/src/config.ts index e6c2e5d0..ff79dbaf 100644 --- a/packages/@best/types/src/config.ts +++ b/packages/@best/types/src/config.ts @@ -75,6 +75,7 @@ export interface NormalizedConfig { benchmarkCustomAssets: string, testMatch: string[], testPathIgnorePatterns: string[], + samplesQuantileThreshold: number; rootDir: string } @@ -106,6 +107,7 @@ export interface ProjectConfig { rootDir: string; testMatch: string[]; testPathIgnorePatterns: string[]; + samplesQuantileThreshold: number; } export type FrozenGlobalConfig = Readonly;