Merge branch 'main' into cleanup/shared

Author: flashdesignory

experimental/README.md

@@ -0,0 +1,31 @@
+# Experimental Workloads
+
+Experimental Speedometer workloads can be used for workloads that do not
+directly meet the criteria for official workloads:
+
+-   Incubation of new workloads
+-   Edge-cases that don't work well in the official setup
+-   Testing features that aren't fully shipped in all browsers
+
+The main purpose is to have a space for low-friction contributions to
+experiment with new ideas. There is no explicit requirement to promote
+experimental workloads to the official selection.
+
+## Review Process
+
+Contributions to experimental workloads can be merged with the approval from a
+single vendor.
+
+This is the opposite from official workloads that are scored, where we require
+approval from all vendors.
+
+## Requirements
+
+-   Stored in the `/experimental` folder
+-   Disabled by default and thus do not contribute to the official score
+-   Use the `experimental` tag in the Suites declaration
+
+## Hosting
+
+Experimental workloads will not be officially hosted on Speedometer releases on
+<https://browserbench.org>, unlike the official workloads.

index.html

@@ -100,7 +100,7 @@ <h2>Non-standard Parameters</h2>
                         </div>
                         <div class="aggregated-metric-result">
                             <h2>Aggregate Metric</h2>
-                            <div id="geomean-chart"></div>
+                            <div id="aggregate-chart"></div>
                             <h2>Test Metrics Overview</h2>
                             <div id="tests-chart"></div>
                         </div>

resources/benchmark-runner.mjs

@@ -1,5 +1,6 @@
 import { Metric } from "./metric.mjs";
 import { params } from "./shared/params.mjs";
+import { forceLayout } from "./shared/helpers.mjs";
 import { SUITE_RUNNER_LOOKUP } from "./suite-runner.mjs";
 
 const performance = globalThis.performance;
@@ -30,8 +31,9 @@ class Page {
     }
 
     layout() {
-        const body = this._frame.contentDocument.body.getBoundingClientRect();
-        this.layout.e = document.elementFromPoint((body.width / 2) | 0, (body.height / 2) | 0);
+        const body = this._frame ? this._frame.contentDocument.body : document.body;
+        const value = forceLayout(body, params.layoutMode);
+        body._leakedLayoutValue = value; // Prevent dead code elimination.
     }
 
     async waitForElement(selector) {
@@ -407,9 +409,8 @@ export class BenchmarkRunner {
         const suites = await this._prepareAllSuites();
         try {
             for (const suite of suites) {
-                if (suite.disabled)
+                if (!suite.enabled)
                     continue;
-
                 try {
                     await this._appendFrame();
                     this._page = new Page(this._frame);
@@ -468,10 +469,10 @@ export class BenchmarkRunner {
 
     _appendIterationMetrics() {
         const getMetric = (name, unit = "ms") => this._metrics[name] || (this._metrics[name] = new Metric(name, unit));
-        const iterationTotalMetric = (i) => {
+        const iterationMetric = (i, name) => {
             if (i >= params.iterationCount)
                 throw new Error(`Requested iteration=${i} does not exist.`);
-            return getMetric(`Iteration-${i}-Total`);
+            return getMetric(`Iteration-${i}-${name}`);
         };
 
         const collectSubMetrics = (prefix, items, parent) => {
@@ -496,20 +497,34 @@ export class BenchmarkRunner {
             // Prepare all iteration metrics so they are listed at the end of
             // of the _metrics object, before "Total" and "Score".
             for (let i = 0; i < this._iterationCount; i++)
-                iterationTotalMetric(i).description = `Test totals for iteration ${i}`;
+                iterationMetric(i, "Total").description = `Test totals for iteration ${i}`;
             getMetric("Geomean", "ms").description = "Geomean of test totals";
             getMetric("Score", "score").description = "Scaled inverse of the Geomean";
+            if (params.measurePrepare)
+                getMetric("Prepare", "ms").description = "Geomean of workload prepare times";
         }
 
         const geomean = getMetric("Geomean");
-        const iterationTotal = iterationTotalMetric(geomean.length);
+        const iteration = geomean.length;
+        const iterationTotal = iterationMetric(iteration, "Total");
         for (const results of Object.values(iterationResults))
             iterationTotal.add(results.total);
         iterationTotal.computeAggregatedMetrics();
         geomean.add(iterationTotal.geomean);
         getMetric("Score").add(geomeanToScore(iterationTotal.geomean));
 
+        if (params.measurePrepare) {
+            const iterationPrepare = iterationMetric(iteration, "Prepare");
+            for (const results of Object.values(iterationResults))
+                iterationPrepare.add(results.prepare);
+            iterationPrepare.computeAggregatedMetrics();
+            const prepare = getMetric("Prepare");
+            prepare.add(iterationPrepare.geomean);
+        }
+
         for (const metric of Object.values(this._metrics))
             metric.computeAggregatedMetrics();
     }
+
+    _initializeMetrics() {}
 }

resources/developer-mode.mjs

@@ -1,5 +1,5 @@
 import { Suites, Tags } from "./tests.mjs";
-import { params } from "./shared/params.mjs";
+import { params, LAYOUT_MODES } from "./shared/params.mjs";
 
 export function createDeveloperModeContainer() {
     const container = document.createElement("div");
@@ -18,10 +18,12 @@ export function createDeveloperModeContainer() {
     const settings = document.createElement("div");
     settings.className = "settings";
     settings.append(createUIForIterationCount());
+    settings.append(createUIForMeasurePrepare());
     settings.append(createUIForWarmupSuite());
     settings.append(createUIForWarmupBeforeSync());
     settings.append(createUIForSyncStepDelay());
     settings.append(createUIForAsyncSteps());
+    settings.append(createUIForLayoutMode());
 
     content.append(document.createElement("hr"));
     content.append(settings);
@@ -46,6 +48,12 @@ function createUIForWarmupSuite() {
     });
 }
 
+function createUIForMeasurePrepare() {
+    return createCheckboxUI("Measure Prepare", params.measurePrepare, (isChecked) => {
+        params.measurePrepare = isChecked;
+    });
+}
+
 function createUIForAsyncSteps() {
     return createCheckboxUI("Use Async Steps", params.useAsyncSteps, (isChecked) => {
         params.useAsyncSteps = isChecked;
@@ -107,12 +115,37 @@ function createTimeRangeUI(labelText, paramKey, unit = "ms", min = 0, max = 1000
     return label;
 }
 
+function createUIForLayoutMode() {
+    return createSelectUI("Force layout mode", params.layoutMode, LAYOUT_MODES, (value) => {
+        params.layoutMode = value;
+    });
+}
+
+function createSelectUI(labelValue, initialValue, choices, paramsUpdateCallback) {
+    const select = document.createElement("select");
+    select.onchange = () => {
+        paramsUpdateCallback(select.value);
+        updateURL();
+    };
+
+    choices.forEach((choice) => {
+        const option = new Option(choice, choice);
+        select.add(option);
+    });
+    select.value = initialValue;
+
+    const label = document.createElement("label");
+    label.append(span(labelValue), select);
+
+    return label;
+}
+
 function createUIForSuites() {
     const control = document.createElement("nav");
     control.className = "suites";
     const checkboxes = [];
     const setSuiteEnabled = (suiteIndex, enabled) => {
-        Suites[suiteIndex].disabled = !enabled;
+        Suites[suiteIndex].enabled = enabled;
         checkboxes[suiteIndex].checked = enabled;
     };
 
@@ -124,9 +157,9 @@ function createUIForSuites() {
         const checkbox = document.createElement("input");
         checkbox.id = suite.name;
         checkbox.type = "checkbox";
-        checkbox.checked = !suite.disabled;
+        checkbox.checked = suite.enabled;
         checkbox.onchange = () => {
-            suite.disabled = !checkbox.checked;
+            suite.enabled = checkbox.checked;
             updateURL();
         };
         checkboxes.push(checkbox);
@@ -219,12 +252,13 @@ function createSuitesTagsButton(setSuiteEnabled) {
 
 function createUIForRun() {
     const stepTestButton = document.createElement("button");
-    stepTestButton.textContent = "Step Test \u23EF";
+    stepTestButton.className = "step-button";
+    stepTestButton.innerHTML = "Step Test<span>\u23EF</span>";
     stepTestButton.onclick = (event) => {
         globalThis.benchmarkClient.step();
     };
     const startTestButton = document.createElement("button");
-    startTestButton.textContent = "Start Test \u23F5";
+    startTestButton.innerHTML = "Start Test<span>\u23F5</span>";
     startTestButton.onclick = (event) => {
         globalThis.benchmarkClient.start();
     };
@@ -241,17 +275,17 @@ function updateParamsSuitesAndTags() {
 
     // If less than all suites are selected then change the URL "Suites" GET parameter
     // to comma separate only the selected
-    const selectedSuites = Suites.filter((suite) => !suite.disabled);
+    const selectedSuites = Suites.filter((suite) => suite.enabled);
     if (!selectedSuites.length)
         return;
 
     // Try finding common tags that would result in the current suite selection.
     let commonTags = new Set(selectedSuites[0].tags);
     for (const suite of Suites) {
-        if (suite.disabled)
-            suite.tags.forEach((tag) => commonTags.delete(tag));
-        else
+        if (suite.enabled)
             commonTags = new Set(suite.tags.filter((tag) => commonTags.has(tag)));
+        else
+            suite.tags.forEach((tag) => commonTags.delete(tag));
     }
     if (selectedSuites.length > 1 && commonTags.size)
         params.tags = [...commonTags];

resources/main.css

@@ -287,15 +287,28 @@ button,
 }
 
 .developer-mode-content button {
+    font-size: 16px;
     background: white;
     flex: auto;
     padding: 4px;
     appearance: button;
     border: 2px solid rgba(255, 255, 255, 0.5);
     border-radius: 10px;
 }
+
+.developer-mode-content button span {
+    padding-left: 10px;
+    font-size: 1.5em;
+    line-height: 10px;
+    position: relative;
+    top: 2px;
+}
+
 .developer-mode-content .tag {
     border-radius: 100vh;
+}
+
+.developer-mode-content .step-button {
     color: white;
     background: rgba(255, 255, 255, 0.1);
 }
@@ -518,6 +531,8 @@ section#details .non-standard-params {
     display: none;
     text-align: center;
     margin-bottom: 2em;
+    /* Increase contrast a bit */
+    filter: brightness(1.5);
 }
 
 section#details .non-standard-params h2 {
@@ -547,6 +562,9 @@ section#details .non-standard-params h2 {
 #non-standard-params-table tbody td {
     padding: 0.1em 0.3em;
 }
+#non-standard-params-table tbody td:nth-child(2) {
+    color: var(--highlight);
+}
 
 section#details .all-metric-results {
     flex: auto;

resources/main.mjs

@@ -66,7 +66,9 @@ class MainBenchmarkClient {
         if (this._isRunning)
             return false;
 
-        if (Suites.every((suite) => suite.disabled)) {
+        const enabledSuites = Suites.filter((suite) => suite.enabled);
+        const totalSuitesCount = enabledSuites.length;
+        if (totalSuitesCount === 0) {
             const message = `No suites selected - "${params.suites}" does not exist.`;
             alert(message);
             console.error(
@@ -75,7 +77,6 @@ class MainBenchmarkClient {
                 "\nValid values:",
                 Suites.map((each) => each.name)
             );
-
             return false;
         }
         if (!this._isStepping())
@@ -93,8 +94,6 @@ class MainBenchmarkClient {
         this._metrics = Object.create(null);
         this._isRunning = true;
 
-        const enabledSuites = Suites.filter((suite) => !suite.disabled);
-        const totalSuitesCount = enabledSuites.length;
         this.stepCount = params.iterationCount * totalSuitesCount;
         this._progressCompleted.max = this.stepCount;
         this.suitesCount = enabledSuites.length;
@@ -250,8 +249,11 @@ class MainBenchmarkClient {
         const trackHeight = 24;
         document.documentElement.style.setProperty("--metrics-line-height", `${trackHeight}px`);
         const plotWidth = (params.viewport.width - 120) / 2;
-        document.getElementById("geomean-chart").innerHTML = renderMetricView({
-            metrics: [metrics.Geomean],
+        const aggregateMetrics = [metrics.Geomean];
+        if (params.measurePrepare)
+            aggregateMetrics.push(metrics.Prepare);
+        document.getElementById("aggregate-chart").innerHTML = renderMetricView({
+            metrics: aggregateMetrics,
             width: plotWidth,
             trackHeight,
             renderChildren: false,

resources/shared/benchmark.mjs

@@ -44,6 +44,7 @@ export class BenchmarkSuite {
     async runAndRecordSuite(params, onProgress) {
         const measuredValues = {
             tests: {},
+            prepare: 0,
             total: 0,
         };
         const suiteStartLabel = `suite-${this.name}-start`;

resources/shared/helpers.mjs

@@ -41,15 +41,15 @@ export function getAllElements(selector, path = [], lookupStartNode = document)
     return elements;
 }
 
-/**
- * Forces a reflow/layout of the document by requesting a DOM property that triggers it.
- * This can be useful for ensuring that styles and positions are up-to-date before
- * performing measurements or animations. It also returns an element from the center of the viewport.
- *
- * @returns {Element|null} An element at the center of the viewport, or null if no element is there.
- */
-export function forceLayout() {
-    const rect = document.body.getBoundingClientRect();
-    const e = document.elementFromPoint((rect.width / 2) | 0, (rect.height / 2) | 0);
-    return e;
+export function forceLayout(body, layoutMode = "getBoundingRectAndElementFromPoint") {
+    body ??= document.body;
+    const rect = body.getBoundingClientRect();
+    switch (layoutMode) {
+        case "getBoundingRectAndElementFromPoint":
+            return document.elementFromPoint((rect.width / 2) | 0, (rect.height / 2) | 0);
+        case "getBoundingClientRect":
+            return rect.height;
+        default:
+            throw Error(`Invalid layoutMode: ${layoutMode}`);
+    }
 }