Skip to content

Commit

Permalink
Performance suite: track TimeToFirstByte in front-end (#47037)
Browse files Browse the repository at this point in the history
Co-authored-by: Sarah Norris <[email protected]>
  • Loading branch information
oandregal and mikachan committed Jan 26, 2023
1 parent 046d404 commit e5b286f
Show file tree
Hide file tree
Showing 7 changed files with 210 additions and 102 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/performance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@ jobs:
run: |
npm ci
- name: Install specific versions of the themes used in tests
run: |
npm run wp-env start
npm run wp-env -- run tests-cli "wp theme update twentytwentyone --version=1.7"
npm run wp-env -- run tests-cli "wp theme update twentytwentythree --version=1.0"
npm run wp-env stop
- name: Compare performance with trunk
if: github.event_name == 'pull_request'
run: ./bin/plugin/cli.js perf $GITHUB_SHA trunk --tests-branch $GITHUB_SHA
Expand Down
183 changes: 81 additions & 102 deletions bin/plugin/commands/performance.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ const config = require( '../config' );
/**
* @typedef WPRawPerformanceResults
*
* @property {number[]} timeToFirstByte Represents the time since the browser started the request until it received a response.
* @property {number[]} serverResponse Represents the time the server takes to respond.
* @property {number[]} firstPaint Represents the time when the user agent first rendered after navigation.
* @property {number[]} domContentLoaded Represents the time immediately after the document's DOMContentLoaded event completes.
Expand All @@ -48,33 +49,35 @@ const config = require( '../config' );
/**
* @typedef WPPerformanceResults
*
* @property {number=} serverResponse Represents the time the server takes to respond.
* @property {number=} firstPaint Represents the time when the user agent first rendered after navigation.
* @property {number=} domContentLoaded Represents the time immediately after the document's DOMContentLoaded event completes.
* @property {number=} loaded Represents the time when the load event of the current document is completed.
* @property {number=} firstContentfulPaint Represents the time when the browser first renders any text or media.
* @property {number=} firstBlock Represents the time when Puppeteer first sees a block selector in the DOM.
* @property {number=} type Average type time.
* @property {number=} minType Minimum type time.
* @property {number=} maxType Maximum type time.
* @property {number=} typeContainer Average type time within a container.
* @property {number=} minTypeContainer Minimum type time within a container.
* @property {number=} maxTypeContainer Maximum type time within a container.
* @property {number=} focus Average block selection time.
* @property {number=} minFocus Min block selection time.
* @property {number=} maxFocus Max block selection time.
* @property {number=} inserterOpen Average time to open global inserter.
* @property {number=} minInserterOpen Min time to open global inserter.
* @property {number=} maxInserterOpen Max time to open global inserter.
* @property {number=} inserterSearch Average time to open global inserter.
* @property {number=} minInserterSearch Min time to open global inserter.
* @property {number=} maxInserterSearch Max time to open global inserter.
* @property {number=} inserterHover Average time to move mouse between two block item in the inserter.
* @property {number=} minInserterHover Min time to move mouse between two block item in the inserter.
* @property {number=} maxInserterHover Max time to move mouse between two block item in the inserter.
* @property {number=} listViewOpen Average time to open list view.
* @property {number=} minListViewOpen Min time to open list view.
* @property {number=} maxListViewOpen Max time to open list view.
* @property {number=} timeToFirstByteMedian Represents the time since the browser started the request until it received a response (median).
* @property {number=} timeToFirstByteP75 Represents the time since the browser started the request until it received a response (75th percentile).
* @property {number=} serverResponse Represents the time the server takes to respond.
* @property {number=} firstPaint Represents the time when the user agent first rendered after navigation.
* @property {number=} domContentLoaded Represents the time immediately after the document's DOMContentLoaded event completes.
* @property {number=} loaded Represents the time when the load event of the current document is completed.
* @property {number=} firstContentfulPaint Represents the time when the browser first renders any text or media.
* @property {number=} firstBlock Represents the time when Puppeteer first sees a block selector in the DOM.
* @property {number=} type Average type time.
* @property {number=} minType Minimum type time.
* @property {number=} maxType Maximum type time.
* @property {number=} typeContainer Average type time within a container.
* @property {number=} minTypeContainer Minimum type time within a container.
* @property {number=} maxTypeContainer Maximum type time within a container.
* @property {number=} focus Average block selection time.
* @property {number=} minFocus Min block selection time.
* @property {number=} maxFocus Max block selection time.
* @property {number=} inserterOpen Average time to open global inserter.
* @property {number=} minInserterOpen Min time to open global inserter.
* @property {number=} maxInserterOpen Max time to open global inserter.
* @property {number=} inserterSearch Average time to open global inserter.
* @property {number=} minInserterSearch Min time to open global inserter.
* @property {number=} maxInserterSearch Max time to open global inserter.
* @property {number=} inserterHover Average time to move mouse between two block item in the inserter.
* @property {number=} minInserterHover Min time to move mouse between two block item in the inserter.
* @property {number=} maxInserterHover Max time to move mouse between two block item in the inserter.
* @property {number=} listViewOpen Average time to open list view.
* @property {number=} minListViewOpen Min time to open list view.
* @property {number=} maxListViewOpen Max time to open list view.
*/

/**
Expand Down Expand Up @@ -103,6 +106,19 @@ function median( array ) {
: ( numbers[ mid - 1 ] + numbers[ mid ] ) / 2;
}

/**
* Computes the 75th percentile from an array of numbers.
*
* @param {number[]} array
*
* @return {number} 75th percentile of the given dataset.
*/
function percentile75( array ) {
const ascending = array.sort( ( a, b ) => a - b );
const position = Math.floor( ( 75 / 100 ) * array.length );
return ascending[ position ];
}

/**
* Rounds and format a time passed in milliseconds.
*
Expand All @@ -118,11 +134,22 @@ function formatTime( number ) {
/**
* Curate the raw performance results.
*
* @param {string} testSuite
* @param {WPRawPerformanceResults} results
*
* @return {WPPerformanceResults} Curated Performance results.
*/
function curateResults( results ) {
function curateResults( testSuite, results ) {
if (
testSuite === 'front-end-classic-theme' ||
testSuite === 'front-end-block-theme'
) {
return {
timeToFirstByteMedian: median( results.timeToFirstByte ),
timeToFirstByteP75: percentile75( results.timeToFirstByte ),
};
}

return {
serverResponse: average( results.serverResponse ),
firstPaint: average( results.firstPaint ),
Expand Down Expand Up @@ -173,7 +200,7 @@ async function runTestSuite( testSuite, performanceTestDirectory ) {
`packages/e2e-tests/specs/performance/${ testSuite }.test.results.json`
)
);
return curateResults( rawResults );
return curateResults( testSuite, rawResults );
}

/**
Expand Down Expand Up @@ -344,7 +371,12 @@ async function runPerformanceTests( branches, options ) {
// 4- Running the tests.
log( '\n>> Running the tests' );

const testSuites = [ 'post-editor', 'site-editor' ];
const testSuites = [
'post-editor',
'site-editor',
'front-end-classic-theme',
'front-end-block-theme',
];

/** @type {Record<string,Record<string, WPPerformanceResults>>} */
const results = {};
Expand Down Expand Up @@ -379,78 +411,25 @@ async function runPerformanceTests( branches, options ) {

// Computing medians.
for ( const branch of branches ) {
const medians = mapValues(
{
serverResponse: rawResults.map(
( r ) => r[ branch ].serverResponse
),
firstPaint: rawResults.map(
( r ) => r[ branch ].firstPaint
),
domContentLoaded: rawResults.map(
( r ) => r[ branch ].domContentLoaded
),
loaded: rawResults.map( ( r ) => r[ branch ].loaded ),
firstContentfulPaint: rawResults.map(
( r ) => r[ branch ].firstContentfulPaint
),
firstBlock: rawResults.map(
( r ) => r[ branch ].firstBlock
),
type: rawResults.map( ( r ) => r[ branch ].type ),
minType: rawResults.map( ( r ) => r[ branch ].minType ),
maxType: rawResults.map( ( r ) => r[ branch ].maxType ),
typeContainer: rawResults.map(
( r ) => r[ branch ].typeContainer
),
minTypeContainer: rawResults.map(
( r ) => r[ branch ].minTypeContainer
),
maxTypeContainer: rawResults.map(
( r ) => r[ branch ].maxTypeContainer
),
focus: rawResults.map( ( r ) => r[ branch ].focus ),
minFocus: rawResults.map( ( r ) => r[ branch ].minFocus ),
maxFocus: rawResults.map( ( r ) => r[ branch ].maxFocus ),
inserterOpen: rawResults.map(
( r ) => r[ branch ].inserterOpen
),
minInserterOpen: rawResults.map(
( r ) => r[ branch ].minInserterOpen
),
maxInserterOpen: rawResults.map(
( r ) => r[ branch ].maxInserterOpen
),
inserterSearch: rawResults.map(
( r ) => r[ branch ].inserterSearch
),
minInserterSearch: rawResults.map(
( r ) => r[ branch ].minInserterSearch
),
maxInserterSearch: rawResults.map(
( r ) => r[ branch ].maxInserterSearch
),
inserterHover: rawResults.map(
( r ) => r[ branch ].inserterHover
),
minInserterHover: rawResults.map(
( r ) => r[ branch ].minInserterHover
),
maxInserterHover: rawResults.map(
( r ) => r[ branch ].maxInserterHover
),
listViewOpen: rawResults.map(
( r ) => r[ branch ].listViewOpen
),
minListViewOpen: rawResults.map(
( r ) => r[ branch ].minListViewOpen
),
maxListViewOpen: rawResults.map(
( r ) => r[ branch ].maxListViewOpen
),
},
median
);
/**
* @type {string[]}
*/
let dataPointsForTestSuite = [];
if ( rawResults.length > 0 ) {
dataPointsForTestSuite = Object.keys(
rawResults[ 0 ][ branch ]
);
}

const resultsByDataPoint = {};
dataPointsForTestSuite.forEach( ( dataPoint ) => {
// @ts-ignore
resultsByDataPoint[ dataPoint ] = rawResults.map(
// @ts-ignore
( r ) => r[ branch ][ dataPoint ]
);
} );
const medians = mapValues( resultsByDataPoint, median );

// Format results as times.
results[ testSuite ][ branch ] = mapValues( medians, formatTime );
Expand Down
4 changes: 4 additions & 0 deletions packages/e2e-test-utils/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,10 @@ _Parameters_
- _username_ `?string`: String to be used as user credential.
- _password_ `?string`: String to be used as user credential.

### logout

Performs log out.

### mockOrTransform

Mocks a request with the supplied mock object, or allows it to run with an optional transform, based on the
Expand Down
1 change: 1 addition & 0 deletions packages/e2e-test-utils/src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ export { installTheme } from './install-theme';
export { isCurrentURL } from './is-current-url';
export { isInDefaultBlock } from './is-in-default-block';
export { loginUser } from './login-user';
export { logout } from './logout';
export { createMenu, deleteAllMenus } from './menus';
export {
enableFocusLossObservation,
Expand Down
25 changes: 25 additions & 0 deletions packages/e2e-test-utils/src/logout.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/**
* Internal dependencies
*/
import { createURL } from './create-url';
import { isCurrentURL } from './is-current-url';

/**
* Performs log out.
*
*/
export async function logout() {
// If it is logged and in a page different than the dashboard,
// move to the dashboard. Some pages may be in full-screen mode,
// so they won't have the log-out button available.
if ( ! isCurrentURL( 'wp-login.php' ) && ! isCurrentURL( 'wp-admin' ) ) {
await page.goto( createURL( 'wp-admin' ) );
}

await Promise.all( [
page.hover( '#wp-admin-bar-my-account' ),
page.waitForSelector( '#wp-admin-bar-logout', { visible: true } ),
] );

await page.click( '#wp-admin-bar-logout' );
}
47 changes: 47 additions & 0 deletions packages/e2e-tests/specs/performance/front-end-block-theme.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/**
* External dependencies
*/
import { basename, join } from 'path';
import { writeFileSync } from 'fs';

/**
* WordPress dependencies
*/
import { activateTheme, createURL, logout } from '@wordpress/e2e-test-utils';

describe( 'Front End Performance', () => {
const results = {
timeToFirstByte: [],
};

beforeAll( async () => {
await activateTheme( 'twentytwentythree' );
await logout();
} );

afterAll( async () => {
await activateTheme( 'twentytwentyone' );
const resultsFilename = basename( __filename, '.js' ) + '.results.json';
writeFileSync(
join( __dirname, resultsFilename ),
JSON.stringify( results, null, 2 )
);
} );

it( 'Time To First Byte (TTFB)', async () => {
// We derive the 75th percentile of the TTFB based on these results.
// By running it 16 times, the percentile value would be (75/100)*16=12,
// meaning that we discard the worst 4 values.
let i = 16;
while ( i-- ) {
await page.goto( createURL( '/' ) );
const navigationTimingJson = await page.evaluate( () =>
JSON.stringify( performance.getEntriesByType( 'navigation' ) )
);
const [ navigationTiming ] = JSON.parse( navigationTimingJson );
results.timeToFirstByte.push(
navigationTiming.responseStart - navigationTiming.startTime
);
}
} );
} );
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/**
* External dependencies
*/
import { basename, join } from 'path';
import { writeFileSync } from 'fs';

/**
* WordPress dependencies
*/
import { createURL, logout } from '@wordpress/e2e-test-utils';

describe( 'Front End Performance', () => {
const results = {
timeToFirstByte: [],
};

beforeAll( async () => {
await logout();
} );

afterAll( async () => {
const resultsFilename = basename( __filename, '.js' ) + '.results.json';
writeFileSync(
join( __dirname, resultsFilename ),
JSON.stringify( results, null, 2 )
);
} );

it( 'Time To First Byte (TTFB)', async () => {
// We derive the 75th percentile of the TTFB based on these results.
// By running it 16 times, the percentile value would be (75/100)*16=12,
// meaning that we discard the worst 4 values.
let i = 16;
while ( i-- ) {
await page.goto( createURL( '/' ) );
const navigationTimingJson = await page.evaluate( () =>
JSON.stringify( performance.getEntriesByType( 'navigation' ) )
);
const [ navigationTiming ] = JSON.parse( navigationTimingJson );
results.timeToFirstByte.push(
navigationTiming.responseStart - navigationTiming.startTime
);
}
} );
} );

0 comments on commit e5b286f

Please sign in to comment.