Repository navigation
chore: refresh benchmarks for v1.27.0 #1041
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| { | ||
| "updated": "2026-07-07T16:22:27.898Z", | ||
| "updated": "2026-07-15T06:18:07.043Z", | ||
| "unit": "ms", | ||
| "managers": [ | ||
| "aube", | ||
|
|
@@ -10,180 +10,180 @@ | |
| "yarn" | ||
| ], | ||
| "versions": { | ||
| "aube": "1.26.0", | ||
| "aube": "1.27.0", | ||
| "bun": "1.3.14", | ||
| "deno": "2.9.1", | ||
| "pnpm": "11.10.0", | ||
| "deno": "2.9.2", | ||
| "pnpm": "11.13.0", | ||
| "npm": "11.9.0", | ||
| "yarn": "4.17.0", | ||
| "yarn": "4.17.1", | ||
| "node": "24.14.0" | ||
| }, | ||
| "rows": [ | ||
| { | ||
| "key": "gvs-warm", | ||
| "label": "Fresh install (warm cache)", | ||
| "values": { | ||
| "aube": 380, | ||
| "bun": 1154, | ||
| "deno": 1353, | ||
| "pnpm": 2212, | ||
| "npm": 8656, | ||
| "yarn": 8864 | ||
| "aube": 263, | ||
| "bun": 2924, | ||
| "deno": 1292, | ||
| "pnpm": 3419, | ||
| "npm": 7825, | ||
| "yarn": 9920 | ||
| }, | ||
| "stats": { | ||
| "aube": { | ||
| "text": "0.380s ± 0.029s", | ||
| "mean": 0.37960803616, | ||
| "stddev": 0.028995153441263417, | ||
| "min": 0.33228807296, | ||
| "max": 0.42753080496 | ||
| "text": "0.263s ± 0.018s", | ||
| "mean": 0.26340452584, | ||
| "stddev": 0.017752895232022173, | ||
| "min": 0.24065752724, | ||
| "max": 0.29142403224 | ||
| }, | ||
| "bun": { | ||
| "text": "1.154s ± 0.143s", | ||
| "mean": 1.1536498483200002, | ||
| "stddev": 0.14330443228429496, | ||
| "min": 0.8957299523200001, | ||
| "max": 1.3249242303200002 | ||
| "text": "2.924s ± 0.261s", | ||
| "mean": 2.9239087655, | ||
| "stddev": 0.26094548627684094, | ||
| "min": 2.6482493493, | ||
| "max": 3.4492324233 | ||
| }, | ||
| "deno": { | ||
| "text": "1.353s ± 0.129s", | ||
| "mean": 1.3533667281, | ||
| "stddev": 0.12879718803228762, | ||
| "min": 1.0377285978, | ||
| "max": 1.5013489328 | ||
| "text": "1.292s ± 0.158s", | ||
| "mean": 1.29186679204, | ||
| "stddev": 0.15795715667232396, | ||
| "min": 1.0006102911399999, | ||
| "max": 1.5238904391399999 | ||
| }, | ||
| "pnpm": { | ||
| "text": "2.212s ± 0.053s", | ||
| "mean": 2.21228314922, | ||
| "stddev": 0.052559484948393374, | ||
| "min": 2.16293807582, | ||
| "max": 2.29254428682 | ||
| "text": "3.419s ± 0.627s", | ||
| "mean": 3.4189730159, | ||
| "stddev": 0.6274974055754965, | ||
| "min": 2.6780925231, | ||
| "max": 4.058610031100001 | ||
| }, | ||
| "npm": { | ||
| "text": "8.656s ± 0.442s", | ||
| "mean": 8.656315280129999, | ||
| "stddev": 0.44225154486324547, | ||
| "min": 8.17292329188, | ||
| "max": 9.113102440879999 | ||
| "text": "7.825s ± 0.387s", | ||
| "mean": 7.825020734210001, | ||
| "stddev": 0.3868661734265498, | ||
| "min": 7.451966108460001, | ||
| "max": 8.30574446846 | ||
| }, | ||
| "yarn": { | ||
| "text": "8.864s ± 0.825s", | ||
| "mean": 8.86428247321, | ||
| "stddev": 0.8248689796421752, | ||
| "min": 8.218083998460001, | ||
| "max": 9.97151420646 | ||
| "text": "9.920s ± 0.214s", | ||
| "mean": 9.91973897257, | ||
| "stddev": 0.21406660036591305, | ||
| "min": 9.70148654982, | ||
| "max": 10.17380474682 | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "key": "gvs-cold", | ||
| "label": "Fresh install (cold cache)", | ||
| "values": { | ||
| "aube": 17450, | ||
| "bun": 7541, | ||
| "deno": 8449, | ||
| "pnpm": 14387, | ||
| "npm": 11424, | ||
| "yarn": 15374 | ||
| "aube": 17149, | ||
| "bun": 6311, | ||
| "deno": 7838, | ||
| "pnpm": 16838, | ||
| "npm": 11588, | ||
| "yarn": 14444 | ||
| }, | ||
| "stats": { | ||
| "aube": { | ||
| "text": "17.450s ± 1.146s", | ||
| "mean": 17.45024752892, | ||
| "stddev": 1.1461897481592471, | ||
| "min": 15.91922600842, | ||
| "max": 19.45928611942 | ||
| "text": "17.149s ± 1.755s", | ||
| "mean": 17.14926471638, | ||
| "stddev": 1.7545333501954121, | ||
| "min": 15.402188523480001, | ||
| "max": 20.90069358848 | ||
| }, | ||
| "bun": { | ||
| "text": "7.541s ± 0.899s", | ||
| "mean": 7.541080236420001, | ||
| "stddev": 0.8987518187225116, | ||
| "min": 6.27302774222, | ||
| "max": 8.91601610122 | ||
| "text": "6.311s ± 0.483s", | ||
| "mean": 6.3110499430800004, | ||
| "stddev": 0.48264064585954547, | ||
| "min": 5.571343366680001, | ||
| "max": 7.09095799868 | ||
| }, | ||
| "deno": { | ||
| "text": "8.449s ± 0.532s", | ||
| "mean": 8.44860676082, | ||
| "stddev": 0.5318088995180958, | ||
| "min": 7.6000775978199995, | ||
| "max": 9.36773796082 | ||
| "text": "7.838s ± 0.418s", | ||
| "mean": 7.83760514404, | ||
| "stddev": 0.41778557043434184, | ||
| "min": 7.1985465176400005, | ||
| "max": 8.46334428364 | ||
| }, | ||
| "pnpm": { | ||
| "text": "14.387s ± 0.411s", | ||
| "mean": 14.38675629034, | ||
| "stddev": 0.4107032298954795, | ||
| "min": 13.97966557134, | ||
| "max": 15.02466579034 | ||
| "text": "16.838s ± 5.149s", | ||
| "mean": 16.83757445338, | ||
| "stddev": 5.149476238887788, | ||
| "min": 12.484147481179999, | ||
|
Comment on lines
+107
to
+115
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The pnpm cold-install stddev jumped from 0.411 s to 5.149 s (12.5×), with a min/max spread of 12.48 s–22.74 s. That 10-second range on a ~16.8 s mean suggests at least one run was seriously impacted by an upstream hiccup rather than a clean cold-cache scenario. The mean itself shifted up (+17 %), but with that variance the measurement is not reliable enough to draw conclusions about pnpm's cold performance. A re-run of the Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time! |
||
| "max": 22.73651995718 | ||
| }, | ||
| "npm": { | ||
| "text": "11.424s ± 0.350s", | ||
| "mean": 11.423710755230001, | ||
| "stddev": 0.35039291063405825, | ||
| "min": 10.97624796498, | ||
| "max": 11.802742375980001 | ||
| "text": "11.588s ± 0.197s", | ||
| "mean": 11.5879297596, | ||
| "stddev": 0.19681414167996067, | ||
| "min": 11.3580772196, | ||
| "max": 11.8361365146 | ||
| }, | ||
| "yarn": { | ||
| "text": "15.374s ± 0.712s", | ||
| "mean": 15.37397946612, | ||
| "stddev": 0.7117824042485366, | ||
| "min": 14.31532494662, | ||
| "max": 15.85723112662 | ||
| "text": "14.444s ± 0.950s", | ||
| "mean": 14.44437189842, | ||
| "stddev": 0.9501010106955466, | ||
| "min": 13.03993888542, | ||
| "max": 15.048036599420001 | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "key": "install-test", | ||
| "label": "npm install && npm run test", | ||
| "values": { | ||
| "aube": 15, | ||
| "bun": 48, | ||
| "deno": 71, | ||
| "pnpm": 419, | ||
| "npm": 849, | ||
| "yarn": 1834 | ||
| "aube": 10, | ||
| "bun": 55, | ||
| "deno": 76, | ||
| "pnpm": 432, | ||
| "npm": 853, | ||
| "yarn": 1965 | ||
| }, | ||
| "stats": { | ||
| "aube": { | ||
| "text": "0.015s ± 0.002s", | ||
| "mean": 0.015385873860000001, | ||
| "stddev": 0.002001640204665079, | ||
| "min": 0.01186476196, | ||
| "max": 0.017969150960000003 | ||
| "text": "0.010s ± 0.001s", | ||
| "mean": 0.009751935480000001, | ||
| "stddev": 0.000968366193967189, | ||
| "min": 0.008464108980000001, | ||
| "max": 0.01157445898 | ||
| }, | ||
| "bun": { | ||
| "text": "0.048s ± 0.004s", | ||
| "mean": 0.04830771988, | ||
| "stddev": 0.003985245367298369, | ||
| "min": 0.04212683918000001, | ||
| "max": 0.055120922180000004 | ||
| "text": "0.055s ± 0.003s", | ||
| "mean": 0.054996691380000004, | ||
| "stddev": 0.002515265745105251, | ||
| "min": 0.05156357628, | ||
| "max": 0.06105625628 | ||
| }, | ||
| "deno": { | ||
| "text": "0.071s ± 0.011s", | ||
| "mean": 0.07117794196000002, | ||
| "stddev": 0.010634504700347651, | ||
| "min": 0.06071075156, | ||
| "max": 0.09840720156 | ||
| "text": "0.076s ± 0.028s", | ||
| "mean": 0.07604115351999999, | ||
| "stddev": 0.027965480915617895, | ||
| "min": 0.04546101452, | ||
| "max": 0.12095088452 | ||
| }, | ||
| "pnpm": { | ||
| "text": "0.419s ± 0.032s", | ||
| "mean": 0.41895784166, | ||
| "stddev": 0.03205284466531346, | ||
| "min": 0.39161411166000004, | ||
| "max": 0.47427068166 | ||
| "text": "0.432s ± 0.010s", | ||
| "mean": 0.4320137144, | ||
| "stddev": 0.01043884855572246, | ||
| "min": 0.42032239700000007, | ||
| "max": 0.4445476930000001 | ||
| }, | ||
| "npm": { | ||
| "text": "0.849s ± 0.024s", | ||
| "mean": 0.8493318313600001, | ||
| "stddev": 0.023871453995838766, | ||
| "min": 0.8345881043600001, | ||
| "max": 0.8848291033600001 | ||
| "text": "0.853s ± 0.036s", | ||
| "mean": 0.85287911853, | ||
| "stddev": 0.035731616097409026, | ||
| "min": 0.81089595928, | ||
| "max": 0.89811591028 | ||
| }, | ||
| "yarn": { | ||
| "text": "1.834s ± 0.122s", | ||
| "mean": 1.8337381562000001, | ||
| "stddev": 0.12155327968868454, | ||
| "min": 1.7330406597, | ||
| "max": 1.9815059597000002 | ||
| "text": "1.965s ± 0.076s", | ||
| "mean": 1.96509949824, | ||
| "stddev": 0.07558838321147573, | ||
| "min": 1.88214116424, | ||
| "max": 2.06281429524 | ||
| } | ||
| } | ||
| } | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Bun's warm-cache time jumped from 1154 ms to 2924 ms (+153 %), which is the primary driver of the headline claim changing from "3x faster than Bun" to "11x faster than Bun". A 153 % regression in a single Bun release seems unlikely to be a real improvement; it's more consistent with a proxy/network hiccup during the Bun run. The stddev for this run (261 ms over a ~2.9 s mean) is tighter than before, which rules out per-sample noise — the whole batch simply ran slow. Retrying just the
gvs-warmrow for Bun before merging would confirm whether this is real or a measurement artifact.Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!