bench: Add benchmarking for program binary size (coral-xyz#2591)

Aursen · Aug 8, 2023 · 4cf447a · 4cf447a
1 parent abfdc4e
commit 4cf447a
Show file tree

Hide file tree

Showing 8 changed files with 528 additions and 442 deletions.
diff --git a/bench/BINARY_SIZE.md b/bench/BINARY_SIZE.md
@@ -0,0 +1,49 @@
+# Binary Size
+
+All notable changes in program binary size will be documented in this file.
+
+The changes are calculated by comparing the current results with the last version's results. Increase in size is shown with 🔴 and decrease is shown with 🟢.
+
+The programs and their tests are located in [/tests/bench](https://github.com/coral-xyz/anchor/tree/master/tests/bench).
+
+> **Note**
+> Results documented in this file are autogenerated. Running the tests will update the current results when necessary, manually editing the results should be avoided.
+
+> **Warning**
+> Results may vary depending on Solana version.
+
+## [Unreleased]
+
+Solana version: 1.16.0
+
+| Program | Binary Size | +/- |
+| ------- | ----------- | --- |
+| bench   | 1,153,736   | -   |
+
+### Notable changes
+
+---
+
+## [0.28.0]
+
+Solana version: 1.16.0
+
+| Program | Binary Size | +/-                    |
+| ------- | ----------- | ---------------------- |
+| bench   | 1,153,736   | 🔴 **+35,000 (3.13%)** |
+
+### Notable changes
+
+- Upgrading Solana to `1.16`. The difference in binary size between `0.27.0` and `0.28.0` is the direct result of upgrading Solana version(both build tools and crates) ([#2512](https://github.com/coral-xyz/anchor/pull/2512)).
+
+---
+
+## [0.27.0]
+
+Solana version: 1.14.16
+
+| Program | Binary Size | +/- |
+| ------- | ----------- | --- |
+| bench   | 1,118,736   | N/A |
+
+---
diff --git a/bench/COMPUTE_UNITS.md b/bench/COMPUTE_UNITS.md
diff --git a/tests/bench/bench.json b/tests/bench/bench.json
@@ -2,6 +2,9 @@
   "0.27.0": {
     "solanaVersion": "1.14.16",
     "result": {
+      "binarySize": {
+        "bench": 1118736
+      },
       "computeUnits": {
         "accountInfo1": 954,
         "accountInfo2": 1567,
@@ -96,6 +99,9 @@
   "0.28.0": {
     "solanaVersion": "1.16.0",
     "result": {
+      "binarySize": {
+        "bench": 1153736
+      },
       "computeUnits": {
         "accountInfo1": 1015,
         "accountInfo2": 1475,
@@ -190,6 +196,9 @@
   "unreleased": {
     "solanaVersion": "1.16.0",
     "result": {
+      "binarySize": {
+        "bench": 1153736
+      },
       "computeUnits": {
         "accountInfo1": 1015,
         "accountInfo2": 1475,
@@ -281,4 +290,4 @@
       }
     }
   }
-}
+}
diff --git a/tests/bench/scripts/sync-markdown.ts b/tests/bench/scripts/sync-markdown.ts
@@ -1,80 +1,69 @@
 /** Sync Markdown files in /bench based on the data from bench.json */
 
-import { BenchData, Markdown } from "./utils";
+import { BenchData, BenchResult, Markdown } from "./utils";
 
 (async () => {
   const bench = await BenchData.open();
 
   await BenchData.forEachMarkdown((markdown, fileName) => {
-    if (fileName === "COMPUTE_UNITS.md") {
-      const versions = bench.getVersions();
+    const resultType = fileName
+      .toLowerCase()
+      .replace(".md", "")
+      .replace(/_\w/g, (match) => match[1].toUpperCase()) as keyof BenchResult;
 
-      // On the first version, compare with itself to update it with no changes
-      versions.unshift(versions[0]);
+    const versions = bench.getVersions();
 
-      for (const i in versions) {
-        const currentVersion = versions[i];
-        const nextVersion = versions[+i + 1];
+    // On the first version, compare with itself to update it with no changes
+    versions.unshift(versions[0]);
 
-        if (currentVersion === "unreleased") {
-          return;
-        }
+    for (const i in versions) {
+      const currentVersion = versions[i];
+      if (currentVersion === "unreleased") return;
 
-        const newData = bench.get(nextVersion);
-        const oldData = bench.get(currentVersion);
+      const nextVersion = versions[+i + 1];
+      const newData = bench.get(nextVersion);
+      const oldData = bench.get(currentVersion);
 
-        // Create table
-        const table = Markdown.createTable(
-          "Instruction",
-          "Compute Units",
-          "+/-"
-        );
+      // Create table
+      const table = Markdown.createTable();
 
-        bench.compareComputeUnits(
-          newData.result.computeUnits,
-          oldData.result.computeUnits,
-          ({ ixName, newComputeUnits, oldComputeUnits }) => {
-            if (newComputeUnits === null) {
-              // Deleted instruction
-              return;
-            }
+      bench.compare({
+        newResult: newData.result[resultType],
+        oldResult: oldData.result[resultType],
+        changeCb: ({ name, newValue, oldValue }) => {
+          if (newValue === null) {
+            // Deleted key
+            return;
+          }
 
-            let changeText;
-            if (oldComputeUnits === null) {
-              // New instruction
-              changeText = "N/A";
-            } else {
-              const delta = newComputeUnits - oldComputeUnits;
-              const percentChange = (
-                (newComputeUnits / oldComputeUnits - 1) *
-                100
-              ).toFixed(2);
+          let changeText: string;
+          if (oldValue === null) {
+            // New key
+            changeText = "N/A";
+          } else {
+            const delta = (newValue - oldValue).toLocaleString();
+            const percentChange = ((newValue / oldValue - 1) * 100).toFixed(2);
 
-              if (+percentChange > 0) {
-                changeText = `🔴 **+${delta} (${percentChange}%)**`;
-              } else {
-                changeText = `🟢 **${delta} (${percentChange.slice(1)}%)**`;
-              }
+            if (+percentChange > 0) {
+              changeText = `🔴 **+${delta} (${percentChange}%)**`;
+            } else {
+              changeText = `🟢 **${delta} (${percentChange.slice(1)}%)**`;
             }
-
-            table.insert(ixName, newComputeUnits.toString(), changeText);
-          },
-          (ixName, computeUnits) => {
-            table.insert(
-              ixName,
-              computeUnits.toString(),
-              +i === 0 ? "N/A" : "-"
-            );
           }
-        );
 
-        // Update version data
-        markdown.updateVersion({
-          version: nextVersion,
-          solanaVersion: newData.solanaVersion,
-          table,
-        });
-      }
+          table.insert(name, newValue.toLocaleString(), changeText);
+        },
+        noChangeCb: ({ name, value }) => {
+          table.insert(name, value.toLocaleString(), +i === 0 ? "N/A" : "-");
+        },
+      });
+
+      // Update version data
+      markdown.updateVersion({
+        version: nextVersion,
+        solanaVersion: newData.solanaVersion,
+        table,
+      });
     }
   });
 })();
diff --git a/tests/bench/scripts/sync.ts b/tests/bench/scripts/sync.ts
@@ -14,6 +14,7 @@ import {
   Toml,
   VersionManager,
   runAnchorTest,
+  spawn,
 } from "./utils";
 
 (async () => {
@@ -75,4 +76,7 @@ import {
       return;
     }
   }
+
+  // Sync markdown files
+  spawn("anchor", ["run", "sync-markdown"]);
 })();