Skip to content

ci(perf): Add benchmark history with JSON-in-comment storage #1289

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yamadashy merged 5 commits into from
Mar 22, 2026
Merged

ci(perf): Add benchmark history with JSON-in-comment storage #1289

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ef865fb
ci(perf): Add benchmark history with JSON-in-comment storage
yamadashy Mar 22, 2026
3828132
fix(ci): Harden benchmark history JSON parsing and HTML escaping
yamadashy Mar 22, 2026
ee092dd
fix(ci): Skip archiving same SHA on benchmark rerun
yamadashy Mar 22, 2026
b34dab8
fix(ci): Suppress shellcheck SC2016 for inline Node scripts
yamadashy Mar 22, 2026
29e5b6e
fix(ci): Add post-pending dependency to comment job
yamadashy Mar 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
266 changes: 189 additions & 77 deletions .github/workflows/perf-benchmark.yml
View file
Open in desktop
Comment thread
yamadashy marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,95 @@ jobs:
GH_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
SHORT_SHA="${COMMIT_SHA:0:7}"
COMMENT_MARKER="<!-- repomix-perf-benchmark -->"
BODY="${COMMENT_MARKER}
## ⚡ Performance Benchmark

<table><tr><td><strong>Latest commit:</strong></td><td><code>${SHORT_SHA}</code></td></tr>
<tr><td><strong>Status:</strong></td><td>⚡ Benchmark in progress...</td></tr></table>

[Workflow run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"

# Find existing comment by marker
COMMENT_ID=$(gh api "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" --paginate --jq ".[] | select(.body | startswith(\"${COMMENT_MARKER}\")) | .id" | head -1)

# Save existing comment body to file for safe parsing
if [ -n "$COMMENT_ID" ]; then
gh api "repos/${GH_REPO}/issues/comments/${COMMENT_ID}" --jq '.body' > "$RUNNER_TEMP/old-comment.txt"
else
echo "" > "$RUNNER_TEMP/old-comment.txt"
fi

# Fetch commit message in shell (avoids escaping issues in Node)
COMMIT_MSG=$(gh api "repos/${GH_REPO}/commits/${COMMIT_SHA}" --jq '.commit.message | split("\n") | .[0]' 2>/dev/null || echo "")

# Generate comment with Node (handles JSON history + rendering)
# shellcheck disable=SC2016
COMMIT_MSG="$COMMIT_MSG" node -e '
const fs = require("fs");
const shortSha = process.env.COMMIT_SHA.slice(0, 7);
const commitMsg = process.env.COMMIT_MSG;
const runUrl = process.env.WORKFLOW_RUN_URL;
const oldBody = fs.readFileSync(process.env.RUNNER_TEMP + "/old-comment.txt", "utf8");

const esc = s => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");

// Extract history JSON from existing comment
const jsonMatch = oldBody.match(/<!-- bench-history-json-start ([\s\S]*?) bench-history-json-end -->/);
let history = [];
if (jsonMatch) { try { history = JSON.parse(jsonMatch[1]); } catch {} }

// If previous comment had completed results, archive them into history
if (oldBody.includes("\u2705 Benchmark complete!")) {
// Extract only the main result table (before any <details> block)
const mainSection = oldBody.split("<details>")[0] || "";
const commitMatch = mainSection.match(/Latest commit:<\/strong><\/td><td><code>([a-f0-9]+)<\/code>\s*(.*?)<\/td>/);
const prevSha = commitMatch ? commitMatch[1] : "";
const prevMsg = commitMatch ? commitMatch[2] : "";
if (prevSha) {
const rowRe = /<tr><td><strong>(Ubuntu|macOS|Windows):<\/strong><\/td><td>(.*?)<\/td><\/tr>/g;
const entry = { sha: prevSha, msg: prevMsg };
let m;
while ((m = rowRe.exec(mainSection)) !== null) {
entry[m[1].toLowerCase()] = m[2];
}
Comment thread
yamadashy marked this conversation as resolved.
if (prevSha !== shortSha && !history.some(h => h.sha === prevSha)) {
history.unshift(entry);
}
}
}

// Keep max 5 entries
history = history.slice(0, 5);

// Render history section
function renderHistory(hist) {
if (hist.length === 0) return "";
return hist.map(h => {
const label = "<code>" + h.sha + "</code>" + (h.msg ? " " + h.msg : "");
const osRows = ["ubuntu", "macos", "windows"]
.filter(os => h[os] && h[os] !== "-")
.map(os => {
const osLabel = os === "ubuntu" ? "Ubuntu" : os === "macos" ? "macOS" : "Windows";
return "<tr><td><strong>" + osLabel + ":</strong></td><td>" + h[os] + "</td></tr>";
})
.join("\n");
return label + "\n<table>\n" + osRows + "\n</table>";
}).join("\n\n");
}

const jsonComment = "<!-- bench-history-json-start " + JSON.stringify(history) + " bench-history-json-end -->";
let body = "<!-- repomix-perf-benchmark -->\n" + jsonComment + "\n";
body += "## \u26a1 Performance Benchmark\n\n";
body += "<table><tr><td><strong>Latest commit:</strong></td><td><code>" + shortSha + "</code> " + esc(commitMsg) + "</td></tr>\n";
body += "<tr><td><strong>Status:</strong></td><td>\u26a1 Benchmark in progress...</td></tr></table>\n\n";
body += "[Workflow run](" + runUrl + ")";
Comment thread
yamadashy marked this conversation as resolved.

const historyHtml = renderHistory(history);
if (historyHtml) {
body += "\n\n<details>\n<summary>History</summary>\n\n" + historyHtml + "\n\n</details>";
}

fs.writeFileSync(process.env.RUNNER_TEMP + "/new-comment.md", body);
'

BODY=$(cat "$RUNNER_TEMP/new-comment.md")

if [ -n "$COMMENT_ID" ]; then
gh api "repos/${GH_REPO}/issues/comments/${COMMENT_ID}" -X PATCH -f body="$BODY"
else
Expand Down Expand Up @@ -153,7 +228,7 @@ jobs:

comment:
name: Comment Results
needs: benchmark
needs: [benchmark, post-pending]
runs-on: ubuntu-latest
if: ${{ always() && !cancelled() }}
permissions:
Expand All @@ -164,86 +239,123 @@ jobs:
path: results
pattern: bench-result-*

- name: Generate benchmark report
id: report
- name: Comment on PR
if: ${{ github.event.pull_request.head.repo.fork == false }}
Comment on lines +242 to +243
Copy link
Copy Markdown
Contributor

@devin-ai-integration devin-ai-integration bot Mar 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Step summary no longer written for fork PRs due to merged steps

In the old code, GITHUB_STEP_SUMMARY was written in the "Generate benchmark report" step which had no if guard — it ran for all PRs, including forks. In the new code, the step summary write (fs.appendFileSync(summaryFile, ...) at line 352) is inside the "Comment on PR" step which is guarded by if: ${{ github.event.pull_request.head.repo.fork == false }} at line 243. This means fork PRs lose the step summary entirely. Previously, the step summary was fork PR authors' only way to view benchmark results (since the PR comment was also fork-guarded).

Prompt for agents 
In .github/workflows/perf-benchmark.yml, the "comment" job (starting around line 229) currently has a single "Comment on PR" step (line 242) guarded by the fork check. To restore the old behavior where GITHUB_STEP_SUMMARY was written regardless of fork status, split this into two steps:

1. A "Generate benchmark report" step with NO `if` condition that runs the Node script to generate the comment body and write to GITHUB_STEP_SUMMARY. This step should write the body to $RUNNER_TEMP/new-comment.md.

2. A "Comment on PR" step with `if: ${{ github.event.pull_request.head.repo.fork == false }}` that reads $RUNNER_TEMP/new-comment.md and posts/updates the PR comment.

This matches the old code's pattern where report generation (including step summary) was unconditional, and only the PR comment posting was fork-guarded.
Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
SHORT_SHA="${COMMIT_SHA:0:7}"

generate_row() {
local os=$1
local label=$2
local file="results/bench-result-${os}/bench-result.json"

if [ ! -f "$file" ]; then
echo "<tr><td><strong>${label}:</strong></td><td>-</td></tr>"
return
fi

local data
data=$(node -e "console.log(JSON.stringify(JSON.parse(require('fs').readFileSync('$file','utf8'))))")

local pr_ms main_ms pr_iqr main_iqr
pr_ms=$(node -e "console.log(JSON.parse('$data').pr)")
main_ms=$(node -e "console.log(JSON.parse('$data').main)")
pr_iqr=$(node -e "console.log(JSON.parse('$data').prIqr)")
main_iqr=$(node -e "console.log(JSON.parse('$data').mainIqr)")

local diff=$((pr_ms - main_ms))
local pr_sec main_sec pr_iqr_sec main_iqr_sec diff_sec diff_pct
pr_sec=$(awk "BEGIN {printf \"%.2f\", $pr_ms / 1000}")
main_sec=$(awk "BEGIN {printf \"%.2f\", $main_ms / 1000}")
pr_iqr_sec=$(awk "BEGIN {printf \"%.2f\", $pr_iqr / 1000}")
main_iqr_sec=$(awk "BEGIN {printf \"%.2f\", $main_iqr / 1000}")
diff_sec=$(awk "BEGIN {printf \"%+.2f\", $diff / 1000}")

if [ "$main_ms" -gt 0 ]; then
diff_pct=$(awk "BEGIN {printf \"%+.1f\", ($diff / $main_ms) * 100}")
else
diff_pct="N/A"
fi

echo "<tr><td><strong>${label}:</strong></td><td>${main_sec}s (±${main_iqr_sec}s) → ${pr_sec}s (±${pr_iqr_sec}s) · ${diff_sec}s (${diff_pct}%)</td></tr>"
}
COMMENT_MARKER="<!-- repomix-perf-benchmark -->"

BODY="## ⚡ Performance Benchmark
# Find existing comment and save old body to file
COMMENT_ID=$(gh api "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" --paginate --jq ".[] | select(.body | startswith(\"${COMMENT_MARKER}\")) | .id" | head -1)

<table><tr><td><strong>Latest commit:</strong></td><td><code>${SHORT_SHA}</code></td></tr>
<tr><td><strong>Status:</strong></td><td>✅ Benchmark complete!</td></tr>
$(generate_row "ubuntu-latest" "Ubuntu")
$(generate_row "macos-latest" "macOS")
$(generate_row "windows-latest" "Windows")
</table>
if [ -n "$COMMENT_ID" ]; then
gh api "repos/${GH_REPO}/issues/comments/${COMMENT_ID}" --jq '.body' > "$RUNNER_TEMP/old-comment.txt"
else
echo "" > "$RUNNER_TEMP/old-comment.txt"
fi

<details>
<summary>Details</summary>
# Fetch commit message in shell
COMMIT_MSG=$(gh api "repos/${GH_REPO}/commits/${COMMIT_SHA}" --jq '.commit.message | split("\n") | .[0]' 2>/dev/null || echo "")

- Packing the repomix repository with \`node bin/repomix.cjs\`
- Warmup: 2 runs (discarded)
- Measurement: 10 runs / 20 on macOS (median ± IQR)
- [Workflow run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})
# Generate complete comment with Node
# shellcheck disable=SC2016
COMMIT_MSG="$COMMIT_MSG" node -e '
const fs = require("fs");
const path = require("path");

</details>"
const shortSha = process.env.COMMIT_SHA.slice(0, 7);
const commitMsg = process.env.COMMIT_MSG;
const runUrl = process.env.WORKFLOW_RUN_URL;
const oldBody = fs.readFileSync(process.env.RUNNER_TEMP + "/old-comment.txt", "utf8");

echo "$BODY" >> "$GITHUB_STEP_SUMMARY"
const esc = s => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");

# Save report for comment step
echo "$BODY" > "$RUNNER_TEMP/benchmark-report.md"
// Extract history JSON from existing comment (set by post-pending)
const jsonMatch = oldBody.match(/<!-- bench-history-json-start ([\s\S]*?) bench-history-json-end -->/);
let history = [];
if (jsonMatch) { try { history = JSON.parse(jsonMatch[1]); } catch {} }

- name: Comment on PR
if: ${{ github.event.pull_request.head.repo.fork == false }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
COMMENT_MARKER="<!-- repomix-perf-benchmark -->"
BODY="${COMMENT_MARKER}
$(cat "$RUNNER_TEMP/benchmark-report.md")"
// Read benchmark results from artifacts
function readResult(os) {
const file = path.join("results", "bench-result-" + os, "bench-result.json");
try {
return JSON.parse(fs.readFileSync(file, "utf8"));
} catch {
return null;
}
}

# Find existing comment by marker
COMMENT_ID=$(gh api "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" --paginate --jq ".[] | select(.body | startswith(\"${COMMENT_MARKER}\")) | .id" | head -1)
function formatResult(data) {
if (!data) return "-";
const prSec = (data.pr / 1000).toFixed(2);
const mainSec = (data.main / 1000).toFixed(2);
const prIqr = (data.prIqr / 1000).toFixed(2);
const mainIqr = (data.mainIqr / 1000).toFixed(2);
const diff = data.pr - data.main;
const diffSec = (diff >= 0 ? "+" : "") + (diff / 1000).toFixed(2);
const diffPct = data.main > 0
? (diff >= 0 ? "+" : "") + ((diff / data.main) * 100).toFixed(1)
: "N/A";
return mainSec + "s (\u00b1" + mainIqr + "s) \u2192 " + prSec + "s (\u00b1" + prIqr + "s) \u00b7 " + diffSec + "s (" + diffPct + "%)";
}

const ubuntuStr = formatResult(readResult("ubuntu-latest"));
const macosStr = formatResult(readResult("macos-latest"));
const windowsStr = formatResult(readResult("windows-latest"));

// Render history section
function renderHistory(hist) {
if (hist.length === 0) return "";
return hist.map(h => {
const label = "<code>" + h.sha + "</code>" + (h.msg ? " " + h.msg : "");
const osRows = ["ubuntu", "macos", "windows"]
.filter(os => h[os] && h[os] !== "-")
.map(os => {
const osLabel = os === "ubuntu" ? "Ubuntu" : os === "macos" ? "macOS" : "Windows";
return "<tr><td><strong>" + osLabel + ":</strong></td><td>" + h[os] + "</td></tr>";
})
.join("\n");
return label + "\n<table>\n" + osRows + "\n</table>";
}).join("\n\n");
}

const jsonComment = "<!-- bench-history-json-start " + JSON.stringify(history) + " bench-history-json-end -->";
let body = "<!-- repomix-perf-benchmark -->\n" + jsonComment + "\n";
body += "## \u26a1 Performance Benchmark\n\n";
body += "<table><tr><td><strong>Latest commit:</strong></td><td><code>" + shortSha + "</code> " + esc(commitMsg) + "</td></tr>\n";
body += "<tr><td><strong>Status:</strong></td><td>\u2705 Benchmark complete!</td></tr>\n";
body += "<tr><td><strong>Ubuntu:</strong></td><td>" + ubuntuStr + "</td></tr>\n";
body += "<tr><td><strong>macOS:</strong></td><td>" + macosStr + "</td></tr>\n";
body += "<tr><td><strong>Windows:</strong></td><td>" + windowsStr + "</td></tr>\n";
body += "</table>\n\n";
body += "<details>\n<summary>Details</summary>\n\n";
body += "- Packing the repomix repository with `node bin/repomix.cjs`\n";
body += "- Warmup: 2 runs (discarded)\n";
body += "- Measurement: 10 runs / 20 on macOS (median \u00b1 IQR)\n";
body += "- [Workflow run](" + runUrl + ")\n\n";
body += "</details>";

const historyHtml = renderHistory(history);
if (historyHtml) {
body += "\n\n<details>\n<summary>History</summary>\n\n" + historyHtml + "\n\n</details>";
}

// Write to step summary (without HTML comments)
const summaryFile = process.env.GITHUB_STEP_SUMMARY;
if (summaryFile) {
const summaryBody = body.split("\n").filter(l => !l.startsWith("<!-- ")).join("\n");
fs.appendFileSync(summaryFile, summaryBody + "\n");
}

fs.writeFileSync(process.env.RUNNER_TEMP + "/new-comment.md", body);
'

BODY=$(cat "$RUNNER_TEMP/new-comment.md")

if [ -n "$COMMENT_ID" ]; then
gh api "repos/${GH_REPO}/issues/comments/${COMMENT_ID}" -X PATCH -f body="$BODY"
Expand Down
Loading