From 6e1dea589fba97bffa1481fb3e1a04f05fc5f564 Mon Sep 17 00:00:00 2001 From: Simo Lin Date: Tue, 10 Feb 2026 08:04:26 -0800 Subject: [PATCH 1/2] chore(e2e): overhaul nightly benchmark summary and trim model list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite nightly_summarize.py for better readability and richer output: - Add glossary explaining all metrics (TTFT, TPOT, E2E, p99, mean), traffic scenarios (D/N/E patterns), and comparison columns - Add auto-generated Key Findings executive summary at the top - Add TPOT (Time Per Output Token) to all summary sections — was parsed from JSON but mostly not displayed - Switch per-model summary and detail tables from mean to p99 metrics which better capture tail latency for production SLAs - Replace confusing +/- percentage format with clear "gRPC X%" / "HTTP X%" / "~" labels using normalized gRPC-advantage direction - Add matplotlib comparison chart generation (aggregate + per-model 2x2 grids of TTFT p99, TPOT p99, E2E p99, Output Throughput vs Concurrency) uploaded as separate artifact - Add error rate tracking section for runs with non-zero errors - Show all gRPC wins >30% instead of capped top-15 at >10% - Add TPOT p99 to aggregate metrics table Remove 5 models that don't add value to the benchmark: - Llama-3.2-1B-Instruct, Qwen2.5-14B-Instruct, DeepSeek-R1-Distill-Qwen-7B (this PR) - Mistral-7B, gpt-oss (removed from matrix + test file) Enable gateway debug logging during nightly benchmarks: - Make log_level and log_dir configurable on Gateway class (was hardcoded to --log-level warn with no --log-dir) - Plumb log_level/log_dir through pytest markers → gateway_config → gateway.start() in setup_backend.py - Set log_level="debug" and log_dir="nightly_gateway_logs" in the nightly test marker so logs are captured as artifacts Workflow changes: - Install matplotlib in summarize job for chart generation - Pass --charts-dir nightly_charts to generate comparison plots - Upload charts as separate artifact - Remove trimmed models from single/multi GPU matrices and TP override JSON Remaining benchmark models: Llama-3.1-8B, Qwen2.5-7B, Qwen3-30B-A3B (single+multi), Llama-4-Maverick (single only) --- .github/workflows/nightly-benchmark.yml | 27 +- e2e_test/benchmarks/nightly_summarize.py | 681 +++++++++++++++++------ e2e_test/benchmarks/test_nightly_perf.py | 7 +- e2e_test/fixtures/setup_backend.py | 8 + e2e_test/infra/gateway.py | 17 +- 5 files changed, 552 insertions(+), 188 deletions(-) diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index acb9f650b6..f5414ee640 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -86,13 +86,8 @@ jobs: matrix: model: - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } - - { id: meta-llama/Llama-3.2-1B-Instruct, slug: meta-llama-Llama-3.2-1B-Instruct, test_class: TestNightlyLlama1bSingle } - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } - - { id: Qwen/Qwen2.5-14B-Instruct, slug: Qwen-Qwen2.5-14B-Instruct, test_class: TestNightlyQwen14bSingle } - - { id: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, slug: deepseek-ai-DeepSeek-R1-Distill-Qwen-7B, test_class: TestNightlyDeepseek7bSingle } - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } - - { id: mistralai/Mistral-7B-Instruct-v0.3, slug: mistralai-Mistral-7B-Instruct-v0.3, test_class: TestNightlyMistral7bSingle } - - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } variant: - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } @@ -147,7 +142,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} GPU_TYPE: H100 - E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"meta-llama/Llama-3.2-1B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen2.5-14B-Instruct":1,"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B":1,"Qwen/Qwen3-30B-A3B":1,"mistralai/Mistral-7B-Instruct-v0.3":1,"openai/gpt-oss-20b":1}' + E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen3-30B-A3B":1}' run: | bash scripts/ci_killall_sglang.sh "nuk_gpus" @@ -190,13 +185,8 @@ jobs: matrix: model: - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bMulti } - - { id: meta-llama/Llama-3.2-1B-Instruct, slug: meta-llama-Llama-3.2-1B-Instruct, test_class: TestNightlyLlama1bMulti } - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bMulti } - - { id: Qwen/Qwen2.5-14B-Instruct, slug: Qwen-Qwen2.5-14B-Instruct, test_class: TestNightlyQwen14bMulti } - - { id: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, slug: deepseek-ai-DeepSeek-R1-Distill-Qwen-7B, test_class: TestNightlyDeepseek7bMulti } - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bMulti } - - { id: mistralai/Mistral-7B-Instruct-v0.3, slug: mistralai-Mistral-7B-Instruct-v0.3, test_class: TestNightlyMistral7bMulti } - - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bMulti } variant: - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } @@ -251,7 +241,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} GPU_TYPE: H100 - E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"meta-llama/Llama-3.2-1B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen2.5-14B-Instruct":1,"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B":1,"Qwen/Qwen3-30B-A3B":1,"mistralai/Mistral-7B-Instruct-v0.3":1,"openai/gpt-oss-20b":1}' + E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen3-30B-A3B":1}' run: | bash scripts/ci_killall_sglang.sh "nuk_gpus" @@ -422,5 +412,16 @@ jobs: pattern: nightly-* merge-multiple: true + - name: Install chart dependencies + run: pip install matplotlib + - name: Create benchmark summary - run: python3 e2e_test/benchmarks/nightly_summarize.py . + run: python3 e2e_test/benchmarks/nightly_summarize.py . --charts-dir nightly_charts + + - name: Upload comparison charts + if: always() + uses: actions/upload-artifact@v6 + with: + name: nightly-comparison-charts-${{ github.run_id }} + path: nightly_charts/ + if-no-files-found: ignore diff --git a/e2e_test/benchmarks/nightly_summarize.py b/e2e_test/benchmarks/nightly_summarize.py index d597f73382..892c51fa04 100755 --- a/e2e_test/benchmarks/nightly_summarize.py +++ b/e2e_test/benchmarks/nightly_summarize.py @@ -18,6 +18,7 @@ from pathlib import Path from statistics import median + # --------------------------------------------------------------------------- # Data types # --------------------------------------------------------------------------- @@ -107,7 +108,11 @@ def parse_folder_name(folder_name: str) -> dict: name = folder_name.replace("nightly_", "") parts = name.rsplit("_", 3) - if len(parts) >= 4 and parts[-1] in _KNOWN_WORKER_TYPES and parts[-2] in _KNOWN_RUNTIMES: + if ( + len(parts) >= 4 + and parts[-1] in _KNOWN_WORKER_TYPES + and parts[-2] in _KNOWN_RUNTIMES + ): info["worker_type"] = parts[-1] info["runtime"] = parts[-2] info["protocol"] = parts[-3] @@ -192,7 +197,9 @@ def parse_experiment(folder: Path) -> ExperimentInfo | None: concurrency=agg.get("num_concurrency", 0) or 0, rps=_get_float(agg, "requests_per_second"), output_throughput=_get_float(agg, "mean_output_throughput_tokens_per_s"), - total_throughput=_get_float(agg, "mean_total_tokens_throughput_tokens_per_s"), + total_throughput=_get_float( + agg, "mean_total_tokens_throughput_tokens_per_s" + ), ttft_mean=_get_float(ttft, "mean"), ttft_p99=_get_float(ttft, "p99"), tpot_mean=_get_float(tpot, "mean"), @@ -260,20 +267,48 @@ def build_comparisons(experiments: list[ExperimentInfo]) -> list[ComparisonPoint # --------------------------------------------------------------------------- # Formatting helpers +# +# All percentages are shown as "gRPC advantage": positive = gRPC is better. +# For latency metrics (lower=better): advantage = (HTTP - gRPC) / HTTP +# For throughput metrics (higher=better): advantage = (gRPC - HTTP) / HTTP # --------------------------------------------------------------------------- -def _pct(grpc_val: float, http_val: float) -> float | None: - """(gRPC - HTTP) / HTTP * 100.""" +def _raw_pct(grpc_val: float, http_val: float) -> float | None: + """Raw (gRPC - HTTP) / HTTP * 100.""" if http_val == 0: return None return (grpc_val - http_val) / http_val * 100 -def _fmt_pct(pct: float | None) -> str: +def _advantage(grpc_val: float, http_val: float, lower_is_better: bool) -> float | None: + """gRPC advantage %: positive = gRPC is better, negative = HTTP is better.""" + pct = _raw_pct(grpc_val, http_val) + if pct is None: + return None + return -pct if lower_is_better else pct + + +def _fmt_winner(pct: float | None, threshold: float = 2.0) -> str: + """Format as 'gRPC X%' or 'HTTP X%' or '~'. pct is gRPC advantage.""" + if pct is None: + return "N/A" + if abs(pct) < threshold: + return "~" + if pct > 0: + return f"gRPC {abs(pct):.1f}%" + return f"HTTP {abs(pct):.1f}%" + + +def _fmt_winner_bold(pct: float | None, threshold: float = 2.0) -> str: + """Like _fmt_winner but bolds the winner name.""" if pct is None: return "N/A" - return f"{pct:+.1f}%" + if abs(pct) < threshold: + return "~" + if pct > 0: + return f"**gRPC** {abs(pct):.1f}%" + return f"**HTTP** {abs(pct):.1f}%" def _fmt_latency_s(val_s: float) -> str: @@ -290,47 +325,294 @@ def _fmt_throughput(val: float) -> str: return f"{val:.0f}" -def _winner(pct: float | None, lower_is_better: bool, threshold: float = 2.0) -> str: - if pct is None: - return "" - if abs(pct) < threshold: - return "~Tie" - if lower_is_better: - return "**gRPC**" if pct < 0 else "HTTP" - return "**gRPC**" if pct > 0 else "HTTP" +# --------------------------------------------------------------------------- +# Chart generation (optional — requires matplotlib) +# --------------------------------------------------------------------------- + +# Metrics used across charts and key findings +_CHART_METRICS = [ + ("TTFT p99", "ttft_p99", True, "ms"), + ("TPOT p99", "tpot_p99", True, "ms"), + ("E2E p99", "e2e_p99", True, "s"), + ("Output Throughput", "output_throughput", False, "tok/s"), +] + + +def generate_charts( + comparisons: list[ComparisonPoint], + experiments: list[ExperimentInfo], + output_dir: Path, +) -> list[str]: + """Generate comparison charts as PNGs. Returns list of generated filenames.""" + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError: + print("matplotlib not available, skipping chart generation", file=sys.stderr) + return [] + + output_dir.mkdir(parents=True, exist_ok=True) + charts: list[str] = [] + + by_conc: dict[int, list[ComparisonPoint]] = defaultdict(list) + for cp in comparisons: + by_conc[cp.concurrency].append(cp) + conc_levels = sorted(by_conc.keys()) + + if not conc_levels: + return [] + # ---- Aggregate comparison (2x2 grid) ---- + fig, axes = plt.subplots(2, 2, figsize=(14, 10)) + fig.suptitle("gRPC vs HTTP — Aggregate Comparison", fontsize=16, fontweight="bold") + + for ax, (title, field, _lower_better, unit) in zip(axes.flat, _CHART_METRICS): + grpc_vals, http_vals = [], [] + for conc in conc_levels: + cps = by_conc[conc] + g = sum(getattr(cp.grpc, field) for cp in cps) / len(cps) + h = sum(getattr(cp.http, field) for cp in cps) / len(cps) + if unit == "ms": + g *= 1000 + h *= 1000 + grpc_vals.append(g) + http_vals.append(h) + + x = range(len(conc_levels)) + ax.plot(x, grpc_vals, "o-", label="gRPC", color="#2196F3", linewidth=2) + ax.plot(x, http_vals, "s--", label="HTTP", color="#FF9800", linewidth=2) + ax.set_title(title, fontsize=12, fontweight="bold") + ax.set_xlabel("Concurrency") + ax.set_ylabel(f"{title} ({unit})") + ax.set_xticks(list(x)) + ax.set_xticklabels([str(c) for c in conc_levels]) + ax.legend() + ax.grid(True, alpha=0.3) + # Use log scale when values span >10x + nz = [v for v in grpc_vals + http_vals if v > 0] + if nz and max(nz) > 10 * min(nz): + ax.set_yscale("log") + + fig.tight_layout() + fname = "aggregate_comparison.png" + fig.savefig(output_dir / fname, dpi=150, bbox_inches="tight") + plt.close(fig) + charts.append(fname) + + # ---- Per-model/config charts ---- + by_model_config: dict[str, list[ComparisonPoint]] = defaultdict(list) + for cp in comparisons: + by_model_config[f"{cp.model}|{cp.runtime}/{cp.worker_type}"].append(cp) -def _safe_avg(vals: list[float]) -> float | None: - return sum(vals) / len(vals) if vals else None + for key, cps in sorted(by_model_config.items()): + model, config = key.split("|", 1) + mc_conc: dict[int, list[ComparisonPoint]] = defaultdict(list) + for cp in cps: + mc_conc[cp.concurrency].append(cp) + mc_levels = sorted(mc_conc.keys()) + if len(mc_levels) < 3: + continue + + fig, axes = plt.subplots(2, 2, figsize=(14, 10)) + display_name = model.split("/")[-1] if "/" in model else model + fig.suptitle( + f"{display_name} ({config}): gRPC vs HTTP", + fontsize=16, + fontweight="bold", + ) + + for ax, (title, field, _lb, unit) in zip(axes.flat, _CHART_METRICS): + gv, hv = [], [] + for conc in mc_levels: + g = sum(getattr(c.grpc, field) for c in mc_conc[conc]) / len( + mc_conc[conc] + ) + h = sum(getattr(c.http, field) for c in mc_conc[conc]) / len( + mc_conc[conc] + ) + if unit == "ms": + g *= 1000 + h *= 1000 + gv.append(g) + hv.append(h) + + x = range(len(mc_levels)) + ax.plot(x, gv, "o-", label="gRPC", color="#2196F3", linewidth=2) + ax.plot(x, hv, "s--", label="HTTP", color="#FF9800", linewidth=2) + ax.set_title(title, fontsize=12, fontweight="bold") + ax.set_xlabel("Concurrency") + ax.set_ylabel(f"{title} ({unit})") + ax.set_xticks(list(x)) + ax.set_xticklabels([str(c) for c in mc_levels]) + ax.legend() + ax.grid(True, alpha=0.3) + nz = [v for v in gv + hv if v > 0] + if nz and max(nz) > 10 * min(nz): + ax.set_yscale("log") + + fig.tight_layout() + safe = model.replace("/", "__") + safe_cfg = config.replace("/", "_") + fname = f"{safe}_{safe_cfg}_comparison.png" + fig.savefig(output_dir / fname, dpi=150, bbox_inches="tight") + plt.close(fig) + charts.append(fname) + + return charts # --------------------------------------------------------------------------- # Section generators # --------------------------------------------------------------------------- - _RUNTIME_DISPLAY = {"sglang": "SGLang", "vllm": "vLLM", "trtllm": "TRT-LLM"} _PROTOCOL_DISPLAY = {"http": "HTTP", "grpc": "gRPC"} -def _section_overview(experiments: list[ExperimentInfo]) -> list[str]: - """Overview table with status per model/config — columns discovered dynamically.""" +def _section_key_findings( + comparisons: list[ComparisonPoint], + experiments: list[ExperimentInfo], +) -> list[str]: + """Auto-generated executive summary of the benchmark results.""" + if not comparisons: + return [] + + lines = ["### Key Findings", ""] + + # 1. Overall verdict per key metric + for label, field, lower_better, _unit in _CHART_METRICS: + advs = [ + a + for cp in comparisons + if ( + a := _advantage( + getattr(cp.grpc, field), getattr(cp.http, field), lower_better + ) + ) + is not None + ] + if not advs: + continue + avg_adv = sum(advs) / len(advs) + grpc_wins = sum(1 for a in advs if a > 2) + http_wins = sum(1 for a in advs if a < -2) + ties = len(advs) - grpc_wins - http_wins + if abs(avg_adv) < 1: + verdict = f"**{label}**: No clear winner — essentially tied across all scenarios" + else: + winner = "gRPC" if avg_adv > 0 else "HTTP" + lines.append( + f"- **{label}**: {winner} wins {max(grpc_wins, http_wins)}/{len(advs)} " + f"comparisons (avg {abs(avg_adv):.1f}% better), " + f"{min(grpc_wins, http_wins)} losses, {ties} ties" + ) + continue + lines.append(f"- {verdict}") + + # 2. Error rates + total_runs = sum(len(e.runs) for e in experiments) + error_runs = [ + (e, r) + for e in experiments + for r in e.runs + if r.error_rate > 0 + ] + if error_runs: + lines.append( + f"- **Errors**: {len(error_runs)}/{total_runs} runs had non-zero error rates" + ) + else: + lines.append(f"- **Errors**: All {total_runs} runs completed with 0% error rate") + + # 3. Biggest outliers + biggest_grpc_win = biggest_http_win = None + biggest_grpc_adv = biggest_http_adv = 0.0 + for cp in comparisons: + for _label, field, lower_better, _ in _CHART_METRICS: + adv = _advantage( + getattr(cp.grpc, field), getattr(cp.http, field), lower_better + ) + if adv is not None and adv > biggest_grpc_adv: + biggest_grpc_adv = adv + biggest_grpc_win = (cp, _label) + if adv is not None and adv < biggest_http_adv: + biggest_http_adv = adv + biggest_http_win = (cp, _label) + + if biggest_grpc_win and biggest_grpc_adv > 10: + cp, metric = biggest_grpc_win + lines.append( + f"- **Largest gRPC win**: {biggest_grpc_adv:.0f}% on {metric} " + f"— {cp.model} `{cp.scenario}` C={cp.concurrency}" + ) + if biggest_http_win and abs(biggest_http_adv) > 10: + cp, metric = biggest_http_win + lines.append( + f"- **Largest HTTP win**: {abs(biggest_http_adv):.0f}% on {metric} " + f"— {cp.model} `{cp.scenario}` C={cp.concurrency}" + ) + + lines.append("") + return lines + + +def _section_error_rates(experiments: list[ExperimentInfo]) -> list[str]: + """Surface any non-zero error rates.""" + errors = [] + for e in experiments: + for r in e.runs: + if r.error_rate > 0: + errors.append((e, r)) + + if not errors: + return [] + + lines = [ + "
", + f"Runs with Errors ({len(errors)} runs)", + "", + "| Model | Protocol | Runtime | Workers | Scenario | C | Error Rate |", + "|-------|----------|---------|---------|----------|--:|-----------:|", + ] + + for e, r in sorted( + errors, key=lambda x: x[1].error_rate, reverse=True + ): + lines.append( + f"| {e.model} | {e.protocol} | {e.runtime} | {e.worker_type} " + f"| `{r.scenario}` | {r.concurrency} | {r.error_rate:.1%} |" + ) + + lines.extend(["", "
", ""]) + return lines + + +def _section_overview( + experiments: list[ExperimentInfo], models_with_data: set[str] +) -> list[str]: + """Overview table — only models that have comparison data.""" by_model: dict[str, dict[str, ExperimentInfo]] = defaultdict(dict) for exp in experiments: by_model[exp.model][exp.table_key] = exp - # Discover all (protocol, runtime, worker_type) combos that actually exist + # Discover columns dynamically all_keys: set[str] = set() - for model_exps in by_model.values(): - all_keys.update(model_exps.keys()) + for model in models_with_data: + if model in by_model: + all_keys.update(by_model[model].keys()) - # Sort: single before multi, then by runtime name, then http before grpc _worker_order = {"single": 0, "multi": 1} _protocol_order = {"http": 0, "grpc": 1} def _col_sort_key(key: str) -> tuple: protocol, runtime, worker = key.split("_") - return (_worker_order.get(worker, 9), runtime, _protocol_order.get(protocol, 9)) + return ( + _worker_order.get(worker, 9), + runtime, + _protocol_order.get(protocol, 9), + ) table_order = [] for key in sorted(all_keys, key=_col_sort_key): @@ -348,7 +630,9 @@ def _col_sort_key(key: str) -> tuple: "|" + "|".join(["---"] * len(header_cols)) + "|", ] - for model in sorted(by_model.keys()): + for model in sorted(models_with_data): + if model not in by_model: + continue model_exps = by_model[model] row = [model] for table_key, _ in table_order: @@ -356,10 +640,21 @@ def _col_sort_key(key: str) -> tuple: row.append("\u2796") else: exp = model_exps[table_key] - has_errors = any(r.rps == 0 or r.output_throughput == 0 for r in exp.runs) + has_errors = any( + r.rps == 0 or r.output_throughput == 0 for r in exp.runs + ) row.append("\u26a0\ufe0f" if has_errors else "\u2705") lines.append("| " + " | ".join(row) + " |") + # Note excluded models + excluded = set(by_model.keys()) - models_with_data + if excluded: + names = ", ".join(sorted(excluded)) + lines.append("") + lines.append( + f"*Excluded from comparison (no matched gRPC/HTTP data): {names}*" + ) + lines.append("") return lines @@ -369,12 +664,14 @@ def _section_aggregate(comparisons: list[ComparisonPoint]) -> list[str]: if not comparisons: return ["*No gRPC vs HTTP comparison data available.*", ""] + # (label, field, lower_is_better) metrics = [ ("TTFT mean", "ttft_mean", True), ("TTFT p99", "ttft_p99", True), ("E2E mean", "e2e_mean", True), ("E2E p99", "e2e_p99", True), ("TPOT mean", "tpot_mean", True), + ("TPOT p99", "tpot_p99", True), ("Output throughput", "output_throughput", False), ("Total throughput", "total_throughput", False), ("RPS", "rps", False), @@ -383,26 +680,28 @@ def _section_aggregate(comparisons: list[ComparisonPoint]) -> list[str]: lines = [ "### Aggregate: gRPC vs HTTP", "", - f"*{len(comparisons)} matched data points (error-free pairs only). " - "Negative % = gRPC lower (better for latency). " - "Positive % = gRPC higher (better for throughput).*", + f"*{len(comparisons)} matched data points. " + "Shows which protocol is better and by how much.*", "", - "| Metric | Avg % | Median % | Winner |", - "|--------|------:|--------:|--------|", + "| Metric | Avg | Median | Verdict |", + "|--------|----:|-------:|---------|", ] for label, fld, lower_better in metrics: - diffs = [] + advs = [] for cp in comparisons: - p = _pct(getattr(cp.grpc, fld), getattr(cp.http, fld)) - if p is not None: - diffs.append(p) - if not diffs: + a = _advantage( + getattr(cp.grpc, fld), getattr(cp.http, fld), lower_better + ) + if a is not None: + advs.append(a) + if not advs: continue - avg = sum(diffs) / len(diffs) - med = median(diffs) + avg = sum(advs) / len(advs) + med = median(advs) lines.append( - f"| {label} | {_fmt_pct(avg)} | {_fmt_pct(med)} | {_winner(avg, lower_better)} |" + f"| {label} | {_fmt_winner(avg)} | {_fmt_winner(med)} | " + f"{_fmt_winner_bold(avg)} |" ) lines.append("") @@ -421,69 +720,46 @@ def _section_by_concurrency(comparisons: list[ComparisonPoint]) -> list[str]: lines = ["### Performance by Concurrency", ""] - # TTFT mean - lines.extend( - [ + def _conc_table( + title: str, field: str, lower_is_better: bool, fmt_val + ) -> list[str]: + tbl = [ "
", - "TTFT Mean by Concurrency", + f"{title}", "", - "| Concurrency | gRPC avg | HTTP avg | Diff % | Winner |", - "|---:|---:|---:|---:|:---|", + "| Concurrency | gRPC | HTTP | Faster |", + "|---:|---:|---:|:---|", ] - ) - for conc in conc_levels: - cps = by_conc[conc] - g_avg = sum(cp.grpc.ttft_mean * 1000 for cp in cps) / len(cps) - h_avg = sum(cp.http.ttft_mean * 1000 for cp in cps) / len(cps) - pct = _pct(g_avg, h_avg) - lines.append( - f"| {conc} | {g_avg:.0f}ms | {h_avg:.0f}ms | {_fmt_pct(pct)} | " - f"{_winner(pct, lower_is_better=True)} |" - ) - lines.extend(["", "
", ""]) + for conc in conc_levels: + cps = by_conc[conc] + g_avg = sum(getattr(cp.grpc, field) for cp in cps) / len(cps) + h_avg = sum(getattr(cp.http, field) for cp in cps) / len(cps) + adv = _advantage(g_avg, h_avg, lower_is_better) + tbl.append( + f"| {conc} | {fmt_val(g_avg)} | {fmt_val(h_avg)} | " + f"{_fmt_winner_bold(adv)} |" + ) + tbl.extend(["", "", ""]) + return tbl - # E2E mean - lines.extend( - [ - "
", - "E2E Latency Mean by Concurrency", - "", - "| Concurrency | gRPC avg | HTTP avg | Diff % | Winner |", - "|---:|---:|---:|---:|:---|", - ] - ) - for conc in conc_levels: - cps = by_conc[conc] - g_avg = sum(cp.grpc.e2e_mean * 1000 for cp in cps) / len(cps) - h_avg = sum(cp.http.e2e_mean * 1000 for cp in cps) / len(cps) - pct = _pct(g_avg, h_avg) - lines.append( - f"| {conc} | {g_avg:.0f}ms | {h_avg:.0f}ms | {_fmt_pct(pct)} | " - f"{_winner(pct, lower_is_better=True)} |" - ) - lines.extend(["", "
", ""]) + def _fmt_ms(v: float) -> str: + ms = v * 1000 + return f"{ms:.0f}ms" if ms < 1000 else f"{v:.2f}s" - # Output throughput - lines.extend( - [ - "
", - "Output Throughput by Concurrency", - "", - "| Concurrency | gRPC avg | HTTP avg | Diff % | Winner |", - "|---:|---:|---:|---:|:---|", - ] - ) - for conc in conc_levels: - cps = by_conc[conc] - g_avg = sum(cp.grpc.output_throughput for cp in cps) / len(cps) - h_avg = sum(cp.http.output_throughput for cp in cps) / len(cps) - pct = _pct(g_avg, h_avg) - lines.append( - f"| {conc} | {_fmt_throughput(g_avg)} tok/s | " - f"{_fmt_throughput(h_avg)} tok/s | {_fmt_pct(pct)} | " - f"{_winner(pct, lower_is_better=False)} |" - ) - lines.extend(["", "
", ""]) + def _fmt_tput(v: float) -> str: + return f"{_fmt_throughput(v)} tok/s" + + def _fmt_tpot(v: float) -> str: + ms = v * 1000 + return f"{ms:.1f}ms" if ms >= 1 else f"{ms:.2f}ms" + + lines.extend(_conc_table("TTFT Mean", "ttft_mean", True, _fmt_ms)) + lines.extend(_conc_table("TTFT p99", "ttft_p99", True, _fmt_ms)) + lines.extend(_conc_table("TPOT Mean", "tpot_mean", True, _fmt_tpot)) + lines.extend(_conc_table("TPOT p99", "tpot_p99", True, _fmt_tpot)) + lines.extend(_conc_table("E2E Latency Mean", "e2e_mean", True, _fmt_ms)) + lines.extend(_conc_table("E2E Latency p99", "e2e_p99", True, _fmt_ms)) + lines.extend(_conc_table("Output Throughput", "output_throughput", False, _fmt_tput)) return lines @@ -496,7 +772,7 @@ def _section_scorecard(comparisons: list[ComparisonPoint]) -> list[str]: lines = [ "### Win/Loss Scorecard", "", - "*How often does gRPC beat HTTP beyond a given threshold?*", + "*How often is one protocol better by > N%?*", "", ] @@ -512,7 +788,7 @@ def _section_scorecard(comparisons: list[ComparisonPoint]) -> list[str]: [ f"**{label}:**", "", - "| Threshold | gRPC wins | HTTP wins | Within |", + "| Threshold | gRPC better | HTTP better | Within |", "|---:|---:|---:|---:|", ] ) @@ -520,23 +796,17 @@ def _section_scorecard(comparisons: list[ComparisonPoint]) -> list[str]: for thresh in thresholds: grpc_w = http_w = within = 0 for cp in comparisons: - pct = _pct(getattr(cp.grpc, fld), getattr(cp.http, fld)) - if pct is None: + adv = _advantage( + getattr(cp.grpc, fld), getattr(cp.http, fld), lower_better + ) + if adv is None: continue - if lower_better: - if pct < -thresh: - grpc_w += 1 - elif pct > thresh: - http_w += 1 - else: - within += 1 + if adv > thresh: + grpc_w += 1 + elif adv < -thresh: + http_w += 1 else: - if pct > thresh: - grpc_w += 1 - elif pct < -thresh: - http_w += 1 - else: - within += 1 + within += 1 g_str = f"**{grpc_w}**" if grpc_w > http_w else str(grpc_w) h_str = f"**{http_w}**" if http_w > grpc_w else str(http_w) @@ -547,8 +817,10 @@ def _section_scorecard(comparisons: list[ComparisonPoint]) -> list[str]: return lines -def _section_top_wins(comparisons: list[ComparisonPoint], top_n: int = 15) -> list[str]: - """Table of largest gRPC wins (>10% improvement).""" +def _section_top_wins( + comparisons: list[ComparisonPoint], threshold: float = 30.0 +) -> list[str]: + """Table of all gRPC wins exceeding the threshold.""" if not comparisons: return [] @@ -557,38 +829,34 @@ def _section_top_wins(comparisons: list[ComparisonPoint], top_n: int = 15) -> li ("TTFT mean", "ttft_mean", True), ("E2E mean", "e2e_mean", True), ("E2E p99", "e2e_p99", True), + ("TPOT p99", "tpot_p99", True), ("Output tput", "output_throughput", False), ("RPS", "rps", False), ] - wins: list[tuple[float, float, str, ComparisonPoint, float, float]] = [] + wins: list[tuple[float, str, ComparisonPoint, float, float]] = [] for cp in comparisons: for label, fld, lower_better in metric_defs: g = getattr(cp.grpc, fld) h = getattr(cp.http, fld) - pct = _pct(g, h) - if pct is None: - continue - if lower_better and pct < -10: - wins.append((abs(pct), pct, label, cp, g, h)) - elif not lower_better and pct > 10: - wins.append((abs(pct), pct, label, cp, g, h)) + adv = _advantage(g, h, lower_better) + if adv is not None and adv > threshold: + wins.append((adv, label, cp, g, h)) wins.sort(key=lambda x: x[0], reverse=True) - wins = wins[:top_n] if not wins: return [] lines = [ "
", - "Top gRPC Wins (>10% improvement)", + f"Top gRPC Wins (>{threshold:.0f}%) — {len(wins)} entries", "", - "| Diff | Model | Config | Scenario | C | Metric | gRPC | HTTP |", - "|-----:|-------|--------|----------|--:|--------|-----:|-----:|", + "| gRPC better by | Model | Config | Scenario | C | Metric | gRPC | HTTP |", + "|---------------:|-------|--------|----------|--:|--------|-----:|-----:|", ] - for _, pct, label, cp, g, h in wins: + for adv, label, cp, g, h in wins: config = f"{cp.runtime}/{cp.worker_type}" is_latency = any(k in label.lower() for k in ("ttft", "e2e", "tpot")) if is_latency: @@ -601,7 +869,7 @@ def _section_top_wins(comparisons: list[ComparisonPoint], top_n: int = 15) -> li g_str = f"{g:.1f}" h_str = f"{h:.1f}" lines.append( - f"| {_fmt_pct(pct)} | {cp.model} | {config} | " + f"| {adv:.1f}% | {cp.model} | {config} | " f"`{cp.scenario}` | {cp.concurrency} | {label} | {g_str} | {h_str} |" ) @@ -610,7 +878,11 @@ def _section_top_wins(comparisons: list[ComparisonPoint], top_n: int = 15) -> li def _section_per_model(comparisons: list[ComparisonPoint]) -> list[str]: - """Per-model summary table plus collapsible detail tables.""" + """Per-model summary table plus collapsible detail tables. + + All values show gRPC advantage: 'gRPC X%' = gRPC is X% better, + 'HTTP X%' = HTTP is X% better, '~' = within 2%. + """ if not comparisons: return [] @@ -618,32 +890,36 @@ def _section_per_model(comparisons: list[ComparisonPoint]) -> list[str]: for cp in comparisons: by_model[cp.model].append(cp) - lines = ["### Per-Model Summary", ""] - - # Compact summary: one row per model - lines.extend( - [ - "| Model | TTFT mean | TTFT p99 | E2E mean | Output tput | N |", - "|-------|--------:|--------:|--------:|--------:|---:|", - ] - ) - - def _avg_or_na(vals: list[float]) -> str: - return _fmt_pct(sum(vals) / len(vals)) if vals else "N/A" + lines = [ + "### Per-Model Summary", + "", + "*Each cell shows which protocol is better and by how much.*", + "", + "| Model | TTFT p99 | TPOT p99 | E2E p99 | Output tput | N |", + "|-------|--------:|--------:|--------:|--------:|---:|", + ] for model in sorted(by_model.keys()): cps = by_model[model] - ttft = [p for cp in cps if (p := _pct(cp.grpc.ttft_mean, cp.http.ttft_mean)) is not None] - ttft_p99 = [p for cp in cps if (p := _pct(cp.grpc.ttft_p99, cp.http.ttft_p99)) is not None] - e2e = [p for cp in cps if (p := _pct(cp.grpc.e2e_mean, cp.http.e2e_mean)) is not None] - tput = [ - p - for cp in cps - if (p := _pct(cp.grpc.output_throughput, cp.http.output_throughput)) is not None - ] + + def _model_avg(field: str, lower_better: bool) -> str: + advs = [ + a + for cp in cps + if (a := _advantage(getattr(cp.grpc, field), getattr(cp.http, field), lower_better)) + is not None + ] + if not advs: + return "N/A" + return _fmt_winner(sum(advs) / len(advs)) + lines.append( - f"| {model} | {_avg_or_na(ttft)} | {_avg_or_na(ttft_p99)} | " - f"{_avg_or_na(e2e)} | {_avg_or_na(tput)} | {len(cps)} |" + f"| {model} " + f"| {_model_avg('ttft_p99', True)} " + f"| {_model_avg('tpot_p99', True)} " + f"| {_model_avg('e2e_p99', True)} " + f"| {_model_avg('output_throughput', False)} " + f"| {len(cps)} |" ) lines.append("") @@ -656,24 +932,27 @@ def _avg_or_na(vals: list[float]) -> str: by_config[f"{cp.runtime}/{cp.worker_type}"].append(cp) for config in sorted(by_config.keys()): - config_cps = sorted(by_config[config], key=lambda x: (x.scenario, x.concurrency)) + config_cps = sorted( + by_config[config], key=lambda x: (x.scenario, x.concurrency) + ) lines.extend( [ "
", - f"{model} ({config}) — {len(config_cps)} data points", + f"{model} ({config}) — " + f"{len(config_cps)} points", "", - "| Scenario | C | TTFT mean | TTFT p99 | E2E mean | Tput |", + "| Scenario | C | TTFT p99 | TPOT p99 | E2E p99 | Tput |", "|----------|--:|--------:|--------:|--------:|--------:|", ] ) for cp in config_cps: lines.append( - f"| `{cp.scenario}` | {cp.concurrency} | " - f"{_fmt_pct(_pct(cp.grpc.ttft_mean, cp.http.ttft_mean))} | " - f"{_fmt_pct(_pct(cp.grpc.ttft_p99, cp.http.ttft_p99))} | " - f"{_fmt_pct(_pct(cp.grpc.e2e_mean, cp.http.e2e_mean))} | " - f"{_fmt_pct(_pct(cp.grpc.output_throughput, cp.http.output_throughput))} |" + f"| `{cp.scenario}` | {cp.concurrency} " + f"| {_fmt_winner(_advantage(cp.grpc.ttft_p99, cp.http.ttft_p99, True))} " + f"| {_fmt_winner(_advantage(cp.grpc.tpot_p99, cp.http.tpot_p99, True))} " + f"| {_fmt_winner(_advantage(cp.grpc.e2e_p99, cp.http.e2e_p99, True))} " + f"| {_fmt_winner(_advantage(cp.grpc.output_throughput, cp.http.output_throughput, False))} |" ) lines.extend(["", "
", ""]) @@ -695,6 +974,9 @@ def generate_summary(base_dir: Path) -> str: comparisons = build_comparisons(experiments) + # Only include models that have comparison data + models_with_data = {cp.model for cp in comparisons} + grpc_count = sum(1 for e in experiments if e.protocol == "grpc") http_count = sum(1 for e in experiments if e.protocol == "http") total_runs = sum(len(e.runs) for e in experiments) @@ -706,29 +988,94 @@ def generate_summary(base_dir: Path) -> str: f"**{total_runs} benchmark runs**, " f"**{len(comparisons)} matched comparison points**", "", + "
", + "Glossary", + "", + "#### Metrics", + "", + "| Metric | Description |", + "|--------|-------------|", + "| **TTFT** | Time To First Token — latency from request sent to first token received |", + "| **TPOT** | Time Per Output Token — average time between consecutive output tokens |", + "| **E2E** | End-to-End latency — total time from request sent to last token received |", + "| **Output tput** | Output throughput — tokens generated per second (tok/s) |", + "| **Total tput** | Total throughput — input + output tokens processed per second |", + "| **RPS** | Requests Per Second — completed requests per second |", + "| **p99** | 99th percentile — the value below which 99% of observations fall (worst 1% of requests) |", + "| **mean** | Arithmetic average across all requests |", + "", + "#### Traffic Scenarios", + "", + "| Pattern | Description |", + "|---------|-------------|", + "| **D(in, out)** | Deterministic — fixed input/output token lengths, e.g. `D(100,100)` = 100 input, 100 output tokens |", + "| **N(μ,σ)/(μ,σ)** | Normal distribution — input and output lengths drawn from Gaussian, e.g. `N(480,240)/(300,150)` |", + "| **E(size)** | Embedding — input of given token length, used for embedding model benchmarks |", + "", + "#### Comparison Columns", + "", + "| Value | Meaning |", + "|-------|---------|", + "| **gRPC X%** | gRPC is X% better than HTTP for this metric |", + "| **HTTP X%** | HTTP is X% better than gRPC for this metric |", + "| **~** | Difference is within 2% — essentially a tie |", + "", + "*Lower is better for latency metrics (TTFT, TPOT, E2E). Higher is better for throughput (tput, RPS).*", + "", + "
", + "", "---", "", ] - lines.extend(_section_overview(experiments)) + lines.extend(_section_key_findings(comparisons, experiments)) + lines.extend(_section_overview(experiments, models_with_data)) lines.extend(_section_aggregate(comparisons)) lines.extend(_section_by_concurrency(comparisons)) lines.extend(_section_scorecard(comparisons)) lines.extend(_section_top_wins(comparisons)) lines.extend(_section_per_model(comparisons)) + lines.extend(_section_error_rates(experiments)) lines.append("---") lines.append( - f"*Generated from {len(experiments)} experiment(s), {len(comparisons)} comparison points*" + f"*Generated from {len(experiments)} experiment(s), " + f"{len(comparisons)} comparison points*" ) - return "\n".join(lines) + return "\n".join(lines), experiments, comparisons def main() -> None: - """Main entry point.""" - base_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else Path.cwd() - summary = generate_summary(base_dir) + """Main entry point. + + Usage: nightly_summarize.py [base_dir] [--charts-dir DIR] + """ + args = sys.argv[1:] + base_dir = Path.cwd() + charts_dir: Path | None = None + + i = 0 + while i < len(args): + if args[i] == "--charts-dir" and i + 1 < len(args): + charts_dir = Path(args[i + 1]) + i += 2 + elif not args[i].startswith("-"): + base_dir = Path(args[i]) + i += 1 + else: + i += 1 + + summary, experiments, comparisons = generate_summary(base_dir) + + # Generate comparison charts if requested + if charts_dir and comparisons: + chart_files = generate_charts(comparisons, experiments, charts_dir) + if chart_files: + print( + f"Generated {len(chart_files)} chart(s) in {charts_dir}", + file=sys.stderr, + ) summary_file = os.environ.get("GITHUB_STEP_SUMMARY") if summary_file: diff --git a/e2e_test/benchmarks/test_nightly_perf.py b/e2e_test/benchmarks/test_nightly_perf.py index ce0613be52..a61847aaaa 100644 --- a/e2e_test/benchmarks/test_nightly_perf.py +++ b/e2e_test/benchmarks/test_nightly_perf.py @@ -97,13 +97,8 @@ def _run_nightly(setup_backend, genai_bench_runner, model_id, worker_count=1, ** _NIGHTLY_MODELS: list[tuple[str, str, int, list[str], dict]] = [ ("meta-llama/Llama-3.1-8B-Instruct", "Llama8b", 4, ["http", "grpc"], {}), - ("meta-llama/Llama-3.2-1B-Instruct", "Llama1b", 4, ["http", "grpc"], {}), ("Qwen/Qwen2.5-7B-Instruct", "Qwen7b", 4, ["http", "grpc"], {}), - ("Qwen/Qwen2.5-14B-Instruct", "Qwen14b", 4, ["http", "grpc"], {}), - ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "Deepseek7b", 4, ["http", "grpc"], {}), ("Qwen/Qwen3-30B-A3B", "Qwen30b", 4, ["http", "grpc"], {}), - ("mistralai/Mistral-7B-Instruct-v0.3", "Mistral7b", 4, ["http", "grpc"], {}), - ("openai/gpt-oss-20b", "GptOss20b", 4, ["http", "grpc"], {}), ( "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "Llama4Maverick", @@ -128,7 +123,7 @@ def _make_test_class(model_id, worker_count, backends, extra_kwargs): @pytest.mark.e2e @pytest.mark.model(model_id) @pytest.mark.workers(count=worker_count) - @pytest.mark.gateway(policy="round_robin") + @pytest.mark.gateway(policy="round_robin", log_level="debug", log_dir="nightly_gateway_logs") @pytest.mark.parametrize("setup_backend", backends, indirect=True) class _NightlyTest: def test_nightly_perf(self, setup_backend, genai_bench_runner): diff --git a/e2e_test/fixtures/setup_backend.py b/e2e_test/fixtures/setup_backend.py index e7bef0fc0b..ec10f583a6 100644 --- a/e2e_test/fixtures/setup_backend.py +++ b/e2e_test/fixtures/setup_backend.py @@ -89,6 +89,8 @@ def _create_backend(request: pytest.FixtureRequest, model_pool: ModelPool): "policy": "round_robin", "timeout": DEFAULT_ROUTER_TIMEOUT, "extra_args": None, + "log_level": None, + "log_dir": None, }, ) @@ -429,6 +431,8 @@ def _setup_pd_backend_common( policy=gateway_config["policy"], timeout=gateway_config["timeout"], extra_args=gateway_config["extra_args"], + log_level=gateway_config.get("log_level"), + log_dir=gateway_config.get("log_dir"), ) except Exception: # Release workers if gateway fails to start @@ -497,6 +501,8 @@ def _setup_grpc_backend( policy=gateway_config["policy"], timeout=gateway_config["timeout"], extra_args=gateway_config["extra_args"], + log_level=gateway_config.get("log_level"), + log_dir=gateway_config.get("log_dir"), ) except Exception: # Release worker if gateway fails to start @@ -609,6 +615,8 @@ def _setup_local_backend( policy=gateway_config["policy"], timeout=gateway_config["timeout"], extra_args=gateway_config["extra_args"], + log_level=gateway_config.get("log_level"), + log_dir=gateway_config.get("log_dir"), ) except Exception: # Release workers if gateway fails to start diff --git a/e2e_test/infra/gateway.py b/e2e_test/infra/gateway.py index 0f96996daf..8280a7e7e8 100644 --- a/e2e_test/infra/gateway.py +++ b/e2e_test/infra/gateway.py @@ -104,6 +104,8 @@ def __init__( self.process: subprocess.Popen | None = None self.model_path: str | None = None self.policy: str = "round_robin" + self.log_level: str = "warn" + self.log_dir: str | None = None self.pd_mode: bool = False self.igw_mode: bool = False self.cloud_mode: bool = False @@ -135,6 +137,8 @@ def start( timeout: float = DEFAULT_ROUTER_TIMEOUT, show_output: bool | None = None, extra_args: list[str] | None = None, + log_level: str | None = None, + log_dir: str | None = None, ) -> None: """Start the gateway. @@ -156,6 +160,8 @@ def start( timeout: Startup timeout in seconds. show_output: Show subprocess output (env var override). extra_args: Additional router arguments. + log_level: Log level override (debug, info, warn, error). + log_dir: Directory to store gateway log files. Raises: RuntimeError: If gateway is already started. @@ -189,6 +195,10 @@ def start( show_output = os.environ.get(ENV_SHOW_ROUTER_LOGS, "0") == "1" self.policy = policy + if log_level: + self.log_level = log_level + if log_dir: + self.log_dir = log_dir if is_igw_mode: # IGW mode: start empty, add workers via API @@ -368,7 +378,7 @@ def shutdown(self) -> None: def _build_base_cmd(self) -> list[str]: """Build the base command for launching the router.""" - return [ + cmd = [ "python3", "-m", "smg.launch_router", @@ -383,8 +393,11 @@ def _build_base_cmd(self) -> list[str]: "--policy", self.policy, "--log-level", - "warn", + self.log_level, ] + if self.log_dir: + cmd.extend(["--log-dir", self.log_dir]) + return cmd # ------------------------------------------------------------------------- # Health & Metrics APIs From b6fc19d91ee886c552e119c2a8f9f3d41a5d2cb9 Mon Sep 17 00:00:00 2001 From: Simo Lin Date: Tue, 10 Feb 2026 08:20:32 -0800 Subject: [PATCH 2/2] refactor(e2e): consolidate nightly summary script and eliminate duplication - Define each metric once in a canonical registry (_M_TTFT_P99, etc.) and compose section-specific lists by reference instead of copy-pasting tuples across 6 separate lists - Merge PER_MODEL_SUMMARY_METRICS and PER_MODEL_DETAIL_METRICS into single PER_MODEL_METRICS used for both summary and detail tables - Replace _raw_pct() + _advantage() with single _advantage() that inlines the percentage calculation - Add _cp_advantage() and _avg_advantage() helpers to eliminate the repeated getattr(cp.grpc, fld) + _advantage() pattern (~15 call sites) - Unify _fmt_winner() and _fmt_winner_bold() into single function with bold parameter - Add _fmt_metric_value() for type-based value formatting, reuse in _section_by_concurrency instead of duplicated if/elif/else dispatch - Extract _group_by_concurrency() helper (was computed 3 times) - Extract _plot_comparison_grid() to deduplicate aggregate vs per-model chart generation (was ~40 lines of identical plotting code) - Add ComparisonPoint.config property to replace f-string duplication - Add SummaryResult dataclass for generate_summary() return type instead of bare tuple - Extract glossary text into _GLOSSARY_LINES constant - Remove unused experiments parameter from generate_charts() - Shorten glossary descriptions to satisfy 120-char line limit --- e2e_test/benchmarks/nightly_summarize.py | 615 ++++++++++------------- 1 file changed, 275 insertions(+), 340 deletions(-) diff --git a/e2e_test/benchmarks/nightly_summarize.py b/e2e_test/benchmarks/nightly_summarize.py index 892c51fa04..b97b9b188b 100755 --- a/e2e_test/benchmarks/nightly_summarize.py +++ b/e2e_test/benchmarks/nightly_summarize.py @@ -18,6 +18,104 @@ from pathlib import Path from statistics import median +# --------------------------------------------------------------------------- +# Metric definitions — single source of truth +# +# Each metric is defined once in _METRIC_REGISTRY. Section-specific lists +# are composed by selecting from the registry, avoiding tuple duplication. +# --------------------------------------------------------------------------- + +# (label, field_name, lower_is_better) +_MetricDef = tuple[str, str, bool] + +# Canonical definitions — every metric appears exactly once. +_M_TTFT_MEAN: _MetricDef = ("TTFT mean", "ttft_mean", True) +_M_TTFT_P99: _MetricDef = ("TTFT p99", "ttft_p99", True) +_M_TPOT_MEAN: _MetricDef = ("TPOT mean", "tpot_mean", True) +_M_TPOT_P99: _MetricDef = ("TPOT p99", "tpot_p99", True) +_M_E2E_MEAN: _MetricDef = ("E2E mean", "e2e_mean", True) +_M_E2E_P99: _MetricDef = ("E2E p99", "e2e_p99", True) +_M_OUT_TPUT: _MetricDef = ("Output tput", "output_throughput", False) +_M_TOT_TPUT: _MetricDef = ("Total throughput", "total_throughput", False) +_M_RPS: _MetricDef = ("RPS", "rps", False) + +# Per-section selections (composed, not copy-pasted). +AGGREGATE_METRICS: list[_MetricDef] = [ + _M_TTFT_MEAN, + _M_TTFT_P99, + _M_E2E_MEAN, + _M_E2E_P99, + _M_TPOT_MEAN, + _M_TPOT_P99, + _M_OUT_TPUT, + _M_TOT_TPUT, + _M_RPS, +] +TOP_WINS_METRICS: list[_MetricDef] = [ + _M_TTFT_P99, + _M_TTFT_MEAN, + _M_E2E_MEAN, + _M_E2E_P99, + _M_TPOT_P99, + _M_OUT_TPUT, + _M_RPS, +] +PER_MODEL_METRICS: list[_MetricDef] = [_M_TTFT_P99, _M_TPOT_P99, _M_E2E_P99, _M_OUT_TPUT] +CONCURRENCY_METRICS: list[_MetricDef] = [ + _M_TTFT_MEAN, + _M_TTFT_P99, + _M_TPOT_MEAN, + _M_TPOT_P99, + _M_E2E_MEAN, + _M_E2E_P99, + _M_OUT_TPUT, +] +SCORECARD_METRICS: list[_MetricDef] = [_M_E2E_MEAN, _M_TTFT_MEAN, _M_OUT_TPUT] + +# Chart metrics carry a unit for axis labels. +CHART_METRICS: list[tuple[str, str, bool, str]] = [ + (*_M_TTFT_P99, "ms"), + (*_M_TPOT_P99, "ms"), + (*_M_E2E_P99, "s"), + (*_M_OUT_TPUT, "tok/s"), +] + +# --------------------------------------------------------------------------- +# Glossary — displayed once in the summary header +# --------------------------------------------------------------------------- + +_GLOSSARY_LINES = [ + "#### Metrics", + "", + "| Metric | Description |", + "|--------|-------------|", + "| **TTFT** | Time To First Token — latency from request sent to first token received |", + "| **TPOT** | Time Per Output Token — average time between consecutive output tokens |", + "| **E2E** | End-to-End latency — total time from request sent to last token received |", + "| **Output tput** | Output throughput — tokens generated per second (tok/s) |", + "| **Total tput** | Total throughput — input + output tokens processed per second |", + "| **RPS** | Requests Per Second — completed requests per second |", + "| **p99** | 99th percentile — the value below which 99% of observations fall (worst 1% of requests) |", + "| **mean** | Arithmetic average across all requests |", + "", + "#### Traffic Scenarios", + "", + "| Pattern | Description |", + "|---------|-------------|", + "| **D(in, out)** | Deterministic — fixed input/output token lengths, e.g. `D(100,100)` |", + "| **N(μ,σ)/(μ,σ)** | Normal distribution — input/output lengths drawn from Gaussian |", + "| **E(size)** | Embedding — input of given token length, used for embedding model benchmarks |", + "", + "#### Comparison Columns", + "", + "| Value | Meaning |", + "|-------|---------|", + "| **gRPC X%** | gRPC is X% better than HTTP for this metric |", + "| **HTTP X%** | HTTP is X% better than gRPC for this metric |", + "| **~** | Difference is within 2% — essentially a tie |", + "", + "*Lower is better for latency metrics (TTFT, TPOT, E2E). Higher is better for throughput (tput, RPS).*", +] # --------------------------------------------------------------------------- # Data types @@ -77,6 +175,19 @@ class ComparisonPoint: grpc: RunResult http: RunResult + @property + def config(self) -> str: + return f"{self.runtime}/{self.worker_type}" + + +@dataclass +class SummaryResult: + """Return value of generate_summary.""" + + markdown: str + experiments: list[ExperimentInfo] + comparisons: list[ComparisonPoint] + # --------------------------------------------------------------------------- # Parsing @@ -108,11 +219,7 @@ def parse_folder_name(folder_name: str) -> dict: name = folder_name.replace("nightly_", "") parts = name.rsplit("_", 3) - if ( - len(parts) >= 4 - and parts[-1] in _KNOWN_WORKER_TYPES - and parts[-2] in _KNOWN_RUNTIMES - ): + if len(parts) >= 4 and parts[-1] in _KNOWN_WORKER_TYPES and parts[-2] in _KNOWN_RUNTIMES: info["worker_type"] = parts[-1] info["runtime"] = parts[-2] info["protocol"] = parts[-3] @@ -197,9 +304,7 @@ def parse_experiment(folder: Path) -> ExperimentInfo | None: concurrency=agg.get("num_concurrency", 0) or 0, rps=_get_float(agg, "requests_per_second"), output_throughput=_get_float(agg, "mean_output_throughput_tokens_per_s"), - total_throughput=_get_float( - agg, "mean_total_tokens_throughput_tokens_per_s" - ), + total_throughput=_get_float(agg, "mean_total_tokens_throughput_tokens_per_s"), ttft_mean=_get_float(ttft, "mean"), ttft_p99=_get_float(ttft, "p99"), tpot_mean=_get_float(tpot, "mean"), @@ -274,106 +379,102 @@ def build_comparisons(experiments: list[ExperimentInfo]) -> list[ComparisonPoint # --------------------------------------------------------------------------- -def _raw_pct(grpc_val: float, http_val: float) -> float | None: - """Raw (gRPC - HTTP) / HTTP * 100.""" - if http_val == 0: - return None - return (grpc_val - http_val) / http_val * 100 - - def _advantage(grpc_val: float, http_val: float, lower_is_better: bool) -> float | None: """gRPC advantage %: positive = gRPC is better, negative = HTTP is better.""" - pct = _raw_pct(grpc_val, http_val) - if pct is None: + if http_val == 0: return None + pct = (grpc_val - http_val) / http_val * 100 return -pct if lower_is_better else pct -def _fmt_winner(pct: float | None, threshold: float = 2.0) -> str: - """Format as 'gRPC X%' or 'HTTP X%' or '~'. pct is gRPC advantage.""" - if pct is None: - return "N/A" - if abs(pct) < threshold: - return "~" - if pct > 0: - return f"gRPC {abs(pct):.1f}%" - return f"HTTP {abs(pct):.1f}%" +def _cp_advantage(cp: ComparisonPoint, fld: str, lower_is_better: bool) -> float | None: + """Shorthand: compute gRPC advantage for a ComparisonPoint and field name.""" + return _advantage(getattr(cp.grpc, fld), getattr(cp.http, fld), lower_is_better) -def _fmt_winner_bold(pct: float | None, threshold: float = 2.0) -> str: - """Like _fmt_winner but bolds the winner name.""" +def _avg_advantage(cps: list[ComparisonPoint], fld: str, lower_is_better: bool) -> float | None: + """Average gRPC advantage across a list of comparison points.""" + advs = [a for cp in cps if (a := _cp_advantage(cp, fld, lower_is_better)) is not None] + return sum(advs) / len(advs) if advs else None + + +def _fmt_winner(pct: float | None, threshold: float = 2.0, bold: bool = False) -> str: + """Format as 'gRPC X%' or 'HTTP X%' or '~'. pct is gRPC advantage.""" if pct is None: return "N/A" if abs(pct) < threshold: return "~" - if pct > 0: - return f"**gRPC** {abs(pct):.1f}%" - return f"**HTTP** {abs(pct):.1f}%" + name = "gRPC" if pct > 0 else "HTTP" + if bold: + name = f"**{name}**" + return f"{name} {abs(pct):.1f}%" def _fmt_latency_s(val_s: float) -> str: """Format latency value (in seconds) for display.""" ms = val_s * 1000 - if ms < 1000: - return f"{ms:.0f}ms" - return f"{val_s:.2f}s" + return f"{ms:.0f}ms" if ms < 1000 else f"{val_s:.2f}s" def _fmt_throughput(val: float) -> str: - if val >= 1000: - return f"{val / 1000:.1f}K" - return f"{val:.0f}" + return f"{val / 1000:.1f}K" if val >= 1000 else f"{val:.0f}" + + +def _fmt_metric_value(val: float, label: str) -> str: + """Format a metric value based on its label/type.""" + label_lower = label.lower() + if any(k in label_lower for k in ("ttft", "e2e")): + return _fmt_latency_s(val) + if "tpot" in label_lower: + ms = val * 1000 + return f"{ms:.1f}ms" if ms >= 1 else f"{ms:.2f}ms" + if "tput" in label_lower or "throughput" in label_lower: + return f"{_fmt_throughput(val)} tok/s" + if "rps" in label_lower: + return f"{val:.1f}" + return f"{val:.1f}" + + +def _group_by_concurrency( + comparisons: list[ComparisonPoint], +) -> tuple[dict[int, list[ComparisonPoint]], list[int]]: + """Group comparisons by concurrency level, return (mapping, sorted_levels).""" + by_conc: dict[int, list[ComparisonPoint]] = defaultdict(list) + for cp in comparisons: + by_conc[cp.concurrency].append(cp) + return by_conc, sorted(by_conc.keys()) # --------------------------------------------------------------------------- # Chart generation (optional — requires matplotlib) # --------------------------------------------------------------------------- -# Metrics used across charts and key findings -_CHART_METRICS = [ - ("TTFT p99", "ttft_p99", True, "ms"), - ("TPOT p99", "tpot_p99", True, "ms"), - ("E2E p99", "e2e_p99", True, "s"), - ("Output Throughput", "output_throughput", False, "tok/s"), -] - -def generate_charts( - comparisons: list[ComparisonPoint], - experiments: list[ExperimentInfo], +def _plot_comparison_grid( + title: str, + by_conc: dict[int, list[ComparisonPoint]], + conc_levels: list[int], output_dir: Path, -) -> list[str]: - """Generate comparison charts as PNGs. Returns list of generated filenames.""" + filename: str, +) -> str | None: + """Plot a 2x2 grid of CHART_METRICS. Returns filename or None on failure.""" try: import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt except ImportError: - print("matplotlib not available, skipping chart generation", file=sys.stderr) - return [] - - output_dir.mkdir(parents=True, exist_ok=True) - charts: list[str] = [] - - by_conc: dict[int, list[ComparisonPoint]] = defaultdict(list) - for cp in comparisons: - by_conc[cp.concurrency].append(cp) - conc_levels = sorted(by_conc.keys()) - - if not conc_levels: - return [] + return None - # ---- Aggregate comparison (2x2 grid) ---- fig, axes = plt.subplots(2, 2, figsize=(14, 10)) - fig.suptitle("gRPC vs HTTP — Aggregate Comparison", fontsize=16, fontweight="bold") + fig.suptitle(title, fontsize=16, fontweight="bold") - for ax, (title, field, _lower_better, unit) in zip(axes.flat, _CHART_METRICS): + for ax, (label, fld, _lower_better, unit) in zip(axes.flat, CHART_METRICS): grpc_vals, http_vals = [], [] for conc in conc_levels: cps = by_conc[conc] - g = sum(getattr(cp.grpc, field) for cp in cps) / len(cps) - h = sum(getattr(cp.http, field) for cp in cps) / len(cps) + g = sum(getattr(cp.grpc, fld) for cp in cps) / len(cps) + h = sum(getattr(cp.http, fld) for cp in cps) / len(cps) if unit == "ms": g *= 1000 h *= 1000 @@ -383,82 +484,69 @@ def generate_charts( x = range(len(conc_levels)) ax.plot(x, grpc_vals, "o-", label="gRPC", color="#2196F3", linewidth=2) ax.plot(x, http_vals, "s--", label="HTTP", color="#FF9800", linewidth=2) - ax.set_title(title, fontsize=12, fontweight="bold") + ax.set_title(label, fontsize=12, fontweight="bold") ax.set_xlabel("Concurrency") - ax.set_ylabel(f"{title} ({unit})") + ax.set_ylabel(f"{label} ({unit})") ax.set_xticks(list(x)) ax.set_xticklabels([str(c) for c in conc_levels]) ax.legend() ax.grid(True, alpha=0.3) - # Use log scale when values span >10x nz = [v for v in grpc_vals + http_vals if v > 0] if nz and max(nz) > 10 * min(nz): ax.set_yscale("log") fig.tight_layout() - fname = "aggregate_comparison.png" - fig.savefig(output_dir / fname, dpi=150, bbox_inches="tight") + fig.savefig(output_dir / filename, dpi=150, bbox_inches="tight") plt.close(fig) - charts.append(fname) + return filename + - # ---- Per-model/config charts ---- +def generate_charts( + comparisons: list[ComparisonPoint], + output_dir: Path, +) -> list[str]: + """Generate comparison charts as PNGs. Returns list of generated filenames.""" + output_dir.mkdir(parents=True, exist_ok=True) + charts: list[str] = [] + + by_conc, conc_levels = _group_by_concurrency(comparisons) + if not conc_levels: + return [] + + # Aggregate comparison + fname = _plot_comparison_grid( + "gRPC vs HTTP — Aggregate Comparison", + by_conc, + conc_levels, + output_dir, + "aggregate_comparison.png", + ) + if fname: + charts.append(fname) + + # Per-model/config charts by_model_config: dict[str, list[ComparisonPoint]] = defaultdict(list) for cp in comparisons: - by_model_config[f"{cp.model}|{cp.runtime}/{cp.worker_type}"].append(cp) + by_model_config[f"{cp.model}|{cp.config}"].append(cp) for key, cps in sorted(by_model_config.items()): model, config = key.split("|", 1) - mc_conc: dict[int, list[ComparisonPoint]] = defaultdict(list) - for cp in cps: - mc_conc[cp.concurrency].append(cp) - mc_levels = sorted(mc_conc.keys()) + mc_conc, mc_levels = _group_by_concurrency(cps) if len(mc_levels) < 3: continue - fig, axes = plt.subplots(2, 2, figsize=(14, 10)) display_name = model.split("/")[-1] if "/" in model else model - fig.suptitle( - f"{display_name} ({config}): gRPC vs HTTP", - fontsize=16, - fontweight="bold", - ) - - for ax, (title, field, _lb, unit) in zip(axes.flat, _CHART_METRICS): - gv, hv = [], [] - for conc in mc_levels: - g = sum(getattr(c.grpc, field) for c in mc_conc[conc]) / len( - mc_conc[conc] - ) - h = sum(getattr(c.http, field) for c in mc_conc[conc]) / len( - mc_conc[conc] - ) - if unit == "ms": - g *= 1000 - h *= 1000 - gv.append(g) - hv.append(h) - - x = range(len(mc_levels)) - ax.plot(x, gv, "o-", label="gRPC", color="#2196F3", linewidth=2) - ax.plot(x, hv, "s--", label="HTTP", color="#FF9800", linewidth=2) - ax.set_title(title, fontsize=12, fontweight="bold") - ax.set_xlabel("Concurrency") - ax.set_ylabel(f"{title} ({unit})") - ax.set_xticks(list(x)) - ax.set_xticklabels([str(c) for c in mc_levels]) - ax.legend() - ax.grid(True, alpha=0.3) - nz = [v for v in gv + hv if v > 0] - if nz and max(nz) > 10 * min(nz): - ax.set_yscale("log") - - fig.tight_layout() safe = model.replace("/", "__") safe_cfg = config.replace("/", "_") - fname = f"{safe}_{safe_cfg}_comparison.png" - fig.savefig(output_dir / fname, dpi=150, bbox_inches="tight") - plt.close(fig) - charts.append(fname) + fname = _plot_comparison_grid( + f"{display_name} ({config}): gRPC vs HTTP", + mc_conc, + mc_levels, + output_dir, + f"{safe}_{safe_cfg}_comparison.png", + ) + if fname: + charts.append(fname) return charts @@ -482,17 +570,8 @@ def _section_key_findings( lines = ["### Key Findings", ""] # 1. Overall verdict per key metric - for label, field, lower_better, _unit in _CHART_METRICS: - advs = [ - a - for cp in comparisons - if ( - a := _advantage( - getattr(cp.grpc, field), getattr(cp.http, field), lower_better - ) - ) - is not None - ] + for label, fld, lower_better, _unit in CHART_METRICS: + advs = [a for cp in comparisons if (a := _cp_advantage(cp, fld, lower_better)) is not None] if not advs: continue avg_adv = sum(advs) / len(advs) @@ -500,7 +579,7 @@ def _section_key_findings( http_wins = sum(1 for a in advs if a < -2) ties = len(advs) - grpc_wins - http_wins if abs(avg_adv) < 1: - verdict = f"**{label}**: No clear winner — essentially tied across all scenarios" + lines.append(f"- **{label}**: No clear winner — essentially tied across all scenarios") else: winner = "gRPC" if avg_adv > 0 else "HTTP" lines.append( @@ -508,21 +587,12 @@ def _section_key_findings( f"comparisons (avg {abs(avg_adv):.1f}% better), " f"{min(grpc_wins, http_wins)} losses, {ties} ties" ) - continue - lines.append(f"- {verdict}") # 2. Error rates total_runs = sum(len(e.runs) for e in experiments) - error_runs = [ - (e, r) - for e in experiments - for r in e.runs - if r.error_rate > 0 - ] + error_runs = [(e, r) for e in experiments for r in e.runs if r.error_rate > 0] if error_runs: - lines.append( - f"- **Errors**: {len(error_runs)}/{total_runs} runs had non-zero error rates" - ) + lines.append(f"- **Errors**: {len(error_runs)}/{total_runs} runs had non-zero error rates") else: lines.append(f"- **Errors**: All {total_runs} runs completed with 0% error rate") @@ -530,16 +600,14 @@ def _section_key_findings( biggest_grpc_win = biggest_http_win = None biggest_grpc_adv = biggest_http_adv = 0.0 for cp in comparisons: - for _label, field, lower_better, _ in _CHART_METRICS: - adv = _advantage( - getattr(cp.grpc, field), getattr(cp.http, field), lower_better - ) + for label, fld, lower_better, _ in CHART_METRICS: + adv = _cp_advantage(cp, fld, lower_better) if adv is not None and adv > biggest_grpc_adv: biggest_grpc_adv = adv - biggest_grpc_win = (cp, _label) + biggest_grpc_win = (cp, label) if adv is not None and adv < biggest_http_adv: biggest_http_adv = adv - biggest_http_win = (cp, _label) + biggest_http_win = (cp, label) if biggest_grpc_win and biggest_grpc_adv > 10: cp, metric = biggest_grpc_win @@ -560,12 +628,7 @@ def _section_key_findings( def _section_error_rates(experiments: list[ExperimentInfo]) -> list[str]: """Surface any non-zero error rates.""" - errors = [] - for e in experiments: - for r in e.runs: - if r.error_rate > 0: - errors.append((e, r)) - + errors = [(e, r) for e in experiments for r in e.runs if r.error_rate > 0] if not errors: return [] @@ -577,9 +640,7 @@ def _section_error_rates(experiments: list[ExperimentInfo]) -> list[str]: "|-------|----------|---------|---------|----------|--:|-----------:|", ] - for e, r in sorted( - errors, key=lambda x: x[1].error_rate, reverse=True - ): + for e, r in sorted(errors, key=lambda x: x[1].error_rate, reverse=True): lines.append( f"| {e.model} | {e.protocol} | {e.runtime} | {e.worker_type} " f"| `{r.scenario}` | {r.concurrency} | {r.error_rate:.1%} |" @@ -589,9 +650,7 @@ def _section_error_rates(experiments: list[ExperimentInfo]) -> list[str]: return lines -def _section_overview( - experiments: list[ExperimentInfo], models_with_data: set[str] -) -> list[str]: +def _section_overview(experiments: list[ExperimentInfo], models_with_data: set[str]) -> list[str]: """Overview table — only models that have comparison data.""" by_model: dict[str, dict[str, ExperimentInfo]] = defaultdict(dict) for exp in experiments: @@ -640,9 +699,7 @@ def _col_sort_key(key: str) -> tuple: row.append("\u2796") else: exp = model_exps[table_key] - has_errors = any( - r.rps == 0 or r.output_throughput == 0 for r in exp.runs - ) + has_errors = any(r.rps == 0 or r.output_throughput == 0 for r in exp.runs) row.append("\u26a0\ufe0f" if has_errors else "\u2705") lines.append("| " + " | ".join(row) + " |") @@ -651,9 +708,7 @@ def _col_sort_key(key: str) -> tuple: if excluded: names = ", ".join(sorted(excluded)) lines.append("") - lines.append( - f"*Excluded from comparison (no matched gRPC/HTTP data): {names}*" - ) + lines.append(f"*Excluded from comparison (no matched gRPC/HTTP data): {names}*") lines.append("") return lines @@ -664,19 +719,6 @@ def _section_aggregate(comparisons: list[ComparisonPoint]) -> list[str]: if not comparisons: return ["*No gRPC vs HTTP comparison data available.*", ""] - # (label, field, lower_is_better) - metrics = [ - ("TTFT mean", "ttft_mean", True), - ("TTFT p99", "ttft_p99", True), - ("E2E mean", "e2e_mean", True), - ("E2E p99", "e2e_p99", True), - ("TPOT mean", "tpot_mean", True), - ("TPOT p99", "tpot_p99", True), - ("Output throughput", "output_throughput", False), - ("Total throughput", "total_throughput", False), - ("RPS", "rps", False), - ] - lines = [ "### Aggregate: gRPC vs HTTP", "", @@ -687,21 +729,14 @@ def _section_aggregate(comparisons: list[ComparisonPoint]) -> list[str]: "|--------|----:|-------:|---------|", ] - for label, fld, lower_better in metrics: - advs = [] - for cp in comparisons: - a = _advantage( - getattr(cp.grpc, fld), getattr(cp.http, fld), lower_better - ) - if a is not None: - advs.append(a) + for label, fld, lower_better in AGGREGATE_METRICS: + advs = [a for cp in comparisons if (a := _cp_advantage(cp, fld, lower_better)) is not None] if not advs: continue avg = sum(advs) / len(advs) med = median(advs) lines.append( - f"| {label} | {_fmt_winner(avg)} | {_fmt_winner(med)} | " - f"{_fmt_winner_bold(avg)} |" + f"| {label} | {_fmt_winner(avg)} | {_fmt_winner(med)} | {_fmt_winner(avg, bold=True)} |" ) lines.append("") @@ -713,16 +748,10 @@ def _section_by_concurrency(comparisons: list[ComparisonPoint]) -> list[str]: if not comparisons: return [] - by_conc: dict[int, list[ComparisonPoint]] = defaultdict(list) - for cp in comparisons: - by_conc[cp.concurrency].append(cp) - conc_levels = sorted(by_conc.keys()) - + by_conc, conc_levels = _group_by_concurrency(comparisons) lines = ["### Performance by Concurrency", ""] - def _conc_table( - title: str, field: str, lower_is_better: bool, fmt_val - ) -> list[str]: + for title, fld, lower_is_better in CONCURRENCY_METRICS: tbl = [ "
", f"{title}", @@ -730,36 +759,19 @@ def _conc_table( "| Concurrency | gRPC | HTTP | Faster |", "|---:|---:|---:|:---|", ] + for conc in conc_levels: cps = by_conc[conc] - g_avg = sum(getattr(cp.grpc, field) for cp in cps) / len(cps) - h_avg = sum(getattr(cp.http, field) for cp in cps) / len(cps) + g_avg = sum(getattr(cp.grpc, fld) for cp in cps) / len(cps) + h_avg = sum(getattr(cp.http, fld) for cp in cps) / len(cps) adv = _advantage(g_avg, h_avg, lower_is_better) tbl.append( - f"| {conc} | {fmt_val(g_avg)} | {fmt_val(h_avg)} | " - f"{_fmt_winner_bold(adv)} |" + f"| {conc} | {_fmt_metric_value(g_avg, title)} " + f"| {_fmt_metric_value(h_avg, title)} " + f"| {_fmt_winner(adv, bold=True)} |" ) tbl.extend(["", "
", ""]) - return tbl - - def _fmt_ms(v: float) -> str: - ms = v * 1000 - return f"{ms:.0f}ms" if ms < 1000 else f"{v:.2f}s" - - def _fmt_tput(v: float) -> str: - return f"{_fmt_throughput(v)} tok/s" - - def _fmt_tpot(v: float) -> str: - ms = v * 1000 - return f"{ms:.1f}ms" if ms >= 1 else f"{ms:.2f}ms" - - lines.extend(_conc_table("TTFT Mean", "ttft_mean", True, _fmt_ms)) - lines.extend(_conc_table("TTFT p99", "ttft_p99", True, _fmt_ms)) - lines.extend(_conc_table("TPOT Mean", "tpot_mean", True, _fmt_tpot)) - lines.extend(_conc_table("TPOT p99", "tpot_p99", True, _fmt_tpot)) - lines.extend(_conc_table("E2E Latency Mean", "e2e_mean", True, _fmt_ms)) - lines.extend(_conc_table("E2E Latency p99", "e2e_p99", True, _fmt_ms)) - lines.extend(_conc_table("Output Throughput", "output_throughput", False, _fmt_tput)) + lines.extend(tbl) return lines @@ -776,14 +788,9 @@ def _section_scorecard(comparisons: list[ComparisonPoint]) -> list[str]: "", ] - metrics = [ - ("E2E mean", "e2e_mean", True), - ("TTFT mean", "ttft_mean", True), - ("Output throughput", "output_throughput", False), - ] thresholds = [1, 2, 5, 10] - for label, fld, lower_better in metrics: + for label, fld, lower_better in SCORECARD_METRICS: lines.extend( [ f"**{label}:**", @@ -796,9 +803,7 @@ def _section_scorecard(comparisons: list[ComparisonPoint]) -> list[str]: for thresh in thresholds: grpc_w = http_w = within = 0 for cp in comparisons: - adv = _advantage( - getattr(cp.grpc, fld), getattr(cp.http, fld), lower_better - ) + adv = _cp_advantage(cp, fld, lower_better) if adv is None: continue if adv > thresh: @@ -817,26 +822,14 @@ def _section_scorecard(comparisons: list[ComparisonPoint]) -> list[str]: return lines -def _section_top_wins( - comparisons: list[ComparisonPoint], threshold: float = 30.0 -) -> list[str]: +def _section_top_wins(comparisons: list[ComparisonPoint], threshold: float = 30.0) -> list[str]: """Table of all gRPC wins exceeding the threshold.""" if not comparisons: return [] - metric_defs = [ - ("TTFT p99", "ttft_p99", True), - ("TTFT mean", "ttft_mean", True), - ("E2E mean", "e2e_mean", True), - ("E2E p99", "e2e_p99", True), - ("TPOT p99", "tpot_p99", True), - ("Output tput", "output_throughput", False), - ("RPS", "rps", False), - ] - wins: list[tuple[float, str, ComparisonPoint, float, float]] = [] for cp in comparisons: - for label, fld, lower_better in metric_defs: + for label, fld, lower_better in TOP_WINS_METRICS: g = getattr(cp.grpc, fld) h = getattr(cp.http, fld) adv = _advantage(g, h, lower_better) @@ -857,19 +850,10 @@ def _section_top_wins( ] for adv, label, cp, g, h in wins: - config = f"{cp.runtime}/{cp.worker_type}" - is_latency = any(k in label.lower() for k in ("ttft", "e2e", "tpot")) - if is_latency: - g_str = _fmt_latency_s(g) - h_str = _fmt_latency_s(h) - elif "tput" in label.lower(): - g_str = f"{_fmt_throughput(g)} tok/s" - h_str = f"{_fmt_throughput(h)} tok/s" - else: - g_str = f"{g:.1f}" - h_str = f"{h:.1f}" + g_str = _fmt_metric_value(g, label) + h_str = _fmt_metric_value(h, label) lines.append( - f"| {adv:.1f}% | {cp.model} | {config} | " + f"| {adv:.1f}% | {cp.model} | {cp.config} | " f"`{cp.scenario}` | {cp.concurrency} | {label} | {g_str} | {h_str} |" ) @@ -878,11 +862,7 @@ def _section_top_wins( def _section_per_model(comparisons: list[ComparisonPoint]) -> list[str]: - """Per-model summary table plus collapsible detail tables. - - All values show gRPC advantage: 'gRPC X%' = gRPC is X% better, - 'HTTP X%' = HTTP is X% better, '~' = within 2%. - """ + """Per-model summary table plus collapsible detail tables.""" if not comparisons: return [] @@ -890,37 +870,20 @@ def _section_per_model(comparisons: list[ComparisonPoint]) -> list[str]: for cp in comparisons: by_model[cp.model].append(cp) + headers = [m[0] for m in PER_MODEL_METRICS] lines = [ "### Per-Model Summary", "", "*Each cell shows which protocol is better and by how much.*", "", - "| Model | TTFT p99 | TPOT p99 | E2E p99 | Output tput | N |", - "|-------|--------:|--------:|--------:|--------:|---:|", + "| Model | " + " | ".join(headers) + " | N |", + "|-------" + "|--------:" * len(headers) + "|---:|", ] for model in sorted(by_model.keys()): cps = by_model[model] - - def _model_avg(field: str, lower_better: bool) -> str: - advs = [ - a - for cp in cps - if (a := _advantage(getattr(cp.grpc, field), getattr(cp.http, field), lower_better)) - is not None - ] - if not advs: - return "N/A" - return _fmt_winner(sum(advs) / len(advs)) - - lines.append( - f"| {model} " - f"| {_model_avg('ttft_p99', True)} " - f"| {_model_avg('tpot_p99', True)} " - f"| {_model_avg('e2e_p99', True)} " - f"| {_model_avg('output_throughput', False)} " - f"| {len(cps)} |" - ) + cells = [_fmt_winner(_avg_advantage(cps, fld, lb)) for _, fld, lb in PER_MODEL_METRICS] + lines.append(f"| {model} | " + " | ".join(cells) + f" | {len(cps)} |") lines.append("") @@ -929,31 +892,25 @@ def _model_avg(field: str, lower_better: bool) -> str: cps = by_model[model] by_config: dict[str, list[ComparisonPoint]] = defaultdict(list) for cp in cps: - by_config[f"{cp.runtime}/{cp.worker_type}"].append(cp) + by_config[cp.config].append(cp) for config in sorted(by_config.keys()): - config_cps = sorted( - by_config[config], key=lambda x: (x.scenario, x.concurrency) - ) + config_cps = sorted(by_config[config], key=lambda x: (x.scenario, x.concurrency)) lines.extend( [ "
", - f"{model} ({config}) — " - f"{len(config_cps)} points", + f"{model} ({config}) — {len(config_cps)} points", "", - "| Scenario | C | TTFT p99 | TPOT p99 | E2E p99 | Tput |", - "|----------|--:|--------:|--------:|--------:|--------:|", + "| Scenario | C | " + " | ".join(headers) + " |", + "|----------|--:" + "|--------:" * len(headers) + "|", ] ) for cp in config_cps: - lines.append( - f"| `{cp.scenario}` | {cp.concurrency} " - f"| {_fmt_winner(_advantage(cp.grpc.ttft_p99, cp.http.ttft_p99, True))} " - f"| {_fmt_winner(_advantage(cp.grpc.tpot_p99, cp.http.tpot_p99, True))} " - f"| {_fmt_winner(_advantage(cp.grpc.e2e_p99, cp.http.e2e_p99, True))} " - f"| {_fmt_winner(_advantage(cp.grpc.output_throughput, cp.http.output_throughput, False))} |" - ) + cells = [ + _fmt_winner(_cp_advantage(cp, fld, lb)) for _, fld, lb in PER_MODEL_METRICS + ] + lines.append(f"| `{cp.scenario}` | {cp.concurrency} | " + " | ".join(cells) + " |") lines.extend(["", "
", ""]) @@ -965,12 +922,16 @@ def _model_avg(field: str, lower_better: bool) -> str: # --------------------------------------------------------------------------- -def generate_summary(base_dir: Path) -> str: +def generate_summary(base_dir: Path) -> SummaryResult: """Generate the full markdown summary.""" experiments = discover_experiments(base_dir) if not experiments: - return "## Nightly Benchmark Summary\n\nNo benchmark results found." + return SummaryResult( + markdown="## Nightly Benchmark Summary\n\nNo benchmark results found.", + experiments=[], + comparisons=[], + ) comparisons = build_comparisons(experiments) @@ -991,36 +952,7 @@ def generate_summary(base_dir: Path) -> str: "
", "Glossary", "", - "#### Metrics", - "", - "| Metric | Description |", - "|--------|-------------|", - "| **TTFT** | Time To First Token — latency from request sent to first token received |", - "| **TPOT** | Time Per Output Token — average time between consecutive output tokens |", - "| **E2E** | End-to-End latency — total time from request sent to last token received |", - "| **Output tput** | Output throughput — tokens generated per second (tok/s) |", - "| **Total tput** | Total throughput — input + output tokens processed per second |", - "| **RPS** | Requests Per Second — completed requests per second |", - "| **p99** | 99th percentile — the value below which 99% of observations fall (worst 1% of requests) |", - "| **mean** | Arithmetic average across all requests |", - "", - "#### Traffic Scenarios", - "", - "| Pattern | Description |", - "|---------|-------------|", - "| **D(in, out)** | Deterministic — fixed input/output token lengths, e.g. `D(100,100)` = 100 input, 100 output tokens |", - "| **N(μ,σ)/(μ,σ)** | Normal distribution — input and output lengths drawn from Gaussian, e.g. `N(480,240)/(300,150)` |", - "| **E(size)** | Embedding — input of given token length, used for embedding model benchmarks |", - "", - "#### Comparison Columns", - "", - "| Value | Meaning |", - "|-------|---------|", - "| **gRPC X%** | gRPC is X% better than HTTP for this metric |", - "| **HTTP X%** | HTTP is X% better than gRPC for this metric |", - "| **~** | Difference is within 2% — essentially a tie |", - "", - "*Lower is better for latency metrics (TTFT, TPOT, E2E). Higher is better for throughput (tput, RPS).*", + *_GLOSSARY_LINES, "", "
", "", @@ -1039,11 +971,14 @@ def generate_summary(base_dir: Path) -> str: lines.append("---") lines.append( - f"*Generated from {len(experiments)} experiment(s), " - f"{len(comparisons)} comparison points*" + f"*Generated from {len(experiments)} experiment(s), {len(comparisons)} comparison points*" ) - return "\n".join(lines), experiments, comparisons + return SummaryResult( + markdown="\n".join(lines), + experiments=experiments, + comparisons=comparisons, + ) def main() -> None: @@ -1066,11 +1001,11 @@ def main() -> None: else: i += 1 - summary, experiments, comparisons = generate_summary(base_dir) + result = generate_summary(base_dir) # Generate comparison charts if requested - if charts_dir and comparisons: - chart_files = generate_charts(comparisons, experiments, charts_dir) + if charts_dir and result.comparisons: + chart_files = generate_charts(result.comparisons, charts_dir) if chart_files: print( f"Generated {len(chart_files)} chart(s) in {charts_dir}", @@ -1080,11 +1015,11 @@ def main() -> None: summary_file = os.environ.get("GITHUB_STEP_SUMMARY") if summary_file: with open(summary_file, "a") as f: - f.write(summary) + f.write(result.markdown) f.write("\n") print(f"Summary written to {summary_file}") else: - print(summary) + print(result.markdown) if __name__ == "__main__":