[codex] Add parse benchmark watchdog

.github/workflows/parse-bench.yml

@@ -0,0 +1,109 @@
+name: Parse Benchmark
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  parse-bench:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+
+    steps:
+      - name: Checkout head
+        # actions/checkout v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          path: head
+          persist-credentials: false
+
+      - name: Checkout base
+        if: github.event_name == 'pull_request'
+        # actions/checkout v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          persist-credentials: false
+
+      - name: Set up Java
+        # actions/setup-java v5
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654
+        with:
+          distribution: temurin
+          java-version: '21'
+
+      - name: Set up Go
+        # actions/setup-go v5
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff
+        with:
+          go-version: '1.23.x'
+
+      - name: Set up Python
+        # actions/setup-python v6
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: '3.11'
+
+      - name: Set up Rust
+        run: |
+          rustup toolchain install 1.95.0 --profile minimal
+          rustup default 1.95.0
+
+      - name: Install Python benchmark dependencies
+        run: python -m pip install -r head/tools/parse-bench/requirements.txt
+
+      - name: Fetch ANTLR benchmark inputs
+        run: |
+          mkdir -p /tmp/antlr-cleanroom/tools
+          curl -fLo /tmp/antlr-cleanroom/tools/antlr-4.13.2-complete.jar \
+            https://www.antlr.org/download/antlr-4.13.2-complete.jar
+          mkdir -p /tmp/antlr-cleanroom/grammars-v4
+          git -C /tmp/antlr-cleanroom/grammars-v4 init -q
+          git -C /tmp/antlr-cleanroom/grammars-v4 remote add origin https://github.com/antlr/grammars-v4.git
+          git -C /tmp/antlr-cleanroom/grammars-v4 sparse-checkout init --cone
+          git -C /tmp/antlr-cleanroom/grammars-v4 sparse-checkout set kotlin/kotlin csharp/v7
+          git -C /tmp/antlr-cleanroom/grammars-v4 fetch --depth 1 origin 284602b3f23ca54dc30778204ab7ae9e969145e9
+          git -C /tmp/antlr-cleanroom/grammars-v4 checkout FETCH_HEAD
+
+      - name: Run base benchmark
+        if: github.event_name == 'pull_request'
+        run: |
+          if [ -f base/tools/parse-bench/run.py ]; then
+            python base/tools/parse-bench/run.py \
+              --quick \
+              --work-dir base/target/parse-bench \
+              --json "$PWD/base-parse-bench.json"
+          else
+            echo "base branch does not have parse benchmark script; skipping base comparison"
+          fi
+
+      - name: Run head benchmark
+        run: |
+          python head/tools/parse-bench/run.py \
+            --quick \
+            --work-dir head/target/parse-bench \
+            --json "$PWD/head-parse-bench.json"
+
+      - name: Compare against base
+        if: github.event_name == 'pull_request'
+        run: |
+          if [ -f base-parse-bench.json ]; then
+            python head/tools/parse-bench/compare.py \
+              --baseline base-parse-bench.json \
+              --current head-parse-bench.json \
+              --max-regression 1.15
+          else
+            echo "no base benchmark report; comparison will start once this workflow is on the base branch"
+          fi
+
+      - name: Upload benchmark reports
+        if: always()
+        # actions/upload-artifact v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: parse-bench-results
+          path: '*-parse-bench.json'

.gitignore

@@ -1,3 +1,4 @@
 /target/
 /.serena/
 *.log
+__pycache__/

tools/parse-bench/README.md

@@ -0,0 +1,82 @@
+# Parse Benchmark
+
+This benchmark compares parse throughput for generated ANTLR parsers and
+tree-sitter parsers on Kotlin and C# fixtures.
+
+The harness is intentionally a standalone script instead of `cargo bench`.
+`cargo bench` is useful for in-process Rust-only measurements, but this check
+has to generate ANTLR parsers, build a Go binary, run Python parsers, and load
+tree-sitter language libraries. Keeping that orchestration outside Cargo makes
+the same command usable locally and in CI.
+
+## Setup
+
+Use the same ANTLR jar and `grammars-v4` checkout described in `AGENTS.md`.
+The benchmark defaults to:
+
+- `ANTLR4_JAR=/tmp/antlr-cleanroom/tools/antlr-4.13.2-complete.jar`
+- `GRAMMARS_V4=/tmp/antlr-cleanroom/grammars-v4`
+
+For C#, the sparse checkout must include `csharp/v7` in addition to Kotlin:
+
+```bash
+git -C /tmp/antlr-cleanroom/grammars-v4 sparse-checkout set kotlin/kotlin csharp/v7
+```
+
+Install the Python dependencies in the interpreter you will use to run the
+benchmark:
+
+```bash
+python3 -m pip install -r tools/parse-bench/requirements.txt
+```
+
+## Run
+
+Quick local smoke:
+
+```bash
+python3 tools/parse-bench/run.py --quick
+```
+
+Longer local run with reports:
+
+```bash
+python3 tools/parse-bench/run.py \
+  --iters 20 \
+  --warmups 3 \
+  --json target/parse-bench/results.json \
+  --markdown target/parse-bench/results.md
+```
+
+The script regenerates parsers into `target/parse-bench`, builds:
+
+- a Rust runner using this runtime and generated `.interp` metadata,
+- a Python ANTLR runner using `antlr4-python3-runtime`,
+- a Go ANTLR runner using `github.com/antlr4-go/antlr/v4`,
+- a tree-sitter runner using `tree-sitter-language-pack`.
+
+The output table reports `min` and `avg` parse time per fixture and a relative
+ratio against `rust-antlr` for the same fixture.
+
+## PR Watchdog
+
+For CI, run the benchmark on the base checkout and the PR checkout on the same
+runner, then compare JSON reports:
+
+```bash
+python3 tools/parse-bench/compare.py \
+  --baseline base-parse-bench.json \
+  --current head-parse-bench.json \
+  --max-regression 1.15
+```
+
+By default the comparator checks `rust-antlr` only. Repeat `--runtime` to add
+other runtimes.
+
+## Fixtures
+
+Fixture metadata lives in `fixtures/manifest.json`. The fixture files are small
+benchmark excerpts that point at independent upstream source patterns:
+
+- Kotlin: JetBrains Kotlin, kotlinx.coroutines, Ktor, Compose Multiplatform.
+- C#: dotnet/runtime, Roslyn, Newtonsoft.Json, dotnet/samples.

tools/parse-bench/compare.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Compare two parse-benchmark JSON reports and fail on Rust regressions."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def result_key(result: dict) -> tuple[str, str, str]:
+    return (
+        str(result["language"]),
+        str(result["fixture"]),
+        str(result["runtime"]),
+    )
+
+
+def load_results(path: Path) -> dict[tuple[str, str, str], dict]:
+    data = json.loads(path.read_text())
+    indexed: dict[tuple[str, str, str], dict] = {}
+    for result in data["results"]:
+        key = result_key(result)
+        if key in indexed:
+            raise ValueError(f"duplicate benchmark result key in {path}: {key}")
+        indexed[key] = result
+    return indexed
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--baseline", required=True, type=Path)
+    parser.add_argument("--current", required=True, type=Path)
+    parser.add_argument("--max-regression", type=float, default=1.15)
+    parser.add_argument(
+        "--runtime",
+        action="append",
+        default=None,
+        help="Runtime to compare; repeat for multiple runtimes.",
+    )
+    args = parser.parse_args()
+
+    baseline = load_results(args.baseline)
+    current = load_results(args.current)
+    runtimes = set(args.runtime or ["rust-antlr"])
+
+    failures: list[str] = []
+    for key, head in sorted(current.items()):
+        language, fixture, runtime = key
+        if runtime not in runtimes or key not in baseline:
+            continue
+        base_avg = float(baseline[key]["avg_ns"])
+        head_avg = float(head["avg_ns"])
+        if base_avg <= 0:
+            continue
+        ratio = head_avg / base_avg
+        if ratio > args.max_regression:
+            failures.append(
+                f"{language}/{fixture} {runtime}: "
+                f"{head_avg / 1_000_000:.3f}ms vs "
+                f"{base_avg / 1_000_000:.3f}ms ({ratio:.2f}x)"
+            )
+
+    compared = sum(
+        1
+        for key in current
+        if key in baseline and key[2] in runtimes
+    )
+    if compared == 0:
+        print(
+            "parse benchmark compare found no matching baseline/current "
+            f"result pairs for runtime(s): {', '.join(sorted(runtimes))}",
+            file=sys.stderr,
+        )
+        return 1
+
+    if failures:
+        print(
+            f"parse benchmark regression exceeds {args.max_regression:.2f}x:",
+            file=sys.stderr,
+        )
+        for failure in failures:
+            print(f"  {failure}", file=sys.stderr)
+        return 1
+
+    print(
+        f"parse benchmark compare passed: {compared} result(s), "
+        f"threshold {args.max_regression:.2f}x"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/parse-bench/fixtures/csharp/dotnet-runtime-boolean.cs

@@ -0,0 +1,39 @@
+// Reference pattern: dotnet/runtime System.Boolean implementation.
+// Source: https://github.com/dotnet/runtime/blob/main/src/libraries/System.Private.CoreLib/src/System/Boolean.cs
+// Upstream license: MIT. This fixture is a compact benchmark excerpt.
+
+using System;
+using System.Runtime.CompilerServices;
+
+namespace System
+{
+    [Serializable]
+    public struct Boolean : IComparable, IComparable<bool>, IEquatable<bool>
+    {
+        private readonly bool m_value;
+
+        public int CompareTo(object obj)
+        {
+            if (obj == null) return 1;
+            if (!(obj is bool)) throw new ArgumentException("Object must be boolean");
+            return CompareTo((bool)obj);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int CompareTo(bool value)
+        {
+            if (m_value == value) return 0;
+            return m_value ? 1 : -1;
+        }
+
+        public bool Equals(bool obj)
+        {
+            return m_value == obj;
+        }
+
+        public override string ToString()
+        {
+            return m_value ? "True" : "False";
+        }
+    }
+}

tools/parse-bench/fixtures/csharp/dotnet-samples-teleprompter.cs

@@ -0,0 +1,44 @@
+// Reference pattern: dotnet/samples console teleprompter.
+// Source: https://github.com/dotnet/samples/blob/main/csharp/getting-started/console-teleprompter/Program.cs
+// Upstream license: MIT. This fixture is a compact benchmark excerpt.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace TeleprompterConsole
+{
+    public class Program
+    {
+        public static async Task Main(string[] args)
+        {
+            await RunTeleprompter();
+        }
+
+        private static async Task RunTeleprompter()
+        {
+            var config = new TelePrompterConfig();
+            var displayTask = ShowTeleprompter(config);
+            var speedTask = GetInput(config);
+            await Task.WhenAny(displayTask, speedTask);
+        }
+
+        private static async Task ShowTeleprompter(TelePrompterConfig config)
+        {
+            var words = ReadFrom("sampleQuotes.txt");
+            foreach (var word in words)
+            {
+                Console.Write(word);
+                if (!string.IsNullOrWhiteSpace(word))
+                {
+                    await Task.Delay(config.DelayInMilliseconds);
+                }
+            }
+        }
+
+        private static IEnumerable<string> ReadFrom(string file)
+        {
+            return new[] { "hello", "from", file };
+        }
+    }
+}