Skip to content
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions .github/workflows/nightly-benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ permissions:
env:
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
GENAI_BENCH_IMAGE: ghcr.io/moirai-internal/genai-bench:0.0.3

jobs:
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -88,9 +89,10 @@ jobs:
- { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle }
- { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle }
- { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle }
- { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle }
variant:
- { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" }
- { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" }
- { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false }
- { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false }

steps:
- name: Check filters
Expand Down Expand Up @@ -135,15 +137,21 @@ jobs:
run: |
pip uninstall -y smg || true
pip install wheel/*.whl
bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }}
bash scripts/ci_install_e2e_deps.sh

- name: Pull genai-bench image
if: steps.filter.outputs.skip != 'true'
run: docker pull ${{ env.GENAI_BENCH_IMAGE }}

- name: Run benchmark
if: steps.filter.outputs.skip != 'true'
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
GPU_TYPE: H100
E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen3-30B-A3B":1}'
E2E_LOG_DIR: nightly_gateway_logs
run: |
mkdir -p nightly_gateway_logs
bash scripts/ci_killall_sglang.sh "nuk_gpus"

K_FILTER="${{ matrix.model.test_class }}"
Expand Down Expand Up @@ -187,6 +195,7 @@ jobs:
- { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bMulti }
- { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bMulti }
- { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bMulti }
- { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bMulti }
variant:
- { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" }
- { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" }
Expand Down Expand Up @@ -235,6 +244,10 @@ jobs:
pip uninstall -y smg || true
pip install wheel/*.whl
bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }}

- name: Pull genai-bench image
if: steps.filter.outputs.skip != 'true'
run: docker pull ${{ env.GENAI_BENCH_IMAGE }}

- name: Run benchmark
if: steps.filter.outputs.skip != 'true'
Expand Down Expand Up @@ -340,6 +353,10 @@ jobs:
pip install wheel/*.whl
bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }}

- name: Pull genai-bench image
if: steps.filter.outputs.skip != 'true'
run: docker pull ${{ env.GENAI_BENCH_IMAGE }}

- name: Run benchmark
if: steps.filter.outputs.skip != 'true'
env:
Expand Down
11 changes: 9 additions & 2 deletions .github/workflows/pr-test-rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ concurrency:
env:
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
GENAI_BENCH_IMAGE: ghcr.io/moirai-internal/genai-bench:0.0.3

jobs:
check-ci:
Expand Down Expand Up @@ -246,7 +247,7 @@ jobs:
- name: benchmarks
timeout: 20
test_dirs: "e2e_test/benchmarks"
extra_deps: "genai-bench==0.0.3"
extra_deps: ""
env_vars: ""
reruns: ""
upload_benchmarks: true
Expand Down Expand Up @@ -367,6 +368,10 @@ jobs:
pip install wheel/*.whl
bash scripts/ci_install_e2e_deps.sh ${{ matrix.extra_deps }}

- name: Pull genai-bench image
if: matrix.upload_benchmarks
run: docker pull ${{ env.GENAI_BENCH_IMAGE }}

- name: Setup Oracle Instant Client
if: matrix.setup_oracle
run: |
Expand Down Expand Up @@ -560,7 +565,9 @@ jobs:
- name: Install test dependencies
run: |
python3 -m pip install pytest pytest-rerunfailures httpx openai grpcio grpcio-health-checking numpy
python3 -m pip --no-cache-dir install --upgrade genai-bench==0.0.3

- name: Pull genai-bench image
run: docker pull ${{ env.GENAI_BENCH_IMAGE }}

- name: Run Go bindings benchmark
run: |
Expand Down
111 changes: 57 additions & 54 deletions e2e_test/benchmarks/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@

logger = logging.getLogger(__name__)

_DEFAULT_IMAGE = "ghcr.io/moirai-internal/genai-bench:0.0.3"
Comment thread
key4ng marked this conversation as resolved.
Resolved


def _build_command(
cli: str,
router_url: str,
model_path: str,
experiment_folder: str,
Expand All @@ -31,31 +32,53 @@ def _build_command(
gpu_type: str | None = None,
gpu_count: int | None = None,
) -> list[str]:
"""Build genai-bench command."""
"""Build genai-bench command via docker run."""
image = os.environ.get("GENAI_BENCH_IMAGE", _DEFAULT_IMAGE)
base_dir = str(Path.cwd())

cmd = [
cli,
"benchmark",
"--api-backend",
"openai",
"--api-base",
router_url,
"--api-key",
"dummy-token",
"--api-model-name",
model_path,
"--model-tokenizer",
model_path,
"--task",
task,
"--max-requests-per-run",
str(max_requests),
"--max-time-per-run",
str(max_time_per_run),
"--experiment-folder-name",
experiment_folder,
"--experiment-base-dir",
str(Path.cwd()),
"docker",
"run",
"--rm",
"--network",
"host",
"-v",
f"{base_dir}:{base_dir}",
"-w",
base_dir,
]

# Pass through environment variables the container may need
for var in ("HF_TOKEN", "HF_HOME"):
if os.environ.get(var):
cmd.extend(["-e", var])

cmd.extend(
[
image,
"benchmark",
"--api-backend",
"openai",
"--api-base",
router_url,
"--api-key",
"dummy-token",
"--api-model-name",
model_path,
"--model-tokenizer",
model_path,
"--task",
task,
"--max-requests-per-run",
str(max_requests),
"--max-time-per-run",
str(max_time_per_run),
"--experiment-folder-name",
experiment_folder,
"--experiment-base-dir",
base_dir,
]
)
if num_concurrency is not None:
cmd.extend(["--num-concurrency", str(num_concurrency)])
if traffic_scenario is not None:
Expand All @@ -66,6 +89,9 @@ def _build_command(
cmd.extend(["--server-gpu-type", gpu_type])
if gpu_count:
cmd.extend(["--server-gpu-count", str(gpu_count)])
log_dir = os.environ.get("E2E_LOG_DIR")
if log_dir:
cmd.extend(["--log-dir", log_dir])
return cmd


Expand Down Expand Up @@ -147,20 +173,16 @@ def _run(
kill_procs: list | None = None,
drain_delay_sec: int = 6,
) -> None:
cli = shutil.which("genai-bench")
if not cli:
pytest.fail("genai-bench CLI not found")
assert cli is not None # for mypy (pytest.fail raises)

# Clean previous results
exp_dir = Path.cwd() / experiment_folder
if exp_dir.exists():
shutil.rmtree(exp_dir, ignore_errors=True)

# Build and run command
max_requests = max_requests_per_run or (num_concurrency or 32) * 5
timeout = timeout_sec or int(os.environ.get("GENAI_BENCH_TEST_TIMEOUT", "240"))

cmd = _build_command(
cli,
router_url,
model_path,
experiment_folder,
Expand All @@ -173,24 +195,18 @@ def _run(
gpu_type=gpu_type,
gpu_count=gpu_count,
)
timeout = timeout_sec or int(os.environ.get("GENAI_BENCH_TEST_TIMEOUT", "120"))

logger.info("Running genai-bench command: %s", " ".join(cmd))

try:
proc = subprocess.Popen(
cmd,
env=os.environ.copy(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
except FileNotFoundError:
pytest.fail(f"genai-bench executable not found at {cli}")
except PermissionError:
pytest.fail(f"Permission denied executing {cli}")
pytest.fail("docker not found — is Docker installed?")
except OSError as e:
pytest.fail(f"Failed to start genai-bench: {e}")
pytest.fail(f"Failed to start genai-bench container: {e}")

# Start GPU monitor if needed
gpu_monitor: GPUMonitor | None = None
Expand All @@ -200,23 +216,17 @@ def _run(
gpu_monitor.start(target_pid=proc.pid)

try:
stdout, stderr = proc.communicate(timeout=timeout)
proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
proc.kill()
stdout, stderr = proc.communicate()
proc.wait()
logger.error("genai-bench timed out after %ds", timeout)

# Fail immediately if genai-bench failed
if proc.returncode != 0:
logger.error(
"genai-bench exited with code %d\nstdout:\n%s\nstderr:\n%s",
proc.returncode,
stdout or "(empty)",
stderr or "(empty)",
)
pytest.fail(
f"genai-bench failed with exit code {proc.returncode}. "
f"Check logs above for details."
f"Check CI logs above for details."
)

try:
Expand All @@ -234,13 +244,6 @@ def _run(
gpu_monitor.assert_thresholds(thresholds)

except AssertionError:
# Log genai-bench output when results not found
logger.error(
"genai-bench output (returncode=%d):\nstdout:\n%s\nstderr:\n%s",
proc.returncode,
stdout or "(empty)",
stderr or "(empty)",
)
raise

finally:
Expand Down
1 change: 1 addition & 0 deletions e2e_test/benchmarks/test_nightly_perf.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ def _run_nightly(setup_backend, genai_bench_runner, model_id, worker_count=1, **
("meta-llama/Llama-3.1-8B-Instruct", "Llama8b", 4, ["http", "grpc"], {}),
("Qwen/Qwen2.5-7B-Instruct", "Qwen7b", 4, ["http", "grpc"], {}),
("Qwen/Qwen3-30B-A3B", "Qwen30b", 4, ["http", "grpc"], {}),
("openai/gpt-oss-20b", "GptOss20b", 1, ["http", "grpc"], {}),
(
"meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
"Llama4Maverick",
Expand Down
6 changes: 5 additions & 1 deletion e2e_test/fixtures/hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,11 @@ def calculate_test_gpus(model_id: str, prefill: int, decode: int, regular: int)
from _pytest.mark.expression import Expression

expr = Expression.compile(keyword)
items_to_scan = [item for item in items if expr.evaluate(lambda x: x in item.keywords)]
items_to_scan = [
item
for item in items
if expr.evaluate(lambda x, **_kw: x in item.keywords) # type: ignore[arg-type]
]
else:
items_to_scan = items

Expand Down
3 changes: 2 additions & 1 deletion e2e_test/fixtures/pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ def model_pool(request: pytest.FixtureRequest) -> ModelPool:

# Create and start the pool
allocator = GPUAllocator()
_model_pool = ModelPool(allocator)
log_dir = os.environ.get("E2E_LOG_DIR")
_model_pool = ModelPool(allocator, log_dir=log_dir)

startup_timeout = int(os.environ.get(ENV_STARTUP_TIMEOUT, "300"))
_model_pool.startup(
Expand Down
1 change: 1 addition & 0 deletions e2e_test/fixtures/setup_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,7 @@ def _setup_cloud_backend(

api_key = os.environ.get(api_key_env) if api_key_env else "not-used"

client: openai.OpenAI | anthropic.Anthropic
if cfg.get("client_type") == "anthropic":
client = anthropic.Anthropic(
base_url=gateway.base_url,
Expand Down
13 changes: 11 additions & 2 deletions e2e_test/infra/gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,11 +341,20 @@ def _launch(
logger.info("Starting %s on port %d", log_msg or "gateway", self.port)
logger.debug("Gateway command: %s", " ".join(cmd))

# Discard subprocess output when not showing it, so the pipe buffer
# never fills up and blocks the router process.
if show_output:
stdout_target = None
stderr_target = None
else:
stdout_target = subprocess.DEVNULL
stderr_target = subprocess.DEVNULL

self.process = subprocess.Popen(
cmd,
env=self._env, # Use custom env if set (e.g., for cloud mode API keys)
stdout=None if show_output else subprocess.PIPE,
stderr=None if show_output else subprocess.PIPE,
stdout=stdout_target,
stderr=stderr_target,
start_new_session=True,
)

Expand Down
Loading