feat(e2e): Add BFCL v3 function calling test infrastructure

.gitignore

@@ -160,3 +160,11 @@ clients/java/target/
 clients/java/build.sbt
 clients/java/gradle.properties
 openapitools.json
+
+# BFCL test data (downloaded from HuggingFace at runtime)
+e2e_test/bfcl/data/*.json
+!e2e_test/bfcl/data/.gitkeep
+
+# BFCL test logs (generated per run)
+e2e_test/bfcl_logs/*
+!e2e_test/bfcl_logs/.gitkeep

e2e_test/bfcl/__init__.py

@@ -0,0 +1,25 @@
+"""BFCL (Berkeley Function Calling Leaderboard) test infrastructure.
+
+Provides data loading, evaluation, and per-test logging for open-source
+BFCL v3 test cases run against the SMG gateway.
+"""
+
+from .converter import bfcl_to_openai_tools
+from .evaluator import BFCLEvaluator, extract_tool_calls, log_file_for_summary
+from .loader import (
+    BFCL_CATEGORIES,
+    BFCLCase,
+    MissingBFCLAnswerFileError,
+    load_bfcl_category,
+)
+
+__all__ = [
+    "BFCL_CATEGORIES",
+    "BFCLCase",
+    "BFCLEvaluator",
+    "MissingBFCLAnswerFileError",
+    "bfcl_to_openai_tools",
+    "extract_tool_calls",
+    "load_bfcl_category",
+    "log_file_for_summary",
+]

e2e_test/bfcl/converter.py

@@ -0,0 +1,47 @@
+"""BFCL-to-OpenAI format converter.
+
+Converts BFCL function definitions to OpenAI-compatible tool calling format,
+handling BFCL-specific schema quirks (non-standard JSON Schema types).
+"""
+
+from __future__ import annotations
+
+
+def _fix_parameter_type(params: dict) -> dict:
+    """Convert BFCL's non-standard types to valid JSON Schema types recursively."""
+    result = dict(params)
+    ptype = result.get("type")
+    if ptype == "dict":
+        result["type"] = "object"
+    elif ptype == "float":
+        result["type"] = "number"
+    elif ptype == "int":
+        result["type"] = "integer"
+    elif ptype in ("list", "tuple"):
+        result["type"] = "array"
+    props = result.get("properties")
+    if isinstance(props, dict):
+        result["properties"] = {
+            k: _fix_parameter_type(v) if isinstance(v, dict) else v for k, v in props.items()
+        }
+    items = result.get("items")
+    if isinstance(items, dict):
+        result["items"] = _fix_parameter_type(items)
+    return result
+
+
+def bfcl_to_openai_tools(bfcl_functions: list[dict]) -> list[dict]:
+    """Convert BFCL function definitions to OpenAI tools format.
+
+    Handles the BFCL-specific quirks:
+      - parameters.type "dict" → "object"
+      - parameters.type "float" → "number"
+      - Wraps in {"type": "function", "function": ...}
+    """
+    tools = []
+    for fn in bfcl_functions:
+        fixed_fn = dict(fn)
+        if "parameters" in fixed_fn:
+            fixed_fn["parameters"] = _fix_parameter_type(fixed_fn["parameters"])
+        tools.append({"type": "function", "function": fixed_fn})
+    return tools

e2e_test/bfcl/download_data.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Download BFCL v3 open-source test data from HuggingFace.
+
+Usage:
+    python e2e_test/bfcl/download_data.py
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import urllib.request
+from pathlib import Path
+
+HF_BASE = (
+    "https://huggingface.co/datasets/gorilla-llm/Berkeley-Function-Calling-Leaderboard/resolve/main"
+)
+
+DATA_DIR = Path(__file__).parent / "data"
+TIMEOUT_SECONDS = 60
+
+FILES = [
+    "BFCL_v3_simple.json",
+    "BFCL_v3_multiple.json",
+    "BFCL_v3_parallel.json",
+    "BFCL_v3_parallel_multiple.json",
+    "BFCL_v3_irrelevance.json",
+]
+
+ANSWER_FILES = [
+    ("possible_answer/BFCL_v3_simple.json", "BFCL_v3_simple_answer.json"),
+    ("possible_answer/BFCL_v3_multiple.json", "BFCL_v3_multiple_answer.json"),
+    ("possible_answer/BFCL_v3_parallel.json", "BFCL_v3_parallel_answer.json"),
+    (
+        "possible_answer/BFCL_v3_parallel_multiple.json",
+        "BFCL_v3_parallel_multiple_answer.json",
+    ),
+]
+
+
+def _count_lines(path: Path) -> int:
+    with path.open(encoding="utf-8") as handle:
+        return sum(1 for _ in handle)
+
+
+def _download_one(remote_path: str, local_name: str, *, force: bool = False) -> None:
+    url = f"{HF_BASE}/{remote_path}"
+    dest = DATA_DIR / local_name
+    if dest.exists() and not force:
+        print(f"Skipping {local_name} (already downloaded)")
+        print(f"  -> {dest.name} ({_count_lines(dest)} entries)")
+        return
+
+    print(f"Downloading {local_name}...")
+    tmp = dest.with_suffix(dest.suffix + ".tmp")
+    try:
+        with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as resp, tmp.open("wb") as out:
+            shutil.copyfileobj(resp, out)
+            out.flush()
+            os.fsync(out.fileno())
+        tmp.replace(dest)
+    finally:
+        if tmp.exists():
+            tmp.unlink()
+
+    lines = _count_lines(dest)
+    print(f"  → {dest.name} ({lines} entries)")
+
+
+def download(*, force_redownload: bool = False) -> None:
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+
+    for fname in FILES:
+        _download_one(fname, fname, force=force_redownload)
+
+    for remote_path, local_name in ANSWER_FILES:
+        _download_one(remote_path, local_name, force=force_redownload)
+
+    print("\nDone. All BFCL v3 data downloaded.")
+
+
+if __name__ == "__main__":
+    download()