diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dc88f8a256d0..bcc26665b746 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -298,3 +298,8 @@ docs/source/performance/perf-benchmarking.md @NVIDIA/trtllm-bench-reviewers # of the NVIDIA/trt-llm-release-branch-approval team, regardless of who else approves the PR. # Without approval from a member of this team, PRs cannot be merged to release branches. # * @NVIDIA/trt-llm-release-branch-approval + +### Telemetry / privacy review +# Golden manifest is the privacy-review artifact; route it and the usage package to the privacy owner. +/tensorrt_llm/usage/llm_args_golden_manifest.json @NVIDIA/trt-llm-oss-compliance +/tensorrt_llm/usage/ @NVIDIA/trt-llm-oss-compliance diff --git a/README.md b/README.md index 1f9e7fa262b7..c60dd106e3ff 100644 --- a/README.md +++ b/README.md @@ -298,9 +298,10 @@ Deprecation is used to inform developers that some APIs and tools are no longer TensorRT-LLM collects anonymous telemetry data by default. This data is used in aggregate to understand usage patterns and prioritize engineering efforts. **This data cannot be traced back to any individual user.** No prompts, -user-identifying information, or persistent identifiers are collected. Any -deployment identifiers are ephemeral, randomly generated per deployment, and -not linked to users. The data we collect includes: +outputs, model weights, model paths, tokenizer paths, user-identifying +information, raw free-form configuration strings, or persistent identifiers are +collected. Any deployment identifiers are ephemeral, randomly generated per +deployment, and not linked to users. The data we collect includes: - Ingress point (e.g., LLM API, CLI, serve command) - Deployment duration (via periodic heartbeats) @@ -309,8 +310,10 @@ not linked to users. The data we collect includes: - Parallelism configuration (TP/PP/CP/MoE-EP/MoE-TP sizes), quantization algorithm, dtype, KV cache dtype - System information (OS platform, Python version, CPU architecture, CPU count) - TRT-LLM version and backend -- Feature flags (LoRA, speculative decoding, prefix caching, CUDA graphs, chunked context, data parallelism) +- Feature summary flags (LoRA, speculative decoding, prefix caching, CUDA graphs, chunked context, data parallelism) - Disaggregated serving metadata (role and deployment ID) +- Selected LLM API configuration values: parallelism, dtype, KV cache, scheduler, CUDA graph, and compile settings +- Capture diagnostics for that payload: a schema checksum (for provenance), the count of captured fields, and whether any free-form value was skipped Telemetry is automatically disabled in CI and test environments. diff --git a/docs/source/_ext/llmapi_config_telemetry.py b/docs/source/_ext/llmapi_config_telemetry.py new file mode 100644 index 000000000000..1c237bb6889c --- /dev/null +++ b/docs/source/_ext/llmapi_config_telemetry.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from pathlib import Path + +_GOLDEN_REL = "tensorrt_llm/usage/llm_args_golden_manifest.json" + +_REFERENCE_PREAMBLE = """\ +# Telemetry + +This page documents TensorRT-LLM usage telemetry. It is generated during the +Sphinx docs build by rendering the committed telemetry manifest +(`tensorrt_llm/usage/llm_args_golden_manifest.json`). + +Start with the +[Telemetry Data Collection section in the root README](source:README.md#telemetry-data-collection) +for the user-facing collection and opt-out overview, and the +[telemetry schema reference](source:tensorrt_llm/usage/schemas/README.md) +for the wire schema. + +**No PII or free-form fields are captured.** LLM API configuration capture is +*type-driven*: fields whose type is categorical (`Literal`/`Enum`/`bool`) or +numeric (`int`/`float`), plus safe collections of those, are captured +automatically. Free-form `str`/`Any`/`Path`/`dict`/`Callable` are never captured +unless a field carries an explicit allowlist (`TelemetryField.categorical(...)`), +and any field may opt out with `telemetry=False`. Every captured field is listed +below; the runtime can capture nothing absent from this list. + +## LLM API Configuration Fields + +A field can still be absent from a specific payload when its parent config is +unset or when the safety sanitizer rejects the runtime value. +""" + + +def _escape(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def _format_values(values: list[str]) -> str: + return ", ".join(f"`{_escape(v)}`" for v in values) if values else "" + + +def _table(rows: list[dict]) -> str: + lines = [ + "| Captured key | Annotation | Kind | Converter | Allowed values |", + "|--------------|------------|------|-----------|----------------|", + ] + for row in rows: + lines.append( + f"| `{_escape(row['path'])}` | `{_escape(row['annotation'])}` | " + f"`{_escape(row['kind'])}` | {_escape(row['converter'])} | " + f"{_format_values(row['allowed_values'])} |" + ) + return "\n".join(lines) + + +def generate_telemetry_reference(repo_root: Path | str, output_path: Path | str) -> None: + repo_root = Path(repo_root) + golden = json.loads((repo_root / _GOLDEN_REL).read_text()) + content = [_REFERENCE_PREAMBLE] + for args_class in ("TorchLlmArgs", "TrtLlmArgs"): + rows = golden.get(args_class, []) + content.extend( + [ + f"### `{args_class}`", + "", + f"{len(rows)} captured fields.", + "", + _table(rows), + "", + ] + ) + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text("\n".join(content)) + + +def _on_builder_inited(app) -> None: + docs_source = Path(app.confdir) + repo_root = docs_source.parents[1] + generate_telemetry_reference(repo_root, docs_source / "developer-guide/telemetry.md") + + +def setup(app) -> dict[str, object]: + app.connect("builder-inited", _on_builder_inited) + return {"version": "0.2", "parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/source/conf.py b/docs/source/conf.py index 34d0e8328257..762e6cb98626 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# # Configuration file for the Sphinx documentation builder. # # For the full list of built-in configuration values, see the documentation: @@ -68,6 +71,7 @@ 'sphinx_togglebutton', 'sphinxcontrib.mermaid', 'trtllm_auto_deploy', + 'llmapi_config_telemetry', 'trtllm_config_selector', ] diff --git a/docs/source/developer-guide/telemetry.md b/docs/source/developer-guide/telemetry.md new file mode 100644 index 000000000000..ec4681397e7e --- /dev/null +++ b/docs/source/developer-guide/telemetry.md @@ -0,0 +1,532 @@ +# Telemetry + +This page documents TensorRT-LLM usage telemetry. It is generated during the +Sphinx docs build by rendering the committed telemetry manifest +(`tensorrt_llm/usage/llm_args_golden_manifest.json`). + +Start with the +[Telemetry Data Collection section in the root README](source:README.md#telemetry-data-collection) +for the user-facing collection and opt-out overview, and the +[telemetry schema reference](source:tensorrt_llm/usage/schemas/README.md) +for the wire schema. + +**No PII or free-form fields are captured.** LLM API configuration capture is +*type-driven*: fields whose type is categorical (`Literal`/`Enum`/`bool`) or +numeric (`int`/`float`), plus safe collections of those, are captured +automatically. Free-form `str`/`Any`/`Path`/`dict`/`Callable` are never captured +unless a field carries an explicit allowlist (`TelemetryField.categorical(...)`), +and any field may opt out with `telemetry=False`. Every captured field is listed +below; the runtime can capture nothing absent from this list. + +## LLM API Configuration Fields + +A field can still be absent from a specific payload when its parent config is +unset or when the safety sanitizer rejects the runtime value. + +### `TorchLlmArgs` + +234 captured fields. + +| Captured key | Annotation | Kind | Converter | Allowed values | +|--------------|------------|------|-----------|----------------| +| `allreduce_strategy` | `Optional[Literal['AUTO', 'NCCL', 'UB', 'MINLATENCY', 'ONESHOT', 'TWOSHOT', 'LOWPRECISION', 'MNNVL', 'NCCL_SYMMETRIC']]` | `categorical` | | `AUTO`, `NCCL`, `UB`, `MINLATENCY`, `ONESHOT`, `TWOSHOT`, `LOWPRECISION`, `MNNVL`, `NCCL_SYMMETRIC` | +| `attention_dp_config.batching_wait_iters` | `` | `value` | | | +| `attention_dp_config.enable_balance` | `` | `value` | | | +| `attention_dp_config.enable_kv_cache_aware_routing` | `` | `value` | | | +| `attention_dp_config.kv_cache_routing_cold_start_warmup` | `` | `value` | | | +| `attention_dp_config.kv_cache_routing_fair_share_multiplier` | `` | `value` | | | +| `attention_dp_config.kv_cache_routing_load_balance_weight` | `` | `value` | | | +| `attention_dp_config.kv_cache_routing_match_rate_threshold` | `` | `value` | | | +| `attention_dp_config.timeout_iters` | `` | `value` | | | +| `attn_backend` | `` | `categorical` | allowlist | `VANILLA`, `TRTLLM`, `FLASHINFER`, `FLASHINFER_STAR_ATTENTION` | +| `backend` | `Literal['pytorch']` | `categorical` | | `pytorch` | +| `batch_wait_max_tokens_ratio` | `` | `value` | | | +| `batch_wait_timeout_iters` | `` | `value` | | | +| `batch_wait_timeout_ms` | `` | `value` | | | +| `cache_transceiver_config.backend` | `Optional[Literal['DEFAULT', 'UCX', 'NIXL', 'MOONCAKE', 'MPI']]` | `categorical` | | `DEFAULT`, `UCX`, `NIXL`, `MOONCAKE`, `MPI` | +| `cache_transceiver_config.kv_transfer_sender_future_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `cache_transceiver_config.kv_transfer_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `cache_transceiver_config.max_tokens_in_buffer` | `Optional[int]` | `value` | | | +| `cache_transceiver_config.transceiver_runtime` | `Optional[Literal['CPP', 'PYTHON']]` | `categorical` | | `CPP`, `PYTHON` | +| `context_parallel_size` | `` | `value` | | | +| `cp_config.block_size` | `Optional[int]` | `value` | | | +| `cp_config.cp_anchor_size` | `Optional[int]` | `value` | | | +| `cp_config.cp_type` | `` | `categorical` | | `ULYSSES`, `STAR`, `RING`, `HELIX` | +| `cp_config.fifo_version` | `Optional[int]` | `value` | | | +| `cp_config.tokens_per_block` | `Optional[int]` | `value` | | | +| `cp_config.use_nccl_for_alltoall` | `Optional[bool]` | `value` | | | +| `cuda_graph_config.batch_sizes` | `Optional[List[int]]` | `value` | | | +| `cuda_graph_config.enable_padding` | `` | `value` | | | +| `cuda_graph_config.max_batch_size` | `` | `value` | | | +| `cuda_graph_config.max_num_token` | `` | `value` | | | +| `cuda_graph_config.max_seq_len` | `` | `value` | | | +| `cuda_graph_config.mode` | `Literal['decode']` | `categorical` | | `decode`, `encode` | +| `cuda_graph_config.num_tokens` | `Optional[List[Annotated[int, Gt(gt=0)]]]` | `value` | | | +| `cuda_graph_config.seq_lens` | `Optional[List[Annotated[int, Gt(gt=0)]]]` | `value` | | | +| `disable_flashinfer_sampling` | `` | `value` | | | +| `disable_overlap_scheduler` | `` | `value` | | | +| `dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32` | +| `dwdp_config.contention_opt` | `` | `value` | | | +| `dwdp_config.dwdp_size` | `` | `value` | | | +| `dwdp_config.num_experts_per_worker` | `` | `value` | | | +| `dwdp_config.num_groups` | `` | `value` | | | +| `dwdp_config.num_prefetch_experts` | `` | `value` | | | +| `enable_attention_dp` | `` | `value` | | | +| `enable_autotuner` | `` | `value` | | | +| `enable_chunked_prefill` | `` | `value` | | | +| `enable_early_first_token_response` | `` | `value` | | | +| `enable_energy_metrics` | `` | `value` | | | +| `enable_iter_perf_stats` | `` | `value` | | | +| `enable_iter_req_stats` | `` | `value` | | | +| `enable_layerwise_nvtx_marker` | `` | `value` | | | +| `enable_lm_head_tp_in_adp` | `` | `value` | | | +| `enable_lora` | `` | `value` | | | +| `enable_min_latency` | `` | `value` | | | +| `enable_resource_governor` | `` | `value` | | | +| `enable_speculative_beam_history_d2h` | `` | `value` | | | +| `encode_only` | `` | `value` | | | +| `force_dynamic_quantization` | `` | `value` | | | +| `garbage_collection_gen0_threshold` | `` | `value` | | | +| `gather_generation_logits` | `` | `value` | | | +| `gms_config.mode` | `Literal['auto', 'rw', 'ro']` | `categorical` | | `auto`, `rw`, `ro` | +| `gpus_per_node` | `Optional[int]` | `value` | | | +| `guided_decoding_backend` | `Optional[Literal['xgrammar', 'llguidance']]` | `categorical` | | `xgrammar`, `llguidance` | +| `iter_stats_max_iterations` | `Optional[int]` | `value` | | | +| `kv_cache_config.attention_dp_events_gather_period_ms` | `` | `value` | | | +| `kv_cache_config.copy_on_partial_reuse` | `` | `value` | | | +| `kv_cache_config.cross_kv_cache_fraction` | `Optional[float]` | `value` | | | +| `kv_cache_config.dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32`, `fp8`, `nvfp4` | +| `kv_cache_config.enable_block_reuse` | `` | `value` | | | +| `kv_cache_config.enable_partial_reuse` | `` | `value` | | | +| `kv_cache_config.event_buffer_max_size` | `` | `value` | | | +| `kv_cache_config.free_gpu_memory_fraction` | `Optional[float]` | `value` | | | +| `kv_cache_config.host_cache_size` | `Optional[int]` | `value` | | | +| `kv_cache_config.iteration_stats_interval` | `` | `value` | | | +| `kv_cache_config.mamba_ssm_cache_dtype` | `Literal['auto', 'float16', 'bfloat16', 'float32']` | `categorical` | | `auto`, `float16`, `bfloat16`, `float32` | +| `kv_cache_config.mamba_ssm_philox_rounds` | `` | `value` | | | +| `kv_cache_config.mamba_ssm_stochastic_rounding` | `` | `value` | | | +| `kv_cache_config.mamba_state_cache_interval` | `` | `value` | | | +| `kv_cache_config.max_attention_window` | `Optional[List[int]]` | `value` | | | +| `kv_cache_config.max_gpu_total_bytes` | `` | `value` | | | +| `kv_cache_config.max_tokens` | `Optional[int]` | `value` | | | +| `kv_cache_config.max_util_for_resume` | `` | `value` | | | +| `kv_cache_config.secondary_offload_min_priority` | `Optional[int]` | `value` | | | +| `kv_cache_config.sink_token_length` | `Optional[int]` | `value` | | | +| `kv_cache_config.tokens_per_block` | `` | `value` | | | +| `kv_cache_config.use_kv_cache_manager_v2` | `` | `value` | | | +| `kv_cache_config.use_uvm` | `` | `value` | | | +| `kv_connector_config.connector` | `Optional[str]` | `categorical` | allowlist | `lmcache`, `lmcache-mp`, `kvbm` | +| `layer_wise_benchmarks_config.calibration_layer_indices` | `Optional[List[int]]` | `value` | | | +| `layer_wise_benchmarks_config.calibration_mode` | `Literal['NONE', 'MARK', 'COLLECT']` | `categorical` | | `NONE`, `MARK`, `COLLECT` | +| `load_format` | `Union[str, tensorrt_llm.llmapi.llm_args.LoadFormat]` | `categorical` | allowlist | `auto`, `dummy`, `vision_only`, `gms` | +| `lora_config.lora_ckpt_source` | `Literal['hf', 'nemo']` | `categorical` | | `hf`, `nemo` | +| `lora_config.max_cpu_loras` | `Optional[int]` | `value` | | | +| `lora_config.max_lora_rank` | `` | `value` | | | +| `lora_config.max_loras` | `Optional[int]` | `value` | | | +| `lora_config.swap_gate_up_proj_lora_b_weight` | `` | `value` | | | +| `max_batch_size` | `Optional[int]` | `value` | | | +| `max_beam_width` | `Optional[int]` | `value` | | | +| `max_input_len` | `Optional[int]` | `value` | | | +| `max_num_tokens` | `Optional[int]` | `value` | | | +| `max_seq_len` | `Optional[int]` | `value` | | | +| `max_stats_len` | `` | `value` | | | +| `mm_encoder_only` | `` | `value` | | | +| `moe_cluster_parallel_size` | `Optional[int]` | `value` | | | +| `moe_config.backend` | `Literal['AUTO', 'CUTLASS', 'CUTEDSL', 'WIDEEP', 'TRTLLM', 'DEEPGEMM', 'DENSEGEMM', 'VANILLA', 'TRITON']` | `categorical` | | `AUTO`, `CUTLASS`, `CUTEDSL`, `WIDEEP`, `TRTLLM`, `DEEPGEMM`, `DENSEGEMM`, `VANILLA`, `TRITON` | +| `moe_config.disable_finalize_fusion` | `` | `value` | | | +| `moe_config.max_num_tokens` | `Optional[int]` | `value` | | | +| `moe_config.use_low_precision_moe_combine` | `` | `value` | | | +| `moe_expert_parallel_size` | `Optional[int]` | `value` | | | +| `moe_tensor_parallel_size` | `Optional[int]` | `value` | | | +| `mx_config.preshard_strategy` | `` | `categorical` | allowlist | `per_module` | +| `mx_config.server_query_timeout_s` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `num_postprocess_workers` | `` | `value` | | | +| `nvfp4_gemm_config.allowed_backends` | `List[Literal['cutlass', 'cublaslt', 'cutedsl', 'cuda_core']]` | `value` | | `cutlass`, `cublaslt`, `cutedsl`, `cuda_core` | +| `orchestrator_type` | `Optional[Literal['rpc', 'ray']]` | `categorical` | | `rpc`, `ray` | +| `peft_cache_config.device_cache_percent` | `` | `value` | | | +| `peft_cache_config.host_cache_size` | `` | `value` | | | +| `peft_cache_config.max_adapter_size` | `` | `value` | | | +| `peft_cache_config.max_pages_per_block_device` | `` | `value` | | | +| `peft_cache_config.max_pages_per_block_host` | `` | `value` | | | +| `peft_cache_config.num_copy_streams` | `` | `value` | | | +| `peft_cache_config.num_device_module_layer` | `` | `value` | | | +| `peft_cache_config.num_ensure_workers` | `` | `value` | | | +| `peft_cache_config.num_host_module_layer` | `` | `value` | | | +| `peft_cache_config.num_put_workers` | `` | `value` | | | +| `peft_cache_config.optimal_adapter_size` | `` | `value` | | | +| `perf_metrics_max_requests` | `` | `value` | | | +| `pipeline_parallel_size` | `` | `value` | | | +| `pp_partition` | `Optional[List[int]]` | `value` | | | +| `print_iter_log` | `` | `value` | | | +| `prometheus_metrics_config.e2e_request_latency_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_decode_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_inference_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_prefill_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_queue_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.time_per_output_token_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.time_to_first_token_buckets` | `Optional[List[float]]` | `value` | | | +| `ray_placement_config.defer_workers_init` | `` | `value` | | | +| `ray_placement_config.per_worker_gpu_share` | `Optional[float]` | `value` | | | +| `ray_placement_config.placement_bundle_indices` | `Optional[List[List[int]]]` | `value` | | | +| `reasoning_parser` | `Optional[str]` | `categorical` | allowlist | `auto`, `deepseek-r1`, `laguna`, `qwen3`, `qwen3_5`, `minimax_m2`, `minimax_m2_append_think`, `nano-v3`, `gemma4`, `kimi_k2`, `kimi_k25` | +| `reorder_policy_config.policy_args.agent_inflight_seq_num` | `` | `value` | | | +| `reorder_policy_config.policy_args.agent_percentage` | `` | `value` | | | +| `reorder_policy_config.policy_name` | `Optional[Literal['AgentTree']]` | `categorical` | | `AgentTree` | +| `request_stats_max_iterations` | `Optional[int]` | `value` | | | +| `return_perf_metrics` | `` | `value` | | | +| `sampler_force_async_worker` | `` | `value` | | | +| `sampler_type` | `Union[str, tensorrt_llm.llmapi.llm_args.SamplerType]` | `categorical` | allowlist | `TRTLLMSampler`, `TorchSampler`, `auto` | +| `scheduler_config.capacity_scheduler_policy` | `` | `categorical` | | `MAX_UTILIZATION`, `GUARANTEED_NO_EVICT`, `STATIC_BATCH` | +| `scheduler_config.context_chunking_policy` | `Optional[tensorrt_llm.llmapi.llm_args.ContextChunkingPolicy]` | `categorical` | | `FIRST_COME_FIRST_SERVED`, `EQUAL_PROGRESS`, `FORCE_CHUNK` | +| `scheduler_config.dynamic_batch_config.dynamic_batch_moving_average_window` | `` | `value` | | | +| `scheduler_config.dynamic_batch_config.enable_batch_size_tuning` | `` | `value` | | | +| `scheduler_config.dynamic_batch_config.enable_max_num_tokens_tuning` | `` | `value` | | | +| `scheduler_config.use_python_scheduler` | `` | `value` | | | +| `scheduler_config.waiting_queue_policy` | `` | `categorical` | | `fcfs`, `priority` | +| `skip_tokenizer_init` | `` | `value` | | | +| `sparse_attention_config.algorithm` | `Literal['dsa']` | `categorical` | | `dsa`, `rocket`, `skip_softmax` | +| `sparse_attention_config.enable_heuristic_topk` | `` | `value` | | | +| `sparse_attention_config.index_head_dim` | `Optional[int]` | `value` | | | +| `sparse_attention_config.index_n_heads` | `Optional[int]` | `value` | | | +| `sparse_attention_config.index_topk` | `Optional[int]` | `value` | | | +| `sparse_attention_config.indexer_k_dtype` | `Literal['fp8', 'fp4']` | `categorical` | | `fp8`, `fp4` | +| `sparse_attention_config.indexer_max_chunk_size` | `Optional[int]` | `value` | | | +| `sparse_attention_config.indexer_rope_interleave` | `` | `value` | | | +| `sparse_attention_config.kernel_size` | `Optional[int]` | `value` | | | +| `sparse_attention_config.kt_cache_dtype` | `Optional[str]` | `categorical` | allowlist | `bfloat16`, `float8_e5m2` | +| `sparse_attention_config.page_size` | `Optional[int]` | `value` | | | +| `sparse_attention_config.prompt_budget` | `Optional[int]` | `value` | | | +| `sparse_attention_config.q_split_threshold` | `` | `value` | | | +| `sparse_attention_config.seq_len_threshold` | `Optional[int]` | `value` | | | +| `sparse_attention_config.skip_indexer_for_short_seqs` | `` | `value` | | | +| `sparse_attention_config.topk` | `Optional[int]` | `value` | | | +| `sparse_attention_config.topr` | `Union[int, float, NoneType]` | `value` | | | +| `sparse_attention_config.use_cute_dsl_paged_mqa_logits` | `` | `value` | | | +| `sparse_attention_config.use_cute_dsl_topk` | `` | `value` | | | +| `sparse_attention_config.window_size` | `Optional[int]` | `value` | | | +| `speculative_config.acceptance_length_threshold` | `Optional[Annotated[float, Ge(ge=0)]]` | `value` | | | +| `speculative_config.acceptance_window` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `speculative_config.allow_advanced_sampling` | `` | `value` | | | +| `speculative_config.begin_thinking_phase_token` | `` | `value` | | | +| `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | +| `speculative_config.dynamic_tree_max_topK` | `Optional[int]` | `value` | | | +| `speculative_config.eagle3_layers_to_capture` | `Optional[Set[int]]` | `value` | | | +| `speculative_config.eagle3_model_arch` | `Literal['llama3', 'mistral_large3']` | `categorical` | | `llama3`, `mistral_large3` | +| `speculative_config.eagle3_one_model` | `Optional[bool]` | `value` | | | +| `speculative_config.eagle_choices` | `Optional[List[List[int]]]` | `value` | | | +| `speculative_config.enable_global_pool` | `` | `value` | | | +| `speculative_config.end_thinking_phase_token` | `` | `value` | | | +| `speculative_config.global_pool_size` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `speculative_config.greedy_sampling` | `Optional[bool]` | `value` | | | +| `speculative_config.is_keep_all` | `` | `value` | | | +| `speculative_config.is_public_pool` | `` | `value` | | | +| `speculative_config.is_use_oldest` | `` | `value` | | | +| `speculative_config.mask_token_id` | `Optional[int]` | `value` | | | +| `speculative_config.max_concurrency` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `speculative_config.max_draft_len` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `speculative_config.max_matching_ngram_size` | `` | `value` | | | +| `speculative_config.max_ngram_size` | `` | `value` | | | +| `speculative_config.max_non_leaves_per_layer` | `Optional[int]` | `value` | | | +| `speculative_config.max_total_draft_tokens` | `Optional[int]` | `value` | | | +| `speculative_config.max_verification_set_size` | `` | `value` | | | +| `speculative_config.max_window_size` | `` | `value` | | | +| `speculative_config.medusa_choices` | `Optional[List[List[int]]]` | `value` | | | +| `speculative_config.mtp_eagle_one_model` | `` | `value` | | | +| `speculative_config.num_eagle_layers` | `Optional[int]` | `value` | | | +| `speculative_config.num_medusa_heads` | `Optional[int]` | `value` | | | +| `speculative_config.num_nextn_predict_layers` | `Optional[int]` | `value` | | | +| `speculative_config.posterior_threshold` | `Optional[float]` | `value` | | | +| `speculative_config.relaxed_delta` | `` | `value` | | | +| `speculative_config.relaxed_topk` | `` | `value` | | | +| `speculative_config.sa_config.enable_global_pool` | `` | `value` | | | +| `speculative_config.sa_config.threshold` | `` | `value` | | | +| `speculative_config.target_layer_ids` | `Optional[List[int]]` | `value` | | | +| `speculative_config.use_dynamic_tree` | `Optional[bool]` | `value` | | | +| `speculative_config.use_mtp_vanilla` | `` | `value` | | | +| `speculative_config.use_rejection_sampling` | `` | `value` | | | +| `speculative_config.use_relaxed_acceptance_for_thinking` | `` | `value` | | | +| `speculative_config.write_interval` | `` | `value` | | | +| `stream_interval` | `` | `value` | | | +| `telemetry_config.disabled` | `` | `value` | | | +| `telemetry_config.usage_context` | `` | `categorical` | | `unknown`, `llm_class`, `cli_serve`, `cli_bench`, `cli_eval` | +| `tensor_parallel_size` | `` | `value` | | | +| `tokenizer_mode` | `Literal['auto', 'slow']` | `categorical` | | `auto`, `slow` | +| `torch_compile_config.capture_num_tokens` | `Optional[List[Annotated[int, Gt(gt=0)]]]` | `value` | | | +| `torch_compile_config.enable_fullgraph` | `` | `value` | | | +| `torch_compile_config.enable_inductor` | `` | `value` | | | +| `torch_compile_config.enable_piecewise_cuda_graph` | `` | `value` | | | +| `torch_compile_config.enable_userbuffers` | `` | `value` | | | +| `torch_compile_config.max_num_streams` | `` | `value` | | | +| `trust_remote_code` | `` | `value` | | | +| `use_cute_dsl_bf16_bmm` | `` | `value` | | | +| `use_cute_dsl_bf16_gemm` | `` | `value` | | | +| `use_cute_dsl_blockscaling_bmm` | `` | `value` | | | +| `use_cute_dsl_blockscaling_mm` | `` | `value` | | | +| `video_pruning_rate` | `Optional[float]` | `value` | | | + +### `TrtLlmArgs` + +260 captured fields. + +| Captured key | Annotation | Kind | Converter | Allowed values | +|--------------|------------|------|-----------|----------------| +| `backend` | `Optional[str]` | `categorical` | allowlist | `pytorch`, `tensorrt`, `_autodeploy` | +| `batching_type` | `Optional[tensorrt_llm.llmapi.llm_args.BatchingType]` | `categorical` | | `STATIC`, `INFLIGHT` | +| `build_config.dry_run` | `` | `value` | | | +| `build_config.enable_debug_output` | `` | `value` | | | +| `build_config.force_num_profiles` | `Optional[int]` | `value` | | | +| `build_config.gather_context_logits` | `` | `value` | | | +| `build_config.gather_generation_logits` | `` | `value` | | | +| `build_config.kv_cache_type` | `Optional[tensorrt_llm.llmapi.kv_cache_type.KVCacheType]` | `categorical` | | `continuous`, `paged`, `disabled` | +| `build_config.lora_config.lora_ckpt_source` | `Literal['hf', 'nemo']` | `categorical` | | `hf`, `nemo` | +| `build_config.lora_config.max_cpu_loras` | `Optional[int]` | `value` | | | +| `build_config.lora_config.max_lora_rank` | `` | `value` | | | +| `build_config.lora_config.max_loras` | `Optional[int]` | `value` | | | +| `build_config.lora_config.swap_gate_up_proj_lora_b_weight` | `` | `value` | | | +| `build_config.max_batch_size` | `` | `value` | | | +| `build_config.max_beam_width` | `` | `value` | | | +| `build_config.max_draft_len` | `` | `value` | | | +| `build_config.max_encoder_input_len` | `` | `value` | | | +| `build_config.max_input_len` | `` | `value` | | | +| `build_config.max_num_tokens` | `` | `value` | | | +| `build_config.max_prompt_embedding_table_size` | `` | `value` | | | +| `build_config.max_seq_len` | `Optional[int]` | `value` | | | +| `build_config.monitor_memory` | `` | `value` | | | +| `build_config.opt_batch_size` | `` | `value` | | | +| `build_config.opt_num_tokens` | `Optional[int]` | `value` | | | +| `build_config.plugin_config.bert_attention_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.bert_context_fmha_fp32_acc` | `` | `value` | | | +| `build_config.plugin_config.context_fmha` | `` | `value` | | | +| `build_config.plugin_config.dora_plugin` | `` | `value` | | | +| `build_config.plugin_config.fp8_rowwise_gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.fuse_fp4_quant` | `` | `value` | | | +| `build_config.plugin_config.gemm_allreduce_plugin` | `Optional[Literal['float16', 'bfloat16', None]]` | `categorical` | | `float16`, `bfloat16`, `None` | +| `build_config.plugin_config.gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', 'fp8', 'nvfp4', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `fp8`, `nvfp4`, `None` | +| `build_config.plugin_config.gemm_swiglu_plugin` | `Optional[Literal['fp8', None]]` | `categorical` | | `fp8`, `None` | +| `build_config.plugin_config.gpt_attention_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.identity_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.layernorm_quantization_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.lora_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.low_latency_gemm_plugin` | `Optional[Literal['fp8', None]]` | `categorical` | | `fp8`, `None` | +| `build_config.plugin_config.low_latency_gemm_swiglu_plugin` | `Optional[Literal['fp8', None]]` | `categorical` | | `fp8`, `None` | +| `build_config.plugin_config.mamba_conv1d_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.manage_weights` | `` | `value` | | | +| `build_config.plugin_config.moe_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.multiple_profiles` | `` | `value` | | | +| `build_config.plugin_config.nccl_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.norm_quant_fusion` | `` | `value` | | | +| `build_config.plugin_config.paged_kv_cache` | `Optional[bool]` | `value` | | | +| `build_config.plugin_config.paged_state` | `` | `value` | | | +| `build_config.plugin_config.pp_reduce_scatter` | `` | `value` | | | +| `build_config.plugin_config.qserve_gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.quantize_per_token_plugin` | `` | `value` | | | +| `build_config.plugin_config.quantize_tensor_plugin` | `` | `value` | | | +| `build_config.plugin_config.reduce_fusion` | `` | `value` | | | +| `build_config.plugin_config.remove_input_padding` | `` | `value` | | | +| `build_config.plugin_config.rmsnorm_quantization_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.smooth_quant_gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.smooth_quant_plugins` | `` | `value` | | | +| `build_config.plugin_config.streamingllm` | `` | `value` | | | +| `build_config.plugin_config.tokens_per_block` | `` | `value` | | | +| `build_config.plugin_config.use_fp8_context_fmha` | `` | `value` | | | +| `build_config.plugin_config.use_fused_mlp` | `` | `value` | | | +| `build_config.plugin_config.use_paged_context_fmha` | `` | `value` | | | +| `build_config.plugin_config.user_buffer` | `` | `value` | | | +| `build_config.plugin_config.weight_only_groupwise_quant_matmul_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.weight_only_quant_matmul_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.speculative_decoding_mode` | `` | `categorical` | | `NONE`, `DRAFT_TOKENS_EXTERNAL`, `MEDUSA`, `LOOKAHEAD_DECODING`, `EXPLICIT_DRAFT_TOKENS`, `EAGLE`, `NGRAM`, `USER_PROVIDED`, `SAVE_HIDDEN_STATES`, `AUTO` | +| `build_config.strongly_typed` | `` | `value` | | | +| `build_config.use_mrope` | `` | `value` | | | +| `build_config.use_refit` | `` | `value` | | | +| `build_config.use_strip_plan` | `` | `value` | | | +| `build_config.weight_sparsity` | `` | `value` | | | +| `build_config.weight_streaming` | `` | `value` | | | +| `cache_transceiver_config.backend` | `Optional[Literal['DEFAULT', 'UCX', 'NIXL', 'MOONCAKE', 'MPI']]` | `categorical` | | `DEFAULT`, `UCX`, `NIXL`, `MOONCAKE`, `MPI` | +| `cache_transceiver_config.kv_transfer_sender_future_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `cache_transceiver_config.kv_transfer_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `cache_transceiver_config.max_tokens_in_buffer` | `Optional[int]` | `value` | | | +| `cache_transceiver_config.transceiver_runtime` | `Optional[Literal['CPP', 'PYTHON']]` | `categorical` | | `CPP`, `PYTHON` | +| `calib_config.calib_batch_size` | `` | `value` | | | +| `calib_config.calib_batches` | `` | `value` | | | +| `calib_config.calib_max_seq_length` | `` | `value` | | | +| `calib_config.device` | `Literal['cuda', 'cpu']` | `categorical` | | `cuda`, `cpu` | +| `calib_config.random_seed` | `` | `value` | | | +| `calib_config.tokenizer_max_seq_length` | `` | `value` | | | +| `context_parallel_size` | `` | `value` | | | +| `cp_config.block_size` | `Optional[int]` | `value` | | | +| `cp_config.cp_anchor_size` | `Optional[int]` | `value` | | | +| `cp_config.cp_type` | `` | `categorical` | | `ULYSSES`, `STAR`, `RING`, `HELIX` | +| `cp_config.fifo_version` | `Optional[int]` | `value` | | | +| `cp_config.tokens_per_block` | `Optional[int]` | `value` | | | +| `cp_config.use_nccl_for_alltoall` | `Optional[bool]` | `value` | | | +| `dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32` | +| `embedding_parallel_mode` | `Literal['NONE', 'SHARDING_ALONG_VOCAB', 'SHARDING_ALONG_HIDDEN']` | `categorical` | | `NONE`, `SHARDING_ALONG_VOCAB`, `SHARDING_ALONG_HIDDEN` | +| `enable_attention_dp` | `` | `value` | | | +| `enable_build_cache.max_cache_storage_gb` | `` | `value` | | | +| `enable_build_cache.max_records` | `` | `value` | | | +| `enable_chunked_prefill` | `` | `value` | | | +| `enable_energy_metrics` | `` | `value` | | | +| `enable_lm_head_tp_in_adp` | `` | `value` | | | +| `enable_lora` | `` | `value` | | | +| `enable_prompt_adapter` | `` | `value` | | | +| `enable_tqdm` | `` | `value` | | | +| `extended_runtime_perf_knob_config.cuda_graph_cache_size` | `` | `value` | | | +| `extended_runtime_perf_knob_config.cuda_graph_mode` | `` | `value` | | | +| `extended_runtime_perf_knob_config.enable_context_fmha_fp32_acc` | `` | `value` | | | +| `extended_runtime_perf_knob_config.multi_block_mode` | `` | `value` | | | +| `fail_fast_on_attention_window_too_large` | `` | `value` | | | +| `fast_build` | `` | `value` | | | +| `gather_generation_logits` | `` | `value` | | | +| `gpus_per_node` | `Optional[int]` | `value` | | | +| `guided_decoding_backend` | `Optional[Literal['xgrammar', 'llguidance']]` | `categorical` | | `xgrammar`, `llguidance` | +| `iter_stats_max_iterations` | `Optional[int]` | `value` | | | +| `kv_cache_config.attention_dp_events_gather_period_ms` | `` | `value` | | | +| `kv_cache_config.copy_on_partial_reuse` | `` | `value` | | | +| `kv_cache_config.cross_kv_cache_fraction` | `Optional[float]` | `value` | | | +| `kv_cache_config.dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32`, `fp8`, `nvfp4` | +| `kv_cache_config.enable_block_reuse` | `` | `value` | | | +| `kv_cache_config.enable_partial_reuse` | `` | `value` | | | +| `kv_cache_config.event_buffer_max_size` | `` | `value` | | | +| `kv_cache_config.free_gpu_memory_fraction` | `Optional[float]` | `value` | | | +| `kv_cache_config.host_cache_size` | `Optional[int]` | `value` | | | +| `kv_cache_config.iteration_stats_interval` | `` | `value` | | | +| `kv_cache_config.mamba_ssm_cache_dtype` | `Literal['auto', 'float16', 'bfloat16', 'float32']` | `categorical` | | `auto`, `float16`, `bfloat16`, `float32` | +| `kv_cache_config.mamba_ssm_philox_rounds` | `` | `value` | | | +| `kv_cache_config.mamba_ssm_stochastic_rounding` | `` | `value` | | | +| `kv_cache_config.mamba_state_cache_interval` | `` | `value` | | | +| `kv_cache_config.max_attention_window` | `Optional[List[int]]` | `value` | | | +| `kv_cache_config.max_gpu_total_bytes` | `` | `value` | | | +| `kv_cache_config.max_tokens` | `Optional[int]` | `value` | | | +| `kv_cache_config.max_util_for_resume` | `` | `value` | | | +| `kv_cache_config.secondary_offload_min_priority` | `Optional[int]` | `value` | | | +| `kv_cache_config.sink_token_length` | `Optional[int]` | `value` | | | +| `kv_cache_config.tokens_per_block` | `` | `value` | | | +| `kv_cache_config.use_kv_cache_manager_v2` | `` | `value` | | | +| `kv_cache_config.use_uvm` | `` | `value` | | | +| `load_format` | `Literal['auto', 'dummy']` | `categorical` | | `auto`, `dummy` | +| `lora_config.lora_ckpt_source` | `Literal['hf', 'nemo']` | `categorical` | | `hf`, `nemo` | +| `lora_config.max_cpu_loras` | `Optional[int]` | `value` | | | +| `lora_config.max_lora_rank` | `` | `value` | | | +| `lora_config.max_loras` | `Optional[int]` | `value` | | | +| `lora_config.swap_gate_up_proj_lora_b_weight` | `` | `value` | | | +| `max_batch_size` | `Optional[int]` | `value` | | | +| `max_beam_width` | `Optional[int]` | `value` | | | +| `max_input_len` | `Optional[int]` | `value` | | | +| `max_num_tokens` | `Optional[int]` | `value` | | | +| `max_prompt_adapter_token` | `` | `value` | | | +| `max_seq_len` | `Optional[int]` | `value` | | | +| `moe_cluster_parallel_size` | `Optional[int]` | `value` | | | +| `moe_expert_parallel_size` | `Optional[int]` | `value` | | | +| `moe_tensor_parallel_size` | `Optional[int]` | `value` | | | +| `normalize_log_probs` | `` | `value` | | | +| `num_postprocess_workers` | `` | `value` | | | +| `orchestrator_type` | `Optional[Literal['rpc', 'ray']]` | `categorical` | | `rpc`, `ray` | +| `peft_cache_config.device_cache_percent` | `` | `value` | | | +| `peft_cache_config.host_cache_size` | `` | `value` | | | +| `peft_cache_config.max_adapter_size` | `` | `value` | | | +| `peft_cache_config.max_pages_per_block_device` | `` | `value` | | | +| `peft_cache_config.max_pages_per_block_host` | `` | `value` | | | +| `peft_cache_config.num_copy_streams` | `` | `value` | | | +| `peft_cache_config.num_device_module_layer` | `` | `value` | | | +| `peft_cache_config.num_ensure_workers` | `` | `value` | | | +| `peft_cache_config.num_host_module_layer` | `` | `value` | | | +| `peft_cache_config.num_put_workers` | `` | `value` | | | +| `peft_cache_config.optimal_adapter_size` | `` | `value` | | | +| `perf_metrics_max_requests` | `` | `value` | | | +| `pipeline_parallel_size` | `` | `value` | | | +| `pp_partition` | `Optional[List[int]]` | `value` | | | +| `prometheus_metrics_config.e2e_request_latency_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_decode_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_inference_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_prefill_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_queue_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.time_per_output_token_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.time_to_first_token_buckets` | `Optional[List[float]]` | `value` | | | +| `quant_config.clamp_val` | `Optional[List[float]]` | `value` | | | +| `quant_config.group_size` | `Optional[int]` | `value` | | | +| `quant_config.has_zero_point` | `` | `value` | | | +| `quant_config.kv_cache_quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | +| `quant_config.mamba_ssm_philox_rounds` | `` | `value` | | | +| `quant_config.mamba_ssm_stochastic_rounding` | `` | `value` | | | +| `quant_config.pre_quant_scale` | `` | `value` | | | +| `quant_config.quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | +| `quant_config.smoothquant_val` | `` | `value` | | | +| `quant_config.use_meta_recipe` | `` | `value` | | | +| `reasoning_parser` | `Optional[str]` | `categorical` | allowlist | `auto`, `deepseek-r1`, `laguna`, `qwen3`, `qwen3_5`, `minimax_m2`, `minimax_m2_append_think`, `nano-v3`, `gemma4`, `kimi_k2`, `kimi_k25` | +| `request_stats_max_iterations` | `Optional[int]` | `value` | | | +| `return_perf_metrics` | `` | `value` | | | +| `scheduler_config.capacity_scheduler_policy` | `` | `categorical` | | `MAX_UTILIZATION`, `GUARANTEED_NO_EVICT`, `STATIC_BATCH` | +| `scheduler_config.context_chunking_policy` | `Optional[tensorrt_llm.llmapi.llm_args.ContextChunkingPolicy]` | `categorical` | | `FIRST_COME_FIRST_SERVED`, `EQUAL_PROGRESS`, `FORCE_CHUNK` | +| `scheduler_config.dynamic_batch_config.dynamic_batch_moving_average_window` | `` | `value` | | | +| `scheduler_config.dynamic_batch_config.enable_batch_size_tuning` | `` | `value` | | | +| `scheduler_config.dynamic_batch_config.enable_max_num_tokens_tuning` | `` | `value` | | | +| `scheduler_config.use_python_scheduler` | `` | `value` | | | +| `scheduler_config.waiting_queue_policy` | `` | `categorical` | | `fcfs`, `priority` | +| `skip_tokenizer_init` | `` | `value` | | | +| `sparse_attention_config.algorithm` | `Literal['dsa']` | `categorical` | | `dsa`, `rocket`, `skip_softmax` | +| `sparse_attention_config.enable_heuristic_topk` | `` | `value` | | | +| `sparse_attention_config.index_head_dim` | `Optional[int]` | `value` | | | +| `sparse_attention_config.index_n_heads` | `Optional[int]` | `value` | | | +| `sparse_attention_config.index_topk` | `Optional[int]` | `value` | | | +| `sparse_attention_config.indexer_k_dtype` | `Literal['fp8', 'fp4']` | `categorical` | | `fp8`, `fp4` | +| `sparse_attention_config.indexer_max_chunk_size` | `Optional[int]` | `value` | | | +| `sparse_attention_config.indexer_rope_interleave` | `` | `value` | | | +| `sparse_attention_config.kernel_size` | `Optional[int]` | `value` | | | +| `sparse_attention_config.kt_cache_dtype` | `Optional[str]` | `categorical` | allowlist | `bfloat16`, `float8_e5m2` | +| `sparse_attention_config.page_size` | `Optional[int]` | `value` | | | +| `sparse_attention_config.prompt_budget` | `Optional[int]` | `value` | | | +| `sparse_attention_config.q_split_threshold` | `` | `value` | | | +| `sparse_attention_config.seq_len_threshold` | `Optional[int]` | `value` | | | +| `sparse_attention_config.skip_indexer_for_short_seqs` | `` | `value` | | | +| `sparse_attention_config.topk` | `Optional[int]` | `value` | | | +| `sparse_attention_config.topr` | `Union[int, float, NoneType]` | `value` | | | +| `sparse_attention_config.use_cute_dsl_paged_mqa_logits` | `` | `value` | | | +| `sparse_attention_config.use_cute_dsl_topk` | `` | `value` | | | +| `sparse_attention_config.window_size` | `Optional[int]` | `value` | | | +| `speculative_config.acceptance_length_threshold` | `Optional[Annotated[float, Ge(ge=0)]]` | `value` | | | +| `speculative_config.acceptance_window` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `speculative_config.allow_advanced_sampling` | `` | `value` | | | +| `speculative_config.begin_thinking_phase_token` | `` | `value` | | | +| `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | +| `speculative_config.dynamic_tree_max_topK` | `Optional[int]` | `value` | | | +| `speculative_config.eagle3_layers_to_capture` | `Optional[Set[int]]` | `value` | | | +| `speculative_config.eagle3_model_arch` | `Literal['llama3', 'mistral_large3']` | `categorical` | | `llama3`, `mistral_large3` | +| `speculative_config.eagle3_one_model` | `Optional[bool]` | `value` | | | +| `speculative_config.eagle_choices` | `Optional[List[List[int]]]` | `value` | | | +| `speculative_config.enable_global_pool` | `` | `value` | | | +| `speculative_config.end_thinking_phase_token` | `` | `value` | | | +| `speculative_config.global_pool_size` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `speculative_config.greedy_sampling` | `Optional[bool]` | `value` | | | +| `speculative_config.is_keep_all` | `` | `value` | | | +| `speculative_config.is_public_pool` | `` | `value` | | | +| `speculative_config.is_use_oldest` | `` | `value` | | | +| `speculative_config.mask_token_id` | `Optional[int]` | `value` | | | +| `speculative_config.max_concurrency` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `speculative_config.max_draft_len` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `speculative_config.max_matching_ngram_size` | `` | `value` | | | +| `speculative_config.max_ngram_size` | `` | `value` | | | +| `speculative_config.max_non_leaves_per_layer` | `Optional[int]` | `value` | | | +| `speculative_config.max_total_draft_tokens` | `Optional[int]` | `value` | | | +| `speculative_config.max_verification_set_size` | `` | `value` | | | +| `speculative_config.max_window_size` | `` | `value` | | | +| `speculative_config.medusa_choices` | `Optional[List[List[int]]]` | `value` | | | +| `speculative_config.mtp_eagle_one_model` | `` | `value` | | | +| `speculative_config.num_eagle_layers` | `Optional[int]` | `value` | | | +| `speculative_config.num_medusa_heads` | `Optional[int]` | `value` | | | +| `speculative_config.num_nextn_predict_layers` | `Optional[int]` | `value` | | | +| `speculative_config.posterior_threshold` | `Optional[float]` | `value` | | | +| `speculative_config.relaxed_delta` | `` | `value` | | | +| `speculative_config.relaxed_topk` | `` | `value` | | | +| `speculative_config.sa_config.enable_global_pool` | `` | `value` | | | +| `speculative_config.sa_config.threshold` | `` | `value` | | | +| `speculative_config.target_layer_ids` | `Optional[List[int]]` | `value` | | | +| `speculative_config.use_dynamic_tree` | `Optional[bool]` | `value` | | | +| `speculative_config.use_mtp_vanilla` | `` | `value` | | | +| `speculative_config.use_rejection_sampling` | `` | `value` | | | +| `speculative_config.use_relaxed_acceptance_for_thinking` | `` | `value` | | | +| `speculative_config.write_interval` | `` | `value` | | | +| `telemetry_config.disabled` | `` | `value` | | | +| `telemetry_config.usage_context` | `` | `categorical` | | `unknown`, `llm_class`, `cli_serve`, `cli_bench`, `cli_eval` | +| `tensor_parallel_size` | `` | `value` | | | +| `tokenizer_mode` | `Literal['auto', 'slow']` | `categorical` | | `auto`, `slow` | +| `trust_remote_code` | `` | `value` | | | diff --git a/docs/source/index.rst b/docs/source/index.rst index 5b5a163278f3..f52c6b6fbfa8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -98,6 +98,7 @@ Welcome to TensorRT LLM's Documentation! developer-guide/dev-containers.md developer-guide/api-change.md developer-guide/kv-transfer.md + developer-guide/telemetry.md .. toctree:: diff --git a/setup.py b/setup.py index 5656531f5237..be0f840caaa5 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,6 @@ import os import platform from pathlib import Path -from typing import List from setuptools import find_packages, setup from setuptools.dist import Distribution @@ -198,7 +197,19 @@ def download_precompiled(workspace: str, version: str) -> str: return wheel_path -def extract_from_precompiled(precompiled_location: str, package_data: List[str], +def should_skip_precompiled_package_data(filename: str) -> bool: + """Return True for source-owned package data kept from local checkout. + + Precompiled wheels own native bits. Source owns telemetry schema JSON. + Skip those wheel files so Python-only schema edits layer over old wheels. + """ + filename = filename.replace("\\", "/") + source_owned_package_data_prefixes = ("tensorrt_llm/usage/schemas/", ) + return filename.endswith(".json") and filename.startswith( + source_owned_package_data_prefixes) + + +def extract_from_precompiled(precompiled_location: str, package_data: list[str], workspace: str) -> None: """Extract package data (binaries and other materials) from a precompiled wheel or local directory to the working directory. This allows skipping the compilation, and repackaging the binaries and Python files in the working directory to a new wheel. @@ -240,6 +251,11 @@ def extract_from_precompiled(precompiled_location: str, package_data: List[str], if dst_file.endswith(".yaml"): continue + # Keep source-owned package data local so Python-only schema edits + # layer over precompiled wheels. + if should_skip_precompiled_package_data(dst_file): + continue + # Skip .py files EXCEPT for generated C++ extension wrappers # (deep_gemm, deep_ep, flash_mla Python files are generated during build) if dst_file.endswith(".py"): @@ -302,6 +318,11 @@ def extract_from_precompiled(precompiled_location: str, package_data: List[str], if file.filename.endswith(".yaml"): continue + # Keep source-owned package data local so Python-only schema edits + # layer over precompiled wheels. + if should_skip_precompiled_package_data(file.filename): + continue + # Skip .py files EXCEPT for generated C++ extension wrappers # (deep_gemm, deep_ep, flash_mla Python files are generated during build) if file.filename.endswith(".py"): diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 7219d59cc76c..6425cb7d2f2d 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -76,13 +76,15 @@ from ..models.modeling_utils import (PretrainedConfig, QuantAlgo, QuantConfig, SpeculativeDecodingMode) from ..sampling_params import BatchedLogitsProcessor -from ..usage.config import TelemetryConfig, UsageContext # noqa: F401 +from ..usage.config import UsageContext # noqa: F401 +from ..usage.config import TelemetryConfig, TelemetryField from .build_cache import BuildCacheConfig from .tokenizer import TokenizerBase, tokenizer_factory from .utils import (StrictBaseModel, generate_api_docs_as_docstring, get_type_repr) TypeBaseModel = TypeVar("T", bound=BaseModel) +_TRTLLM_JSON_SCHEMA_EXTRA_ATTR = "_trtllm_json_schema_extra" if TYPE_CHECKING: from tensorrt_llm._torch.virtual_memory import \ @@ -94,8 +96,10 @@ def Field(default: Any = ..., *, status: Optional[Literal["prototype", "beta", "deprecated"]] = None, + telemetry: Optional[Union[bool, Dict[str, Any], + TelemetryField]] = None, **kwargs: Any) -> Any: - """Custom Field wrapper that adds status to json_schema_extra. + """Custom Field wrapper that adds status and telemetry metadata. Args: default: The default value for the field @@ -103,24 +107,70 @@ def Field(default: Any = ..., - None: Stable. - "beta": Recommended for use per the latest documentation. - "prototype": Not yet stable and subject to breaking changes; intended for experimentation only. + telemetry: Optional field-local telemetry override for LLM API config + capture. Type-safe fields (categorical/numeric) auto-enroll; pass + telemetry=TelemetryField.categorical(...) to opt a free-form str/Any field + in via an allowlist, or telemetry=False to opt a type-safe field out. **kwargs: All other arguments passed to the original Pydantic Field Returns: - A Pydantic FieldInfo object with the status added to json_schema_extra if provided + A Pydantic FieldInfo object with extra metadata added to + json_schema_extra if provided. """ + telemetry_explicit_exclude = telemetry is False + telemetry_requested = telemetry is not None and not telemetry_explicit_exclude - if status is not None: + if status is not None or telemetry_requested or telemetry_explicit_exclude: + trtllm_schema_extra: dict[str, Any] = {} json_schema_extra = kwargs.get('json_schema_extra', {}) + if status is not None: + trtllm_schema_extra['status'] = status + if telemetry_explicit_exclude: + # Honored opt-out sentinel: excludes a type-safe-but-sensitive field + # from capture. Consumed by build_capture_manifest's selection rule. + trtllm_schema_extra['telemetry'] = {"exclude": True} + elif telemetry_requested: + if isinstance(telemetry, TelemetryField): + telemetry_metadata = telemetry.as_json_schema_extra() + elif telemetry is True: + telemetry_metadata = {"kind": "value"} + elif isinstance(telemetry, dict): + telemetry_metadata = dict(telemetry) + else: + raise TypeError( + "telemetry must be bool, dict, or TelemetryField") + trtllm_schema_extra['telemetry'] = telemetry_metadata if isinstance(json_schema_extra, dict): - json_schema_extra['status'] = status + json_schema_extra = {**json_schema_extra, **trtllm_schema_extra} + elif callable(json_schema_extra): + original_json_schema_extra = json_schema_extra + + def merged_json_schema_extra(schema: dict[str, Any]) -> None: + original_extra = original_json_schema_extra(schema) + if isinstance(original_extra, dict): + schema.update(original_extra) + schema.update(trtllm_schema_extra) + + setattr(merged_json_schema_extra, _TRTLLM_JSON_SCHEMA_EXTRA_ATTR, + trtllm_schema_extra) + json_schema_extra = merged_json_schema_extra else: - # If json_schema_extra is not a dict, create a new dict with the status - json_schema_extra = {'status': status} + json_schema_extra = trtllm_schema_extra kwargs['json_schema_extra'] = json_schema_extra return PydanticField(default, **kwargs) +def _get_trtllm_json_schema_extra(field_info: Any) -> dict[str, Any]: + json_schema_extra = getattr(field_info, "json_schema_extra", None) + if callable(json_schema_extra): + json_schema_extra = getattr(json_schema_extra, + _TRTLLM_JSON_SCHEMA_EXTRA_ATTR, None) + if isinstance(json_schema_extra, dict): + return json_schema_extra + return {} + + class BaseCudaGraphConfig(StrictBaseModel): """Common configuration for CUDA graphs.""" # List of batch sizes to create CUDA graphs for. @@ -460,7 +510,7 @@ def needs_separate_short_long_cuda_graphs(self) -> bool: class RocketSparseAttentionConfig(BaseSparseAttentionConfig): """Configuration for RocketKV sparse attention.""" - algorithm: Literal["rocket"] = "rocket" + algorithm: Literal["rocket"] = Field(default="rocket") window_size: Optional[int] = Field( default=32, description="The window size for RocketKV.") kernel_size: Optional[int] = Field( @@ -470,11 +520,11 @@ class RocketSparseAttentionConfig(BaseSparseAttentionConfig): prompt_budget: Optional[int] = Field(default=2048, description="Prompt budget") page_size: Optional[int] = Field(default=4, description="Page size") - kt_cache_dtype: Optional[str] = Field( - default='float8_e5m2', - choices=['bfloat16', 'float8_e5m2'], - description="KT cache dtype", - ) + kt_cache_dtype: Optional[str] = Field(default='float8_e5m2', + choices=['bfloat16', 'float8_e5m2'], + description="KT cache dtype", + telemetry=TelemetryField.categorical( + 'bfloat16', 'float8_e5m2')) def supports_backend(self, backend: str) -> bool: return backend == "pytorch" @@ -485,7 +535,7 @@ def get_indices_block_size(self) -> int: class DeepSeekSparseAttentionConfig(BaseSparseAttentionConfig): """Configuration for DeepSeek Sparse Attention.""" - algorithm: Literal["dsa"] = "dsa" + algorithm: Literal["dsa"] = Field(default="dsa") index_n_heads: Optional[int] = Field( default=None, description="The number of heads for the indexer.") index_head_dim: Optional[int] = Field( @@ -530,8 +580,7 @@ class DeepSeekSparseAttentionConfig(BaseSparseAttentionConfig): description= "Data type used for the indexer K cache. `fp4` requires Blackwell+ " "(SM>=100) and index_head_dim=128, it can halve the indexer K cache " - "per-token footprint from 132 B to 68 B.", - ) + "per-token footprint from 132 B to 68 B.") @model_validator(mode="after") def _validate_indexer_k_dtype(self): @@ -575,7 +624,7 @@ def needs_separate_short_long_cuda_graphs(self) -> bool: class SkipSoftmaxAttentionConfig(BaseSparseAttentionConfig): """Configuration for skip softmax attention.""" - algorithm: Literal["skip_softmax"] = "skip_softmax" + algorithm: Literal["skip_softmax"] = Field(default="skip_softmax") threshold_scale_factor: Optional[Union[float, Dict[str, float]]] = Field( default=None, description="The threshold scale factor for skip softmax attention.") @@ -1220,7 +1269,8 @@ class KvCacheConnectorConfig(StrictBaseModel): None, description="Named connector preset (e.g. 'lmcache'). " "When set, connector_module/scheduler_class/worker_class are " - "auto-populated from the preset registry.") + "auto-populated from the preset registry.", + telemetry=TelemetryField.categorical('lmcache', 'lmcache-mp', 'kvbm')) connector_module: Optional[str] = Field( None, description= @@ -1290,7 +1340,7 @@ def validate_calibration_file_path(self) -> 'LayerwiseBenchmarksConfig': class MedusaDecodingConfig(DecodingBaseConfig): - decoding_type: Literal["Medusa"] = "Medusa" + decoding_type: Literal["Medusa"] = Field(default="Medusa") medusa_choices: Optional[List[List[int]]] = Field( default=None, description= @@ -1313,7 +1363,7 @@ def supports_backend(self, backend: str) -> bool: class EagleDecodingConfig(DecodingBaseConfig): - decoding_type: Literal["Eagle"] = "Eagle" + decoding_type: Literal["Eagle"] = Field(default="Eagle") eagle_choices: Optional[List[List[int]]] = Field( default=None, description= @@ -1525,7 +1575,7 @@ class SAEnhancerConfig(StrictBaseModel): class Eagle3DecodingConfig(EagleDecodingConfig): - decoding_type: Literal["Eagle3"] = "Eagle3" + decoding_type: Literal["Eagle3"] = Field(default="Eagle3") # Backs the dynamic-tree worker's pre-allocated, batch-indexed CUDA buffers # (draft_tokens_buffer, history_*_buffer, tree_mask_buffer, etc. in @@ -1546,7 +1596,7 @@ class Eagle3DecodingConfig(EagleDecodingConfig): class SaveHiddenStatesDecodingConfig(DecodingBaseConfig): - decoding_type: Literal["SaveState"] = "SaveState" + decoding_type: Literal["SaveState"] = Field(default="SaveState") output_directory: str = Field( description= "Directory path where hidden states data files will be saved. The directory is created if it does not exist." @@ -1622,7 +1672,7 @@ def num_capture_layers(self): class UserProvidedDecodingConfig(DecodingBaseConfig): - decoding_type: Literal["User_Provided"] = "User_Provided" + decoding_type: Literal["User_Provided"] = Field(default="User_Provided") # Cannot use real type annotations due to circular imports drafter: object = Field( description= @@ -1644,7 +1694,7 @@ def set_max_total_draft_tokens(self): class NGramDecodingConfig(DecodingBaseConfig): """Configuration for NGram drafter speculative decoding.""" - decoding_type: Literal["NGram"] = "NGram" + decoding_type: Literal["NGram"] = Field(default="NGram") max_matching_ngram_size: PositiveInt = Field( default=2, description= @@ -1684,7 +1734,7 @@ class SADecodingConfig(DecodingBaseConfig): To combine SA with a neural drafter (Eagle3, MTP, PARD) instead of using it standalone, pass :class:`SAEnhancerConfig` via ``sa_config``. """ - decoding_type: Literal["SA"] = "SA" + decoding_type: Literal["SA"] = Field(default="SA") max_matching_ngram_size: int = Field( default=-1, description="Positive value (e.g., 3): fixed-size ngram matching. " @@ -1736,7 +1786,7 @@ def supports_backend(self, backend: str) -> bool: class DraftTargetDecodingConfig(DecodingBaseConfig): - decoding_type: Literal["Draft_Target"] = "Draft_Target" + decoding_type: Literal["Draft_Target"] = Field(default="Draft_Target") _draft_target_one_model: bool = PrivateAttr(True) @model_validator(mode="after") @@ -1762,7 +1812,7 @@ def spec_dec_mode(self): class MTPDecodingConfig(DecodingBaseConfig): - decoding_type: Literal["MTP"] = "MTP" + decoding_type: Literal["MTP"] = Field(default="MTP") use_relaxed_acceptance_for_thinking: bool = Field( default=False, description= @@ -1903,7 +1953,7 @@ class PARDDecodingConfig(DecodingBaseConfig): "If None, it will be read from the draft model config (typically vocab_size)." ) - decoding_type: Literal["PARD"] = "PARD" + decoding_type: Literal["PARD"] = Field(default="PARD") sa_config: Optional[SAEnhancerConfig] = Field( default=None, @@ -1959,7 +2009,7 @@ class DFlashDecodingConfig(DecodingBaseConfig): "for cross-attention in the draft model. If None, read from the draft " "model config (dflash_config.target_layer_ids).") - decoding_type: Literal["DFlash"] = "DFlash" + decoding_type: Literal["DFlash"] = Field(default="DFlash") @model_validator(mode="after") def set_max_total_draft_tokens(self): @@ -1994,7 +2044,7 @@ class AutoDecodingConfig(DecodingBaseConfig): Attributes that are inherited from the base class are ignored. """ - decoding_type: Literal["AUTO"] = "AUTO" + decoding_type: Literal["AUTO"] = Field(default="AUTO") @model_validator(mode="after") def set_max_total_draft_tokens(self): @@ -2597,7 +2647,7 @@ def _to_pybind(self): class LookaheadDecodingConfig(DecodingBaseConfig, PybindMirror): """Configuration for lookahead speculative decoding.""" - decoding_type: Literal["Lookahead"] = "Lookahead" + decoding_type: Literal["Lookahead"] = Field(default="Lookahead") max_window_size: PositiveInt = Field( default=_LookaheadDecodingConfig.get_default_lookahead_decoding_window( ), @@ -2793,8 +2843,9 @@ class KvCacheConfig(StrictBaseModel, PybindMirror): dtype: str = Field( default="auto", description= - "The data type to use for the KV cache. Use 'auto' to follow checkpoint metadata, otherwise force the specified dtype." - ) + "The data type to use for the KV cache. Use 'auto' to follow checkpoint metadata, otherwise force the specified dtype.", + telemetry=TelemetryField.categorical("auto", "float16", "bfloat16", + "float32", "fp8", "nvfp4")) # This is a pure python field, not a pybind field. It is only for the Pytorch backend. mamba_ssm_cache_dtype: Literal[ @@ -3129,7 +3180,9 @@ class BaseLlmArgs(StrictBaseModel): description="The tensor parallel size.") dtype: str = Field(default="auto", - description="The data type to use for the model.") + description="The data type to use for the model.", + telemetry=TelemetryField.categorical( + "auto", "float16", "bfloat16", "float32")) revision: Optional[str] = Field( default=None, description="The revision to use for the model.") @@ -3292,7 +3345,12 @@ class BaseLlmArgs(StrictBaseModel): reasoning_parser: Optional[str] = Field( default=None, description="The parser to separate reasoning content from output.", - status="prototype") + status="prototype", + telemetry=TelemetryField.categorical('auto', 'deepseek-r1', 'laguna', + 'qwen3', 'qwen3_5', 'minimax_m2', + 'minimax_m2_append_think', + 'nano-v3', 'gemma4', 'kimi_k2', + 'kimi_k25')) # TODO[Superjomn]: To deprecate this config. decoding_config: Optional[object] = Field( @@ -3324,7 +3382,8 @@ class BaseLlmArgs(StrictBaseModel): exclude_json_schema=True, # hide from API references validate_default=True, status="deprecated", - ) + telemetry=TelemetryField.categorical('pytorch', 'tensorrt', + '_autodeploy')) return_perf_metrics: bool = Field(default=False, description="Return perf metrics.", @@ -3898,8 +3957,7 @@ class ModelExpressConfig(StrictBaseModel): "discovery. When unset, TRT-LLM first probes for existing sources: " "no source uses a short 30-second fallback cap, while an existing " "source uses modelexpress's default wait for long donor loads.", - status="prototype", - ) + status="prototype") preshard_strategy: str = Field( default="per_module", @@ -3907,7 +3965,7 @@ class ModelExpressConfig(StrictBaseModel): "TP-sharded for the local rank. Only 'per_module' is supported in " "this MX-only PR; 'global' requires LoadFormat.PRESHARDED.", status="prototype", - ) + telemetry=TelemetryField.categorical('per_module')) @model_validator(mode="after") def validate_preshard_strategy(self) -> 'ModelExpressConfig': @@ -3958,8 +4016,7 @@ class GmsConfig(StrictBaseModel): default="auto", description="GMS operating mode: 'auto' requests RW or RO, 'rw' " "requires writer mode, and 'ro' requires read-only mode.", - status="prototype", - ) + status="prototype") tag: str = Field( default="weights", @@ -4100,9 +4157,14 @@ def infer_cuda_graph_config_mode(cls, v): description="DWDP (Distributed Weight Data Parallelism) config.", status="prototype") - attn_backend: str = Field(default='TRTLLM', - description="Attention backend to use.", - status="beta") + attn_backend: str = Field( + default='TRTLLM', + description="Attention backend to use.", + status="beta", + # Recognized values mirror get_attention_backend dispatch in + # tensorrt_llm/_torch/attention_backend/utils.py. + telemetry=TelemetryField.categorical("VANILLA", "TRTLLM", "FLASHINFER", + "FLASHINFER_STAR_ATTENTION")) sampler_type: Union[str, SamplerType] = Field( default=SamplerType.auto, @@ -4111,8 +4173,9 @@ def infer_cuda_graph_config_mode(cls, v): "TRTLLMSampler is deprecated and will be removed in release 1.4.", status="deprecated", deprecated= - "This parameter will be removed in release 1.4. TorchSampler will be the default sampler." - ) + "This parameter will be removed in release 1.4. TorchSampler will be the default sampler.", + telemetry=TelemetryField.categorical('TRTLLMSampler', 'TorchSampler', + 'auto')) sampler_force_async_worker: bool = Field( default=False, @@ -4194,29 +4257,28 @@ def infer_cuda_graph_config_mode(cls, v): load_format: Union[str, LoadFormat] = Field( default=LoadFormat.AUTO, description= - "How to load the model weights. By default, detect the weight type from the model checkpoint." - ) + "How to load the model weights. By default, detect the weight type from the model checkpoint.", + telemetry=TelemetryField.categorical("auto", "dummy", "vision_only", + "gms")) enable_min_latency: bool = Field( default=False, description= "If true, enable min-latency mode. Currently only used for Llama4.", - status="beta", - ) + status="beta") # TODO: make this a per-request parameter stream_interval: PositiveInt = Field( default=1, description= "The iteration interval to create responses under the streaming mode. " - "Set this to a larger value when the batch size is large, which helps reduce the streaming overhead.", + "Set this to a larger value when the batch size is large, which helps reduce the streaming overhead." ) force_dynamic_quantization: bool = Field( default=False, description="If true, force dynamic quantization. Defaults to False.", - status="prototype", - ) + status="prototype") allreduce_strategy: Optional[Literal[ 'AUTO', 'NCCL', 'UB', 'MINLATENCY', 'ONESHOT', 'TWOSHOT', @@ -4274,8 +4336,7 @@ def infer_cuda_graph_config_mode(cls, v): default=False, description= "Only load/execute the vision encoder part of the full model. Defaults to False.", - status="prototype", - ) + status="prototype") encode_only: bool = Field( default=False, @@ -4286,8 +4347,7 @@ def infer_cuda_graph_config_mode(cls, v): "models (BERT, RoBERTa, reward models) and decoder models used in " "single-prefill mode (e.g., extracting embeddings). When False " "(default), uses the standard generate() path.", - status="prototype", - ) + status="prototype") ray_worker_extension_cls: Optional[str] = Field( default=None, @@ -4326,33 +4386,28 @@ def infer_cuda_graph_config_mode(cls, v): description="Enable the resource governor for runtime cache management " "operations such as KV cache truncation. This adds a per-iteration " "broadcast collective.", - status="prototype", - ) + status="prototype") # fp8 cute dsl configs use_cute_dsl_blockscaling_mm: bool = Field( default=False, description="If true, use CuTe DSL fp8 blockscaling mm implementation.", - status="prototype", - ) + status="prototype") use_cute_dsl_blockscaling_bmm: bool = Field( default=False, description="If true, use CuTe DSL fp8 blockscaling bmm implementation.", - status="prototype", - ) + status="prototype") # bf16 cute dsl configs use_cute_dsl_bf16_bmm: bool = Field( default=False, description= "If true, use CuTe DSL bf16 persistent GEMM for BMM on Blackwell.", - status="prototype", - ) + status="prototype") use_cute_dsl_bf16_gemm: bool = Field( default=False, description= "If true, use CuTe DSL bf16 persistent GEMM for Linear layers on Blackwell.", - status="prototype", - ) + status="prototype") # PrivateVars _quant_config: Optional[QuantConfig] = PrivateAttr(default=None) @@ -4361,14 +4416,12 @@ def infer_cuda_graph_config_mode(cls, v): default=False, description= "Disable the use of FlashInfer.sampling. This option is likely to be removed in the future.", - status="prototype", - ) + status="prototype") max_stats_len: int = Field( default=1000, description="The max number of performance statistic entries.", - status="prototype", - ) + status="prototype") layer_wise_benchmarks_config: LayerwiseBenchmarksConfig = Field( default_factory=LayerwiseBenchmarksConfig, @@ -4762,10 +4815,11 @@ def warn_on_unstable_feature_usage(self) -> 'TorchLlmArgs': for field_name in set_fields: field_info = self.model_fields.get(field_name) - if not field_info or not field_info.json_schema_extra: + if not field_info: continue - status = field_info.json_schema_extra.get('status', None) + status = _get_trtllm_json_schema_extra(field_info).get( + 'status', None) if status in ('beta', 'prototype'): logger.warning( diff --git a/tensorrt_llm/models/modeling_utils.py b/tensorrt_llm/models/modeling_utils.py index 66ff49d8db4c..df8d559ea9e5 100644 --- a/tensorrt_llm/models/modeling_utils.py +++ b/tensorrt_llm/models/modeling_utils.py @@ -133,7 +133,9 @@ class QuantConfig(StrictBaseModel): """Serializable quantization configuration class, part of the PretrainedConfig.""" quant_algo: Optional[QuantAlgo] = Field( - default=None, description="Quantization algorithm.") + default=None, + description="Quantization algorithm.", + json_schema_extra={"telemetry": True}) kv_cache_quant_algo: Optional[QuantAlgo] = Field( default=None, description="KV cache quantization algorithm.") group_size: Optional[int] = Field( diff --git a/tensorrt_llm/usage/__init__.py b/tensorrt_llm/usage/__init__.py index 5a753234ff44..447054c2f106 100644 --- a/tensorrt_llm/usage/__init__.py +++ b/tensorrt_llm/usage/__init__.py @@ -22,7 +22,7 @@ - Set environment variable TELEMETRY_DISABLED=true or TELEMETRY_DISABLED=1 - Set environment variable DO_NOT_TRACK=1 - Create file ~/.config/trtllm/do_not_track - - Pass TelemetryConfig(disabled=True) to LLM() or --telemetry-disabled via CLI + - Pass TelemetryConfig(disabled=True) to LLM() or --no-telemetry via CLI - Automatically disabled in CI/test environments (override with TRTLLM_USAGE_FORCE_ENABLED=1) """ @@ -30,12 +30,14 @@ from tensorrt_llm.usage import usage_lib as _usage_lib TelemetryConfig = _config.TelemetryConfig +TelemetryField = _config.TelemetryField UsageContext = _config.UsageContext report_usage = _usage_lib.report_usage is_usage_stats_enabled = _usage_lib.is_usage_stats_enabled __all__ = [ "TelemetryConfig", + "TelemetryField", "UsageContext", "report_usage", "is_usage_stats_enabled", diff --git a/tensorrt_llm/usage/config.py b/tensorrt_llm/usage/config.py index f08359f1ed68..2ee680141908 100644 --- a/tensorrt_llm/usage/config.py +++ b/tensorrt_llm/usage/config.py @@ -22,11 +22,19 @@ Imported by tensorrt_llm.llmapi.llm_args for use in BaseLlmArgs. """ +from dataclasses import dataclass from enum import Enum +from typing import Any, Literal, Optional -from pydantic import Field +from pydantic import BaseModel, ConfigDict, Field -from tensorrt_llm.llmapi.utils import StrictBaseModel + +class _StrictUsageBaseModel(BaseModel): + """Strict usage-local base. Same extra=forbid contract as llmapi StrictBaseModel.""" + + # Keep usage import light. Do not import llmapi.utils here: it pulls torch + # and HF deps. Needed contract stays same: extra fields forbidden. + model_config = ConfigDict(extra="forbid") class UsageContext(str, Enum): @@ -39,7 +47,38 @@ class UsageContext(str, Enum): CLI_EVAL = "cli_eval" -class TelemetryConfig(StrictBaseModel): +@dataclass(frozen=True) +class TelemetryField: + """Field-local opt-in metadata for LLM API config telemetry capture.""" + + kind: Literal["value", "categorical"] = "value" + converter: Optional[Literal["allowlist"]] = None + allowed_values: Optional[tuple[Any, ...]] = None + + @classmethod + def categorical(cls, *allowed_values: Any) -> "TelemetryField": + """Build a categorical allowlist field from the recognized values. + + Shorthand for the common bare-string allowlist case: marks the field + categorical and pins capture to the explicit allowed values via the + allowlist converter. + """ + return cls( + kind="categorical", + converter="allowlist", + allowed_values=tuple(allowed_values), + ) + + def as_json_schema_extra(self) -> dict[str, Any]: + data: dict[str, Any] = {"kind": self.kind} + if self.converter is not None: + data["converter"] = self.converter + if self.allowed_values is not None: + data["allowed_values"] = list(self.allowed_values) + return data + + +class TelemetryConfig(_StrictUsageBaseModel): """Telemetry configuration for usage data collection. Controls opt-out behavior and tracks which entry point invoked TRT-LLM. diff --git a/tensorrt_llm/usage/llm_args_golden_manifest.json b/tensorrt_llm/usage/llm_args_golden_manifest.json new file mode 100644 index 000000000000..93090ffa5e73 --- /dev/null +++ b/tensorrt_llm/usage/llm_args_golden_manifest.json @@ -0,0 +1,3931 @@ +{ + "TorchLlmArgs": [ + { + "allowed_values": [ + "AUTO", + "NCCL", + "UB", + "MINLATENCY", + "ONESHOT", + "TWOSHOT", + "LOWPRECISION", + "MNNVL", + "NCCL_SYMMETRIC" + ], + "annotation": "Optional[Literal['AUTO', 'NCCL', 'UB', 'MINLATENCY', 'ONESHOT', 'TWOSHOT', 'LOWPRECISION', 'MNNVL', 'NCCL_SYMMETRIC']]", + "converter": "", + "kind": "categorical", + "path": "allreduce_strategy" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "attention_dp_config.batching_wait_iters" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "attention_dp_config.enable_balance" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "attention_dp_config.enable_kv_cache_aware_routing" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "attention_dp_config.kv_cache_routing_cold_start_warmup" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "attention_dp_config.kv_cache_routing_fair_share_multiplier" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "attention_dp_config.kv_cache_routing_load_balance_weight" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "attention_dp_config.kv_cache_routing_match_rate_threshold" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "attention_dp_config.timeout_iters" + }, + { + "allowed_values": [ + "VANILLA", + "TRTLLM", + "FLASHINFER", + "FLASHINFER_STAR_ATTENTION" + ], + "annotation": "", + "converter": "allowlist", + "kind": "categorical", + "path": "attn_backend" + }, + { + "allowed_values": [ + "pytorch" + ], + "annotation": "Literal['pytorch']", + "converter": "", + "kind": "categorical", + "path": "backend" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "batch_wait_max_tokens_ratio" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "batch_wait_timeout_iters" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "batch_wait_timeout_ms" + }, + { + "allowed_values": [ + "DEFAULT", + "UCX", + "NIXL", + "MOONCAKE", + "MPI" + ], + "annotation": "Optional[Literal['DEFAULT', 'UCX', 'NIXL', 'MOONCAKE', 'MPI']]", + "converter": "", + "kind": "categorical", + "path": "cache_transceiver_config.backend" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Gt(gt=0)]]", + "converter": "", + "kind": "value", + "path": "cache_transceiver_config.kv_transfer_sender_future_timeout_ms" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Gt(gt=0)]]", + "converter": "", + "kind": "value", + "path": "cache_transceiver_config.kv_transfer_timeout_ms" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cache_transceiver_config.max_tokens_in_buffer" + }, + { + "allowed_values": [ + "CPP", + "PYTHON" + ], + "annotation": "Optional[Literal['CPP', 'PYTHON']]", + "converter": "", + "kind": "categorical", + "path": "cache_transceiver_config.transceiver_runtime" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "context_parallel_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cp_config.block_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cp_config.cp_anchor_size" + }, + { + "allowed_values": [ + "ULYSSES", + "STAR", + "RING", + "HELIX" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "cp_config.cp_type" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cp_config.fifo_version" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cp_config.tokens_per_block" + }, + { + "allowed_values": [], + "annotation": "Optional[bool]", + "converter": "", + "kind": "value", + "path": "cp_config.use_nccl_for_alltoall" + }, + { + "allowed_values": [], + "annotation": "Optional[List[int]]", + "converter": "", + "kind": "value", + "path": "cuda_graph_config.batch_sizes" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "cuda_graph_config.enable_padding" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "cuda_graph_config.max_batch_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "cuda_graph_config.max_num_token" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "cuda_graph_config.max_seq_len" + }, + { + "allowed_values": [ + "decode", + "encode" + ], + "annotation": "Literal['decode']", + "converter": "", + "kind": "categorical", + "path": "cuda_graph_config.mode" + }, + { + "allowed_values": [], + "annotation": "Optional[List[Annotated[int, Gt(gt=0)]]]", + "converter": "", + "kind": "value", + "path": "cuda_graph_config.num_tokens" + }, + { + "allowed_values": [], + "annotation": "Optional[List[Annotated[int, Gt(gt=0)]]]", + "converter": "", + "kind": "value", + "path": "cuda_graph_config.seq_lens" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "disable_flashinfer_sampling" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "disable_overlap_scheduler" + }, + { + "allowed_values": [ + "auto", + "float16", + "bfloat16", + "float32" + ], + "annotation": "", + "converter": "allowlist", + "kind": "categorical", + "path": "dtype" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "dwdp_config.contention_opt" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "dwdp_config.dwdp_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "dwdp_config.num_experts_per_worker" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "dwdp_config.num_groups" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "dwdp_config.num_prefetch_experts" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_attention_dp" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_autotuner" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_chunked_prefill" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_early_first_token_response" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_energy_metrics" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_iter_perf_stats" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_iter_req_stats" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_layerwise_nvtx_marker" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_lm_head_tp_in_adp" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_lora" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_min_latency" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_resource_governor" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_speculative_beam_history_d2h" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "encode_only" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "force_dynamic_quantization" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "garbage_collection_gen0_threshold" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "gather_generation_logits" + }, + { + "allowed_values": [ + "auto", + "rw", + "ro" + ], + "annotation": "Literal['auto', 'rw', 'ro']", + "converter": "", + "kind": "categorical", + "path": "gms_config.mode" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "gpus_per_node" + }, + { + "allowed_values": [ + "xgrammar", + "llguidance" + ], + "annotation": "Optional[Literal['xgrammar', 'llguidance']]", + "converter": "", + "kind": "categorical", + "path": "guided_decoding_backend" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "iter_stats_max_iterations" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.attention_dp_events_gather_period_ms" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.copy_on_partial_reuse" + }, + { + "allowed_values": [], + "annotation": "Optional[float]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.cross_kv_cache_fraction" + }, + { + "allowed_values": [ + "auto", + "float16", + "bfloat16", + "float32", + "fp8", + "nvfp4" + ], + "annotation": "", + "converter": "allowlist", + "kind": "categorical", + "path": "kv_cache_config.dtype" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.enable_block_reuse" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.enable_partial_reuse" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.event_buffer_max_size" + }, + { + "allowed_values": [], + "annotation": "Optional[float]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.free_gpu_memory_fraction" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.host_cache_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.iteration_stats_interval" + }, + { + "allowed_values": [ + "auto", + "float16", + "bfloat16", + "float32" + ], + "annotation": "Literal['auto', 'float16', 'bfloat16', 'float32']", + "converter": "", + "kind": "categorical", + "path": "kv_cache_config.mamba_ssm_cache_dtype" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.mamba_ssm_philox_rounds" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.mamba_ssm_stochastic_rounding" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.mamba_state_cache_interval" + }, + { + "allowed_values": [], + "annotation": "Optional[List[int]]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.max_attention_window" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.max_gpu_total_bytes" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.max_tokens" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.max_util_for_resume" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.secondary_offload_min_priority" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.sink_token_length" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.tokens_per_block" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.use_kv_cache_manager_v2" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.use_uvm" + }, + { + "allowed_values": [ + "lmcache", + "lmcache-mp", + "kvbm" + ], + "annotation": "Optional[str]", + "converter": "allowlist", + "kind": "categorical", + "path": "kv_connector_config.connector" + }, + { + "allowed_values": [], + "annotation": "Optional[List[int]]", + "converter": "", + "kind": "value", + "path": "layer_wise_benchmarks_config.calibration_layer_indices" + }, + { + "allowed_values": [ + "NONE", + "MARK", + "COLLECT" + ], + "annotation": "Literal['NONE', 'MARK', 'COLLECT']", + "converter": "", + "kind": "categorical", + "path": "layer_wise_benchmarks_config.calibration_mode" + }, + { + "allowed_values": [ + "auto", + "dummy", + "vision_only", + "gms" + ], + "annotation": "Union[str, tensorrt_llm.llmapi.llm_args.LoadFormat]", + "converter": "allowlist", + "kind": "categorical", + "path": "load_format" + }, + { + "allowed_values": [ + "hf", + "nemo" + ], + "annotation": "Literal['hf', 'nemo']", + "converter": "", + "kind": "categorical", + "path": "lora_config.lora_ckpt_source" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "lora_config.max_cpu_loras" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "lora_config.max_lora_rank" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "lora_config.max_loras" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "lora_config.swap_gate_up_proj_lora_b_weight" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_batch_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_beam_width" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_input_len" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_num_tokens" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_seq_len" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "max_stats_len" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "mm_encoder_only" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "moe_cluster_parallel_size" + }, + { + "allowed_values": [ + "AUTO", + "CUTLASS", + "CUTEDSL", + "WIDEEP", + "TRTLLM", + "DEEPGEMM", + "DENSEGEMM", + "VANILLA", + "TRITON" + ], + "annotation": "Literal['AUTO', 'CUTLASS', 'CUTEDSL', 'WIDEEP', 'TRTLLM', 'DEEPGEMM', 'DENSEGEMM', 'VANILLA', 'TRITON']", + "converter": "", + "kind": "categorical", + "path": "moe_config.backend" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "moe_config.disable_finalize_fusion" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "moe_config.max_num_tokens" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "moe_config.use_low_precision_moe_combine" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "moe_expert_parallel_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "moe_tensor_parallel_size" + }, + { + "allowed_values": [ + "per_module" + ], + "annotation": "", + "converter": "allowlist", + "kind": "categorical", + "path": "mx_config.preshard_strategy" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Ge(ge=0)]]", + "converter": "", + "kind": "value", + "path": "mx_config.server_query_timeout_s" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "num_postprocess_workers" + }, + { + "allowed_values": [ + "cutlass", + "cublaslt", + "cutedsl", + "cuda_core" + ], + "annotation": "List[Literal['cutlass', 'cublaslt', 'cutedsl', 'cuda_core']]", + "converter": "", + "kind": "value", + "path": "nvfp4_gemm_config.allowed_backends" + }, + { + "allowed_values": [ + "rpc", + "ray" + ], + "annotation": "Optional[Literal['rpc', 'ray']]", + "converter": "", + "kind": "categorical", + "path": "orchestrator_type" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.device_cache_percent" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.host_cache_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.max_adapter_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.max_pages_per_block_device" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.max_pages_per_block_host" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_copy_streams" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_device_module_layer" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_ensure_workers" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_host_module_layer" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_put_workers" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.optimal_adapter_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "perf_metrics_max_requests" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "pipeline_parallel_size" + }, + { + "allowed_values": [], + "annotation": "Optional[List[int]]", + "converter": "", + "kind": "value", + "path": "pp_partition" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "print_iter_log" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.e2e_request_latency_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.request_decode_time_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.request_inference_time_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.request_prefill_time_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.request_queue_time_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.time_per_output_token_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.time_to_first_token_buckets" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "ray_placement_config.defer_workers_init" + }, + { + "allowed_values": [], + "annotation": "Optional[float]", + "converter": "", + "kind": "value", + "path": "ray_placement_config.per_worker_gpu_share" + }, + { + "allowed_values": [], + "annotation": "Optional[List[List[int]]]", + "converter": "", + "kind": "value", + "path": "ray_placement_config.placement_bundle_indices" + }, + { + "allowed_values": [ + "auto", + "deepseek-r1", + "laguna", + "qwen3", + "qwen3_5", + "minimax_m2", + "minimax_m2_append_think", + "nano-v3", + "gemma4", + "kimi_k2", + "kimi_k25" + ], + "annotation": "Optional[str]", + "converter": "allowlist", + "kind": "categorical", + "path": "reasoning_parser" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "reorder_policy_config.policy_args.agent_inflight_seq_num" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "reorder_policy_config.policy_args.agent_percentage" + }, + { + "allowed_values": [ + "AgentTree" + ], + "annotation": "Optional[Literal['AgentTree']]", + "converter": "", + "kind": "categorical", + "path": "reorder_policy_config.policy_name" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "request_stats_max_iterations" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "return_perf_metrics" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sampler_force_async_worker" + }, + { + "allowed_values": [ + "TRTLLMSampler", + "TorchSampler", + "auto" + ], + "annotation": "Union[str, tensorrt_llm.llmapi.llm_args.SamplerType]", + "converter": "allowlist", + "kind": "categorical", + "path": "sampler_type" + }, + { + "allowed_values": [ + "MAX_UTILIZATION", + "GUARANTEED_NO_EVICT", + "STATIC_BATCH" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "scheduler_config.capacity_scheduler_policy" + }, + { + "allowed_values": [ + "FIRST_COME_FIRST_SERVED", + "EQUAL_PROGRESS", + "FORCE_CHUNK" + ], + "annotation": "Optional[tensorrt_llm.llmapi.llm_args.ContextChunkingPolicy]", + "converter": "", + "kind": "categorical", + "path": "scheduler_config.context_chunking_policy" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "scheduler_config.dynamic_batch_config.dynamic_batch_moving_average_window" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "scheduler_config.dynamic_batch_config.enable_batch_size_tuning" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "scheduler_config.dynamic_batch_config.enable_max_num_tokens_tuning" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "scheduler_config.use_python_scheduler" + }, + { + "allowed_values": [ + "fcfs", + "priority" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "scheduler_config.waiting_queue_policy" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "skip_tokenizer_init" + }, + { + "allowed_values": [ + "dsa", + "rocket", + "skip_softmax" + ], + "annotation": "Literal['dsa']", + "converter": "", + "kind": "categorical", + "path": "sparse_attention_config.algorithm" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.enable_heuristic_topk" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.index_head_dim" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.index_n_heads" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.index_topk" + }, + { + "allowed_values": [ + "fp8", + "fp4" + ], + "annotation": "Literal['fp8', 'fp4']", + "converter": "", + "kind": "categorical", + "path": "sparse_attention_config.indexer_k_dtype" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.indexer_max_chunk_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.indexer_rope_interleave" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.kernel_size" + }, + { + "allowed_values": [ + "bfloat16", + "float8_e5m2" + ], + "annotation": "Optional[str]", + "converter": "allowlist", + "kind": "categorical", + "path": "sparse_attention_config.kt_cache_dtype" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.page_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.prompt_budget" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.q_split_threshold" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.seq_len_threshold" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.skip_indexer_for_short_seqs" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.topk" + }, + { + "allowed_values": [], + "annotation": "Union[int, float, NoneType]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.topr" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.use_cute_dsl_paged_mqa_logits" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.use_cute_dsl_topk" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.window_size" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[float, Ge(ge=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.acceptance_length_threshold" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Ge(ge=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.acceptance_window" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.allow_advanced_sampling" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.begin_thinking_phase_token" + }, + { + "allowed_values": [ + "AUTO", + "DFlash", + "Draft_Target", + "Eagle3", + "Eagle", + "Lookahead", + "MTP", + "Medusa", + "NGram", + "PARD", + "SA", + "SaveState", + "User_Provided" + ], + "annotation": "Literal['AUTO']", + "converter": "", + "kind": "categorical", + "path": "speculative_config.decoding_type" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.dynamic_tree_max_topK" + }, + { + "allowed_values": [], + "annotation": "Optional[Set[int]]", + "converter": "", + "kind": "value", + "path": "speculative_config.eagle3_layers_to_capture" + }, + { + "allowed_values": [ + "llama3", + "mistral_large3" + ], + "annotation": "Literal['llama3', 'mistral_large3']", + "converter": "", + "kind": "categorical", + "path": "speculative_config.eagle3_model_arch" + }, + { + "allowed_values": [], + "annotation": "Optional[bool]", + "converter": "", + "kind": "value", + "path": "speculative_config.eagle3_one_model" + }, + { + "allowed_values": [], + "annotation": "Optional[List[List[int]]]", + "converter": "", + "kind": "value", + "path": "speculative_config.eagle_choices" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.enable_global_pool" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.end_thinking_phase_token" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Gt(gt=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.global_pool_size" + }, + { + "allowed_values": [], + "annotation": "Optional[bool]", + "converter": "", + "kind": "value", + "path": "speculative_config.greedy_sampling" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.is_keep_all" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.is_public_pool" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.is_use_oldest" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.mask_token_id" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Gt(gt=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.max_concurrency" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Ge(ge=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.max_draft_len" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.max_matching_ngram_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.max_ngram_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.max_non_leaves_per_layer" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.max_total_draft_tokens" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.max_verification_set_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.max_window_size" + }, + { + "allowed_values": [], + "annotation": "Optional[List[List[int]]]", + "converter": "", + "kind": "value", + "path": "speculative_config.medusa_choices" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.mtp_eagle_one_model" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.num_eagle_layers" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.num_medusa_heads" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.num_nextn_predict_layers" + }, + { + "allowed_values": [], + "annotation": "Optional[float]", + "converter": "", + "kind": "value", + "path": "speculative_config.posterior_threshold" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.relaxed_delta" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.relaxed_topk" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.sa_config.enable_global_pool" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.sa_config.threshold" + }, + { + "allowed_values": [], + "annotation": "Optional[List[int]]", + "converter": "", + "kind": "value", + "path": "speculative_config.target_layer_ids" + }, + { + "allowed_values": [], + "annotation": "Optional[bool]", + "converter": "", + "kind": "value", + "path": "speculative_config.use_dynamic_tree" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.use_mtp_vanilla" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.use_rejection_sampling" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.use_relaxed_acceptance_for_thinking" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.write_interval" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "stream_interval" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "telemetry_config.disabled" + }, + { + "allowed_values": [ + "unknown", + "llm_class", + "cli_serve", + "cli_bench", + "cli_eval" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "telemetry_config.usage_context" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "tensor_parallel_size" + }, + { + "allowed_values": [ + "auto", + "slow" + ], + "annotation": "Literal['auto', 'slow']", + "converter": "", + "kind": "categorical", + "path": "tokenizer_mode" + }, + { + "allowed_values": [], + "annotation": "Optional[List[Annotated[int, Gt(gt=0)]]]", + "converter": "", + "kind": "value", + "path": "torch_compile_config.capture_num_tokens" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "torch_compile_config.enable_fullgraph" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "torch_compile_config.enable_inductor" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "torch_compile_config.enable_piecewise_cuda_graph" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "torch_compile_config.enable_userbuffers" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "torch_compile_config.max_num_streams" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "trust_remote_code" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "use_cute_dsl_bf16_bmm" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "use_cute_dsl_bf16_gemm" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "use_cute_dsl_blockscaling_bmm" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "use_cute_dsl_blockscaling_mm" + }, + { + "allowed_values": [], + "annotation": "Optional[float]", + "converter": "", + "kind": "value", + "path": "video_pruning_rate" + } + ], + "TrtLlmArgs": [ + { + "allowed_values": [ + "pytorch", + "tensorrt", + "_autodeploy" + ], + "annotation": "Optional[str]", + "converter": "allowlist", + "kind": "categorical", + "path": "backend" + }, + { + "allowed_values": [ + "STATIC", + "INFLIGHT" + ], + "annotation": "Optional[tensorrt_llm.llmapi.llm_args.BatchingType]", + "converter": "", + "kind": "categorical", + "path": "batching_type" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.dry_run" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.enable_debug_output" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "build_config.force_num_profiles" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.gather_context_logits" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.gather_generation_logits" + }, + { + "allowed_values": [ + "continuous", + "paged", + "disabled" + ], + "annotation": "Optional[tensorrt_llm.llmapi.kv_cache_type.KVCacheType]", + "converter": "", + "kind": "categorical", + "path": "build_config.kv_cache_type" + }, + { + "allowed_values": [ + "hf", + "nemo" + ], + "annotation": "Literal['hf', 'nemo']", + "converter": "", + "kind": "categorical", + "path": "build_config.lora_config.lora_ckpt_source" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "build_config.lora_config.max_cpu_loras" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.lora_config.max_lora_rank" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "build_config.lora_config.max_loras" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.lora_config.swap_gate_up_proj_lora_b_weight" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.max_batch_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.max_beam_width" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.max_draft_len" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.max_encoder_input_len" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.max_input_len" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.max_num_tokens" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.max_prompt_embedding_table_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "build_config.max_seq_len" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.monitor_memory" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.opt_batch_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "build_config.opt_num_tokens" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.bert_attention_plugin" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.bert_context_fmha_fp32_acc" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.context_fmha" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.dora_plugin" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.fp8_rowwise_gemm_plugin" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.fuse_fp4_quant" + }, + { + "allowed_values": [ + "float16", + "bfloat16", + "None" + ], + "annotation": "Optional[Literal['float16', 'bfloat16', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.gemm_allreduce_plugin" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "fp8", + "nvfp4", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', 'fp8', 'nvfp4', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.gemm_plugin" + }, + { + "allowed_values": [ + "fp8", + "None" + ], + "annotation": "Optional[Literal['fp8', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.gemm_swiglu_plugin" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.gpt_attention_plugin" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.identity_plugin" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.layernorm_quantization_plugin" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.lora_plugin" + }, + { + "allowed_values": [ + "fp8", + "None" + ], + "annotation": "Optional[Literal['fp8', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.low_latency_gemm_plugin" + }, + { + "allowed_values": [ + "fp8", + "None" + ], + "annotation": "Optional[Literal['fp8', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.low_latency_gemm_swiglu_plugin" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.mamba_conv1d_plugin" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.manage_weights" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.moe_plugin" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.multiple_profiles" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.nccl_plugin" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.norm_quant_fusion" + }, + { + "allowed_values": [], + "annotation": "Optional[bool]", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.paged_kv_cache" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.paged_state" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.pp_reduce_scatter" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.qserve_gemm_plugin" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.quantize_per_token_plugin" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.quantize_tensor_plugin" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.reduce_fusion" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.remove_input_padding" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.rmsnorm_quantization_plugin" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.smooth_quant_gemm_plugin" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.smooth_quant_plugins" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.streamingllm" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.tokens_per_block" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.use_fp8_context_fmha" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.use_fused_mlp" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.use_paged_context_fmha" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.plugin_config.user_buffer" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.weight_only_groupwise_quant_matmul_plugin" + }, + { + "allowed_values": [ + "auto", + "float16", + "float32", + "bfloat16", + "int32", + "None" + ], + "annotation": "Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]", + "converter": "", + "kind": "categorical", + "path": "build_config.plugin_config.weight_only_quant_matmul_plugin" + }, + { + "allowed_values": [ + "NONE", + "DRAFT_TOKENS_EXTERNAL", + "MEDUSA", + "LOOKAHEAD_DECODING", + "EXPLICIT_DRAFT_TOKENS", + "EAGLE", + "NGRAM", + "USER_PROVIDED", + "SAVE_HIDDEN_STATES", + "AUTO" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "build_config.speculative_decoding_mode" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.strongly_typed" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.use_mrope" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.use_refit" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.use_strip_plan" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.weight_sparsity" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "build_config.weight_streaming" + }, + { + "allowed_values": [ + "DEFAULT", + "UCX", + "NIXL", + "MOONCAKE", + "MPI" + ], + "annotation": "Optional[Literal['DEFAULT', 'UCX', 'NIXL', 'MOONCAKE', 'MPI']]", + "converter": "", + "kind": "categorical", + "path": "cache_transceiver_config.backend" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Gt(gt=0)]]", + "converter": "", + "kind": "value", + "path": "cache_transceiver_config.kv_transfer_sender_future_timeout_ms" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Gt(gt=0)]]", + "converter": "", + "kind": "value", + "path": "cache_transceiver_config.kv_transfer_timeout_ms" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cache_transceiver_config.max_tokens_in_buffer" + }, + { + "allowed_values": [ + "CPP", + "PYTHON" + ], + "annotation": "Optional[Literal['CPP', 'PYTHON']]", + "converter": "", + "kind": "categorical", + "path": "cache_transceiver_config.transceiver_runtime" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "calib_config.calib_batch_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "calib_config.calib_batches" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "calib_config.calib_max_seq_length" + }, + { + "allowed_values": [ + "cuda", + "cpu" + ], + "annotation": "Literal['cuda', 'cpu']", + "converter": "", + "kind": "categorical", + "path": "calib_config.device" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "calib_config.random_seed" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "calib_config.tokenizer_max_seq_length" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "context_parallel_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cp_config.block_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cp_config.cp_anchor_size" + }, + { + "allowed_values": [ + "ULYSSES", + "STAR", + "RING", + "HELIX" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "cp_config.cp_type" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cp_config.fifo_version" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "cp_config.tokens_per_block" + }, + { + "allowed_values": [], + "annotation": "Optional[bool]", + "converter": "", + "kind": "value", + "path": "cp_config.use_nccl_for_alltoall" + }, + { + "allowed_values": [ + "auto", + "float16", + "bfloat16", + "float32" + ], + "annotation": "", + "converter": "allowlist", + "kind": "categorical", + "path": "dtype" + }, + { + "allowed_values": [ + "NONE", + "SHARDING_ALONG_VOCAB", + "SHARDING_ALONG_HIDDEN" + ], + "annotation": "Literal['NONE', 'SHARDING_ALONG_VOCAB', 'SHARDING_ALONG_HIDDEN']", + "converter": "", + "kind": "categorical", + "path": "embedding_parallel_mode" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_attention_dp" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_build_cache.max_cache_storage_gb" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_build_cache.max_records" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_chunked_prefill" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_energy_metrics" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_lm_head_tp_in_adp" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_lora" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_prompt_adapter" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "enable_tqdm" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "extended_runtime_perf_knob_config.cuda_graph_cache_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "extended_runtime_perf_knob_config.cuda_graph_mode" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "extended_runtime_perf_knob_config.enable_context_fmha_fp32_acc" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "extended_runtime_perf_knob_config.multi_block_mode" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "fail_fast_on_attention_window_too_large" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "fast_build" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "gather_generation_logits" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "gpus_per_node" + }, + { + "allowed_values": [ + "xgrammar", + "llguidance" + ], + "annotation": "Optional[Literal['xgrammar', 'llguidance']]", + "converter": "", + "kind": "categorical", + "path": "guided_decoding_backend" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "iter_stats_max_iterations" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.attention_dp_events_gather_period_ms" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.copy_on_partial_reuse" + }, + { + "allowed_values": [], + "annotation": "Optional[float]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.cross_kv_cache_fraction" + }, + { + "allowed_values": [ + "auto", + "float16", + "bfloat16", + "float32", + "fp8", + "nvfp4" + ], + "annotation": "", + "converter": "allowlist", + "kind": "categorical", + "path": "kv_cache_config.dtype" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.enable_block_reuse" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.enable_partial_reuse" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.event_buffer_max_size" + }, + { + "allowed_values": [], + "annotation": "Optional[float]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.free_gpu_memory_fraction" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.host_cache_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.iteration_stats_interval" + }, + { + "allowed_values": [ + "auto", + "float16", + "bfloat16", + "float32" + ], + "annotation": "Literal['auto', 'float16', 'bfloat16', 'float32']", + "converter": "", + "kind": "categorical", + "path": "kv_cache_config.mamba_ssm_cache_dtype" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.mamba_ssm_philox_rounds" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.mamba_ssm_stochastic_rounding" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.mamba_state_cache_interval" + }, + { + "allowed_values": [], + "annotation": "Optional[List[int]]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.max_attention_window" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.max_gpu_total_bytes" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.max_tokens" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.max_util_for_resume" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.secondary_offload_min_priority" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "kv_cache_config.sink_token_length" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.tokens_per_block" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.use_kv_cache_manager_v2" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "kv_cache_config.use_uvm" + }, + { + "allowed_values": [ + "auto", + "dummy" + ], + "annotation": "Literal['auto', 'dummy']", + "converter": "", + "kind": "categorical", + "path": "load_format" + }, + { + "allowed_values": [ + "hf", + "nemo" + ], + "annotation": "Literal['hf', 'nemo']", + "converter": "", + "kind": "categorical", + "path": "lora_config.lora_ckpt_source" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "lora_config.max_cpu_loras" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "lora_config.max_lora_rank" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "lora_config.max_loras" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "lora_config.swap_gate_up_proj_lora_b_weight" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_batch_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_beam_width" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_input_len" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_num_tokens" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "max_prompt_adapter_token" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "max_seq_len" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "moe_cluster_parallel_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "moe_expert_parallel_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "moe_tensor_parallel_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "normalize_log_probs" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "num_postprocess_workers" + }, + { + "allowed_values": [ + "rpc", + "ray" + ], + "annotation": "Optional[Literal['rpc', 'ray']]", + "converter": "", + "kind": "categorical", + "path": "orchestrator_type" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.device_cache_percent" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.host_cache_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.max_adapter_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.max_pages_per_block_device" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.max_pages_per_block_host" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_copy_streams" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_device_module_layer" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_ensure_workers" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_host_module_layer" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.num_put_workers" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "peft_cache_config.optimal_adapter_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "perf_metrics_max_requests" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "pipeline_parallel_size" + }, + { + "allowed_values": [], + "annotation": "Optional[List[int]]", + "converter": "", + "kind": "value", + "path": "pp_partition" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.e2e_request_latency_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.request_decode_time_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.request_inference_time_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.request_prefill_time_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.request_queue_time_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.time_per_output_token_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "prometheus_metrics_config.time_to_first_token_buckets" + }, + { + "allowed_values": [], + "annotation": "Optional[List[float]]", + "converter": "", + "kind": "value", + "path": "quant_config.clamp_val" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "quant_config.group_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "quant_config.has_zero_point" + }, + { + "allowed_values": [ + "W8A16", + "W4A16", + "W4A16_AWQ", + "W4A8_AWQ", + "W8A16_GPTQ", + "W4A16_GPTQ", + "W8A8_SQ_PER_CHANNEL", + "W8A8_SQ_PER_TENSOR_PLUGIN", + "W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN", + "W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN", + "W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN", + "W4A8_QSERVE_PER_GROUP", + "W4A8_QSERVE_PER_CHANNEL", + "FP8", + "FP8_PER_CHANNEL_PER_TOKEN", + "FP8_BLOCK_SCALES", + "INT8", + "MIXED_PRECISION", + "NVFP4", + "W4A8_NVFP4_FP8", + "W4A8_MXFP4_FP8", + "W4A8_MXFP4_MXFP8", + "W4A16_MXFP4", + "NVFP4_AWQ", + "NVFP4_ARC", + "NO_QUANT" + ], + "annotation": "Optional[tensorrt_llm.quantization.mode.QuantAlgo]", + "converter": "", + "kind": "categorical", + "path": "quant_config.kv_cache_quant_algo" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "quant_config.mamba_ssm_philox_rounds" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "quant_config.mamba_ssm_stochastic_rounding" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "quant_config.pre_quant_scale" + }, + { + "allowed_values": [ + "W8A16", + "W4A16", + "W4A16_AWQ", + "W4A8_AWQ", + "W8A16_GPTQ", + "W4A16_GPTQ", + "W8A8_SQ_PER_CHANNEL", + "W8A8_SQ_PER_TENSOR_PLUGIN", + "W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN", + "W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN", + "W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN", + "W4A8_QSERVE_PER_GROUP", + "W4A8_QSERVE_PER_CHANNEL", + "FP8", + "FP8_PER_CHANNEL_PER_TOKEN", + "FP8_BLOCK_SCALES", + "INT8", + "MIXED_PRECISION", + "NVFP4", + "W4A8_NVFP4_FP8", + "W4A8_MXFP4_FP8", + "W4A8_MXFP4_MXFP8", + "W4A16_MXFP4", + "NVFP4_AWQ", + "NVFP4_ARC", + "NO_QUANT" + ], + "annotation": "Optional[tensorrt_llm.quantization.mode.QuantAlgo]", + "converter": "", + "kind": "categorical", + "path": "quant_config.quant_algo" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "quant_config.smoothquant_val" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "quant_config.use_meta_recipe" + }, + { + "allowed_values": [ + "auto", + "deepseek-r1", + "laguna", + "qwen3", + "qwen3_5", + "minimax_m2", + "minimax_m2_append_think", + "nano-v3", + "gemma4", + "kimi_k2", + "kimi_k25" + ], + "annotation": "Optional[str]", + "converter": "allowlist", + "kind": "categorical", + "path": "reasoning_parser" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "request_stats_max_iterations" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "return_perf_metrics" + }, + { + "allowed_values": [ + "MAX_UTILIZATION", + "GUARANTEED_NO_EVICT", + "STATIC_BATCH" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "scheduler_config.capacity_scheduler_policy" + }, + { + "allowed_values": [ + "FIRST_COME_FIRST_SERVED", + "EQUAL_PROGRESS", + "FORCE_CHUNK" + ], + "annotation": "Optional[tensorrt_llm.llmapi.llm_args.ContextChunkingPolicy]", + "converter": "", + "kind": "categorical", + "path": "scheduler_config.context_chunking_policy" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "scheduler_config.dynamic_batch_config.dynamic_batch_moving_average_window" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "scheduler_config.dynamic_batch_config.enable_batch_size_tuning" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "scheduler_config.dynamic_batch_config.enable_max_num_tokens_tuning" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "scheduler_config.use_python_scheduler" + }, + { + "allowed_values": [ + "fcfs", + "priority" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "scheduler_config.waiting_queue_policy" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "skip_tokenizer_init" + }, + { + "allowed_values": [ + "dsa", + "rocket", + "skip_softmax" + ], + "annotation": "Literal['dsa']", + "converter": "", + "kind": "categorical", + "path": "sparse_attention_config.algorithm" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.enable_heuristic_topk" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.index_head_dim" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.index_n_heads" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.index_topk" + }, + { + "allowed_values": [ + "fp8", + "fp4" + ], + "annotation": "Literal['fp8', 'fp4']", + "converter": "", + "kind": "categorical", + "path": "sparse_attention_config.indexer_k_dtype" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.indexer_max_chunk_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.indexer_rope_interleave" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.kernel_size" + }, + { + "allowed_values": [ + "bfloat16", + "float8_e5m2" + ], + "annotation": "Optional[str]", + "converter": "allowlist", + "kind": "categorical", + "path": "sparse_attention_config.kt_cache_dtype" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.page_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.prompt_budget" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.q_split_threshold" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.seq_len_threshold" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.skip_indexer_for_short_seqs" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.topk" + }, + { + "allowed_values": [], + "annotation": "Union[int, float, NoneType]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.topr" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.use_cute_dsl_paged_mqa_logits" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.use_cute_dsl_topk" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "sparse_attention_config.window_size" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[float, Ge(ge=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.acceptance_length_threshold" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Ge(ge=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.acceptance_window" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.allow_advanced_sampling" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.begin_thinking_phase_token" + }, + { + "allowed_values": [ + "AUTO", + "DFlash", + "Draft_Target", + "Eagle3", + "Eagle", + "Lookahead", + "MTP", + "Medusa", + "NGram", + "PARD", + "SA", + "SaveState", + "User_Provided" + ], + "annotation": "Literal['AUTO']", + "converter": "", + "kind": "categorical", + "path": "speculative_config.decoding_type" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.dynamic_tree_max_topK" + }, + { + "allowed_values": [], + "annotation": "Optional[Set[int]]", + "converter": "", + "kind": "value", + "path": "speculative_config.eagle3_layers_to_capture" + }, + { + "allowed_values": [ + "llama3", + "mistral_large3" + ], + "annotation": "Literal['llama3', 'mistral_large3']", + "converter": "", + "kind": "categorical", + "path": "speculative_config.eagle3_model_arch" + }, + { + "allowed_values": [], + "annotation": "Optional[bool]", + "converter": "", + "kind": "value", + "path": "speculative_config.eagle3_one_model" + }, + { + "allowed_values": [], + "annotation": "Optional[List[List[int]]]", + "converter": "", + "kind": "value", + "path": "speculative_config.eagle_choices" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.enable_global_pool" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.end_thinking_phase_token" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Gt(gt=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.global_pool_size" + }, + { + "allowed_values": [], + "annotation": "Optional[bool]", + "converter": "", + "kind": "value", + "path": "speculative_config.greedy_sampling" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.is_keep_all" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.is_public_pool" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.is_use_oldest" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.mask_token_id" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Gt(gt=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.max_concurrency" + }, + { + "allowed_values": [], + "annotation": "Optional[Annotated[int, Ge(ge=0)]]", + "converter": "", + "kind": "value", + "path": "speculative_config.max_draft_len" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.max_matching_ngram_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.max_ngram_size" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.max_non_leaves_per_layer" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.max_total_draft_tokens" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.max_verification_set_size" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.max_window_size" + }, + { + "allowed_values": [], + "annotation": "Optional[List[List[int]]]", + "converter": "", + "kind": "value", + "path": "speculative_config.medusa_choices" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.mtp_eagle_one_model" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.num_eagle_layers" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.num_medusa_heads" + }, + { + "allowed_values": [], + "annotation": "Optional[int]", + "converter": "", + "kind": "value", + "path": "speculative_config.num_nextn_predict_layers" + }, + { + "allowed_values": [], + "annotation": "Optional[float]", + "converter": "", + "kind": "value", + "path": "speculative_config.posterior_threshold" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.relaxed_delta" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.relaxed_topk" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.sa_config.enable_global_pool" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.sa_config.threshold" + }, + { + "allowed_values": [], + "annotation": "Optional[List[int]]", + "converter": "", + "kind": "value", + "path": "speculative_config.target_layer_ids" + }, + { + "allowed_values": [], + "annotation": "Optional[bool]", + "converter": "", + "kind": "value", + "path": "speculative_config.use_dynamic_tree" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.use_mtp_vanilla" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.use_rejection_sampling" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.use_relaxed_acceptance_for_thinking" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "speculative_config.write_interval" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "telemetry_config.disabled" + }, + { + "allowed_values": [ + "unknown", + "llm_class", + "cli_serve", + "cli_bench", + "cli_eval" + ], + "annotation": "", + "converter": "", + "kind": "categorical", + "path": "telemetry_config.usage_context" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "tensor_parallel_size" + }, + { + "allowed_values": [ + "auto", + "slow" + ], + "annotation": "Literal['auto', 'slow']", + "converter": "", + "kind": "categorical", + "path": "tokenizer_mode" + }, + { + "allowed_values": [], + "annotation": "", + "converter": "", + "kind": "value", + "path": "trust_remote_code" + } + ] +} diff --git a/tensorrt_llm/usage/llmapi_config.py b/tensorrt_llm/usage/llmapi_config.py new file mode 100644 index 000000000000..02c0751b1ded --- /dev/null +++ b/tensorrt_llm/usage/llmapi_config.py @@ -0,0 +1,650 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LLM API configuration capture for usage telemetry.""" + +from __future__ import annotations + +import hashlib +import json +import math +import types +from collections.abc import Callable +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Annotated, Any, Literal, Union, get_args, get_origin + +from pydantic import BaseModel + +from tensorrt_llm.usage.config import TelemetryField + +CAPTURE_VERSION = "2" +FIELD_POLICY_VERSION = "2" +API_CONTRACT_VERSION = "0.2.0" +CAPTURE_SOURCE = "effective_validated_llm_args" + +# Cap total serialized bytes of llmApiConfigJson. The wire field is unbounded and +# the reporter is fail-silent, so an oversized payload is dropped whole by the +# endpoint; truncate and flag instead. Conservative bound until the endpoint limit +# is confirmed. +MAX_CONFIG_BYTES = 16384 + +_TELEMETRY_EXTRA_KEY = "telemetry" +_TRTLLM_JSON_SCHEMA_EXTRA_ATTR = "_trtllm_json_schema_extra" +_APPROVED_CONVERTERS = {"allowlist"} + +# Per-sequence cap, applied recursively so each inner list of a nested +# List[List[int]] is bounded independently. 256 sits above the longest realistic +# captured sequence (~200 per-layer entries) yet still bounds a runaway list. +MAX_SEQ_ITEMS = 256 + + +class _CaptureState: + def __init__(self) -> None: + self.values: dict[str, Any] = {} + self.excluded_field_count = 0 + self.unsafe_excluded = False + self.sequence_truncated = False + self.payload_truncated = False + + +def _canonical_json(data: Any) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":")) + + +def _digest(data: Any) -> str: + return hashlib.sha256(_canonical_json(data).encode("utf-8")).hexdigest() + + +def _is_pydantic_model(value: Any) -> bool: + return isinstance(value, BaseModel) + + +def _none_type() -> type[None]: + return type(None) + + +def _unwrap_annotated(annotation: Any) -> Any: + while get_origin(annotation) is Annotated: + annotation = get_args(annotation)[0] + return annotation + + +def _is_union(annotation: Any) -> bool: + return get_origin(annotation) in {Union, types.UnionType} + + +def _unwrap_optional(annotation: Any) -> Any: + annotation = _unwrap_annotated(annotation) + if not _is_union(annotation): + return annotation + branches = [arg for arg in get_args(annotation) if arg is not _none_type()] + if len(branches) == 1: + return branches[0] + return annotation + + +def _is_literal(annotation: Any) -> bool: + return get_origin(annotation) is Literal + + +def _is_enum_annotation(annotation: Any) -> bool: + try: + return isinstance(annotation, type) and issubclass(annotation, Enum) + except TypeError: + return False + + +def _is_path_annotation(annotation: Any) -> bool: + try: + return isinstance(annotation, type) and issubclass(annotation, Path) + except TypeError: + return False + + +def _is_callable_annotation(annotation: Any) -> bool: + origin = get_origin(annotation) + return annotation is Callable or origin is Callable + + +def _is_safe_annotation_branch(annotation: Any) -> bool: + annotation = _unwrap_annotated(annotation) + origin = get_origin(annotation) + + if annotation is Any: + return False + if annotation is _none_type(): + return True + if _is_literal(annotation): + return True + if _is_enum_annotation(annotation): + return True + if annotation in {bool, int, float}: + return True + if annotation is str or annotation is object: + return False + if _is_path_annotation(annotation) or _is_callable_annotation(annotation): + return False + if _is_union(annotation): + return all(_is_safe_annotation_branch(arg) for arg in get_args(annotation)) + if origin in {list, tuple, set}: + args = get_args(annotation) + return bool(args) and all(_is_safe_annotation_branch(arg) for arg in args) + if origin is dict: + return False + return False + + +def _union_needs_converter(annotation: Any) -> bool: + annotation = _unwrap_annotated(annotation) + if not _is_union(annotation): + return False + branches = [arg for arg in get_args(annotation) if arg is not _none_type()] + if len(branches) <= 1: + return False + return not all(_is_safe_annotation_branch(arg) for arg in branches) + + +def _normalize_metadata(metadata: Any) -> dict[str, Any] | None: + if metadata is None: + return None + if metadata is False: + return {"exclude": True} + if metadata is True: + return {"kind": "value"} + if isinstance(metadata, TelemetryField): + return metadata.as_json_schema_extra() + if isinstance(metadata, dict): + return dict(metadata) + return None + + +def _get_telemetry_metadata(field_info: Any) -> dict[str, Any] | None: + json_schema_extra = getattr(field_info, "json_schema_extra", None) + if callable(json_schema_extra): + json_schema_extra = getattr(json_schema_extra, _TRTLLM_JSON_SCHEMA_EXTRA_ATTR, None) + if not isinstance(json_schema_extra, dict): + return None + return _normalize_metadata(json_schema_extra.get(_TELEMETRY_EXTRA_KEY)) + + +def _converter_is_approved(metadata: dict[str, Any]) -> bool: + return metadata.get("converter") in _APPROVED_CONVERTERS + + +def _is_explicit_exclude(metadata: dict[str, Any] | None) -> bool: + return bool(metadata) and metadata.get("exclude") is True + + +def _metadata_has_allowlist(metadata: dict[str, Any]) -> bool: + return _converter_is_approved(metadata) or metadata.get("allowed_values") is not None + + +def derive_kind(annotation: Any, metadata: dict[str, Any]) -> str: + """Derive a telemetry field's kind from its annotation and metadata. + + Categorical iff the Optional-unwrapped annotation is a Literal or Enum, or it + carries an allowlist; otherwise a plain value. Any registered kind is ignored + so kind stays annotation-driven. + """ + if _metadata_has_allowlist(metadata): + return "categorical" + unwrapped = _unwrap_optional(annotation) + if _is_literal(unwrapped) or _is_enum_annotation(unwrapped): + return "categorical" + return "value" + + +def _annotation_is_capture_safe(annotation: Any) -> bool: + return _is_safe_annotation_branch(annotation) + + +@dataclass(frozen=True) +class _ManifestEntry: + path: str + annotation: Any # real type object — needed by the sanitizer + kind: str + converter: str + allowed_values: tuple[str, ...] + metadata: dict[str, Any] # normalized metadata (allowlist) for the sanitizer + + +def _domain_values(annotation: Any, metadata: dict[str, Any]) -> list[str]: + """Compute a field's allowed-value domain. + + Explicit allowlist wins; else Literal args AND Enum members found anywhere in + the annotation tree, order-preserving + deduped. Mirrors what _sanitize_value + would emit for an Enum (str value, else name). + """ + allowlist = metadata.get("allowed_values") if metadata else None + if isinstance(allowlist, (list, tuple, set)): + return [str(v) for v in allowlist] + + values: list[str] = [] + + def rec(ann: Any) -> None: + ann = _unwrap_annotated(ann) + if _is_literal(ann): + for v in get_args(ann): + if isinstance(v, (bool, int, float, str)) or v is None: + values.append(str(v)) + return + if _is_enum_annotation(ann): + for member in ann: + values.append(member.value if isinstance(member.value, str) else member.name) + return + if _is_union(ann) or get_origin(ann) in {list, tuple, set}: + for arg in get_args(ann): + rec(arg) + + rec(annotation) + seen: list[str] = [] + for v in values: + if v not in seen: + seen.append(v) + return seen + + +def _nested_models(annotation: Any) -> list[type]: + """Every BaseModel reachable in an annotation tree. + + Covers Optional / Union / discriminated-union arms / list|tuple|set element + types. dict is NOT traversed (keys/values are not captured). + """ + out: list[type] = [] + + def rec(ann: Any) -> None: + ann = _unwrap_annotated(ann) + if isinstance(ann, type) and issubclass(ann, BaseModel): + out.append(ann) + return + if _is_union(ann) or get_origin(ann) in {list, tuple, set}: + for arg in get_args(ann): + rec(arg) + + rec(annotation) + deduped: list[type] = [] + for m in out: + if m not in deduped: + deduped.append(m) + return deduped + + +def _defining_class(cls: type, field_name: str) -> str: + for klass in cls.__mro__: + if field_name in getattr(klass, "__annotations__", {}): + return f"{klass.__name__}.{field_name}" + return f"{cls.__name__}.{field_name}" + + +def _field_is_selected(annotation: Any, metadata: dict[str, Any] | None) -> bool: + if _is_explicit_exclude(metadata): + return False + if _annotation_is_capture_safe(annotation): + return True + if metadata is not None and _converter_is_approved(metadata): + return True + return False + + +def build_capture_manifest(model_cls: type[BaseModel]) -> list[_ManifestEntry]: + """Walk real type objects and emit the complete capturable manifest. + + The single source of truth. Type-safe annotations auto-enroll; str/Any + allowlist escape hatches opt in; telemetry=False opts out. Recurses into + statically reachable nested BaseModels with a cycle guard. Collapses + duplicate keys (shared union-arm base fields): keeps the first by + (key, defining_class), unions allowed_values across arms, and FAILS if two + arms give a key a different kind. + """ + rows: list[dict[str, Any]] = [] + + def walk(cls: type, prefix: str, stack: tuple) -> None: + if cls in stack: + return + for fname, finfo in cls.model_fields.items(): + key = f"{prefix}.{fname}" if prefix else fname + ann = finfo.annotation + meta = _get_telemetry_metadata(finfo) + if _field_is_selected(ann, meta): + normalized = meta if (meta and not _is_explicit_exclude(meta)) else {} + rows.append( + { + "key": key, + "defining": _defining_class(cls, fname), + "annotation": ann, + "kind": derive_kind(ann, normalized), + "converter": str(normalized.get("converter", "")), + "allowed": _domain_values(ann, normalized), + "metadata": normalized, + } + ) + if not _is_explicit_exclude(meta): + for sub in _nested_models(ann): + walk(sub, key, (*stack, cls)) + + walk(model_cls, "", ()) + + rows.sort(key=lambda r: (r["key"], r["defining"])) + first: dict[str, dict] = {} + union_allowed: dict[str, list[str]] = {} + for r in rows: + if r["key"] not in first: + first[r["key"]] = r + elif first[r["key"]]["kind"] != r["kind"]: + raise ValueError( + f"telemetry manifest: key '{r['key']}' has conflicting kinds " + f"across union arms: {first[r['key']]['kind']} vs {r['kind']}" + ) + seen = union_allowed.setdefault(r["key"], []) + for v in r["allowed"]: + if v not in seen: + seen.append(v) + + entries = [ + _ManifestEntry( + path=key, + annotation=r["annotation"], + kind=r["kind"], + converter=r["converter"], + allowed_values=tuple(union_allowed[key]), + metadata=r["metadata"], + ) + for key, r in first.items() + ] + entries.sort(key=lambda e: e.path) + return entries + + +def manifest_rows(model_cls: type[BaseModel]) -> list[dict[str, Any]]: + """Serializable, human-legible projection of build_capture_manifest. + + Used by the committed golden, the docs renderer, and the + capture_manifest_digest. + """ + return [ + { + "path": e.path, + "annotation": _annotation_repr(e.annotation), + "kind": e.kind, + "converter": e.converter, + "allowed_values": list(e.allowed_values), + } + for e in build_capture_manifest(model_cls) + ] + + +def golden_manifest() -> dict[str, list[dict[str, Any]]]: + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs, TrtLlmArgs + + return { + "TorchLlmArgs": manifest_rows(TorchLlmArgs), + "TrtLlmArgs": manifest_rows(TrtLlmArgs), + } + + +def _sanitize_allowlist(value: Any, metadata: dict[str, Any]) -> tuple[bool, Any]: + """Capture value only when it matches field-owned allowed values.""" + allowed_values = metadata.get("allowed_values") + if not isinstance(allowed_values, (list, tuple, set)): + return False, None + candidates = [value] + if isinstance(value, Enum): + candidates.append(value.value) + candidates.append(value.name) + candidates.append(value.name.lower()) + + for candidate in candidates: + if candidate in allowed_values and ( + isinstance(candidate, (bool, int, float, str)) or candidate is None + ): + return True, candidate + return False, None + + +def _sanitize_literal(value: Any, annotation: Any) -> tuple[bool, Any]: + """Capture only values declared by Literal annotation.""" + allowed = get_args(annotation) + if value in allowed: + return True, value + return False, None + + +def _sanitize_sequence( + value: Any, + annotation: Any, + metadata: dict[str, Any], + state: _CaptureState | None = None, +) -> tuple[bool, Any]: + """Sanitize homogeneous sequence values. Reject whole sequence on one bad item. + + Captured items are capped at MAX_SEQ_ITEMS. The cap rides the recursive + _sanitize_value call, so each inner list of a nested sequence is bounded + independently. When any sequence is clipped, state.sequence_truncated is + set so the metadata reports the truncation honestly. + """ + annotation = _unwrap_optional(annotation) + origin = get_origin(annotation) + if origin not in {list, tuple, set}: + element_annotation = Any + else: + args = get_args(annotation) + # Homogeneous only. tuple[int, str] uses first annotation and rejects on + # mismatch. Fail closed if future telemetry field adds heterogeneous tuple. + element_annotation = args[0] if args else Any + + sanitized = [] + for item in value: + item_safe, item_value = _sanitize_value(item, element_annotation, metadata, state) + if not item_safe: + return False, None + sanitized.append(item_value) + if origin is set: + sanitized = sorted(sanitized, key=_canonical_json) + if len(sanitized) > MAX_SEQ_ITEMS: + sanitized = sanitized[:MAX_SEQ_ITEMS] + if state is not None: + state.sequence_truncated = True + return True, sanitized + + +def _sanitize_value( + value: Any, + annotation: Any, + metadata: dict[str, Any], + state: _CaptureState | None = None, +) -> tuple[bool, Any]: + """Return telemetry-safe primitive value, else exclude it. + + Bare strings are unsafe unless Literal or allowlist-converted. Exclusion is + visible through unsafe_excluded metadata. + """ + has_converter = _converter_is_approved(metadata) + if _union_needs_converter(annotation) and not has_converter: + return False, None + if not has_converter and not _annotation_is_capture_safe(annotation): + return False, None + annotation = _unwrap_optional(annotation) + + # None: Optional field unset -> capture as null, regardless of converter. + # Must precede the allowlist branch, else None on an Optional allowlist field + # fails the allowlist and falsely flips unsafe_excluded. + if value is None: + return True, None + + if has_converter: + return _sanitize_allowlist(value, metadata) + if isinstance(value, Enum): + enum_value = value.value + if isinstance(enum_value, str): + return True, enum_value + # Enum.name always str. Prefer stable names for int-valued enums. + return True, value.name + # bool before int. Python bool is int subclass; keep True/False not 1/0. + if isinstance(value, bool): + return True, value + if isinstance(value, int) and not isinstance(value, bool): + return True, value + if isinstance(value, float): + # Reject nan/inf. json.dumps emits the bare NaN/Infinity tokens for + # non-finite floats, which are invalid JSON and break downstream + # parsing and digest stability. + if not math.isfinite(value): + return False, None + return True, value + if isinstance(value, str): + if _is_literal(annotation): + return _sanitize_literal(value, annotation) + # Bare str can be path/secret/user text. Require Literal or allowlist. + return False, None + if isinstance(value, Path): + return False, None + if isinstance(value, (list, tuple, set)): + return _sanitize_sequence(value, annotation, metadata, state) + return False, None + + +def _annotation_repr(annotation: Any) -> str: + text = repr(annotation) + return text.replace("typing.", "") + + +def _schema_digest(model_cls: type[BaseModel]) -> str: + schema_fields = [] + for field_name, field_info in sorted(model_cls.model_fields.items()): + schema_fields.append( + { + "path": field_name, + "annotation": _annotation_repr(field_info.annotation), + "required": field_info.is_required(), + } + ) + return _digest({"class": model_cls.__name__, "fields": schema_fields}) + + +def _resolve_path(instance: BaseModel, path: str) -> tuple[bool, Any]: + """Resolve a dotted manifest path against a live instance. + + Returns (present, value). Skips when a parent segment is missing/None or is + not a pydantic model (unset config, or a discriminated-union arm that isn't + the active one). A present leaf whose value is None resolves as (True, None). + """ + segments = path.split(".") + obj: Any = instance + for seg in segments[:-1]: + if not _is_pydantic_model(obj): + return False, None + if seg not in obj.__class__.model_fields: + return False, None + obj = getattr(obj, seg, None) + if obj is None: + return False, None + leaf = segments[-1] + if not _is_pydantic_model(obj) or leaf not in obj.__class__.model_fields: + return False, None + return True, getattr(obj, leaf, None) + + +def _truncate_to_budget(values: dict[str, Any]) -> tuple[dict[str, Any], str]: + """Keep deterministically (sorted keys) as many fields as fit MAX_CONFIG_BYTES.""" + kept: dict[str, Any] = {} + for key in sorted(values): + trial = dict(kept) + trial[key] = values[key] + if len(_canonical_json(trial).encode("utf-8")) > MAX_CONFIG_BYTES: + break + kept = trial + return kept, _canonical_json(kept) + + +def _failure_meta(args_class: str = "") -> dict[str, Any]: + """Metadata for capture failure. One shape used by collector and reporter.""" + return { + "api_contract_version": API_CONTRACT_VERSION, + "args_class": args_class, + "capture_manifest_digest": "", + "capture_succeeded": False, + "capture_version": CAPTURE_VERSION, + "capturable_field_count": 0, + "captured_field_count": 0, + "excluded_field_count": 0, + "field_policy_version": FIELD_POLICY_VERSION, + "payload_truncated": False, + "schema_digest": "", + "sequence_truncated": False, + "source": CAPTURE_SOURCE, + "unsafe_excluded": False, + } + + +def _failure_llm_api_config_payloads(args_class: str = "") -> tuple[str, str]: + """Return empty config plus canonical failure metadata JSON.""" + return "{}", _canonical_json(_failure_meta(args_class=args_class)) + + +def collect_llm_api_config_payloads(llm_args: Any) -> tuple[str, str]: + """Return sanitized LLM API config and capture metadata JSON strings. + + Manifest-driven: capture exactly the keys build_capture_manifest lists for + this class, so the runtime can never emit a key absent from the committed + golden (runtime_keys subset of manifest_keys, by construction). + """ + try: + if not _is_pydantic_model(llm_args): + return _failure_llm_api_config_payloads() + + cls = llm_args.__class__ + entries = build_capture_manifest(cls) + state = _CaptureState() + for entry in entries: + present, value = _resolve_path(llm_args, entry.path) + if not present: + continue + is_safe, sanitized = _sanitize_value(value, entry.annotation, entry.metadata, state) + if is_safe: + state.values[entry.path] = sanitized + else: + state.excluded_field_count += 1 + state.unsafe_excluded = True + + config_json = _canonical_json(state.values) + if len(config_json.encode("utf-8")) > MAX_CONFIG_BYTES: + state.values, config_json = _truncate_to_budget(state.values) + state.payload_truncated = True + + rows = manifest_rows(cls) + metadata = { + "api_contract_version": API_CONTRACT_VERSION, + "args_class": cls.__name__, + "capture_manifest_digest": _digest({"args_class": cls.__name__, "fields": rows}), + "capture_succeeded": True, + "capture_version": CAPTURE_VERSION, + "capturable_field_count": len(entries), + "captured_field_count": len(state.values), + "excluded_field_count": state.excluded_field_count, + "field_policy_version": FIELD_POLICY_VERSION, + "payload_truncated": state.payload_truncated, + "schema_digest": _schema_digest(cls), + "sequence_truncated": state.sequence_truncated, + "source": CAPTURE_SOURCE, + "unsafe_excluded": state.unsafe_excluded, + } + return config_json, _canonical_json(metadata) + except (AttributeError, TypeError, ValueError, KeyError): + # Stay fail-silent only for the sanitizer/walk error family we expect. + # Unexpected exceptions propagate to the daemon-thread guard in + # usage_lib so genuine collector bugs are not silently masked. + args_class = type(llm_args).__name__ if llm_args is not None else "" + return _failure_llm_api_config_payloads(args_class=args_class) diff --git a/tensorrt_llm/usage/schema.py b/tensorrt_llm/usage/schema.py index be9f9127cc7a..5d5405c9e499 100644 --- a/tensorrt_llm/usage/schema.py +++ b/tensorrt_llm/usage/schema.py @@ -39,7 +39,7 @@ CLIENT_ID = "616561816355034" EVENT_PROTOCOL = "1.6" -EVENT_SCHEMA_VER = "0.1" +EVENT_SCHEMA_VER = "0.2" EVENT_SYS_VER = "trtllm-telemetry/1.0" CLIENT_TYPE = "Native" CLIENT_VARIANT = "Release" @@ -117,8 +117,10 @@ class TrtllmInitialReport(BaseModel): disagg_role: str = Field(default="", max_length=_SHORT_STR, alias="disaggRole") deployment_id: str = Field(default="", max_length=_SHORT_STR, alias="deploymentId") - # Feature flags (JSON-serialized dict of enabled features) + # Legacy feature summary and sanitized opt-in LLM API config capture. features_json: str = Field(default="{}", alias="featuresJson") + llm_api_config_json: str = Field(default="{}", alias="llmApiConfigJson") + llm_api_config_meta_json: str = Field(default="{}", alias="llmApiConfigMetaJson") model_config = {"populate_by_name": True} diff --git a/tensorrt_llm/usage/schemas/README.md b/tensorrt_llm/usage/schemas/README.md index 8bc49551fe38..0eb66f2f09fa 100644 --- a/tensorrt_llm/usage/schemas/README.md +++ b/tensorrt_llm/usage/schemas/README.md @@ -1,12 +1,13 @@ # TRT-LLM Telemetry Schema Reference -Schema version: **0.1** | Client ID: `616561816355034` | Protocol: GXT Event Protocol v1.6 +Schema version: **0.2** | Client ID: `616561816355034` | Protocol: GXT Event Protocol v1.6 ## Overview TRT-LLM collects anonymous, session-level deployment telemetry to understand how the library is used in production (GPU types, parallelism configs, model -architectures). No PII, model weights, prompts, or outputs are collected. +architectures). No PII, model weights, prompts, outputs, model paths, tokenizer +paths, or raw free-form configuration strings are collected. **Opt-out** (any one of these disables telemetry): - `TRTLLM_NO_USAGE_STATS=1` @@ -30,7 +31,7 @@ these top-level fields in Kibana alongside the event parameters. | `clientType` | string | Always `"Native"`. | | `clientVer` | string | TRT-LLM version, e.g. `"1.3.0rc9"`. | | `eventProtocol` | string | Always `"1.6"`. | -| `eventSchemaVer` | string | Schema version, currently `"0.1"`. | +| `eventSchemaVer` | string | Schema version, currently `"0.2"`. | | `eventSysVer` | string | Always `"trtllm-telemetry/1.0"`. | | `sessionId` | string | Unique hex UUID per server lifetime. Use this to correlate initial report with heartbeats. | | `sentTs` | string | ISO 8601 UTC timestamp of when the payload was sent. | @@ -89,7 +90,9 @@ Sent once at server startup. Contains system info and serving configuration. | Field | Type | Description | Example | |-------|------|-------------|---------| | `ingressPoint` | ShortString | How TRT-LLM was invoked. See [Ingress point values](#ingress-point-values). | `"cli_serve"` | -| `featuresJson` | string | JSON-serialized dict of feature flags. See [featuresJson keys](#featuresjson-keys). | `'{"lora":false,...}'` | +| `featuresJson` | string | Legacy JSON-serialized summary of feature flags. See [featuresJson keys](#featuresjson-keys). | `'{"lora":false,...}'` | +| `llmApiConfigJson` | string | JSON-serialized sanitized, type-driven effective LLM API configuration. See [LLM API config capture](#llm-api-config-capture). | `'{"tensor_parallel_size":2,...}'` | +| `llmApiConfigMetaJson` | string | JSON-serialized metadata for LLM API configuration capture. | `'{"capture_succeeded":true,...}'` | | `disaggRole` | ShortString | Disaggregated serving role. Empty if not disaggregated. | `""`, `"context"`, `"generation"` | | `deploymentId` | ShortString | Shared ID across disaggregated workers. Empty if not disaggregated. | `""`, `"dep-abc123"` | @@ -127,6 +130,9 @@ The `ingressPoint` field identifies which TRT-LLM entry point started the sessio The `featuresJson` field is a JSON-serialized dict. All keys are always present with safe defaults. This list may evolve as features are added. +TODO: Deduplicate `featuresJson` with `llmApiConfigJson` after derived-only +flags such as LoRA/speculative decoding have explicit safe config fields. + | Key | Type | Default | Description | |-----|------|---------|-------------| | `lora` | bool | `false` | LoRA adapter enabled (`enable_lora=True` or `lora_config` provided). | @@ -136,6 +142,65 @@ with safe defaults. This list may evolve as features are added. | `chunked_context` | bool | `false` | Chunked prefill enabled (`enable_chunked_prefill=True`). | | `data_parallel_size` | int | `1` | Data parallel degree. `1` = no data parallelism. Derived from `tp_size` when attention DP is enabled. | +## LLM API Config Capture + +The `llmApiConfigJson` field is a JSON-serialized dict containing a type-driven +subset of the validated, effective LLM API configuration. Capture is +**type-driven**: a field is captured automatically when its type is categorical +(`Literal`/`Enum`/`bool`) or numeric (`int`/`float`), or a safe collection of +those. Free-form `str`/`Any`/`Path`/`dict`/`Callable` are not captured unless the +field carries an explicit allowlist (`TelemetryField.categorical(...)`). Any field +can opt out with `telemetry=False`. + +Captured values must be safe primitives. Raw strings are excluded unless the +field is a `Literal[...]` or uses an explicit `allowlist` converter. Paths, +tokenizer locations, dicts, objects, callables, raw `Any` values, non-finite +floats (`nan`/`inf`), and unsafe or heterogeneous sequences are excluded. +Captured sequences are capped at a fixed length and any clipping is reported in +`llmApiConfigMetaJson`. Exclusion is fail-closed: the value is omitted instead +of being serialized, and `llmApiConfigMetaJson` reports whether any resolved field +was excluded as unsafe. + +The table below is a non-exhaustive set of examples for readers building +dashboards. The exhaustive source of truth is +`tensorrt_llm/usage/llm_args_golden_manifest.json` (regenerated from +`build_capture_manifest`), after the safety sanitizer has excluded unsafe values. +Use `llmApiConfigMetaJson` digests and field counts to track the exact capture +manifest for a given release. The rendered documentation generates the +exhaustive field table at docs build time under **Developer Guide > Telemetry**. + +| Key | Description | +|-----|-------------| +| `tensor_parallel_size` | Tensor parallelism degree from the effective LLM args. | +| `pipeline_parallel_size` | Pipeline parallelism degree from the effective LLM args. | +| `context_parallel_size` | Context parallelism degree from the effective LLM args. | +| `moe_expert_parallel_size` | MoE expert parallelism degree (None/unset when runtime decides). | +| `moe_tensor_parallel_size` | MoE tensor parallelism degree (None/unset when runtime decides). | +| `moe_cluster_parallel_size` | MoE cluster parallelism degree (None/unset when runtime decides). | +| `backend` | Execution backend. Captured as the `Literal["pytorch"]` value on the PyTorch args, and through an explicit allowlist (`pytorch`, `tensorrt`, `_autodeploy`) on the base/TRT args. | +| `dtype` | Model dtype, captured through an explicit allowlist. | +| `load_format` | Weight load format, captured as a low-cardinality enum/string value. | +| `quant_config.quant_algo` | Quantization algorithm, captured as a closed `QuantAlgo` enum value (TRT args only). Empty/absent when unquantized. | +| `kv_cache_config.dtype` | KV cache dtype, captured through an explicit allowlist. | +| `kv_cache_config.enable_block_reuse` | Whether KV cache block reuse/prefix caching is enabled. | +| `cuda_graph_config.batch_sizes` | CUDA graph batch sizes when configured. | +| `scheduler_config.capacity_scheduler_policy` | Scheduler capacity policy. | +| `torch_compile_config.enable_inductor` | Whether Torch Inductor compilation is enabled. | +| `moe_config.backend` | MoE backend selection (`AUTO`, `CUTLASS`, `TRTLLM`, ...), an annotation-derived categorical. | +| `speculative_config.decoding_type` | Speculative decoding mode discriminator (e.g. `User_Provided`); other arms expose their own numeric/boolean knobs under `speculative_config.*`. | +| `sparse_attention_config.algorithm` | Sparse attention algorithm discriminator; arm-specific knobs appear under `sparse_attention_config.*`. | +| `reasoning_parser` | Reasoning parser selection, captured through an allowlist mirroring the `ReasoningParserFactory` registry. | +| `sampler_type` | Sampler selection, captured through an allowlist mirroring the `SamplerType` enum. | + +`llmApiConfigMetaJson` describes the capture process itself. It includes +contract/version fields, schema and manifest digests, source args class, field +counts (`capturable_field_count`, `captured_field_count`, `excluded_field_count`), capture +success, unsafe-exclusion status, a `sequence_truncated` flag set when any captured +sequence was clipped to the length cap, and a `payload_truncated` flag set when the +total serialized config exceeded the size budget and fields were dropped. The metadata +is intended to make dashboards robust when the safe capture manifest changes +over time. + ## Environment Variables | Variable | Default | Description | @@ -162,6 +227,42 @@ Checklist for adding a telemetry field: 7. **SMS schema upload** — Upload the updated JSON schema to the NvTelemetry Schema Management Service and toggle "on stage" / "on prod". 8. **Update this README** — Add the field to the appropriate table above. +Checklist for adding an LLM API config capture field inside `llmApiConfigJson`: + +1. **Add the field with its natural type.** If it is categorical + (`Literal`/`Enum`/`bool`) or numeric (`int`/`float`) — or a safe collection of + those — it is captured automatically; no marker is needed. +2. **Bounded bare-string fields opt in via an allowlist.** If a free-form + `str`/`Any` field should be captured, mark it + `telemetry=TelemetryField.categorical()`, mirroring its real + recognized domain. **Prefer tightening the type (e.g. `str` -> `Literal`) over + an allowlist** when the API contract allows it; the allowlist is the fallback + when the annotation cannot be narrowed without a breaking validation change. +3. **Type-safe but sensitive? Opt out with `telemetry=False`.** This honored + exclusion sentinel keeps a categorical/numeric field out of capture. +4. **Do not capture unsafe data.** No model/tokenizer/file paths, prompts, + outputs, secrets/tokens/URLs/hostnames, free-form user strings, raw + dict/object payloads, or callables. The sanitizer fails closed regardless: + bare `str`, `Any`, `object`, `Path`, `dict`, callables, permissive unions, and + non-finite floats are dropped unless an approved `allowlist` converter applies. +5. **`tests/unittest/usage/test_llmapi_config_capture.py`** — Add behavior + coverage: assert the value is captured, and for a categorical bare-string + field assert that an out-of-allowlist value is redacted (dropped) while an + in-allowlist value is captured. +6. **Regenerate the manifest golden** from `build_capture_manifest`: + `python -c "import json; from tensorrt_llm.usage.llmapi_config import golden_manifest; open('tensorrt_llm/usage/llm_args_golden_manifest.json','w').write(json.dumps(golden_manifest(), indent=2, sort_keys=True)+'\n')"` + Review the golden diff — **it is the privacy review.** A newly captured field + requires sign-off from the GitHub telemetry/privacy CODEOWNER (`.github/CODEOWNERS`). +7. **`docs/source/developer-guide/telemetry.md` is generated** from the committed + golden at docs-build time; do not hand-edit it. +8. **Update this README** — Add a common-key row above when the field is + important enough for dashboard users to know by name. + +Dashboard note: payloads carry `capture_version` and `field_policy_version` in +`llmApiConfigMetaJson`. During release adoption, v1 (opt-in) and v2 (type-driven) +payloads coexist in the same index — **bucket by these before aggregating** +`captured_field_count` or any `llmApiConfigJson.`. + ### Conventions - Use **camelCase** aliases for JSON wire format (Pydantic `alias=`). @@ -170,5 +271,11 @@ Checklist for adding a telemetry field: - Integer fields: use `PositiveInt` (0–4B). Use `0` for "auto/unset" semantics. - All fields must be **required** in the JSON schema (no optional fields). - Empty string `""` is the sentinel for "not applicable" string fields. -- The telemetry code is **fail-silent** — exceptions are caught and swallowed. -- No PII. No model weights. No prompts. No outputs. Architecture class names only. +- The telemetry code is **fail-silent in two layers.** The LLM API config + collector catches only the expected sanitizer/walk error family + (`AttributeError`, `TypeError`, `ValueError`, `KeyError`) and emits an empty + config plus `capture_succeeded=false`; unexpected exceptions are left to + propagate so genuine collector bugs are not masked. They are then caught by + the outer daemon-thread reporter guard in `usage_lib.py`, which keeps the + reporting thread from ever taking down the host process. +- No PII. No model weights. No prompts. No outputs. No model/tokenizer paths. diff --git a/tensorrt_llm/usage/schemas/trtllm_usage_event_schema.json b/tensorrt_llm/usage/schemas/trtllm_usage_event_schema.json index e175b939eeeb..5a0e98172fdb 100644 --- a/tensorrt_llm/usage/schemas/trtllm_usage_event_schema.json +++ b/tensorrt_llm/usage/schemas/trtllm_usage_event_schema.json @@ -9,7 +9,7 @@ ], "$schema": "http://json-schema.org/draft-07/schema#", "schemaMeta": { - "schemaVersion": "0.1", + "schemaVersion": "0.2", "clientId": "616561816355034", "clientName": "TrtllmTelemetry", "definitionVersion": "2.0" @@ -107,7 +107,15 @@ }, "featuresJson": { "type": "string", - "description": "JSON-serialized dict of feature flags (lora, speculative_decoding, prefix_caching, cuda_graphs, chunked_context, data_parallel_size)" + "description": "Legacy JSON-serialized summary of feature flags (lora, speculative_decoding, prefix_caching, cuda_graphs, chunked_context, data_parallel_size)" + }, + "llmApiConfigJson": { + "type": "string", + "description": "JSON-serialized sanitized, opt-in subset of effective LLM API configuration" + }, + "llmApiConfigMetaJson": { + "type": "string", + "description": "JSON-serialized metadata for LLM API configuration capture, including digests, field counts, capture success, and unsafe-exclusion status" }, "disaggRole": { "$ref": "#/definitions/types/ShortString" @@ -138,6 +146,8 @@ "kvCacheDtype", "ingressPoint", "featuresJson", + "llmApiConfigJson", + "llmApiConfigMetaJson", "disaggRole", "deploymentId" ] diff --git a/tensorrt_llm/usage/usage_lib.py b/tensorrt_llm/usage/usage_lib.py index 22bed2bd677b..9622af83cc06 100644 --- a/tensorrt_llm/usage/usage_lib.py +++ b/tensorrt_llm/usage/usage_lib.py @@ -54,6 +54,10 @@ from typing import Any, Dict, Optional from tensorrt_llm.usage import schema +from tensorrt_llm.usage.llmapi_config import _failure_llm_api_config_payloads +from tensorrt_llm.usage.llmapi_config import ( + collect_llm_api_config_payloads as _collect_llm_api_config_payloads, +) logger = logging.getLogger("tensorrt_llm") @@ -356,6 +360,10 @@ def _extract_trtllm_config(llm_args: Any) -> Dict[str, Any]: Returns: Dict of config values, with None for unavailable fields. """ + # TODO: Consolidate with llmApiConfigJson, which now captures a near-superset + # of these columns (backend, quant_config.quant_algo, MoE parallel sizes). + # Blocker is downstream SMS/dashboard migration: dropping them is a breaking + # wire-schema change, not a code-only refactor. if llm_args is None: return {} @@ -460,6 +468,8 @@ def _collect_features(llm_args: Any) -> str: Compact JSON string, e.g. '{"lora":false,"speculative_decoding":false,...}' """ features = dict(_FEATURES_DEFAULTS) + # TODO: Deduplicate featuresJson with llmApiConfigJson once remaining + # derived-only flags have explicit safe fields in LLM API config telemetry. if llm_args is None: return json.dumps(features, separators=(",", ":")) @@ -590,6 +600,20 @@ def _background_reporter( arch_class_name = _extract_architecture_class_name(pretrained_config) trtllm_config = _extract_trtllm_config(llm_args) features_json = _collect_features(llm_args) + try: + llm_api_config_json, llm_api_config_meta_json = _collect_llm_api_config_payloads( + llm_args + ) + except Exception: + # Double net: reporter must not die if collector call breaks. + # Intentionally broad -- this is the daemon-thread fail-silent guard; + # telemetry must never crash user inference, so anything the narrowed + # collector net lets propagate is contained here. Shared failure + # payload keeps metadata shape in one place. + args_class = type(llm_args).__name__ if llm_args is not None else "" + llm_api_config_json, llm_api_config_meta_json = _failure_llm_api_config_payloads( + args_class=args_class + ) # Disaggregated serving metadata (set by serve.py orchestrator) disagg_role = os.environ.get(_DISAGG_ROLE_ENV, "") @@ -631,6 +655,8 @@ def _background_reporter( ingressPoint=_clamp_str(usage_context or "", _S), # Feature flags featuresJson=features_json, + llmApiConfigJson=llm_api_config_json, + llmApiConfigMetaJson=llm_api_config_meta_json, # Disaggregated serving disaggRole=_clamp_str(disagg_role, _S), deploymentId=_clamp_str(deployment_id, _S), diff --git a/tests/unittest/llmapi/test_disagg_telemetry_launcher.py b/tests/unittest/llmapi/test_disagg_telemetry_launcher.py new file mode 100644 index 000000000000..d67a3f84faae --- /dev/null +++ b/tests/unittest/llmapi/test_disagg_telemetry_launcher.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Smoke tests for disaggregated launcher telemetry environment propagation.""" + +import os +import sys +from collections.abc import Callable +from types import SimpleNamespace +from unittest import mock + +import pytest + +from tensorrt_llm.commands import serve + + +def _mock_llmapi_modules( + monkeypatch: pytest.MonkeyPatch, + split_mpi_env: Callable[[], tuple[dict[str, str], dict[str, str]]], +) -> None: + monkeypatch.setitem( + sys.modules, + "tensorrt_llm.llmapi.mgmn_leader_node", + SimpleNamespace(launch_server_main=lambda sub_comm: None), + ) + monkeypatch.setitem( + sys.modules, + "tensorrt_llm.llmapi.mpi_session", + SimpleNamespace(split_mpi_env=split_mpi_env), + ) + + +def test_disaggregated_command_sets_shared_deployment_id(monkeypatch) -> None: + """The top-level disagg launcher assigns one deployment id for child workers.""" + monkeypatch.delenv( + serve.DisaggLauncherEnvs.TLLM_DISAGG_DEPLOYMENT_ID, + raising=False, + ) + + disagg_config = SimpleNamespace(hostname="127.0.0.1", port=0, schedule_style=None) + fake_socket = mock.MagicMock() + fake_socket.__enter__.return_value = fake_socket + deployment_id = SimpleNamespace(hex="deploy123") + + with ( + mock.patch.object(serve.uuid, "uuid4", return_value=deployment_id), + mock.patch.object(serve, "parse_disagg_config_file", return_value=disagg_config), + mock.patch.object(serve.socket, "socket", return_value=fake_socket), + mock.patch.object(serve, "parse_metadata_server_config_file", return_value=None), + mock.patch.object(serve, "OpenAIDisaggServer"), + mock.patch.object(serve.asyncio, "run"), + ): + serve.disaggregated.callback( + config_file="disagg.yaml", + metadata_server_config_file=None, + server_start_timeout=180, + request_timeout=180, + log_level="info", + metrics_log_interval=0, + schedule_style=None, + ) + + assert os.environ[serve.DisaggLauncherEnvs.TLLM_DISAGG_DEPLOYMENT_ID] == "deploy123" + fake_socket.bind.assert_called_once_with(("127.0.0.1", 0)) + + +def test_launch_disaggregated_leader_propagates_deployment_id(monkeypatch) -> None: + """Leader subprocess env keeps the shared telemetry deployment id.""" + observed = {} + + class _FakeComm: + def Get_rank(self): + return 0 + + class _FakePopen: + pid = 12345 + + def __init__(self, command, **kwargs): + observed["command"] = command + observed["env"] = kwargs["env"] + self._status = None + + def poll(self): + return self._status + + def terminate(self): + self._status = -15 + + def wait(self, timeout=None): + self._status = -15 + return self._status + + def kill(self): + self._status = -9 + + def _fake_split_mpi_env(): + return dict(os.environ), {} + + monkeypatch.setenv( + serve.DisaggLauncherEnvs.TLLM_DISAGG_DEPLOYMENT_ID, + "deploy123", + ) + monkeypatch.setattr(serve, "find_free_ipc_addr", lambda: "ipc://fake-proxy") + monkeypatch.setattr(serve.subprocess, "Popen", _FakePopen) + monkeypatch.setattr(serve.sys, "argv", ["trtllm-serve"]) + _mock_llmapi_modules(monkeypatch, _fake_split_mpi_env) + + serve._launch_disaggregated_leader(_FakeComm(), 2, "disagg.yaml", "info") + + assert observed["env"][serve.DisaggLauncherEnvs.TLLM_DISAGG_DEPLOYMENT_ID] == "deploy123" + assert observed["env"][serve.DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX] == "2" + assert ( + observed["env"][serve.DisaggLauncherEnvs.TLLM_DISAGG_RUN_REMOTE_MPI_SESSION_CLIENT] == "1" + ) + assert observed["command"] == [ + "python3", + "trtllm-serve", + "disaggregated_mpi_worker", + "-c", + "disagg.yaml", + "--log_level", + "info", + ] + + +@pytest.mark.parametrize( + ("server_type", "expected_role"), + [ + ("ctx", "context"), + ("gen", "generation"), + ], +) +def test_launch_disaggregated_server_sets_worker_role( + monkeypatch, + server_type, + expected_role, +) -> None: + """Worker launch maps disagg server type to telemetry role env.""" + monkeypatch.setenv(serve.DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX, "0") + monkeypatch.delenv(serve.DisaggLauncherEnvs.TLLM_DISAGG_ROLE, raising=False) + + llm_args = {"model": "dummy/model"} + server_config = SimpleNamespace(type=server_type, hostname="127.0.0.1", port=8000) + disagg_config = SimpleNamespace(server_configs=[server_config]) + + with ( + mock.patch.object(serve, "parse_disagg_config_file", return_value=disagg_config), + mock.patch.object(serve, "mpi_rank", return_value=0), + mock.patch.object(serve, "launch_server") as mock_launch_server, + ): + serve._launch_disaggregated_server("disagg.yaml", llm_args) + + assert os.environ[serve.DisaggLauncherEnvs.TLLM_DISAGG_ROLE] == expected_role + mock_launch_server.assert_called_once_with( + host="127.0.0.1", + port=8000, + llm_args=llm_args, + ) diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index 6eb88d5b65cd..5b73c0e60f7f 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -2171,11 +2171,16 @@ def _is_allowed_type(self, annotation, model_cls: type, annotation, _ALLOWED_CLASS_BASES): return False, f"class '{annotation.__name__}' is not a Pydantic model (convert to StrictBaseModel)" - # Require user-facing Pydantic models to inherit from StrictBaseModel - if isinstance(annotation, type) and issubclass( - annotation, - BaseModel) and not issubclass(annotation, StrictBaseModel): - return False, f"Pydantic model '{annotation.__name__}' is not a StrictBaseModel (convert to StrictBaseModel)" + # Require user-facing Pydantic models to forbid extra fields. StrictBaseModel + # enforces this; usage-local models (e.g. TelemetryConfig) may instead set + # model_config extra="forbid" directly so they stay importable without the + # heavy llmapi.utils dependency chain (which would otherwise pull torch/HF and + # create a circular import via llm_args). + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + is_strict = issubclass(annotation, StrictBaseModel) + forbids_extra = annotation.model_config.get("extra") == "forbid" + if not (is_strict or forbids_extra): + return False, f"Pydantic model '{annotation.__name__}' does not forbid extra fields (inherit StrictBaseModel or set model_config extra='forbid')" # Recursively check generic type arguments for disallowed types origin = get_origin(annotation) diff --git a/tests/unittest/usage/test_config.py b/tests/unittest/usage/test_config.py index 03092e244ee9..9083aba69825 100644 --- a/tests/unittest/usage/test_config.py +++ b/tests/unittest/usage/test_config.py @@ -104,3 +104,47 @@ def test_same_types_both_locations(self): assert config.TelemetryConfig is llm_args.TelemetryConfig assert config.UsageContext is llm_args.UsageContext + + +class TestFieldTelemetryMetadata: + """Verify llm_args.Field telemetry metadata handling.""" + + def test_telemetry_false_records_exclude_marker(self): + """Field(telemetry=False) records an honored exclude sentinel. + + Under type-driven auto-enroll, telemetry=False is no longer a no-op: it + is the explicit opt-out for a type-safe-but-sensitive field, recorded as + json_schema_extra['telemetry'] = {"exclude": True} and honored by + build_capture_manifest's selection rule. + """ + from tensorrt_llm.llmapi import llm_args + + field = llm_args.Field(default=0, telemetry=False) + + assert field.json_schema_extra == {"telemetry": {"exclude": True}} + + def test_telemetry_false_preserves_status_and_records_exclude_marker(self): + """Field(telemetry=False) preserves unrelated json schema metadata.""" + from tensorrt_llm.llmapi import llm_args + + field = llm_args.Field(default=0, status="beta", telemetry=False) + + assert field.json_schema_extra == { + "status": "beta", + "telemetry": {"exclude": True}, + } + + +class TestTelemetryFieldCategorical: + """Verify TelemetryField.categorical(*values) allowlist shorthand.""" + + def test_categorical_builds_allowlist_metadata(self): + from tensorrt_llm.usage.config import TelemetryField + + field = TelemetryField.categorical("a", "b") + + assert field.as_json_schema_extra() == { + "kind": "categorical", + "converter": "allowlist", + "allowed_values": ["a", "b"], + } diff --git a/tests/unittest/usage/test_e2e_capture.py b/tests/unittest/usage/test_e2e_capture.py index ddaf9b955bfe..c34d87a03698 100644 --- a/tests/unittest/usage/test_e2e_capture.py +++ b/tests/unittest/usage/test_e2e_capture.py @@ -113,6 +113,24 @@ def capture_server(): server.shutdown() +def _assert_llm_api_config_capture(params): + """Assert that the public LLM entrypoint populated config telemetry.""" + assert "llmApiConfigJson" in params + assert "llmApiConfigMetaJson" in params + + config = json.loads(params["llmApiConfigJson"]) + meta = json.loads(params["llmApiConfigMetaJson"]) + + assert config["tensor_parallel_size"] == 1 + assert config["pipeline_parallel_size"] == 1 + # Sensitive identifiers must never be captured. + assert "model" not in config + assert "tokenizer" not in config + assert meta["args_class"] == "TorchLlmArgs" + assert meta["capture_succeeded"] is True + assert meta["captured_field_count"] > 0 + + # --------------------------------------------------------------------------- # E2E test # --------------------------------------------------------------------------- @@ -226,12 +244,14 @@ def test_initial_report_captured(self, capture_server, monkeypatch): assert set(features.keys()) == expected_keys # Schema version - assert payload["eventSchemaVer"] == "0.1" + assert payload["eventSchemaVer"] == "0.2" # Disagg fields present (may be empty strings) assert "disaggRole" in params assert "deploymentId" in params + _assert_llm_api_config_capture(params) + def test_cli_serve_context_e2e(self, capture_server, monkeypatch): """Verify CLI_SERVE context flows through to the captured payload.""" import tensorrt_llm.usage.usage_lib as usage_lib @@ -261,3 +281,4 @@ def test_cli_serve_context_e2e(self, capture_server, monkeypatch): payload = CaptureHandler.captured_payloads[0] params = payload["events"][0]["parameters"] assert params["ingressPoint"] == "cli_serve" + _assert_llm_api_config_capture(params) diff --git a/tests/unittest/usage/test_llmapi_config_capture.py b/tests/unittest/usage/test_llmapi_config_capture.py new file mode 100644 index 000000000000..aca86c43b02e --- /dev/null +++ b/tests/unittest/usage/test_llmapi_config_capture.py @@ -0,0 +1,923 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from enum import Enum +from pathlib import Path +from typing import Any, Literal, Optional, Union + +from tensorrt_llm.llmapi.llm_args import ( + CudaGraphConfig, + Field, + KvCacheConfig, + TorchCompileConfig, + TorchLlmArgs, +) +from tensorrt_llm.llmapi.utils import StrictBaseModel +from tensorrt_llm.usage import usage_lib +from tensorrt_llm.usage.llmapi_config import collect_llm_api_config_payloads + + +class _NestedConfig(StrictBaseModel): + marked: int = Field(default=7, telemetry={"kind": "value"}) + unmarked: int = Field(default=11) + + +class _ExampleConfig(StrictBaseModel): + safe_marked: int = Field(default=3, telemetry={"kind": "value"}) + safe_unmarked: int = Field(default=5) + private_path: str = Field(default="/customer/private/model", telemetry={"kind": "value"}) + mode: Literal["auto", "slow"] = Field(default="auto", telemetry={"kind": "categorical"}) + nested: _NestedConfig = Field(default_factory=_NestedConfig) + unsafe_union: Optional[Union[str, Path]] = Field( + default="/customer/tokenizer", telemetry={"kind": "categorical"} + ) + + +def _loads_payloads(args) -> tuple[dict, dict]: + config_json, meta_json = collect_llm_api_config_payloads(args) + return json.loads(config_json), json.loads(meta_json) + + +def test_collect_llm_api_config_uses_type_driven_autoenroll_and_safety_vetoes(): + # Renamed from ..._uses_strict_opt_in_...: under auto-enroll, unmarked + # type-safe ints (safe_unmarked, nested.unmarked) are now captured; bare + # str / Union[str,Path] without an approved allowlist remain uncapturable. + config, meta = _loads_payloads(_ExampleConfig()) + + assert config == { + "mode": "auto", + "nested.marked": 7, + "nested.unmarked": 11, + "safe_marked": 3, + "safe_unmarked": 5, + } + assert "private_path" not in config # bare str, no allowlist -> not capturable + assert "unsafe_union" not in config # Union[str,Path], no allowlist -> not capturable + assert meta["source"] == "effective_validated_llm_args" + assert meta["args_class"] == "_ExampleConfig" + assert meta["capturable_field_count"] == 5 + assert meta["captured_field_count"] == 5 + assert meta["excluded_field_count"] == 0 + assert meta["unsafe_excluded"] is False + assert meta["payload_truncated"] is False + + +def test_collect_llm_api_config_allows_approved_string_converters_only(): + # The union_backend / union_path fixtures below are Union[str, Path] solely + # to exercise the value-fail-closed allowlist seam: union_path defaults to a + # Path, which is dropped because it is not an allowlisted scalar, while + # union_backend's allowlisted str is captured. No production telemetry field + # is Union[str, Path]; the only real Union allowlist fields are + # Union[str, Enum] (sampler_type, load_format). See CR-E (declined). + class _StringConfig(StrictBaseModel): + backend: Optional[str] = Field( + default="pytorch", + telemetry={ + "kind": "categorical", + "converter": "allowlist", + "allowed_values": ["pytorch", "tensorrt"], + }, + ) + unsafe_backend: Optional[str] = Field( + default="file:///customer/private", + telemetry={ + "kind": "categorical", + "converter": "allowlist", + "allowed_values": ["pytorch", "tensorrt"], + }, + ) + unconverted: Optional[str] = Field( + default="arbitrary-user-string", telemetry={"kind": "categorical"} + ) + union_backend: Union[str, Path] = Field( + default="tensorrt", + telemetry={ + "kind": "categorical", + "converter": "allowlist", + "allowed_values": ["pytorch", "tensorrt"], + }, + ) + union_path: Union[str, Path] = Field( + default=Path("/customer/private"), + telemetry={ + "kind": "categorical", + "converter": "allowlist", + "allowed_values": ["pytorch", "tensorrt"], + }, + ) + + config, meta = _loads_payloads(_StringConfig()) + + assert config == {"backend": "pytorch", "union_backend": "tensorrt"} + assert meta["captured_field_count"] == 2 + # unconverted (kind=categorical, no converter) is not capturable -> not in + # manifest; unsafe_backend + union_path resolve but fail the allowlist sanitizer. + assert meta["capturable_field_count"] == 4 + assert meta["excluded_field_count"] == 2 + assert meta["unsafe_excluded"] is True + + +def test_sanitize_allowlist_is_value_fail_closed_for_non_scalars(): + """The allowlist only emits scalars, even if a non-scalar is allowlisted. + + Documents the CR-E decision (declined): the sanitizer is value-fail-closed, + so a Path or arbitrary object cannot leak through the allowlist converter + even if it were placed in allowed_values. _sanitize_allowlist returns a + candidate only when it is BOTH in allowed_values AND a scalar + (bool/int/float/str) or None. Excluding Union-with-Any/Path from allowlist + eligibility at the type level is therefore unnecessary for safety, and a + coarse rule would also break legitimate Union[str, Enum] allowlist fields + such as sampler_type and load_format (verified captured elsewhere). + """ + from tensorrt_llm.usage import llmapi_config + + secret = Path("/customer/secret") + metadata = {"converter": "allowlist", "allowed_values": [secret]} + # A Path placed in the allowlist is still rejected: not a scalar. + assert llmapi_config._sanitize_allowlist(secret, metadata) == (False, None) + # An allowlisted scalar is captured. + str_metadata = {"converter": "allowlist", "allowed_values": ["pytorch"]} + assert llmapi_config._sanitize_allowlist("pytorch", str_metadata) == (True, "pytorch") + + +def test_collect_llm_api_config_walks_only_declared_pydantic_fields(): + class _DeclaredFieldsOnlyConfig(StrictBaseModel): + safe_value: int = Field(default=3, telemetry={"kind": "value"}) + + @property + def leaked_value(self): + raise AssertionError("collector must not inspect arbitrary attributes") + + config, meta = _loads_payloads(_DeclaredFieldsOnlyConfig()) + + assert config == {"safe_value": 3} + assert meta["capturable_field_count"] == 1 + assert meta["excluded_field_count"] == 0 + + +def test_collect_llm_api_config_rejects_unsafe_annotations_even_for_safe_values(): + class _UnsafeAnnotationConfig(StrictBaseModel): + safe_value: int = Field(default=3, telemetry={"kind": "value"}) + raw_any: Any = Field(default=11, telemetry={"kind": "value"}) + object_like: object = Field(default=True, telemetry={"kind": "value"}) + raw_dict: dict[str, Any] = Field(default_factory=dict, telemetry={"kind": "value"}) + converted_any: Any = Field( + default="known", + telemetry={ + "kind": "categorical", + "converter": "allowlist", + "allowed_values": ["known"], + }, + ) + + config, meta = _loads_payloads(_UnsafeAnnotationConfig()) + + assert config == {"converted_any": "known", "safe_value": 3} + # raw_any/object_like/raw_dict have unsafe annotations -> excluded at the + # MANIFEST level (never selected), so the sanitizer never sees them. + assert "raw_any" not in config + assert "object_like" not in config + assert "raw_dict" not in config + assert meta["capturable_field_count"] == 2 + assert meta["captured_field_count"] == 2 + assert meta["excluded_field_count"] == 0 + assert meta["unsafe_excluded"] is False + + +def test_collect_llm_api_config_is_deterministic_for_effective_torch_args(): + args = TorchLlmArgs( + model="/customer/private/Llama", + tokenizer="/customer/private/tokenizer", + skip_tokenizer_init=True, + tensor_parallel_size=2, + pipeline_parallel_size=1, + context_parallel_size=1, + dtype="float16", + load_format="dummy", + enable_chunked_prefill=True, + max_num_tokens=4096, + kv_cache_config=KvCacheConfig( + enable_block_reuse=False, + free_gpu_memory_fraction=0.5, + dtype="bfloat16", + tokens_per_block=64, + ), + cuda_graph_config=CudaGraphConfig(batch_sizes=[4, 1], max_batch_size=4), + torch_compile_config=TorchCompileConfig( + enable_inductor=True, + enable_piecewise_cuda_graph=True, + capture_num_tokens=[128, 64], + ), + ) + + first_config_json, first_meta_json = collect_llm_api_config_payloads(args) + second_config_json, second_meta_json = collect_llm_api_config_payloads(args) + config = json.loads(first_config_json) + meta = json.loads(first_meta_json) + + assert first_config_json == second_config_json + assert first_meta_json == second_meta_json + assert config["tensor_parallel_size"] == 2 + assert config["dtype"] == "float16" + assert config["load_format"] == "dummy" + assert config["enable_chunked_prefill"] is True + assert config["max_num_tokens"] == 4096 + assert config["kv_cache_config.enable_block_reuse"] is False + assert config["kv_cache_config.free_gpu_memory_fraction"] == 0.5 + assert config["kv_cache_config.dtype"] == "bfloat16" + assert config["kv_cache_config.tokens_per_block"] == 64 + assert config["cuda_graph_config.batch_sizes"] == [1, 4] + assert config["cuda_graph_config.max_batch_size"] == 4 + assert config["torch_compile_config.enable_inductor"] is True + assert config["torch_compile_config.capture_num_tokens"] == [128, 64] + assert "model" not in config + assert "tokenizer" not in config + assert meta["capture_succeeded"] is True + assert meta["args_class"] == "TorchLlmArgs" + assert meta["schema_digest"] + assert meta["capture_manifest_digest"] + + +def test_collect_llm_api_config_captures_nvfp4_kv_cache_dtype(): + args = TorchLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + kv_cache_config=KvCacheConfig(dtype="nvfp4"), + ) + + config, meta = _loads_payloads(args) + + assert config["kv_cache_config.dtype"] == "nvfp4" + assert meta["capture_succeeded"] is True + + +def test_collect_llm_api_config_captures_int_enum_name(): + class _LoadMode(Enum): + AUTO = 0 + + class _PrecisionMode(Enum): + FP8 = "fp8" + + class _EnumConfig(StrictBaseModel): + mode: _LoadMode = Field(default=_LoadMode.AUTO, telemetry={"kind": "categorical"}) + precision: _PrecisionMode = Field( + default=_PrecisionMode.FP8, telemetry={"kind": "categorical"} + ) + + config, meta = _loads_payloads(_EnumConfig()) + + assert config == {"mode": "AUTO", "precision": "fp8"} + assert meta["unsafe_excluded"] is False + + +def test_collect_llm_api_config_keeps_bool_values_boolean(): + class _BoolConfig(StrictBaseModel): + enabled: bool = Field(default=True, telemetry={"kind": "value"}) + + config, _ = _loads_payloads(_BoolConfig()) + + assert config["enabled"] is True + assert type(config["enabled"]) is bool + + +def test_collect_llm_api_config_rejects_non_finite_floats(): + """Non-finite floats (nan/inf) are excluded; finite floats are captured. + + json.dumps emits the bare NaN/Infinity tokens for non-finite floats, which + are invalid JSON and break downstream parsing and digest stability. Guard + the float branch with math.isfinite so a marked float set to inf/nan is + dropped (unsafe_excluded) while a finite float is still captured. + """ + + class _FloatConfig(StrictBaseModel): + finite: float = Field(default=0.5, telemetry={"kind": "value"}) + infinite: float = Field(default=float("inf"), telemetry={"kind": "value"}) + not_a_number: float = Field(default=float("nan"), telemetry={"kind": "value"}) + + config, meta = _loads_payloads(_FloatConfig()) + + assert config == {"finite": 0.5} + assert "infinite" not in config + assert "not_a_number" not in config + assert meta["excluded_field_count"] == 2 + assert meta["unsafe_excluded"] is True + + +def test_collect_llm_api_config_rejects_non_finite_floats_in_sequence(): + """A single non-finite float poisons the whole marked sequence. + + The sequence sanitizer fails closed on one bad item, so a list containing + inf/nan is dropped entirely rather than emitting invalid JSON tokens. + """ + + class _FloatSeqConfig(StrictBaseModel): + finite_buckets: list[float] = Field( + default_factory=lambda: [0.1, 0.5, 1.0], telemetry={"kind": "value"} + ) + poisoned_buckets: list[float] = Field( + default_factory=lambda: [0.1, float("inf"), 1.0], telemetry={"kind": "value"} + ) + + config, meta = _loads_payloads(_FloatSeqConfig()) + + assert config == {"finite_buckets": [0.1, 0.5, 1.0]} + assert "poisoned_buckets" not in config + assert meta["unsafe_excluded"] is True + + +def test_collect_llm_api_config_caps_long_sequences_and_flags_truncation(): + """A marked sequence longer than MAX_SEQ_ITEMS is clipped and flagged. + + llmApiConfigJson is unbounded on the wire and the reporter is fail-silent, + so a pathological user-sized list could silently drop the whole payload. + Cap captured sequences to MAX_SEQ_ITEMS and record a single honest + sequence_truncated boolean in the metadata. + """ + from tensorrt_llm.usage import llmapi_config + + cap = llmapi_config.MAX_SEQ_ITEMS + + class _LongSeqConfig(StrictBaseModel): + values: list[int] = Field( + default_factory=lambda: list(range(cap + 50)), telemetry={"kind": "value"} + ) + + config, meta = _loads_payloads(_LongSeqConfig()) + + assert len(config["values"]) == cap + assert config["values"] == list(range(cap)) + assert meta["sequence_truncated"] is True + + +def test_collect_llm_api_config_caps_nested_inner_sequences(): + """Each inner list of a nested List[List[int]] is capped independently.""" + from tensorrt_llm.usage import llmapi_config + + cap = llmapi_config.MAX_SEQ_ITEMS + + class _NestedSeqConfig(StrictBaseModel): + rows: list[list[int]] = Field( + default_factory=lambda: [list(range(cap + 10)), list(range(cap + 20))], + telemetry={"kind": "value"}, + ) + + config, meta = _loads_payloads(_NestedSeqConfig()) + + assert len(config["rows"]) == 2 + assert all(len(inner) == cap for inner in config["rows"]) + assert config["rows"][0] == list(range(cap)) + assert meta["sequence_truncated"] is True + + +def test_collect_llm_api_config_caps_outer_nested_sequence(): + """The outer list of a nested List[List[int]] is also capped.""" + from tensorrt_llm.usage import llmapi_config + + cap = llmapi_config.MAX_SEQ_ITEMS + + class _WideNestedSeqConfig(StrictBaseModel): + rows: list[list[int]] = Field( + default_factory=lambda: [[0, 1] for _ in range(cap + 30)], + telemetry={"kind": "value"}, + ) + + config, meta = _loads_payloads(_WideNestedSeqConfig()) + + assert len(config["rows"]) == cap + assert meta["sequence_truncated"] is True + + +def test_collect_llm_api_config_small_sequence_not_truncated(): + """A sequence within the cap is captured whole and the flag stays false.""" + + class _SmallSeqConfig(StrictBaseModel): + values: list[int] = Field(default_factory=lambda: [1, 2, 3], telemetry={"kind": "value"}) + + config, meta = _loads_payloads(_SmallSeqConfig()) + + assert config["values"] == [1, 2, 3] + assert meta["sequence_truncated"] is False + + +def test_collect_llm_api_config_failure_meta_has_truncation_key(): + """Failure metadata carries sequence_truncated for shape parity.""" + from tensorrt_llm.usage import llmapi_config + + meta = llmapi_config._failure_meta(args_class="Foo") + assert meta["sequence_truncated"] is False + + +def test_failure_meta_uses_new_contract_keys_and_versions(): + from tensorrt_llm.usage import llmapi_config as rc + + meta = rc._failure_meta(args_class="X") + assert meta["capture_version"] == "2" + assert meta["api_contract_version"] == "0.2.0" + assert meta["field_policy_version"] == "2" + assert meta["excluded_field_count"] == 0 # renamed from the old marked-count key + assert meta["payload_truncated"] is False + # The pre-migration keys must be gone from the new contract; assert by literal + # so a regression that reintroduces them fails loudly. + assert "excluded_marked_field_count" not in meta + assert "included_field_count" not in meta + + +def test_collect_llm_api_config_rejects_heterogeneous_tuples(): + class _TupleConfig(StrictBaseModel): + pair: tuple[int, Literal["safe"]] = Field(default=(1, "safe"), telemetry={"kind": "value"}) + + config, meta = _loads_payloads(_TupleConfig()) + + assert config == {} + assert meta["excluded_field_count"] == 1 + assert meta["unsafe_excluded"] is True + + +def test_collect_llm_api_config_derives_manifest_kind_from_annotation(): + """Manifest 'kind' is derived per D1, not taken from the registered value. + + Categorical iff (Optional-unwrapped) annotation is Literal/Enum OR an + allowlist is present; otherwise 'value'. The registered kind is ignored. + """ + + class _Mode(Enum): + AUTO = "auto" + + class _KindConfig(StrictBaseModel): + # Literal but deliberately registered with the wrong kind -> categorical. + literal_field: Literal["a", "b"] = Field(default="a", telemetry={"kind": "value"}) + # Enum -> categorical. + enum_field: _Mode = Field(default=_Mode.AUTO, telemetry=True) + # Bare str + allowlist -> categorical. + allowlist_field: str = Field( + default="x", + telemetry={ + "kind": "value", + "converter": "allowlist", + "allowed_values": ["x", "y"], + }, + ) + # Plain int -> value. + int_field: int = Field(default=3, telemetry={"kind": "categorical"}) + + from tensorrt_llm.usage.llmapi_config import build_capture_manifest + + by_path = {e.path: e for e in build_capture_manifest(_KindConfig)} + assert by_path["literal_field"].kind == "categorical" + assert by_path["enum_field"].kind == "categorical" + assert by_path["allowlist_field"].kind == "categorical" + assert by_path["int_field"].kind == "value" + + +def test_collect_llm_api_config_captures_star_attention_backend(): + """attn_backend allowlist recognizes the real FLASHINFER_STAR_ATTENTION value. + + The recognized set mirrors get_attention_backend dispatch; the previously + listed FLASH_ATTENTION is not a real backend and is removed. + """ + args = TorchLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + attn_backend="FLASHINFER_STAR_ATTENTION", + ) + + config, meta = _loads_payloads(args) + + assert config["attn_backend"] == "FLASHINFER_STAR_ATTENTION" + assert meta["capture_succeeded"] is True + + +def test_collect_llm_api_config_swallows_expected_capture_errors(monkeypatch): + """The inner net stays fail-silent for the expected sanitizer error family.""" + from tensorrt_llm.usage import llmapi_config + + def raise_value_error(*_args, **_kwargs): + raise ValueError("synthetic sanitizer error") + + monkeypatch.setattr(llmapi_config, "build_capture_manifest", raise_value_error) + + config, meta = _loads_payloads(_ExampleConfig()) + + assert config == {} + assert meta["capture_succeeded"] is False + assert meta["args_class"] == "_ExampleConfig" + + +def test_collect_llm_api_config_propagates_unexpected_errors(monkeypatch): + """Unexpected errors must propagate, not get swallowed by the inner net.""" + import pytest + + from tensorrt_llm.usage import llmapi_config + + def raise_runtime_error(*_args, **_kwargs): + raise RuntimeError("unexpected collector bug") + + monkeypatch.setattr(llmapi_config, "build_capture_manifest", raise_runtime_error) + + with pytest.raises(RuntimeError, match="unexpected collector bug"): + collect_llm_api_config_payloads(_ExampleConfig()) + + +def test_collect_llm_api_config_captures_expanded_value_fields(): + """Representative newly-marked value fields are captured on TorchLlmArgs.""" + args = TorchLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + moe_expert_parallel_size=2, + moe_tensor_parallel_size=1, + moe_cluster_parallel_size=1, + num_postprocess_workers=3, + stream_interval=4, + trust_remote_code=True, + ) + + config, meta = _loads_payloads(args) + + assert config["moe_expert_parallel_size"] == 2 + assert config["moe_tensor_parallel_size"] == 1 + assert config["moe_cluster_parallel_size"] == 1 + assert config["num_postprocess_workers"] == 3 + assert config["stream_interval"] == 4 + assert config["trust_remote_code"] is True + # backend on TorchLlmArgs is the Literal["pytorch"] override -> value capture. + assert config["backend"] == "pytorch" + assert meta["capture_succeeded"] is True + + +def test_collect_llm_api_config_captures_nested_config_value_fields(): + """Newly-marked nested-config value fields are captured via recursion.""" + from tensorrt_llm.llmapi.llm_args import MoeConfig + + args = TorchLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + moe_config=MoeConfig(max_num_tokens=8192, disable_finalize_fusion=True), + ) + + config, _ = _loads_payloads(args) + + assert config["moe_config.max_num_tokens"] == 8192 + assert config["moe_config.disable_finalize_fusion"] is True + # MoeConfig.backend is a Literal -> derived categorical, value captured. + assert config["moe_config.backend"] == "AUTO" + + +def test_collect_llm_api_config_captures_quant_algo_cross_module(): + """quant_config.quant_algo (QuantConfig in modeling_utils.py) is captured. + + QuantConfig is defined outside llm_args.py but the collector recurses into + any nested pydantic model, so marking the field at its definition site is + sufficient. quant_config is a real model field only on TrtLlmArgs. + """ + from tensorrt_llm.llmapi.llm_args import TrtLlmArgs + from tensorrt_llm.models.modeling_utils import QuantConfig + from tensorrt_llm.quantization.mode import QuantAlgo + + args = TrtLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + quant_config=QuantConfig(quant_algo=QuantAlgo.FP8), + ) + + config, meta = _loads_payloads(args) + + assert config["quant_config.quant_algo"] == "FP8" + assert meta["capture_succeeded"] is True + + +def test_field_wrapper_preserves_callable_json_schema_extra_with_metadata(): + """Preserve callable json_schema_extra when adding status/telemetry metadata. + + The schema callable still runs for Pydantic JSON schema generation, while + the collector can read the attached TRT-LLM metadata without executing it. + """ + + def mark_schema(schema: dict[str, object]) -> dict[str, object]: + schema["original"] = True + return {"returned": True} + + field = Field( + default=1, + status="beta", + telemetry=True, + json_schema_extra=mark_schema, + ) + schema: dict[str, object] = {} + assert callable(field.json_schema_extra) + field.json_schema_extra(schema) + + assert schema == { + "original": True, + "returned": True, + "status": "beta", + "telemetry": {"kind": "value"}, + } + + class _CallableExtraConfig(StrictBaseModel): + value: int = Field( + default=1, + telemetry=True, + json_schema_extra=mark_schema, + ) + + config, meta = _loads_payloads(_CallableExtraConfig()) + assert config == {"value": 1} + assert meta["capture_succeeded"] is True + + +def test_collect_llm_api_config_captures_none_on_optional_allowlist_field(): + """None on an Optional allowlist field is captured as null, not excluded. + + Regression: the None check must precede the allowlist branch, else a None + default (e.g. reasoning_parser, TrtLlmArgs.backend) fails the allowlist and + permanently flips unsafe_excluded on default configs. + """ + + class _C(StrictBaseModel): + backend: Optional[str] = Field( + default=None, + telemetry={ + "kind": "categorical", + "converter": "allowlist", + "allowed_values": ["pytorch", "tensorrt"], + }, + ) + + config, meta = _loads_payloads(_C()) + assert config == {"backend": None} + assert meta["captured_field_count"] == 1 + assert meta["excluded_field_count"] == 0 + assert meta["unsafe_excluded"] is False + + +def test_collect_llm_api_config_captures_quant_algo_none(): + """quant_config.quant_algo defaults to None and is captured as null.""" + from tensorrt_llm.llmapi.llm_args import TrtLlmArgs + + args = TrtLlmArgs(model="/customer/private/Llama", skip_tokenizer_init=True) + + config, _ = _loads_payloads(args) + + assert config["quant_config.quant_algo"] is None + + +def test_collect_llm_api_config_captures_sampler_type_categorical(): + """sampler_type is a bounded Union[str, SamplerType] categorical allowlist.""" + args = TorchLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + sampler_type="TorchSampler", + ) + + config, meta = _loads_payloads(args) + + assert config["sampler_type"] == "TorchSampler" + assert meta["capture_succeeded"] is True + + +def test_collect_llm_api_config_redacts_out_of_allowlist_categorical_str(): + """An out-of-allowlist value on a categorical bare-str field is dropped. + + reasoning_parser is captured via TelemetryField.categorical mirroring the + ReasoningParserFactory registry; any value outside that recognized domain + (e.g. injected free-form text) must be excluded, not captured. + """ + args = TorchLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + reasoning_parser="not-a-real-parser-/customer/secret", + ) + + config, meta = _loads_payloads(args) + + assert "reasoning_parser" not in config + assert meta["unsafe_excluded"] is True + # An in-allowlist value is captured. + args_ok = TorchLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + reasoning_parser="deepseek-r1", + ) + config_ok, _ = _loads_payloads(args_ok) + assert config_ok["reasoning_parser"] == "deepseek-r1" + + +def test_collect_llm_api_config_captures_gms_load_format(): + """load_format=GMS is captured as 'gms' (was dropped before the allowlist fix). + + LoadFormat.GMS is a real, accepted value (convert_load_format maps the + string 'gms' to the enum), but it was missing from the load_format telemetry + allowlist, so GMS deployments were silently excluded from llmApiConfigJson. + """ + args = TorchLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + load_format="gms", + ) + + config, meta = _loads_payloads(args) + + assert config["load_format"] == "gms" + assert meta["capture_succeeded"] is True + + +def test_collect_llm_api_config_captures_backend_allowlist_on_trt_args(): + """TrtLlmArgs inherits backend as Optional[str] -> bounded categorical allowlist.""" + from tensorrt_llm.llmapi.llm_args import TrtLlmArgs + + args = TrtLlmArgs( + model="/customer/private/Llama", + skip_tokenizer_init=True, + backend="tensorrt", + ) + + config, meta = _loads_payloads(args) + + assert config["backend"] == "tensorrt" + assert meta["capture_succeeded"] is True + + +def _walk_captured_keys(model) -> set[str]: + """Capture a single nested config model and return its captured keys.""" + config, _ = _loads_payloads(model) + return set(config) + + +def test_collect_llm_api_config_captures_decoding_type_for_every_arm(): + """The speculative discriminator decoding_type is captured for every arm. + + decoding_type is the single most valuable categorical (it identifies which + speculative mode is active). The runtime collector walks the concrete active + arm's model_fields, so marking it on only one arm drops it for the others. + Assert representative non-UserProvided arms capture it. + """ + from tensorrt_llm.llmapi.llm_args import ( + AutoDecodingConfig, + MedusaDecodingConfig, + MTPDecodingConfig, + NGramDecodingConfig, + ) + + mtp = MTPDecodingConfig(num_nextn_predict_layers=1) + assert _walk_captured_keys(mtp) >= {"decoding_type"} + + medusa = MedusaDecodingConfig(max_draft_len=1, medusa_choices=[[0]]) + assert _walk_captured_keys(medusa) >= {"decoding_type"} + + ngram = NGramDecodingConfig(max_draft_len=1, max_matching_ngram_size=2) + assert _walk_captured_keys(ngram) >= {"decoding_type"} + + auto = AutoDecodingConfig() + assert _walk_captured_keys(auto) >= {"decoding_type"} + + +def test_collect_llm_api_config_captures_max_total_draft_tokens_for_every_arm(): + """max_total_draft_tokens is a safe value field on the shared base. + + Marking it only on a single override (SaveHiddenStates) drops it for every + other arm at runtime even though the doc-gen union-collapse advertises it. + Mark it on DecodingBaseConfig so all arms capture it. + """ + from tensorrt_llm.llmapi.llm_args import MTPDecodingConfig + + mtp = MTPDecodingConfig(num_nextn_predict_layers=1) + assert "max_total_draft_tokens" in _walk_captured_keys(mtp) + + +def test_collect_llm_api_config_captures_sparse_algorithm_for_every_arm(): + """The sparse-attention discriminator algorithm is captured for every arm. + + algorithm identifies the active sparse algorithm (rocket, dsa, skip_softmax). + Marking it on only one arm drops it for the others at runtime even though the + doc-gen union-collapse advertises sparse_attention_config.algorithm. + """ + from tensorrt_llm.llmapi.llm_args import ( + DeepSeekSparseAttentionConfig, + RocketSparseAttentionConfig, + SkipSoftmaxAttentionConfig, + ) + + assert _walk_captured_keys(RocketSparseAttentionConfig()) >= {"algorithm"} + assert _walk_captured_keys(DeepSeekSparseAttentionConfig()) >= {"algorithm"} + assert _walk_captured_keys(SkipSoftmaxAttentionConfig()) >= {"algorithm"} + + +def test_background_reporter_keeps_initial_report_when_config_capture_fails(monkeypatch): + sent_payloads = [] + + monkeypatch.setattr(usage_lib, "_MAX_HEARTBEATS", 0) + monkeypatch.setattr(usage_lib, "_get_trtllm_version", lambda: "0.0.0-test") + monkeypatch.setattr( + usage_lib, + "_collect_system_info", + lambda: { + "platform": "linux", + "python_version": "3", + "cpu_architecture": "x86", + "cpu_count": 1, + }, + ) + monkeypatch.setattr( + usage_lib, + "_collect_gpu_info", + lambda: {"gpu_count": 0, "gpu_name": "", "gpu_memory_mb": 0, "cuda_version": ""}, + ) + monkeypatch.setattr(usage_lib, "_send_to_gxt", sent_payloads.append) + + def raise_capture_error(_): + raise RuntimeError("capture failed") + + monkeypatch.setattr(usage_lib, "_collect_llm_api_config_payloads", raise_capture_error) + + usage_lib._background_reporter( + llm_args=object(), pretrained_config=None, usage_context="llm_class" + ) + + params = sent_payloads[0]["events"][0]["parameters"] + assert json.loads(params["llmApiConfigJson"]) == {} + meta = json.loads(params["llmApiConfigMetaJson"]) + assert meta["capture_succeeded"] is False + assert meta["args_class"] == "object" + assert params["featuresJson"] + + +def test_field_wrapper_records_explicit_exclude_marker(): + from tensorrt_llm.usage.llmapi_config import _get_telemetry_metadata + + class _C(StrictBaseModel): + a: int = Field(default=1, telemetry=False) + + meta = _get_telemetry_metadata(_C.model_fields["a"]) + assert meta == {"exclude": True} + + +def test_collect_llm_api_config_honors_explicit_exclude_sentinel(): + class _ExcludeConfig(StrictBaseModel): + kept: int = Field(default=1) + secret_seed: int = Field(default=42, telemetry=False) + + config, meta = _loads_payloads(_ExcludeConfig()) + assert config == {"kept": 1} + assert "secret_seed" not in config + assert meta["capturable_field_count"] == 1 + + +def test_collect_llm_api_config_honors_raw_json_schema_extra_exclude(): + # Cross-module models use bare pydantic Field with json_schema_extra={"telemetry": ...}. + # A raw {"telemetry": False} must be honored as an exclude, like the wrapper telemetry=False. + from pydantic import Field as PydField + + class _RawExcludeConfig(StrictBaseModel): + kept: int = 1 + secret: int = PydField(default=2, json_schema_extra={"telemetry": False}) + + config, meta = _loads_payloads(_RawExcludeConfig()) + assert "secret" not in config + assert config == {"kept": 1} + + +def test_runtime_keys_are_subset_of_manifest_for_fixture(): + from tensorrt_llm.usage.llmapi_config import build_capture_manifest + + inst = _ExampleConfig() + manifest_paths = {e.path for e in build_capture_manifest(_ExampleConfig)} + config, _ = _loads_payloads(inst) + assert set(config) <= manifest_paths + + +def test_manifest_excludes_loosely_typed_model_children(): + # B-1 regression: moe_config.load_balancer is Optional[Union[object, str]]; + # a validator coerces it into a MoeLoadBalancerConfig at runtime, but the + # annotation names no BaseModel, so its children must NOT be capturable. + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + from tensorrt_llm.usage.llmapi_config import build_capture_manifest + + paths = {e.path for e in build_capture_manifest(TorchLlmArgs)} + assert not any(p.startswith("moe_config.load_balancer.") for p in paths) + + +def test_collect_llm_api_config_caps_total_payload_size(monkeypatch): + from tensorrt_llm.usage import llmapi_config as rc + + class _BigConfig(StrictBaseModel): + a: int = 11111111 + b: int = 22222222 + c: int = 33333333 + + monkeypatch.setattr(rc, "MAX_CONFIG_BYTES", 20) # force truncation + config, meta = _loads_payloads(_BigConfig()) + assert meta["payload_truncated"] is True + assert len(rc._canonical_json(config).encode("utf-8")) <= 20 diff --git a/tests/unittest/usage/test_llmapi_config_telemetry_docs.py b/tests/unittest/usage/test_llmapi_config_telemetry_docs.py new file mode 100644 index 000000000000..cbfe456288ee --- /dev/null +++ b/tests/unittest/usage/test_llmapi_config_telemetry_docs.py @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def _load_generator() -> ModuleType: + module_path = _repo_root() / "docs/source/_ext/llmapi_config_telemetry.py" + spec = importlib.util.spec_from_file_location("llmapi_config_telemetry", module_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + # exec_module needs the module registered (frozen dataclasses resolve their + # owning module via sys.modules), but restore the prior state afterwards so + # the loader does not leak its temporary module on success or failure. + sentinel = object() + previous = sys.modules.get(spec.name, sentinel) + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + finally: + if previous is sentinel: + sys.modules.pop(spec.name, None) + else: + sys.modules[spec.name] = previous + return module + + +def _golden_path() -> Path: + return _repo_root() / "tensorrt_llm/usage/llm_args_golden_manifest.json" + + +def test_build_capture_manifest_matches_committed_golden(): + """The CI privacy gate (closes TRTLLM-12872). + + Regenerate in-memory and diff against the committed golden; any drift + ('field X now phones home') must be a deliberate, privacy-reviewed golden + update committed in the same change. + """ + from tensorrt_llm.usage.llmapi_config import golden_manifest + + golden = json.loads(_golden_path().read_text()) + assert golden_manifest() == golden + + +def test_load_generator_does_not_leak_sys_modules(): + """_load_generator must not leak its temporary module into sys.modules. + + The loader needs the module registered while exec_module runs (frozen + dataclasses resolve their module via sys.modules), but it must restore the + prior state on both success and failure. + """ + name = "llmapi_config_telemetry" + sys.modules.pop(name, None) + + _load_generator() + assert name not in sys.modules + + import importlib.util as _util + + real_spec_from_file_location = _util.spec_from_file_location + + def _boom(*args, **kwargs): + spec = real_spec_from_file_location(*args, **kwargs) + + class _BoomLoader: + name = spec.name + + def create_module(self, spec): + return None + + def exec_module(self, module): + raise RuntimeError("synthetic load failure") + + spec.loader = _BoomLoader() + return spec + + _util.spec_from_file_location = _boom + try: + try: + _load_generator() + except RuntimeError: + pass + assert name not in sys.modules + finally: + _util.spec_from_file_location = real_spec_from_file_location + + +def test_domain_values_cover_literal_and_enum(): + from enum import Enum + from typing import Literal, Optional + + from tensorrt_llm.usage import llmapi_config as rc + + class _Color(str, Enum): + RED = "red" + BLUE = "blue" + + assert rc._domain_values(Optional[Literal["a", "b"]], {}) == ["a", "b"] + assert rc._domain_values(Optional[_Color], {}) == ["red", "blue"] + assert rc._domain_values(int, {"converter": "allowlist", "allowed_values": ["x", "y"]}) == [ + "x", + "y", + ] + + +def _small_models(): + from enum import Enum + + from pydantic import BaseModel + + from tensorrt_llm.llmapi.llm_args import Field + + class Mode(str, Enum): + A = "a" + B = "b" + + class Nested(BaseModel): + n_int: int = 0 + n_str: str = "secret" # bare str -> OUT + n_secret: int = Field(default=0, telemetry=False) # honored exclude + + class Root(BaseModel): + flag: bool = True + mode: Mode = Mode.A + sizes: list[int] = Field(default_factory=list) + path_like: str = "x" # bare str -> OUT + allow: str = Field( + default="a", + telemetry={ + "kind": "categorical", + "converter": "allowlist", + "allowed_values": ["a", "b"], + }, + ) + nested: Nested | None = None + loose: object | str | None = None # no BaseModel arm -> not recursed + + return Root + + +def test_build_capture_manifest_selection_and_recursion(): + from tensorrt_llm.usage.llmapi_config import build_capture_manifest + + Root = _small_models() + paths = {e.path for e in build_capture_manifest(Root)} + assert paths == {"flag", "mode", "sizes", "allow", "nested.n_int"} + assert "path_like" not in paths # bare str OUT + assert "nested.n_str" not in paths # bare str OUT + assert "nested.n_secret" not in paths # honored telemetry=False + assert not any(p.startswith("loose") for p in paths) # loose has no model arm + + +def test_build_capture_manifest_kinds_and_domains(): + from tensorrt_llm.usage.llmapi_config import build_capture_manifest + + Root = _small_models() + by_path = {e.path: e for e in build_capture_manifest(Root)} + assert by_path["flag"].kind == "value" + assert by_path["mode"].kind == "categorical" + assert list(by_path["mode"].allowed_values) == ["a", "b"] # Enum domain + assert by_path["allow"].kind == "categorical" + assert by_path["allow"].converter == "allowlist" + assert list(by_path["allow"].allowed_values) == ["a", "b"] + + +def test_renderer_emits_table_from_committed_golden(tmp_path): + generator = _load_generator() + out = tmp_path / "telemetry.md" + generator.generate_telemetry_reference(_repo_root(), out) + text = out.read_text() + assert "## LLM API Configuration Fields" in text + assert "explicitly marked" not in text # opt-in prose must be gone + assert "`backend`" in text # a known captured key renders + + +def test_build_capture_manifest_fails_on_divergent_kind_across_union_arms(): + from typing import Literal, Union + + import pytest + from pydantic import BaseModel + + from tensorrt_llm.usage.llmapi_config import build_capture_manifest + + class ArmA(BaseModel): + tag: Literal["a"] = "a" + shared: int = 0 # kind=value + + class ArmB(BaseModel): + tag: Literal["b"] = "b" + shared: Literal["x", "y"] = "x" # kind=categorical -> conflict on "arm.shared" + + class Root(BaseModel): + arm: Union[ArmA, ArmB] = ArmA() + + with pytest.raises(ValueError, match="conflicting kinds"): + build_capture_manifest(Root) + + +def test_build_capture_manifest_cycle_guard_terminates_on_self_reference(): + from typing import Optional + + from pydantic import BaseModel + + from tensorrt_llm.usage.llmapi_config import build_capture_manifest + + class Node(BaseModel): + value: int = 0 + child: Optional["Node"] = None + + Node.model_rebuild() + entries = build_capture_manifest(Node) # must TERMINATE (cycle guard), not infinite-recurse + paths = {e.path for e in entries} + assert "value" in paths + assert "child.value" not in paths diff --git a/tests/unittest/usage/test_reporter.py b/tests/unittest/usage/test_reporter.py index e498be3013e4..130b08a54b68 100644 --- a/tests/unittest/usage/test_reporter.py +++ b/tests/unittest/usage/test_reporter.py @@ -14,11 +14,14 @@ # limitations under the License. """Tests for report_usage(), background reporter, thread lifecycle, and heartbeat.""" +import json import logging import threading from types import SimpleNamespace from unittest.mock import MagicMock, patch +from pydantic import BaseModel, Field + from tensorrt_llm.usage import usage_lib # --------------------------------------------------------------------------- @@ -396,6 +399,40 @@ def fake_send(payload): assert params["disaggRole"] == "context" assert params["deploymentId"] == "abc123" + def test_disagg_payload_includes_llm_api_config_json(self, monkeypatch): + """Disagg payloads retain sanitized LLM API config JSON fields.""" + + class _DisaggTelemetryArgs(BaseModel): + tensor_parallel_size: int = Field( + default=2, json_schema_extra={"telemetry": {"kind": "value"}} + ) + + monkeypatch.setenv("TRTLLM_DISAGG_ROLE", "generation") + monkeypatch.setenv("TRTLLM_DISAGG_DEPLOYMENT_ID", "deploy123") + + captured = {} + + def fake_send(payload): + captured.update(payload) + + stop_event = threading.Event() + stop_event.set() + + with ( + patch.object(usage_lib, "_send_to_gxt", side_effect=fake_send), + patch.object(usage_lib, "_REPORTER_STOP", stop_event), + ): + usage_lib._background_reporter(_DisaggTelemetryArgs(), None, "cli_serve") + + assert captured, "No payload was captured" + params = captured["events"][0]["parameters"] + assert params["disaggRole"] == "generation" + assert params["deploymentId"] == "deploy123" + assert json.loads(params["llmApiConfigJson"]) == {"tensor_parallel_size": 2} + meta = json.loads(params["llmApiConfigMetaJson"]) + assert meta["capture_succeeded"] is True + assert meta["args_class"] == "_DisaggTelemetryArgs" + class TestDisaggMetadataEmpty: """Verify empty defaults when disagg env vars are unset (non-disagg mode).""" diff --git a/tests/unittest/usage/test_schema.py b/tests/unittest/usage/test_schema.py index f65fed9e34af..1603b3550762 100644 --- a/tests/unittest/usage/test_schema.py +++ b/tests/unittest/usage/test_schema.py @@ -46,7 +46,13 @@ def test_initial_report_contains_features_json(self): def test_features_json_round_trips_through_gxt_payload(self): """FeaturesJson survives full JSON serialization round-trip.""" features = '{"lora":true,"speculative_decoding":false,"data_parallel_size":4}' - report = schema.TrtllmInitialReport(featuresJson=features) + llm_api_config = '{"tensor_parallel_size":2}' + llm_api_config_meta = '{"capture_succeeded":true}' + report = schema.TrtllmInitialReport( + featuresJson=features, + llmApiConfigJson=llm_api_config, + llmApiConfigMetaJson=llm_api_config_meta, + ) payload = schema.build_gxt_payload( event=report, session_id="test", @@ -58,12 +64,16 @@ def test_features_json_round_trips_through_gxt_payload(self): assert inner["lora"] is True assert inner["speculative_decoding"] is False assert inner["data_parallel_size"] == 4 + assert parsed["events"][0]["parameters"]["llmApiConfigJson"] == llm_api_config + assert parsed["events"][0]["parameters"]["llmApiConfigMetaJson"] == llm_api_config_meta def test_features_json_default_is_empty_object(self): """Default featuresJson value is '{}'.""" report = schema.TrtllmInitialReport() data = report.model_dump(by_alias=True) assert data["featuresJson"] == "{}" + assert data["llmApiConfigJson"] == "{}" + assert data["llmApiConfigMetaJson"] == "{}" # --------------------------------------------------------------------------- @@ -203,8 +213,8 @@ def test_event_protocol(self): assert schema.EVENT_PROTOCOL == "1.6" def test_event_schema_ver(self): - """EVENT_SCHEMA_VER is 0.1 (matches SMS schema schemaVersion).""" - assert schema.EVENT_SCHEMA_VER == "0.1" + """EVENT_SCHEMA_VER is 0.2 (matches SMS schema schemaVersion).""" + assert schema.EVENT_SCHEMA_VER == "0.2" def test_event_sys_ver(self): """EVENT_SYS_VER identifies the telemetry subsystem.""" @@ -442,6 +452,8 @@ def test_initial_report_has_all_expected_fields(self): "kvCacheDtype", "ingressPoint", "featuresJson", + "llmApiConfigJson", + "llmApiConfigMetaJson", "disaggRole", "deploymentId", } @@ -655,6 +667,8 @@ def test_initial_report_validates_against_json_schema(self): kvCacheDtype="", ingressPoint="llm_class", featuresJson='{"lora":false}', + llmApiConfigJson='{"tensor_parallel_size":1}', + llmApiConfigMetaJson='{"capture_succeeded":true}', disaggRole="", deploymentId="", )