diff --git a/examples/rl/README.md b/examples/rl/README.md index 9c2de3ec088..b66b2abbc32 100644 --- a/examples/rl/README.md +++ b/examples/rl/README.md @@ -172,7 +172,6 @@ torchrun \ --save $CHECKPOINT_DIR \ --load $CHECKPOINT_DIR \ --tensorboard-dir $TB_DIR \ - --langrl-inference-server-type inplace_megatron \ --seed $SEED \ --sequence-parallel \ --finetune \ diff --git a/examples/rl/environment_configs/gsm8k_nanov3.yaml b/examples/rl/environment_configs/gsm8k_nanov3.yaml index 30403ed052b..b759423ee5b 100644 --- a/examples/rl/environment_configs/gsm8k_nanov3.yaml +++ b/examples/rl/environment_configs/gsm8k_nanov3.yaml @@ -2,8 +2,6 @@ agent_args: answer_format: "boxed" format_reward: 0.5 - assistant_suffix: "Assistant: " - chat_mode: true negative_reward: 0.0 partial_end_reward: 0.75 weight: 1.0 diff --git a/examples/rl/environments/countdown/countdown_agent.py b/examples/rl/environments/countdown/countdown_agent.py index bd9413a19d0..e995602b0a2 100644 --- a/examples/rl/environments/countdown/countdown_agent.py +++ b/examples/rl/environments/countdown/countdown_agent.py @@ -12,15 +12,8 @@ class CountdownAgent(RewardOnlyAgent, HFDatasetAgent): def make_prefix(self, target, nums) -> str: - if self.chat_mode: - prefix = f"""Using the numbers {nums}, create an equation that equals {target}. You can use basic arithmetic operations (+, -, *, /) and each number can only be used once. + prefix = f"""Using the numbers {nums}, create an equation that equals {target}. You can use basic arithmetic operations (+, -, *, /) and each number can only be used once. Return the final answer in tags, for example (1 + 2) / 3 . Do not include an = sign.""" - else: - prefix = f"""A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. - User: Using the numbers {nums}, create an equation that equals {target}. You can use basic arithmetic operations (+, -, *, /) and each number can only be used once. Show your work in tags. - And return the final answer in tags, for example (1 + 2) / 3 . Do not include an = sign. - Assistant: Let me solve this step by step. - """ return prefix def get_dataset(self, validation: bool = False): diff --git a/examples/rl/environments/math/gsm8k_agent.py b/examples/rl/environments/math/gsm8k_agent.py index 3bb39bc09f9..6cdfb4f926e 100644 --- a/examples/rl/environments/math/gsm8k_agent.py +++ b/examples/rl/environments/math/gsm8k_agent.py @@ -25,16 +25,12 @@ class GSM8KAgent(MathAgent): def __init__(self, answer_format: str = "boxed", - chat_mode: bool = False, - assistant_suffix: str = "Assistant: Let me solve this step by step.\n", format_reward: float = 0.0, negative_reward: float = 0.0, partial_end_reward: float = 0.0, **kwargs): super().__init__( answer_format=answer_format, - chat_mode=chat_mode, - assistant_suffix=assistant_suffix, format_reward=format_reward, negative_reward=negative_reward, partial_end_reward=partial_end_reward, diff --git a/examples/rl/environments/math/math_agent.py b/examples/rl/environments/math/math_agent.py index bdf322561eb..027ef242285 100644 --- a/examples/rl/environments/math/math_agent.py +++ b/examples/rl/environments/math/math_agent.py @@ -3,7 +3,6 @@ import re import traceback -from megatron.rl.agent.pass_at_evaluation_agent import PassAtEvaluationAgent from megatron.rl.agent.reward_only_agent import RewardOnlyAgent try: @@ -25,8 +24,6 @@ class MathAgent(RewardOnlyAgent): def __init__(self, format_reward: float = 0.0, answer_format: str = "tagged", - assistant_suffix: str = "Assistant: Let me solve this step by step.\n", - chat_mode: bool = False, negative_reward: float = 0.0, partial_end_reward: float = 0.0, **kwargs): @@ -36,9 +33,6 @@ def __init__(self, even if the answer is incorrect or is missing the end-of-text token. answer_format (str): Which answer format is expected: "tagged" for tags, or "boxed" for \boxed{} LaTeX formatting. - assistant_suffix (str): The suffix string included in the assistant's response, typically to - guide the assistant's output format and "persona". For example, "Let me solve this step by step." - chat_mode (bool): If True, agent operates in a chat (conversational) context. negative_reward (float): Reward assigned for a clearly incorrect or unparseable answer. partial_end_reward (float): Reward when the answer is correct but an expected end token is not matched exactly. **kwargs: Additional arguments for the base RewardOnlyAgent. @@ -49,8 +43,6 @@ def __init__(self, self.format_reward = format_reward self.answer_format = answer_format - self.assistant_suffix = assistant_suffix - self.chat_mode = chat_mode self.negative_reward = negative_reward self.partial_end_reward = partial_end_reward @@ -134,12 +126,6 @@ def make_prefix(self, problem_key: str = "problem", **kwargs) -> str: else: raise ValueError(f"Invalid answer format: {self.answer_format}") - if self.chat_mode: - prefix = f"""{kwargs[problem_key]}\n{answer_format}""" - else: - prefix = f"""A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. - The question will be a word math problem. Show your work in tags. - {answer_format} - User: {kwargs[problem_key]} - {self.assistant_suffix}""" + prefix = f"""{kwargs[problem_key]}\n{answer_format}""" + return prefix diff --git a/examples/rl/model_configs/common.sh b/examples/rl/model_configs/common.sh index 4f6ca0e18cf..c37d88fb4df 100644 --- a/examples/rl/model_configs/common.sh +++ b/examples/rl/model_configs/common.sh @@ -22,7 +22,7 @@ COMMON_OPTIONS="\ --attention-backend flash \ --timing-log-level 1 \ --log-timers-to-tensorboard \ - --save-retain-interval 120 \ + --save-retain-interval 160 \ --inference-dynamic-batching-num-cuda-graphs 1 \ --inference-dynamic-batching-unified-memory-level 1 \ --adam-beta1 0.9 \ diff --git a/examples/rl/model_configs/llama3p1_8b_instruct.sh b/examples/rl/model_configs/llama3p1_8b_instruct.sh index 5398dad1a4e..325c1d80617 100644 --- a/examples/rl/model_configs/llama3p1_8b_instruct.sh +++ b/examples/rl/model_configs/llama3p1_8b_instruct.sh @@ -101,9 +101,6 @@ MODEL_OPTIONS="\ --max-position-embeddings 131072 \ --tokenizer-type HuggingFaceTokenizer \ --tokenizer-model unsloth/Meta-Llama-3.1-8B-Instruct \ - --legacy-tokenizer \ - --langrl-inference-server-type "inplace_megatron_chat" \ - --langrl-inference-server-conversation-template "unsloth/Meta-Llama-3.1-8B-Instruct" \ --lr 3e-7 \ --make-vocab-size-divisible-by 128 \ --clip-grad 1.0 \ diff --git a/examples/rl/model_configs/qwen3_4b.sh b/examples/rl/model_configs/qwen3_4b.sh index 6f6c6b6bf57..81899f2ea7b 100644 --- a/examples/rl/model_configs/qwen3_4b.sh +++ b/examples/rl/model_configs/qwen3_4b.sh @@ -63,8 +63,6 @@ MODEL_OPTIONS="\ --attention-softmax-in-fp32 \ --tokenizer-type HuggingFaceTokenizer \ --tokenizer-model Qwen/Qwen3-4B \ - --langrl-inference-server-type "inplace_megatron_chat" \ - --langrl-inference-server-conversation-template "Qwen/Qwen3-4B" \ --vocab-size 151936 \ --make-vocab-size-divisible-by 128 \ --optimizer adam \ diff --git a/examples/rl/model_configs/qwen3_8b.sh b/examples/rl/model_configs/qwen3_8b.sh index 54ff7385331..4b5e1103f0e 100644 --- a/examples/rl/model_configs/qwen3_8b.sh +++ b/examples/rl/model_configs/qwen3_8b.sh @@ -64,8 +64,6 @@ MODEL_OPTIONS="\ --attention-softmax-in-fp32 \ --tokenizer-type HuggingFaceTokenizer \ --tokenizer-model Qwen/Qwen3-8B \ - --langrl-inference-server-type "inplace_megatron_chat" \ - --langrl-inference-server-conversation-template "Qwen/Qwen3-8B" \ --vocab-size 151936 \ --make-vocab-size-divisible-by 128 \ --optimizer adam \ diff --git a/examples/rl/model_configs/qwen_2p5_3b.sh b/examples/rl/model_configs/qwen_2p5_3b.sh index f3250f39ecc..647023d3050 100644 --- a/examples/rl/model_configs/qwen_2p5_3b.sh +++ b/examples/rl/model_configs/qwen_2p5_3b.sh @@ -22,7 +22,7 @@ if [ "$(basename "$ENV_CONFIG")" = "dapo.yaml" ]; then GRPO_KL_BETA=${GRPO_KL_BETA:-"0.0"} ENTROPY_WEIGHT=${ENTROPY_WEIGHT:-"0.0"} TRAINING_BATCH_SIZE=${TRAINING_BATCH_SIZE:-1024} - MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-2} + MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-1} MAX_SEQ_LENGTH=${MAX_SEQ_LENGTH:-8192} EXIT_INTERVAL=${EXIT_INTERVAL:-16} CHKPT_SAVE_INTERVAL=${CHKPT_SAVE_INTERVAL:-16} @@ -38,7 +38,7 @@ else GRPO_KL_BETA=${GRPO_KL_BETA:-"0.0"} ENTROPY_WEIGHT=${ENTROPY_WEIGHT:-"0.0"} TRAINING_BATCH_SIZE=${TRAINING_BATCH_SIZE:-512} - MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-2} + MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-1} MAX_SEQ_LENGTH=${MAX_SEQ_LENGTH:-8192} EXIT_INTERVAL=${EXIT_INTERVAL:-16} CHKPT_SAVE_INTERVAL=${CHKPT_SAVE_INTERVAL:-16} diff --git a/examples/rl/model_configs/qwen_2p5_distill_7b.sh b/examples/rl/model_configs/qwen_2p5_distill_7b.sh index 1438bca0726..ed214b3aae9 100644 --- a/examples/rl/model_configs/qwen_2p5_distill_7b.sh +++ b/examples/rl/model_configs/qwen_2p5_distill_7b.sh @@ -70,8 +70,6 @@ MODEL_OPTIONS="\ --max-position-embeddings 131072 \ --tokenizer-type HuggingFaceTokenizer \ --tokenizer-model "unsloth/DeepSeek-R1-Distill-Qwen-7B" \ - --langrl-inference-server-type "inplace_megatron_chat" \ - --langrl-inference-server-conversation-template "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" \ --lr 0.000001 \ --lr-warmup-samples 0 \ --make-vocab-size-divisible-by 128 \ diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 34c3b954074..8d3ecba235f 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -3,8 +3,10 @@ import asyncio import logging import time +import traceback from megatron.core.inference.sampling_params import SamplingParams +from megatron.core.tokenizers.text.parsers import PARSER_MAPPING logger = logging.getLogger(__name__) @@ -19,6 +21,7 @@ async def chat_completions(): """Handles async POST requests for chat completions.""" client = current_app.config['client'] tokenizer = current_app.config['tokenizer'] + parsers = current_app.config['parsers'] req = request.get_json() @@ -31,14 +34,17 @@ async def chat_completions(): try: prompt_tokens = tokenizer.apply_chat_template( - messages, tokenize=True, add_generation_prompt=True + messages, tokenize=True, add_generation_prompt=True, tools=req.get("tools", None) + ) + except (AttributeError, AssertionError): + logger.warning( + "Tokenizer does not support 'apply_chat_template'. Using tokenize instead." + ) + prompt_tokens = tokenizer.tokenize( + "\n".join([message["content"] for message in messages]) ) - except AttributeError: - return ( - "Tokenizer does not support 'apply_chat_template'. " - "Chat completions requires a tokenizer with a configured chat template." - ), 500 except Exception as e: + logger.error(f"{traceback.format_exc()}") return f"Error processing 'messages': {e}", 500 # --- 2. Parse Sampling Params --- @@ -55,6 +61,17 @@ async def chat_completions(): # Check for 'logprobs' (bool) and 'top_logprobs' (int) return_log_probs = bool(req.get("logprobs", False)) top_n_logprobs = int(req.get("top_logprobs", 0)) if return_log_probs else 0 + skip_prompt_log_probs = bool(req.get("skip_prompt_log_probs", False)) + add_BOS = bool(req.get("add_BOS", False)) + + # The engine only handles add_BOS for string prompts, not pre-tokenized + # input. Since we pre-tokenize via apply_chat_template, we must handle + # BOS ourselves, matching the logic in tokenize_prompt(). + if hasattr(tokenizer, 'bos') and tokenizer.bos is not None: + while prompt_tokens and prompt_tokens[0] == tokenizer.bos: + prompt_tokens.pop(0) + if add_BOS: + prompt_tokens = [tokenizer.bos] + prompt_tokens sampling_params = SamplingParams( temperature=temperature, @@ -62,7 +79,13 @@ async def chat_completions(): top_p=top_p, return_log_probs=return_log_probs, top_n_logprobs=top_n_logprobs, - num_tokens_to_generate=int(req.get("max_tokens", 16)), + num_tokens_to_generate=( + int(max_tokens) + if ((max_tokens := req.get("max_tokens", None)) is not None) + else None + ), + skip_prompt_log_probs=skip_prompt_log_probs, + add_BOS=add_BOS, ) except ValueError as e: return f"Invalid sampling parameter: {e}", 400 @@ -71,20 +94,13 @@ async def chat_completions(): # For chat, we run the *same* prompt 'n' times. tasks = [] for _ in range(n): - per_req_params = SamplingParams( - temperature=sampling_params.temperature, - top_k=sampling_params.top_k, - top_p=sampling_params.top_p, - return_log_probs=sampling_params.return_log_probs, - top_n_logprobs=sampling_params.top_n_logprobs, - num_tokens_to_generate=sampling_params.num_tokens_to_generate, - ) - tasks.append(client.add_request(prompt_tokens, per_req_params)) + tasks.append(client.add_request(prompt_tokens, sampling_params)) start_time = time.perf_counter() try: batch_results = await asyncio.gather(*tasks) except Exception as e: + logger.error(f"Error during inference: {e}") return f"Error during inference: {e}", 500 logger.info( @@ -95,21 +111,29 @@ async def chat_completions(): # --- 4. Format OpenAI Response --- choices = [] total_completion_tokens = 0 - prompt_token_count = len(prompt_tokens) # Calculated once + prompt_tokens_counts = [] request_idx = 0 for record in batch_results: assert len(record.requests) == 1, "Each record should contain one request result." - result = record.merge() - text_output = result.generated_text + result = record.merge().serialize() + # Unwrap ("tensor", [...]) tuples from serialize() into plain lists. + result = { + k: v[1] if isinstance(v, (list, tuple)) and len(v) == 2 and v[0] == "tensor" else v + for k, v in result.items() + } + prompt_tokens = result["prompt_tokens"] # The engine can modify prompt_tokens. + text_output = result["generated_text"] + prompt_tokens_count = len(prompt_tokens) if prompt_tokens is not None else 0 + prompt_tokens_counts.append(prompt_tokens_count) logprobs_content = None if sampling_params.return_log_probs: - token_logprobs = getattr(result, 'log_probs', []) - tokens = [tokenizer.detokenize([tok]) for tok in result.generated_tokens] + token_logprobs = result.get('log_probs', []) + tokens = [tokenizer.detokenize([tok]) for tok in result["generated_tokens"]] # Get top_n_logprobs if available - generated_top_n_logprobs = getattr(result, 'generated_top_n_logprobs', None) + generated_top_n_logprobs = result.get('generated_top_n_logprobs') logprobs_content = [] for i, (tok, lp) in enumerate(zip(tokens, token_logprobs)): @@ -134,25 +158,50 @@ async def chat_completions(): } logprobs_content.append(entry) + metadata = {} + message_text = text_output + if parsers: + for parser in parsers: + if parser not in PARSER_MAPPING: + raise ValueError(f"Parser {parser} not found in PARSER_MAPPING") + message_text, new_info = PARSER_MAPPING[parser].parse( + message_text, tools=req.get("tools", None) + ) + assert not ( + metadata.keys() & new_info.keys() + ), "Multiple parsers found the same information." + metadata.update(new_info) + message = {"role": "assistant", "content": message_text} + if "tool_calls" in metadata: + message["tool_calls"] = metadata["tool_calls"] + if "reasoning" in metadata: + message["reasoning"] = metadata["reasoning"] + choice_data = { - "index": 0, - "message": {"role": "assistant", "content": text_output}, + "index": request_idx, + "message": message, + "prompt_token_ids": result["prompt_tokens"], + "generation_token_ids": result["generated_tokens"], + "generation_log_probs": result["generated_log_probs"], + "raw_text": result["prompt"] + result["generated_text"], # 'logprobs' in chat API is an object containing 'content' "logprobs": {"content": logprobs_content} if logprobs_content else None, - "finish_reason": "length", # Original code hardcoded this. + "finish_reason": ( + "tool_calls" if metadata.get("tool_calls", []) else "stop" + ), # Original code hardcoded this. } logging.info(result) - if result.routing_indices is not None: - choice_data["moe_topk_indices"] = result.routing_indices.tolist() - prompt_length = len(result.prompt_tokens) if result.prompt_tokens is not None else 0 - if prompt_length: - choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[ - :prompt_length - ].tolist() + if result["routing_indices"] is not None: + choice_data["moe_topk_indices"] = result["routing_indices"] + if prompt_tokens_count: + choices[-1]["prompt_moe_topk_indices"] = result["routing_indices"][ + :prompt_tokens_count + ] choices.append(choice_data) - total_completion_tokens += len(result.generated_tokens) - request_idx += 0 + total_completion_tokens += len(result["generated_tokens"]) + request_idx += 1 + prompt_token_count = max(prompt_tokens_counts) response = { "choices": choices, "usage": { diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py index 1701ff63c36..b4bcfb3513d 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py @@ -33,8 +33,10 @@ def temp_log_level(level, logger=None): @trace_async_exceptions -async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_port: int): - """Initializes and runs the async Flask server.""" +async def run_flask_server_on_client( + client: InferenceClient, tokenizer, flask_port: int, parsers: list[str] = None +): + """Initializes and runs the async Flask server using the provided InferenceClient.""" if not HAS_FLASK: raise RuntimeError(f"Flask not available") @@ -44,15 +46,12 @@ async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_po logger.warning(f"Could not get hostname: {e}") hostname = "0.0.0.0" - inference_client = InferenceClient(coordinator_addr) - await inference_client.start() - logger.info(f"Rank {rank}: InferenceClient connected.") - app = Flask(__name__) # Store client and tokenizer in app config for Blueprints to use - app.config['client'] = inference_client + app.config['client'] = client app.config['tokenizer'] = tokenizer + app.config['parsers'] = parsers # Register all blueprints from the 'endpoints' package for endpoint in endpoints.__all__: @@ -68,9 +67,23 @@ def health_check(): # Force logging level to INFO to ensure that hostname is printed with temp_log_level(logging.INFO, logger): logger.info(f"Starting Flask server on http://{hostname}:{flask_port}") + logger.info(f"Using tokenizer: {type(tokenizer)}") + logger.info(f"Using parsers: {parsers}") + + await serve(app, config) + +@trace_async_exceptions +async def run_flask_server( + coordinator_addr: str, tokenizer, rank: int, flask_port: int, parsers: list[str] = None +): + """Initializes and runs the async Flask server + starting an InferenceClient with the provided coordinator address.""" + inference_client = InferenceClient(coordinator_addr) + await inference_client.start() + logger.info(f"Rank {rank}: InferenceClient connected.") try: - await serve(app, config) + await run_flask_server_on_client(inference_client, tokenizer, flask_port, parsers) finally: await inference_client.stop() logger.info(f"Rank {rank}: Flask server and client shut down.") diff --git a/megatron/core/tokenizers/text/parsers/__init__.py b/megatron/core/tokenizers/text/parsers/__init__.py new file mode 100644 index 00000000000..dc27763f905 --- /dev/null +++ b/megatron/core/tokenizers/text/parsers/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from megatron.core.tokenizers.text.parsers.deepseek_r1_reasoning_parser import ( + DeepSeekR1ReasoningParser, +) +from megatron.core.tokenizers.text.parsers.qwen3_coder_tool_parser import Qwen3CoderToolParser + +PARSER_MAPPING = { + "deepseek-r1-reasoning": DeepSeekR1ReasoningParser, + "qwen3-coder-tool": Qwen3CoderToolParser, +} + +__all__ = ["PARSER_MAPPING"] diff --git a/megatron/core/tokenizers/text/parsers/base_parser.py b/megatron/core/tokenizers/text/parsers/base_parser.py new file mode 100644 index 00000000000..afd847d1bfc --- /dev/null +++ b/megatron/core/tokenizers/text/parsers/base_parser.py @@ -0,0 +1,21 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from typing import Any + + +class BaseParser: + """Base class for text parsers.""" + + @staticmethod + def parse(text: str, **kwargs) -> tuple[str, dict[str, Any]]: + """ + Parses the text into a tuple containing extracted content + and a dictionary of additional information. + + Args: + text (str): The text to parse. + + Returns: + tuple[str, dict[str, Any]]: A tuple containing the unprocessed text + and a dictionary with the extracted information. + """ + return text, {} diff --git a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py new file mode 100644 index 00000000000..17952c61daf --- /dev/null +++ b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py @@ -0,0 +1,33 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from megatron.core.tokenizers.text.parsers.base_parser import BaseParser + + +class DeepSeekR1ReasoningParser(BaseParser): + """Parser for DeepSeek R1 style reasoning output.""" + + @staticmethod + def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: + """ + Extracts the reasoning content from the text using ... tags. + Only extracts the first set of think tags. + If an initial tag is not present but a tag is, + it will infer a tag at the beginning of the text. + + Args: + text (str): The text to parse. + + Returns: + tuple[str, dict[str, str]]: A tuple containing the unprocessed text + and a dictionary with the extracted reasoning content. + """ + + if "" in text: + if "" in text: + # Strip the prefix (it might not be present if it was part of the prompt) + pre_text, text = text.split("", maxsplit=1) + else: + pre_text = "" + reasoning_content, remaining_text = text.split("", maxsplit=1) + return pre_text + remaining_text, {'reasoning': reasoning_content} + else: + return text, {} diff --git a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py new file mode 100644 index 00000000000..1d1f20a3a5c --- /dev/null +++ b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py @@ -0,0 +1,282 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import ast +import json +import logging +import re +import uuid +from types import SimpleNamespace +from typing import Any + +from megatron.core.tokenizers.text.parsers.base_parser import BaseParser + +logger = logging.getLogger(__name__) + +# These map to vLLM types but we just use dictionaries for now +ToolCall = dict[str, Any] +FunctionCall = dict[str, Any] +ChatCompletionToolsParam = dict[str, Any] +ChatCompletionRequest = dict[str, Any] +ExtractedToolCallInformation = dict + + +class _Qwen3CoderToolParser: + + # Sentinel tokens for streaming mode + tool_call_start_token: str = "" + tool_call_end_token: str = "" + tool_call_prefix: str = "(.*?)", re.DOTALL) + tool_call_regex = re.compile(r"(.*?)|(.*?)$", re.DOTALL) + tool_call_function_regex = re.compile(r"||(?=)|$)", re.DOTALL + ) + + def _generate_tool_call_id(self) -> str: + """Generate a unique tool call ID.""" + return f"call_{uuid.uuid4().hex[:24]}" + + def _get_arguments_config( + self, func_name: str, tools: list[ChatCompletionToolsParam] | None + ) -> dict: + """Extract argument configuration for a function.""" + if tools is None: + return {} + for config in tools: + config = SimpleNamespace(**config) # Convert to SimpleNamespace for ease of access + if not hasattr(config, "type") or not ( + hasattr(config, "function") and hasattr(config.function, "name") + ): + continue + if config.type == "function" and config.function.name == func_name: + if not hasattr(config.function, "parameters"): + return {} + params = config.function.parameters + if isinstance(params, dict) and "properties" in params: + return params["properties"] + elif isinstance(params, dict): + return params + else: + return {} + logger.debug("Tool '%s' is not defined in the tools list.", func_name) + return {} + + def _convert_param_value( + self, param_value: str, param_name: str, param_config: dict, func_name: str + ) -> Any: + """Convert parameter value based on its type in the schema.""" + # Handle null value for any type + if param_value.lower() == "null": + return None + + if param_name not in param_config: + if param_config != {}: + logger.debug( + "Parsed parameter '%s' is not defined in the tool " + "parameters for tool '%s', directly returning the " + "string value.", + param_name, + func_name, + ) + return param_value + + if isinstance(param_config[param_name], dict) and "type" in param_config[param_name]: + param_type = str(param_config[param_name]["type"]).strip().lower() + else: + param_type = "string" + if param_type in ["string", "str", "text", "varchar", "char", "enum"]: + return param_value + elif ( + param_type.startswith("int") + or param_type.startswith("uint") + or param_type.startswith("long") + or param_type.startswith("short") + or param_type.startswith("unsigned") + ): + try: + return int(param_value) + except (ValueError, TypeError): + logger.debug( + "Parsed value '%s' of parameter '%s' is not an " + "integer in tool '%s', degenerating to string.", + param_value, + param_name, + func_name, + ) + return param_value + elif param_type.startswith("num") or param_type.startswith("float"): + try: + float_param_value = float(param_value) + return ( + float_param_value + if float_param_value - int(float_param_value) != 0 + else int(float_param_value) + ) + except (ValueError, TypeError): + logger.debug( + "Parsed value '%s' of parameter '%s' is not a float " + "in tool '%s', degenerating to string.", + param_value, + param_name, + func_name, + ) + return param_value + elif param_type in ["boolean", "bool", "binary"]: + param_value = param_value.lower() + if param_value not in ["true", "false"]: + logger.debug( + "Parsed value '%s' of parameter '%s' is not a boolean " + "(`true` or `false`) in tool '%s', degenerating to " + "false.", + param_value, + param_name, + func_name, + ) + return param_value == "true" + else: + if ( + param_type in ["object", "array", "arr"] + or param_type.startswith("dict") + or param_type.startswith("list") + ): + try: + param_value = json.loads(param_value) + return param_value + except (json.JSONDecodeError, TypeError, ValueError): + logger.debug( + "Parsed value '%s' of parameter '%s' cannot be " + "parsed with json.loads in tool '%s', will try " + "other methods to parse it.", + param_value, + param_name, + func_name, + ) + try: + param_value = ast.literal_eval(param_value) # safer + except (ValueError, SyntaxError, TypeError): + logger.debug( + "Parsed value '%s' of parameter '%s' cannot be " + "converted via Python `ast.literal_eval()` in tool " + "'%s', degenerating to string.", + param_value, + param_name, + func_name, + ) + return param_value + + def _parse_xml_function_call( + self, function_call_str: str, tools: list[ChatCompletionToolsParam] | None + ) -> ToolCall | None: + # Extract function name + end_index = function_call_str.index(">") + function_name = function_call_str[:end_index] + param_config = self._get_arguments_config(function_name, tools) + parameters = function_call_str[end_index + 1 :] + param_dict = {} + for match_text in self.tool_call_parameter_regex.findall(parameters): + idx = match_text.index(">") + param_name = match_text[:idx] + param_value = str(match_text[idx + 1 :]) + # Remove prefix and trailing \n + if param_value.startswith("\n"): + param_value = param_value[1:] + if param_value.endswith("\n"): + param_value = param_value[:-1] + + param_dict[param_name] = self._convert_param_value( + param_value, param_name, param_config, function_name + ) + return ToolCall( + type="function", + id=self._generate_tool_call_id(), + function=FunctionCall( + name=function_name, arguments=json.dumps(param_dict, ensure_ascii=False) + ), + ) + + def _get_function_calls(self, model_output: str) -> list[str]: + # Find all tool calls + matched_ranges = self.tool_call_regex.findall(model_output) + raw_tool_calls = [match[0] if match[0] else match[1] for match in matched_ranges] + + # Back-off strategy if no tool_call tags found + if len(raw_tool_calls) == 0: + raw_tool_calls = [model_output] + + raw_function_calls = [] + for tool_call in raw_tool_calls: + raw_function_calls.extend(self.tool_call_function_regex.findall(tool_call)) + + function_calls = [match[0] if match[0] else match[1] for match in raw_function_calls] + return function_calls + + def extract_tool_calls( + self, model_output: str, tools: list[ChatCompletionToolsParam] | None + ) -> ExtractedToolCallInformation: + """Extracts the tool calls from the text using ... tags.""" + # Quick check to avoid unnecessary processing + if self.tool_call_prefix not in model_output: + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + try: + function_calls = self._get_function_calls(model_output) + if len(function_calls) == 0: + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + tool_calls = [ + self._parse_xml_function_call(function_call_str, tools) + for function_call_str in function_calls + ] + + # Extract content before tool calls + content_index = model_output.find(self.tool_call_start_token) + idx = model_output.find(self.tool_call_prefix) + content_index = content_index if content_index >= 0 else idx + content = model_output[:content_index] # .rstrip() + + return ExtractedToolCallInformation( + tools_called=(len(tool_calls) > 0), + tool_calls=tool_calls, + content=content if content else None, + ) + + except Exception: + logger.exception("Error in extracting tool call from response.") + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + +class Qwen3CoderToolParser(BaseParser): + """Parser for Qwen3 Coder style tool calls.""" + + @staticmethod + def parse(text: str, **kwargs) -> tuple[str, dict[str, list[dict]]]: + """ + Extracts the tool calls from the text using ... tags. + Uses the _Qwen3CoderToolParser class (copied from vLLM) to extract the tool calls. + + Args: + text (str): The text to parse. + + Returns: + tuple[str, dict[str, str]]: A tuple containing the unprocessed text + and a dictionary with the extracted tool calls. + """ + + information = _Qwen3CoderToolParser().extract_tool_calls( + text, tools=kwargs.get("tools", []) + ) + if information.get("tools_called", False): + return information.get("content", ""), {"tool_calls": information.get("tool_calls", [])} + else: + return text, {} diff --git a/megatron/rl/agent/api.py b/megatron/rl/agent/api.py index 9568db3a54d..643c43197b6 100644 --- a/megatron/rl/agent/api.py +++ b/megatron/rl/agent/api.py @@ -12,10 +12,7 @@ from ..__init__ import Request, TypeLookupable from ..inference import ( - ChatInferenceInterface, - ChatInferenceRequest, InferenceInterface, - InferenceRequest, LLMChatMessage, ReturnsRaw, ) @@ -124,11 +121,6 @@ async def get_reward_rollouts(self, request: RolloutRequest) -> list[Rollout]: request.inference_interface, ReturnsRaw ), "InferenceInterface must support raw_text return to provide rollouts." - if isinstance(request.inference_interface, ChatInferenceInterface): - self.chat_mode = True - else: - self.chat_mode = False - return await asyncio.gather( *[self.rollout(request=request) for _ in range(request.num_rollouts)] ) @@ -158,11 +150,6 @@ async def get_reward_rollouts(self, request: RolloutRequest) -> list[TokenRollou request.inference_interface, ReturnsRaw ), "InferenceInterface must support raw_text return to provide rollouts." - if isinstance(request.inference_interface, ChatInferenceInterface): - self.chat_mode = True - else: - self.chat_mode = False - return await asyncio.gather( *[self.rollout(request=request) for _ in range(request.num_rollouts)] ) @@ -187,11 +174,6 @@ async def get_grouped_rollouts(self, request: GroupedRolloutRequest): request.inference_interface, ReturnsRaw ), "InferenceInterface must support raw_text return to provide rollouts." - if isinstance(request.inference_interface, ChatInferenceInterface): - self.chat_mode = True - else: - self.chat_mode = False - # If num_groups is -1, we generate a stream of groups. # The buffer size is used to create backpressure for each agent in order to balance group generation in a multi-task setting. grouped_rollouts: asyncio.Queue[list[Rollout]] = asyncio.Queue( diff --git a/megatron/rl/agent/pass_at_evaluation_agent.py b/megatron/rl/agent/pass_at_evaluation_agent.py index b10e3b897c8..c04e1c2772f 100644 --- a/megatron/rl/agent/pass_at_evaluation_agent.py +++ b/megatron/rl/agent/pass_at_evaluation_agent.py @@ -7,7 +7,7 @@ import numpy as np from ..__init__ import GenericGenerationArgs -from ..inference import ChatInferenceResponse, LLMChatMessage +from ..inference import LLMChatMessage from .api import EvaluationAgent, EvaluationRequest, EvaluationResponse, RewardEvaluationResult diff --git a/megatron/rl/agent/reward_only_agent.py b/megatron/rl/agent/reward_only_agent.py index 53b1f7407b2..4099406a98a 100644 --- a/megatron/rl/agent/reward_only_agent.py +++ b/megatron/rl/agent/reward_only_agent.py @@ -7,8 +7,6 @@ from tqdm.asyncio import tqdm from ..inference import ( - ChatInferenceInterface, - ChatInferenceResponse, InferenceResponse, LLMChatMessage, ReturnsRaw, @@ -91,11 +89,7 @@ async def rollout_from_response( ), "InferenceInterface must support raw_text return to provide rollouts." raw_text = response.raw_text - response_text = ( - response.response.content - if isinstance(response, ChatInferenceResponse) - else response.response - ) + response_text = response.response.content if isinstance(request.inference_interface, ReturnsTokens): logprobs = response.logprobs @@ -126,14 +120,10 @@ async def rollout(self, request: RolloutRequest) -> Rollout: prompt, golden = await self.get_prompt(validation=request.validation) inference_request = request.inference_interface.prepare_request( - [prompt], request.generation_args + prompt, request.generation_args ) - responses = await request.inference_interface.agenerate(inference_request) - assert ( - len(responses) == 1 - ), "get_reward_rollouts only requested a single response but got multiple responses" - response = responses[0] + response = await request.inference_interface.agenerate(inference_request) return await self.rollout_from_response(request, response, golden) @@ -142,41 +132,22 @@ async def group_rollout(self, request: GroupedRolloutRequest) -> list[Rollout]: prompt, golden = await self.get_prompt(validation=request.validation) inference_request = request.inference_interface.prepare_request( - [prompt], request.generation_args - ) - inference_request.n = request.rollouts_per_group - - groups = await request.inference_interface.agenerate(inference_request) - assert ( - len(groups) == 1 - ), "get_grouped_rollouts only requested a single group but got multiple groups" - responses = groups[0].responses - - rollouts = await asyncio.gather( - *[self.rollout_from_response(request, response, golden) for response in responses] + prompt, request.generation_args ) - return rollouts + responses = await asyncio.gather(*[request.inference_interface.agenerate(inference_request) for _ in range(request.rollouts_per_group)]) + return [await self.rollout_from_response(request, response, golden) for response in responses] async def _evaluation( self, prompt: str, golden: Any, request: EvaluationRequest ) -> RewardOnlyEvaluationResponse: inference_request = request.inference_interface.prepare_request( - [prompt], request.generation_args + prompt, request.generation_args ) - responses = await request.inference_interface.agenerate(inference_request) - assert ( - len(responses) == 1 - ), "evaluation only requested a single response but got multiple responses" - response = responses[0] - - response_text = ( - response.response.content - if isinstance(response, ChatInferenceResponse) - else response.response - ) + response = await request.inference_interface.agenerate(inference_request) + response_text = response.response.content result = RewardEvaluationResult( env_id=self.env_id, @@ -190,11 +161,6 @@ async def _evaluation( async def run_evaluation(self, request: EvaluationRequest): - if isinstance(request.inference_interface, ChatInferenceInterface): - self.chat_mode = True - else: - self.chat_mode = False - # Get all prompts first all_prompts = list( await self.evaluation_prompts( diff --git a/megatron/rl/inference/api.py b/megatron/rl/inference/api.py index ae19380842e..87f6b87f908 100644 --- a/megatron/rl/inference/api.py +++ b/megatron/rl/inference/api.py @@ -11,42 +11,15 @@ class LLMChatMessage(BaseModel): class InferenceRequest(Request): - prompt: list[str] - n: int | None = None - - -class ChatInferenceRequest(InferenceRequest): - prompt: list[list[LLMChatMessage]] + prompt: list[LLMChatMessage] tools: list[dict] | None = None -class GroupedInferenceRequest(InferenceRequest): - group_size: int = 1 - - class InferenceResponse(BaseModel): """The minimum required response for an inference interface.""" - response: str + response: LLMChatMessage raw_text: str | None = None token_ids: list[int] | None = None prompt_length: int | None = None logprobs: list[float] | None = None - - -class GroupedInferenceResponse(BaseModel): - """An inference response which includes a list of responses.""" - - responses: list[InferenceResponse] - - -class ChatInferenceResponse(InferenceResponse): - """The minimum required response for a chat inference interface.""" - - response: LLMChatMessage - - -class GroupedChatInferenceResponse(GroupedInferenceResponse): - """A chat inference response which includes a list of responses.""" - - responses: list[ChatInferenceResponse] diff --git a/megatron/rl/inference/chat_templates.py b/megatron/rl/inference/chat_templates.py deleted file mode 100644 index 3e464842859..00000000000 --- a/megatron/rl/inference/chat_templates.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -import warnings - -from pydantic import BaseModel, ConfigDict, Field -from transformers import AutoTokenizer -from transformers.tokenization_utils import PreTrainedTokenizer -from transformers.tokenization_utils_fast import PreTrainedTokenizerFast - -from .api import InferenceResponse, LLMChatMessage - - -class ConversationTemplate(BaseModel): - """Transformers tokenizer based template.""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast = Field(repr=False) - stop_words: list[str] = [] - - def format(self, messages: list[LLMChatMessage], tools: list[dict] | None = None) -> str: - return self.tokenizer.apply_chat_template( - messages, add_generation_prompt=True, tokenize=False, tools=tools - ) - - def parse_response(self, responses: list[InferenceResponse]) -> list[LLMChatMessage]: - return [ - LLMChatMessage(role="assistant", content=response.response) for response in responses - ] - - @classmethod - def from_string(cls, tokenizer_name: str) -> 'ConversationTemplate': - if tokenizer_name == "null": - warnings.warn( - "Using NullConversationTemplate. This provides no chat templating to Chat requests." - ) - return NullConversationTemplate() - return cls(tokenizer=AutoTokenizer.from_pretrained(tokenizer_name)) - - -class NullConversationTemplate(ConversationTemplate): - - tokenizer: None = None - - def format(self, messages: list[LLMChatMessage], tools: list[dict] | None = None) -> str: - return "\n".join([f"{message.content}" for message in messages]) + "\n" - - def parse_response(self, responses: list[InferenceResponse]) -> list[LLMChatMessage]: - return [ - LLMChatMessage(role="assistant", content=response.response) for response in responses - ] diff --git a/megatron/rl/inference/inference_interface.py b/megatron/rl/inference/inference_interface.py index e950792e72b..715f5ada9d4 100644 --- a/megatron/rl/inference/inference_interface.py +++ b/megatron/rl/inference/inference_interface.py @@ -1,83 +1,40 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import asyncio -from abc import abstractmethod -from itertools import zip_longest -from typing import Annotated, Any, ClassVar -from pydantic import BaseModel, BeforeValidator, ValidationError +from pydantic import BaseModel from ..__init__ import GenericGenerationArgs from ..inference.api import ( - ChatInferenceRequest, - ChatInferenceResponse, - GroupedChatInferenceResponse, - GroupedInferenceResponse, InferenceRequest, InferenceResponse, LLMChatMessage, ) -from ..inference.chat_templates import ConversationTemplate - - -# Used when generating n resposnes for a single prompt -def grouper(iterable, n, fillvalue=None): - """Fold an iterable into a list of lists of size n.""" - args = [iter(iterable)] * n - return zip_longest(*args, fillvalue=fillvalue) class InferenceInterface(BaseModel): - """Inference interface that for base language models.""" + """Inference interface for chat models.""" class Config: arbitrary_types_allowed = True - supports_n: ClassVar[bool] = False - def prepare_request( - self, prompts: list[str], generation_args: GenericGenerationArgs + self, prompt: str | list[LLMChatMessage], generation_args: GenericGenerationArgs ) -> InferenceRequest: - assert all(isinstance(p, str) for p in prompts), "Prompt must be a list of strings" - return InferenceRequest(prompt=prompts, generation_args=generation_args) - - async def base_generate(self, request: InferenceRequest) -> list[InferenceResponse]: - raise NotImplementedError( - "Direct Inference Classes must implement the base_generate method." - ) - - def duplicate_requests(self, request: InferenceRequest, n: int) -> list[InferenceRequest]: - return request.model_copy(update={'prompt': request.prompt * n}) + prompt = [LLMChatMessage(role='user', content=prompt)] if isinstance(prompt, str) else prompt + return InferenceRequest(prompt=prompt, generation_args=generation_args) - def fold_responses( - self, responses: list[InferenceResponse], n: int - ) -> list[GroupedInferenceResponse]: - return [GroupedInferenceResponse(responses=x) for x in list(grouper(responses, n))] + async def base_generate(self, request: InferenceRequest) -> InferenceResponse: + assert NotImplementedError("Direct Inference Classes must implement the base_generate method.") async def agenerate( self, request: InferenceRequest - ) -> list[InferenceResponse] | list[GroupedInferenceResponse]: - if not self.supports_n and request.n is not None: - request = self.duplicate_requests(request, request.n) - - generations = await self.base_generate(request) - - if request.n is not None: - if self.supports_n: - assert ( - len(generations) == len(request.prompt) * request.n - ), f"Number of generations ({len(generations)}) does not match number of prompts ({len(request.prompt)} * {request.n})." - else: - assert len(generations) == len( - request.prompt - ), f"Number of generations ({len(generations)}) does not match number of prompts ({len(request.prompt)})." - generations = self.fold_responses(generations, request.n) - - return generations + ) -> InferenceResponse: + return await self.base_generate(request) def generate( self, request: InferenceRequest - ) -> list[InferenceResponse] | list[GroupedInferenceResponse]: + ) -> InferenceResponse: try: loop = asyncio.get_running_loop() except RuntimeError: @@ -85,58 +42,6 @@ def generate( else: return loop.run_until_complete(self.agenerate(request)) - -def ensure_template(value: Any) -> ConversationTemplate: - if isinstance(value, ConversationTemplate): - return value - elif isinstance(value, str): - return ConversationTemplate.from_string(value) - else: - raise ValueError(f"Invalid conversation template: {value}") - - -class ChatInferenceInterface(InferenceInterface): - """Inference interface for chat models.""" - - conversation_template: Annotated[ConversationTemplate, BeforeValidator(ensure_template)] - - def prepare_request( - self, prompts: list[str | list[LLMChatMessage]], generation_args: GenericGenerationArgs - ) -> ChatInferenceRequest: - prompt = [ - [LLMChatMessage(role='user', content=p)] if isinstance(p, str) else p for p in prompts - ] - return ChatInferenceRequest(prompt=prompt, generation_args=generation_args) - - async def base_generate(self, request: ChatInferenceRequest) -> list[ChatInferenceResponse]: - base_generate_results = await super().base_generate( - InferenceRequest( - prompt=[ - self.conversation_template.format(messages, request.tools) - for messages in request.prompt - ], - generation_args=request.generation_args, - ) - ) - chat_message_results = self.conversation_template.parse_response(base_generate_results) - return [ - ChatInferenceResponse( - response=chat_message, **response.model_dump(exclude={'response'}) - ) - for chat_message, response in zip(chat_message_results, base_generate_results) - ] - - def generate( - self, request: ChatInferenceRequest - ) -> list[ChatInferenceResponse] | list[GroupedChatInferenceResponse]: - return super().generate(request) - - async def agenerate( - self, request: ChatInferenceRequest - ) -> list[ChatInferenceResponse] | list[GroupedChatInferenceResponse]: - return await super().agenerate(request) - - class ReturnsRaw(InferenceInterface): """Mix-In for interface that supports returning complete string fed to the LLM.""" diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index 602ff4f7450..61eb4602d02 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -2,31 +2,17 @@ import asyncio import logging -from argparse import Namespace import torch.distributed as dist from pydantic import PrivateAttr -from megatron.core.inference.contexts.dynamic_context import DynamicInferenceContext -from megatron.core.inference.engines.abstract_engine import AbstractEngine from megatron.core.inference.engines.dynamic_engine import DynamicInferenceEngine -from megatron.core.inference.engines.mcore_engine import MCoreEngine from megatron.core.inference.inference_client import InferenceClient -from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import ( - GPTInferenceWrapper, -) -from megatron.core.inference.sampling_params import SamplingParams -from megatron.core.inference.text_generation_controllers.text_generation_controller import ( - TextGenerationController, -) from megatron.core.models.gpt.gpt_model import GPTModel -from megatron.core.transformer.module import MegatronModule -from megatron.core.utils import get_attr_wrapped_model, log_single_rank -from megatron.training import get_wandb_writer +from megatron.core.utils import log_single_rank from megatron.training.global_vars import get_args, get_tokenizer from ..inference.inference_interface import ( - ChatInferenceInterface, InferenceRequest, InferenceResponse, LLMChatMessage, @@ -38,86 +24,51 @@ logger = logging.getLogger(__name__) -## This code is copied from tools/run_text_generation_server.py -def get_static_inference_engine(args: Namespace, model: MegatronModule) -> AbstractEngine: - """Get the relevant backend for running inference. - - This function will automatically choose the TRTLLMBackend when possible, - and default to Mcore backend if the user does not specify any backends. - TRTLLMBackend is not implmented yet. - - Args: - args (Namespace): The user arguments parsed from command line - model (MegatronModule): The megatron model. - - Returns: - AbstractBackend: The chosen backend - """ - tokenizer = get_tokenizer() - - inference_wrapped_model = GPTInferenceWrapper(model) - pg_collection = get_attr_wrapped_model(model, "pg_collection") - pp_group = pg_collection.pp - text_generation_controller = TextGenerationController( - inference_wrapped_model=inference_wrapped_model, tokenizer=tokenizer, pp_group=pp_group - ) - return MCoreEngine( - text_generation_controller=text_generation_controller, - max_batch_size=( - args.inference_max_requests if args.inference_max_requests is not None else 1 - ), - ) - - class MegatronLocal(InferenceServer, ReturnsTokens, ReturnsRaw): """Interface to use MCoreEngine directly as an inference engine.""" + host: str + port: int + + _server_task: asyncio.Task = PrivateAttr(None) _client: InferenceClient = PrivateAttr(None) _inference_engine: DynamicInferenceEngine = PrivateAttr(None) - async def base_generate(self, request: InferenceRequest): - - if any(isinstance(p, LLMChatMessage) for p in request.prompt): - raise ValueError( - "MegatronLocal does not support chat requests." - "Use MegatronChatLocal to apply chat templating." - ) - assert all( - isinstance(p, str) for p in request.prompt - ), "MegatronLocal only supports string prompts." - - assert self._client is not None, "Client is not initialized" + async def base_generate(self, request: InferenceRequest) -> InferenceResponse: + assert self._server_task is not None, "Inference server is not initialized" tokenizer = get_tokenizer() args = get_args() - sampling_params = SamplingParams( - num_tokens_to_generate=None, - num_tokens_total=request.generation_args.max_tokens, + from openai import AsyncOpenAI + client = AsyncOpenAI(base_url=f"http://{self.host}:{self.port}", api_key="NONE") + + # Things that may be problematic when doign this switch + # - Add BOS token + # - Skip prompt logprobs + response = await client.chat.completions.create( + model="", + messages=[message.model_dump() for message in request.prompt], temperature=request.generation_args.temperature or 1.0, - top_k=request.generation_args.top_k or 0, top_p=request.generation_args.top_p or 0.0, - termination_id=self._inference_engine.controller.tokenizer.eod, - return_log_probs=True, - skip_prompt_log_probs=True, - add_BOS=(not args.rl_skip_bos_token and tokenizer.bos is not None), + n=1, + logprobs=True, + extra_body={ + "skip_prompt_log_probs": True, + "add_BOS": (not args.rl_skip_bos_token and tokenizer.bos is not None), + }, + ) + + choice = response.choices[0] + + return InferenceResponse( + # TODO: Handle tool calls and reasoning in LLMChatMessage + response=LLMChatMessage(**choice.message.model_dump(include={'role', 'content'})), + raw_text=choice.raw_text, + token_ids=choice.prompt_token_ids + choice.generation_token_ids, + logprobs=choice.generation_log_probs, + prompt_length=len(choice.prompt_token_ids), ) - requests = [ - self._client.add_request(prompt=prompt, sampling_params=sampling_params) - for prompt in request.prompt - ] - records = await asyncio.gather(*requests) - responses = [record[-1] for record in records] - return [ - InferenceResponse( - response=r.generated_text, - raw_text=p + r.generated_text, - token_ids=r.prompt_tokens.tolist() + r.generated_tokens, - logprobs=r.generated_log_probs, - prompt_length=len(r.prompt_tokens), - ) - for p, r in zip(request.prompt, responses) - ] @classmethod async def launch(cls, model: GPTModel, **kwargs): @@ -138,14 +89,25 @@ async def launch(cls, model: GPTModel, **kwargs): dp_addr = await inference_engine.start_listening_to_data_parallel_coordinator( inference_coordinator_port=41521, launch_inference_coordinator=True, ) + if dist.get_rank() == 0: - # TODO: We have to do this only on the rank 0 process, should be fixed in the future when we have support for multiple inference clients. !2278 + from megatron.core.inference.text_generation_server.dynamic_text_gen_server.flask_server import run_flask_server_on_client + loop = asyncio.get_event_loop() client = InferenceClient(inference_coordinator_address=dp_addr) await client.start() + server_task = loop.create_task(run_flask_server_on_client( + client=client, + tokenizer=inference_engine.controller.tokenizer, + flask_port=kwargs.get('port', 8294), + parsers=[] + )) else: client = None + server_task = None + launched_server = cls(**kwargs) launched_server._client = client + launched_server._server_task = server_task launched_server._inference_engine = inference_engine return launched_server @@ -164,6 +126,3 @@ async def resume(self): if dist.get_rank() == 0: self._client.unpause_engines() await self._inference_engine.running.wait() - - -class MegatronChatLocal(ChatInferenceInterface, MegatronLocal): ... diff --git a/megatron/rl/rl_utils.py b/megatron/rl/rl_utils.py index 71730daa323..4919107733b 100644 --- a/megatron/rl/rl_utils.py +++ b/megatron/rl/rl_utils.py @@ -8,6 +8,7 @@ import itertools import math import logging +import os import pickle from collections import Counter, defaultdict from contextlib import contextmanager, nullcontext @@ -60,7 +61,7 @@ TokenRollout, ) from megatron.rl.agent.weighted_multi_task import WeightedMultiTask -from megatron.rl.inference.megatron import MegatronChatLocal, MegatronLocal +from megatron.rl.inference.megatron import MegatronLocal from megatron.rl.logging import LOG_DIR as lang_rl_log_dir from megatron.rl.logging import log as lang_rl_log from megatron.rl.server.inference.inference_interface_server import InferenceInterfaceServer @@ -430,36 +431,12 @@ def get_agent(args, parallel_generation_tasks: int | None = None): def get_inference_interface(args, loop, model): global _INFERENCE_INTERFACE if _INFERENCE_INTERFACE is None: - rank = torch.distributed.get_rank() - if rank == 0 and args.langrl_external_server: - if args.langrl_inference_server_type == 'inplace_megatron': - _INFERENCE_INTERFACE = loop.run_until_complete( - InferenceInterfaceServer.launch(MegatronLocal, model=model[0]) - ) - elif args.langrl_inference_server_type == 'inplace_megatron_chat': - _INFERENCE_INTERFACE = loop.run_until_complete( - InferenceInterfaceServer.launch( - MegatronChatLocal, - model=model[0], - conversation_template=args.langrl_inference_server_conversation_template, - ) - ) - else: - raise ValueError(f"Unknown inference_server_type {args.inference_server_type}") - else: - if args.langrl_inference_server_type == 'inplace_megatron': - _INFERENCE_INTERFACE = loop.run_until_complete(MegatronLocal.launch(model[0])) - elif args.langrl_inference_server_type == 'inplace_megatron_chat': - _INFERENCE_INTERFACE = loop.run_until_complete( - MegatronChatLocal.launch( - model[0], - conversation_template=args.langrl_inference_server_conversation_template, - ) - ) - else: - raise ValueError( - f"Unknown inference_server_type {args.langrl_inference_server_type}" - ) + _INFERENCE_INTERFACE = loop.run_until_complete( + MegatronLocal.launch( + model[0], + host='0.0.0.0', + port=8294) + ) return _INFERENCE_INTERFACE @@ -1817,6 +1794,12 @@ def rl_inference_interface_shutdown(): else: logger.warning("No inference interface to shutdown. This should not happen.") + # TODO(rkirby): This is a hack to hard exit. There is a bug that is preventing us from using sys.exit(0). + # It seem the Flask server has non-daemon threads that are preventing the program from exiting. + # We need to find a way to gracefully complete all in progress requests and shutdown the Flask server. + import os + os._exit(0) + def get_iteration_sequence_count(args): """Get the total number of sequences processed in this iteration across all ranks.""" diff --git a/megatron/rl/server/inference/inference_interface_server.py b/megatron/rl/server/inference/inference_interface_server.py index ba595c3ca0e..ceac4a6cab7 100644 --- a/megatron/rl/server/inference/inference_interface_server.py +++ b/megatron/rl/server/inference/inference_interface_server.py @@ -11,10 +11,10 @@ from typing_extensions import Self from uvicorn import Config, Server -from ...inference.api import ChatInferenceRequest, ChatInferenceResponse from ...inference.inference_interface import ( - ChatInferenceInterface, InferenceInterface, + InferenceRequest, + InferenceResponse, ReturnsRaw, ReturnsTokens, ) @@ -22,20 +22,17 @@ @InferenceServer.register_subclass -class InferenceInterfaceClient(ChatInferenceInterface, InferenceServer): +class InferenceInterfaceClient(InferenceServer): type_name: str = Field(default='InferenceInterfaceClient', frozen=True) env_server_host_port: str conversation_template: None = None - async def base_generate(self, request: ChatInferenceRequest) -> list[ChatInferenceResponse]: + async def base_generate(self, request: InferenceRequest) -> InferenceResponse: async with httpx.AsyncClient(timeout=None) as client: response = await client.post( f"http://{self.env_server_host_port}/base_generate/", json=request.model_dump() ) - return [ - ChatInferenceResponse.model_validate(inference_response) - for inference_response in response.json() - ] + return InferenceResponse.model_validate(response.json()) @InferenceServer.register_subclass @@ -69,7 +66,7 @@ async def launch(cls, interface_cls: type[InferenceInterface], **kwargs) -> Self server_ref = weakref.ref(launched_server) @app.post("/base_generate/") - async def base_generate(request: ChatInferenceRequest): + async def base_generate(request: InferenceRequest) -> InferenceResponse: server = server_ref() if server is None: raise RuntimeError("Server has been garbage collected") diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index d2c6d29f4c7..51e92325604 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1987,12 +1987,6 @@ def _add_rl_args(parser): help="Entropy term weight in GRPO loss.") group.add_argument('--grpo-filter-groups-with-same-reward', action='store_true', help="Filter groups with same reward.") - group.add_argument('--langrl-inference-server-type', type=str, - choices=['inplace_megatron', 'inplace_megatron_chat'], default='inplace_megatron', - help="Type of inference server to use.") - group.add_argument('--langrl-inference-server-conversation-template', type=str, default=None, - help="Conversation template, if using a chat server.") - group.add_argument('--langrl-external-server', action=argparse.BooleanOptionalAction, required=False, default=False) group.add_argument('--langrl-env-config', type=str, default=None, help="Path to YAML config file for RL environment configuration.") group.add_argument('--rl-default-temperature', type=float, default=1.0, diff --git a/megatron/training/checkpointing.py b/megatron/training/checkpointing.py index f964b8dd32e..8ced7d267d6 100644 --- a/megatron/training/checkpointing.py +++ b/megatron/training/checkpointing.py @@ -1458,8 +1458,8 @@ def _set_arg(arg_name, old_arg_name=None, force=False): _set_arg('moe_latent_size', force=True) # Tokenizer args. - # Using checkpoint version might not always be safe (e.g., if running on different cluster). if args.use_tokenizer_model_from_checkpoint_args: + # Using checkpoint version might not always be safe (e.g., if running on different cluster). _set_arg('tokenizer_model', force=True) _set_arg('tokenizer_type', force=True) _set_arg('tiktoken_pattern', force=True) diff --git a/pyproject.toml b/pyproject.toml index fc1227b84d0..9b75fcf3596 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,9 @@ dev = [ "fastapi~=0.50", # Forcing a little bit more recent version of fastapi to be compatible with pydantic 2.0 "datasets", "emerging_optimizers", + "flask[async]", + "hypercorn", + "openai", ] lts = [ diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_basic_function/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_basic_function/model_config.yaml index 76fccecb827..0143a39f017 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_basic_function/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_basic_function/model_config.yaml @@ -43,8 +43,6 @@ MODEL_ARGS: --attention-softmax-in-fp32: true --tokenizer-type: HuggingFaceTokenizer --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer - --langrl-inference-server-type: inplace_megatron_chat - --langrl-inference-server-conversation-template: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer --vocab-size: 151936 --make-vocab-size-divisible-by: 128 --optimizer: adam diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest/model_config.yaml index b12911358f0..4f9be214289 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest/model_config.yaml @@ -43,7 +43,6 @@ MODEL_ARGS: --straggler-minmax-count: 16 --tensorboard-log-interval: 1 --empty-unused-memory-level: 2 - --langrl-inference-server-type: inplace_megatron --seed: 42 --calculate-per-token-loss: true --rl-use-sequence-packing: true diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest_github/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest_github/model_config.yaml index 97e40ee096f..c8fa19d0500 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest_github/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest_github/model_config.yaml @@ -43,7 +43,6 @@ MODEL_ARGS: --straggler-minmax-count: 16 --tensorboard-log-interval: 1 --empty-unused-memory-level: 2 - --langrl-inference-server-type: inplace_megatron --seed: 42 --calculate-per-token-loss: true --rl-use-sequence-packing: true diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput/model_config.yaml index 46b0474056f..f3c0c4ecc5b 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput/model_config.yaml @@ -40,8 +40,6 @@ MODEL_ARGS: --attention-softmax-in-fp32: true --tokenizer-type: HuggingFaceTokenizer --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer - --langrl-inference-server-type: inplace_megatron_chat - --langrl-inference-server-conversation-template: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer --vocab-size: 151936 --make-vocab-size-divisible-by: 128 --optimizer: adam diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput/model_config.yaml index ada0350b876..80664dcdc59 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput/model_config.yaml @@ -40,8 +40,6 @@ MODEL_ARGS: --attention-softmax-in-fp32: true --tokenizer-type: HuggingFaceTokenizer --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer - --langrl-inference-server-type: inplace_megatron_chat - --langrl-inference-server-conversation-template: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer --vocab-size: 151936 --make-vocab-size-divisible-by: 128 --optimizer: adam diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput_github/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput_github/model_config.yaml index 4490ced3988..cc25f3ab90e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput_github/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput_github/model_config.yaml @@ -40,8 +40,6 @@ MODEL_ARGS: --attention-softmax-in-fp32: true --tokenizer-type: HuggingFaceTokenizer --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer - --langrl-inference-server-type: inplace_megatron_chat - --langrl-inference-server-conversation-template: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer --vocab-size: 151936 --make-vocab-size-divisible-by: 128 --optimizer: adam diff --git a/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/model_config.yaml index c7dcfa594d8..139c5a82e57 100644 --- a/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/model_config.yaml @@ -90,7 +90,6 @@ MODEL_ARGS: --mock-data: true --max-tokens-to-oom: 3600000 --inference-max-seq-length: 256 - --langrl-inference-server-type: inplace_megatron --calculate-per-token-loss: true --rl-use-sequence-packing: true --rl-sequence-packing-algo: fifo diff --git a/tools/run_dynamic_text_generation_server.py b/tools/run_dynamic_text_generation_server.py index c09c788ca8e..b6092f36732 100644 --- a/tools/run_dynamic_text_generation_server.py +++ b/tools/run_dynamic_text_generation_server.py @@ -19,6 +19,7 @@ def add_text_generation_server_args(parser: argparse.ArgumentParser): parser = add_modelopt_args(parser) parser = add_inference_args(parser) parser.add_argument("--port", type=int, default=5000, help="Port for Flask server to run on") + parser.add_argument("--parsers", type=str, nargs="+", default=[], help="Parsers to use for parsing the response") return parser @@ -46,6 +47,7 @@ async def run_text_generation_server( run_flask_server( coordinator_addr=coordinator_addr, tokenizer=engine.controller.tokenizer, + parsers=args.parsers, rank=rank, flask_port=flask_port, ) diff --git a/train_rl.py b/train_rl.py index 4b5cec5fcc8..645e78ba986 100644 --- a/train_rl.py +++ b/train_rl.py @@ -33,7 +33,6 @@ logging.basicConfig(level=logging.INFO, force=True) - def _gpt_builder(args, pre_process, post_process, vp_stage=None, config=None, pg_collection=None): # TODO(Peter): This is a hack to get around the fact that we are activation recomputation for training but not # for inference with cuda graphs. Without out this the post checks in the transformer config will assert error. diff --git a/uv.lock b/uv.lock index 46d1a349aa6..13a344ac365 100644 --- a/uv.lock +++ b/uv.lock @@ -291,6 +291,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, ] +[[package]] +name = "asgiref" +version = "3.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.330676Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039625Z" }, +] + [[package]] name = "apache-tvm-ffi" version = "0.1.8.post2" @@ -1215,6 +1227,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + [[package]] name = "docker" version = "7.1.0" @@ -1437,6 +1458,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/f9/7f9263c5695f4bd0023734af91bedb2ff8209e8de6ead162f35d8dc762fd/flask-3.1.2-py3-none-any.whl", hash = "sha256:ca1d8112ec8a6158cc29ea4858963350011b5c846a414cdb7a954aa9e967d03c", size = 103308, upload-time = "2025-08-19T21:03:19.499Z" }, ] +[package.optional-dependencies] +async = [ + { name = "asgiref" }, +] + [[package]] name = "flask-restful" version = "0.3.10" @@ -1819,6 +1845,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, ] +[[package]] +name = "hypercorn" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "h11" }, + { name = "h2" }, + { name = "priority" }, + { name = "taskgroup", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/01/39f41a014b83dd5c795217362f2ca9071cf243e6a75bdcd6cd5b944658cc/hypercorn-0.18.0.tar.gz", hash = "sha256:d63267548939c46b0247dc8e5b45a9947590e35e64ee73a23c074aa3cf88e9da", size = 68420, upload-time = "2025-11-08T13:54:04.780563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/35/850277d1b17b206bd10874c8a9a3f52e059452fb49bb0d22cbb908f6038b/hypercorn-0.18.0-py3-none-any.whl", hash = "sha256:225e268f2c1c2f28f6d8f6db8f40cb8c992963610c5725e13ccfcddccb24b1cd", size = 61640, upload-time = "2025-11-08T13:54:03.202784Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1910,6 +1955,103 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jiter" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/5e/4ec91646aee381d01cdb9974e30882c9cd3b8c5d1079d6b5ff4af522439a/jiter-0.13.0.tar.gz", hash = "sha256:f2839f9c2c7e2dffc1bc5929a510e14ce0a946be9365fd1219e7ef342dae14f4", size = 164847, upload-time = "2026-02-02T12:37:56.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/5a/41da76c5ea07bec1b0472b6b2fdb1b651074d504b19374d7e130e0cdfb25/jiter-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2ffc63785fd6c7977defe49b9824ae6ce2b2e2b77ce539bdaf006c26da06342e", size = 311164, upload-time = "2026-02-02T12:35:17.688Z" }, + { url = "https://files.pythonhosted.org/packages/40/cb/4a1bf994a3e869f0d39d10e11efb471b76d0ad70ecbfb591427a46c880c2/jiter-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4a638816427006c1e3f0013eb66d391d7a3acda99a7b0cf091eff4497ccea33a", size = 320296, upload-time = "2026-02-02T12:35:19.828Z" }, + { url = "https://files.pythonhosted.org/packages/09/82/acd71ca9b50ecebadc3979c541cd717cce2fe2bc86236f4fa597565d8f1a/jiter-0.13.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19928b5d1ce0ff8c1ee1b9bdef3b5bfc19e8304f1b904e436caf30bc15dc6cf5", size = 352742, upload-time = "2026-02-02T12:35:21.258Z" }, + { url = "https://files.pythonhosted.org/packages/71/03/d1fc996f3aecfd42eb70922edecfb6dd26421c874503e241153ad41df94f/jiter-0.13.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:309549b778b949d731a2f0e1594a3f805716be704a73bf3ad9a807eed5eb5721", size = 363145, upload-time = "2026-02-02T12:35:24.653Z" }, + { url = "https://files.pythonhosted.org/packages/f1/61/a30492366378cc7a93088858f8991acd7d959759fe6138c12a4644e58e81/jiter-0.13.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bcdabaea26cb04e25df3103ce47f97466627999260290349a88c8136ecae0060", size = 487683, upload-time = "2026-02-02T12:35:26.162Z" }, + { url = "https://files.pythonhosted.org/packages/20/4e/4223cffa9dbbbc96ed821c5aeb6bca510848c72c02086d1ed3f1da3d58a7/jiter-0.13.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a3a377af27b236abbf665a69b2bdd680e3b5a0bd2af825cd3b81245279a7606c", size = 373579, upload-time = "2026-02-02T12:35:27.582Z" }, + { url = "https://files.pythonhosted.org/packages/fe/c9/b0489a01329ab07a83812d9ebcffe7820a38163c6d9e7da644f926ff877c/jiter-0.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe49d3ff6db74321f144dff9addd4a5874d3105ac5ba7c5b77fac099cfae31ae", size = 362904, upload-time = "2026-02-02T12:35:28.925Z" }, + { url = "https://files.pythonhosted.org/packages/05/af/53e561352a44afcba9a9bc67ee1d320b05a370aed8df54eafe714c4e454d/jiter-0.13.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2113c17c9a67071b0f820733c0893ed1d467b5fcf4414068169e5c2cabddb1e2", size = 392380, upload-time = "2026-02-02T12:35:30.385Z" }, + { url = "https://files.pythonhosted.org/packages/76/2a/dd805c3afb8ed5b326c5ae49e725d1b1255b9754b1b77dbecdc621b20773/jiter-0.13.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ab1185ca5c8b9491b55ebf6c1e8866b8f68258612899693e24a92c5fdb9455d5", size = 517939, upload-time = "2026-02-02T12:35:31.865Z" }, + { url = "https://files.pythonhosted.org/packages/20/2a/7b67d76f55b8fe14c937e7640389612f05f9a4145fc28ae128aaa5e62257/jiter-0.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9621ca242547edc16400981ca3231e0c91c0c4c1ab8573a596cd9bb3575d5c2b", size = 551696, upload-time = "2026-02-02T12:35:33.306Z" }, + { url = "https://files.pythonhosted.org/packages/85/9c/57cdd64dac8f4c6ab8f994fe0eb04dc9fd1db102856a4458fcf8a99dfa62/jiter-0.13.0-cp310-cp310-win32.whl", hash = "sha256:a7637d92b1c9d7a771e8c56f445c7f84396d48f2e756e5978840ecba2fac0894", size = 204592, upload-time = "2026-02-02T12:35:34.58Z" }, + { url = "https://files.pythonhosted.org/packages/a7/38/f4f3ea5788b8a5bae7510a678cdc747eda0c45ffe534f9878ff37e7cf3b3/jiter-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c1b609e5cbd2f52bb74fb721515745b407df26d7b800458bd97cb3b972c29e7d", size = 206016, upload-time = "2026-02-02T12:35:36.435Z" }, + { url = "https://files.pythonhosted.org/packages/71/29/499f8c9eaa8a16751b1c0e45e6f5f1761d180da873d417996cc7bddc8eef/jiter-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ea026e70a9a28ebbdddcbcf0f1323128a8db66898a06eaad3a4e62d2f554d096", size = 311157, upload-time = "2026-02-02T12:35:37.758Z" }, + { url = "https://files.pythonhosted.org/packages/50/f6/566364c777d2ab450b92100bea11333c64c38d32caf8dc378b48e5b20c46/jiter-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66aa3e663840152d18cc8ff1e4faad3dd181373491b9cfdc6004b92198d67911", size = 319729, upload-time = "2026-02-02T12:35:39.246Z" }, + { url = "https://files.pythonhosted.org/packages/73/dd/560f13ec5e4f116d8ad2658781646cca91b617ae3b8758d4a5076b278f70/jiter-0.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3524798e70655ff19aec58c7d05adb1f074fecff62da857ea9be2b908b6d701", size = 354766, upload-time = "2026-02-02T12:35:40.662Z" }, + { url = "https://files.pythonhosted.org/packages/7c/0d/061faffcfe94608cbc28a0d42a77a74222bdf5055ccdbe5fd2292b94f510/jiter-0.13.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ec7e287d7fbd02cb6e22f9a00dd9c9cd504c40a61f2c61e7e1f9690a82726b4c", size = 362587, upload-time = "2026-02-02T12:35:42.025Z" }, + { url = "https://files.pythonhosted.org/packages/92/c9/c66a7864982fd38a9773ec6e932e0398d1262677b8c60faecd02ffb67bf3/jiter-0.13.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:47455245307e4debf2ce6c6e65a717550a0244231240dcf3b8f7d64e4c2f22f4", size = 487537, upload-time = "2026-02-02T12:35:43.459Z" }, + { url = "https://files.pythonhosted.org/packages/6c/86/84eb4352cd3668f16d1a88929b5888a3fe0418ea8c1dfc2ad4e7bf6e069a/jiter-0.13.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ee9da221dca6e0429c2704c1b3655fe7b025204a71d4d9b73390c759d776d165", size = 373717, upload-time = "2026-02-02T12:35:44.928Z" }, + { url = "https://files.pythonhosted.org/packages/6e/09/9fe4c159358176f82d4390407a03f506a8659ed13ca3ac93a843402acecf/jiter-0.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24ab43126d5e05f3d53a36a8e11eb2f23304c6c1117844aaaf9a0aa5e40b5018", size = 362683, upload-time = "2026-02-02T12:35:46.636Z" }, + { url = "https://files.pythonhosted.org/packages/c9/5e/85f3ab9caca0c1d0897937d378b4a515cae9e119730563572361ea0c48ae/jiter-0.13.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9da38b4fedde4fb528c740c2564628fbab737166a0e73d6d46cb4bb5463ff411", size = 392345, upload-time = "2026-02-02T12:35:48.088Z" }, + { url = "https://files.pythonhosted.org/packages/12/4c/05b8629ad546191939e6f0c2f17e29f542a398f4a52fb987bc70b6d1eb8b/jiter-0.13.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0b34c519e17658ed88d5047999a93547f8889f3c1824120c26ad6be5f27b6cf5", size = 517775, upload-time = "2026-02-02T12:35:49.482Z" }, + { url = "https://files.pythonhosted.org/packages/4d/88/367ea2eb6bc582c7052e4baf5ddf57ebe5ab924a88e0e09830dfb585c02d/jiter-0.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2a6394e6af690d462310a86b53c47ad75ac8c21dc79f120714ea449979cb1d3", size = 551325, upload-time = "2026-02-02T12:35:51.104Z" }, + { url = "https://files.pythonhosted.org/packages/f3/12/fa377ffb94a2f28c41afaed093e0d70cfe512035d5ecb0cad0ae4792d35e/jiter-0.13.0-cp311-cp311-win32.whl", hash = "sha256:0f0c065695f616a27c920a56ad0d4fc46415ef8b806bf8fc1cacf25002bd24e1", size = 204709, upload-time = "2026-02-02T12:35:52.467Z" }, + { url = "https://files.pythonhosted.org/packages/cb/16/8e8203ce92f844dfcd3d9d6a5a7322c77077248dbb12da52d23193a839cd/jiter-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:0733312953b909688ae3c2d58d043aa040f9f1a6a75693defed7bc2cc4bf2654", size = 204560, upload-time = "2026-02-02T12:35:53.925Z" }, + { url = "https://files.pythonhosted.org/packages/44/26/97cc40663deb17b9e13c3a5cf29251788c271b18ee4d262c8f94798b8336/jiter-0.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:5d9b34ad56761b3bf0fbe8f7e55468704107608512350962d3317ffd7a4382d5", size = 189608, upload-time = "2026-02-02T12:35:55.304Z" }, + { url = "https://files.pythonhosted.org/packages/2e/30/7687e4f87086829955013ca12a9233523349767f69653ebc27036313def9/jiter-0.13.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0a2bd69fc1d902e89925fc34d1da51b2128019423d7b339a45d9e99c894e0663", size = 307958, upload-time = "2026-02-02T12:35:57.165Z" }, + { url = "https://files.pythonhosted.org/packages/c3/27/e57f9a783246ed95481e6749cc5002a8a767a73177a83c63ea71f0528b90/jiter-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f917a04240ef31898182f76a332f508f2cc4b57d2b4d7ad2dbfebbfe167eb505", size = 318597, upload-time = "2026-02-02T12:35:58.591Z" }, + { url = "https://files.pythonhosted.org/packages/cf/52/e5719a60ac5d4d7c5995461a94ad5ef962a37c8bf5b088390e6fad59b2ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1e2b199f446d3e82246b4fd9236d7cb502dc2222b18698ba0d986d2fecc6152", size = 348821, upload-time = "2026-02-02T12:36:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/61/db/c1efc32b8ba4c740ab3fc2d037d8753f67685f475e26b9d6536a4322bcdd/jiter-0.13.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04670992b576fa65bd056dbac0c39fe8bd67681c380cb2b48efa885711d9d726", size = 364163, upload-time = "2026-02-02T12:36:01.937Z" }, + { url = "https://files.pythonhosted.org/packages/55/8a/fb75556236047c8806995671a18e4a0ad646ed255276f51a20f32dceaeec/jiter-0.13.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a1aff1fbdb803a376d4d22a8f63f8e7ccbce0b4890c26cc7af9e501ab339ef0", size = 483709, upload-time = "2026-02-02T12:36:03.41Z" }, + { url = "https://files.pythonhosted.org/packages/7e/16/43512e6ee863875693a8e6f6d532e19d650779d6ba9a81593ae40a9088ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b3fb8c2053acaef8580809ac1d1f7481a0a0bdc012fd7f5d8b18fb696a5a089", size = 370480, upload-time = "2026-02-02T12:36:04.791Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4c/09b93e30e984a187bc8aaa3510e1ec8dcbdcd71ca05d2f56aac0492453aa/jiter-0.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdaba7d87e66f26a2c45d8cbadcbfc4bf7884182317907baf39cfe9775bb4d93", size = 360735, upload-time = "2026-02-02T12:36:06.994Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1b/46c5e349019874ec5dfa508c14c37e29864ea108d376ae26d90bee238cd7/jiter-0.13.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7b88d649135aca526da172e48083da915ec086b54e8e73a425ba50999468cc08", size = 391814, upload-time = "2026-02-02T12:36:08.368Z" }, + { url = "https://files.pythonhosted.org/packages/15/9e/26184760e85baee7162ad37b7912797d2077718476bf91517641c92b3639/jiter-0.13.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e404ea551d35438013c64b4f357b0474c7abf9f781c06d44fcaf7a14c69ff9e2", size = 513990, upload-time = "2026-02-02T12:36:09.993Z" }, + { url = "https://files.pythonhosted.org/packages/e9/34/2c9355247d6debad57a0a15e76ab1566ab799388042743656e566b3b7de1/jiter-0.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1f4748aad1b4a93c8bdd70f604d0f748cdc0e8744c5547798acfa52f10e79228", size = 548021, upload-time = "2026-02-02T12:36:11.376Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4a/9f2c23255d04a834398b9c2e0e665382116911dc4d06b795710503cdad25/jiter-0.13.0-cp312-cp312-win32.whl", hash = "sha256:0bf670e3b1445fc4d31612199f1744f67f889ee1bbae703c4b54dc097e5dd394", size = 203024, upload-time = "2026-02-02T12:36:12.682Z" }, + { url = "https://files.pythonhosted.org/packages/09/ee/f0ae675a957ae5a8f160be3e87acea6b11dc7b89f6b7ab057e77b2d2b13a/jiter-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:15db60e121e11fe186c0b15236bd5d18381b9ddacdcf4e659feb96fc6c969c92", size = 205424, upload-time = "2026-02-02T12:36:13.93Z" }, + { url = "https://files.pythonhosted.org/packages/1b/02/ae611edf913d3cbf02c97cdb90374af2082c48d7190d74c1111dde08bcdd/jiter-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:41f92313d17989102f3cb5dd533a02787cdb99454d494344b0361355da52fcb9", size = 186818, upload-time = "2026-02-02T12:36:15.308Z" }, + { url = "https://files.pythonhosted.org/packages/91/9c/7ee5a6ff4b9991e1a45263bfc46731634c4a2bde27dfda6c8251df2d958c/jiter-0.13.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1f8a55b848cbabf97d861495cd65f1e5c590246fabca8b48e1747c4dfc8f85bf", size = 306897, upload-time = "2026-02-02T12:36:16.748Z" }, + { url = "https://files.pythonhosted.org/packages/7c/02/be5b870d1d2be5dd6a91bdfb90f248fbb7dcbd21338f092c6b89817c3dbf/jiter-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f556aa591c00f2c45eb1b89f68f52441a016034d18b65da60e2d2875bbbf344a", size = 317507, upload-time = "2026-02-02T12:36:18.351Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/b25d2ec333615f5f284f3a4024f7ce68cfa0604c322c6808b2344c7f5d2b/jiter-0.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7e1d61da332ec412350463891923f960c3073cf1aae93b538f0bb4c8cd46efb", size = 350560, upload-time = "2026-02-02T12:36:19.746Z" }, + { url = "https://files.pythonhosted.org/packages/be/ec/74dcb99fef0aca9fbe56b303bf79f6bd839010cb18ad41000bf6cc71eec0/jiter-0.13.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3097d665a27bc96fd9bbf7f86178037db139f319f785e4757ce7ccbf390db6c2", size = 363232, upload-time = "2026-02-02T12:36:21.243Z" }, + { url = "https://files.pythonhosted.org/packages/1b/37/f17375e0bb2f6a812d4dd92d7616e41917f740f3e71343627da9db2824ce/jiter-0.13.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d01ecc3a8cbdb6f25a37bd500510550b64ddf9f7d64a107d92f3ccb25035d0f", size = 483727, upload-time = "2026-02-02T12:36:22.688Z" }, + { url = "https://files.pythonhosted.org/packages/77/d2/a71160a5ae1a1e66c1395b37ef77da67513b0adba73b993a27fbe47eb048/jiter-0.13.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed9bbc30f5d60a3bdf63ae76beb3f9db280d7f195dfcfa61af792d6ce912d159", size = 370799, upload-time = "2026-02-02T12:36:24.106Z" }, + { url = "https://files.pythonhosted.org/packages/01/99/ed5e478ff0eb4e8aa5fd998f9d69603c9fd3f32de3bd16c2b1194f68361c/jiter-0.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98fbafb6e88256f4454de33c1f40203d09fc33ed19162a68b3b257b29ca7f663", size = 359120, upload-time = "2026-02-02T12:36:25.519Z" }, + { url = "https://files.pythonhosted.org/packages/16/be/7ffd08203277a813f732ba897352797fa9493faf8dc7995b31f3d9cb9488/jiter-0.13.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5467696f6b827f1116556cb0db620440380434591e93ecee7fd14d1a491b6daa", size = 390664, upload-time = "2026-02-02T12:36:26.866Z" }, + { url = "https://files.pythonhosted.org/packages/d1/84/e0787856196d6d346264d6dcccb01f741e5f0bd014c1d9a2ebe149caf4f3/jiter-0.13.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:2d08c9475d48b92892583df9da592a0e2ac49bcd41fae1fec4f39ba6cf107820", size = 513543, upload-time = "2026-02-02T12:36:28.217Z" }, + { url = "https://files.pythonhosted.org/packages/65/50/ecbd258181c4313cf79bca6c88fb63207d04d5bf5e4f65174114d072aa55/jiter-0.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:aed40e099404721d7fcaf5b89bd3b4568a4666358bcac7b6b15c09fb6252ab68", size = 547262, upload-time = "2026-02-02T12:36:29.678Z" }, + { url = "https://files.pythonhosted.org/packages/27/da/68f38d12e7111d2016cd198161b36e1f042bd115c169255bcb7ec823a3bf/jiter-0.13.0-cp313-cp313-win32.whl", hash = "sha256:36ebfbcffafb146d0e6ffb3e74d51e03d9c35ce7c625c8066cdbfc7b953bdc72", size = 200630, upload-time = "2026-02-02T12:36:31.808Z" }, + { url = "https://files.pythonhosted.org/packages/25/65/3bd1a972c9a08ecd22eb3b08a95d1941ebe6938aea620c246cf426ae09c2/jiter-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:8d76029f077379374cf0dbc78dbe45b38dec4a2eb78b08b5194ce836b2517afc", size = 202602, upload-time = "2026-02-02T12:36:33.679Z" }, + { url = "https://files.pythonhosted.org/packages/15/fe/13bd3678a311aa67686bb303654792c48206a112068f8b0b21426eb6851e/jiter-0.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:bb7613e1a427cfcb6ea4544f9ac566b93d5bf67e0d48c787eca673ff9c9dff2b", size = 185939, upload-time = "2026-02-02T12:36:35.065Z" }, + { url = "https://files.pythonhosted.org/packages/49/19/a929ec002ad3228bc97ca01dbb14f7632fffdc84a95ec92ceaf4145688ae/jiter-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fa476ab5dd49f3bf3a168e05f89358c75a17608dbabb080ef65f96b27c19ab10", size = 316616, upload-time = "2026-02-02T12:36:36.579Z" }, + { url = "https://files.pythonhosted.org/packages/52/56/d19a9a194afa37c1728831e5fb81b7722c3de18a3109e8f282bfc23e587a/jiter-0.13.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ade8cb6ff5632a62b7dbd4757d8c5573f7a2e9ae285d6b5b841707d8363205ef", size = 346850, upload-time = "2026-02-02T12:36:38.058Z" }, + { url = "https://files.pythonhosted.org/packages/36/4a/94e831c6bf287754a8a019cb966ed39ff8be6ab78cadecf08df3bb02d505/jiter-0.13.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9950290340acc1adaded363edd94baebcee7dabdfa8bee4790794cd5cfad2af6", size = 358551, upload-time = "2026-02-02T12:36:39.417Z" }, + { url = "https://files.pythonhosted.org/packages/a2/ec/a4c72c822695fa80e55d2b4142b73f0012035d9fcf90eccc56bc060db37c/jiter-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2b4972c6df33731aac0742b64fd0d18e0a69bc7d6e03108ce7d40c85fd9e3e6d", size = 201950, upload-time = "2026-02-02T12:36:40.791Z" }, + { url = "https://files.pythonhosted.org/packages/b6/00/393553ec27b824fbc29047e9c7cd4a3951d7fbe4a76743f17e44034fa4e4/jiter-0.13.0-cp313-cp313t-win_arm64.whl", hash = "sha256:701a1e77d1e593c1b435315ff625fd071f0998c5f02792038a5ca98899261b7d", size = 185852, upload-time = "2026-02-02T12:36:42.077Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f5/f1997e987211f6f9bd71b8083047b316208b4aca0b529bb5f8c96c89ef3e/jiter-0.13.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cc5223ab19fe25e2f0bf2643204ad7318896fe3729bf12fde41b77bfc4fafff0", size = 308804, upload-time = "2026-02-02T12:36:43.496Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8f/5482a7677731fd44881f0204981ce2d7175db271f82cba2085dd2212e095/jiter-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9776ebe51713acf438fd9b4405fcd86893ae5d03487546dae7f34993217f8a91", size = 318787, upload-time = "2026-02-02T12:36:45.071Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b9/7257ac59778f1cd025b26a23c5520a36a424f7f1b068f2442a5b499b7464/jiter-0.13.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:879e768938e7b49b5e90b7e3fecc0dbec01b8cb89595861fb39a8967c5220d09", size = 353880, upload-time = "2026-02-02T12:36:47.365Z" }, + { url = "https://files.pythonhosted.org/packages/c3/87/719eec4a3f0841dad99e3d3604ee4cba36af4419a76f3cb0b8e2e691ad67/jiter-0.13.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:682161a67adea11e3aae9038c06c8b4a9a71023228767477d683f69903ebc607", size = 366702, upload-time = "2026-02-02T12:36:48.871Z" }, + { url = "https://files.pythonhosted.org/packages/d2/65/415f0a75cf6921e43365a1bc227c565cb949caca8b7532776e430cbaa530/jiter-0.13.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a13b68cd1cd8cc9de8f244ebae18ccb3e4067ad205220ef324c39181e23bbf66", size = 486319, upload-time = "2026-02-02T12:36:53.006Z" }, + { url = "https://files.pythonhosted.org/packages/54/a2/9e12b48e82c6bbc6081fd81abf915e1443add1b13d8fc586e1d90bb02bb8/jiter-0.13.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87ce0f14c6c08892b610686ae8be350bf368467b6acd5085a5b65441e2bf36d2", size = 372289, upload-time = "2026-02-02T12:36:54.593Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c1/e4693f107a1789a239c759a432e9afc592366f04e901470c2af89cfd28e1/jiter-0.13.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c365005b05505a90d1c47856420980d0237adf82f70c4aff7aebd3c1cc143ad", size = 360165, upload-time = "2026-02-02T12:36:56.112Z" }, + { url = "https://files.pythonhosted.org/packages/17/08/91b9ea976c1c758240614bd88442681a87672eebc3d9a6dde476874e706b/jiter-0.13.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1317fdffd16f5873e46ce27d0e0f7f4f90f0cdf1d86bf6abeaea9f63ca2c401d", size = 389634, upload-time = "2026-02-02T12:36:57.495Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/58325ef99390d6d40427ed6005bf1ad54f2577866594bcf13ce55675f87d/jiter-0.13.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c05b450d37ba0c9e21c77fef1f205f56bcee2330bddca68d344baebfc55ae0df", size = 514933, upload-time = "2026-02-02T12:36:58.909Z" }, + { url = "https://files.pythonhosted.org/packages/5b/25/69f1120c7c395fd276c3996bb8adefa9c6b84c12bb7111e5c6ccdcd8526d/jiter-0.13.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:775e10de3849d0631a97c603f996f518159272db00fdda0a780f81752255ee9d", size = 548842, upload-time = "2026-02-02T12:37:00.433Z" }, + { url = "https://files.pythonhosted.org/packages/18/05/981c9669d86850c5fbb0d9e62bba144787f9fba84546ba43d624ee27ef29/jiter-0.13.0-cp314-cp314-win32.whl", hash = "sha256:632bf7c1d28421c00dd8bbb8a3bac5663e1f57d5cd5ed962bce3c73bf62608e6", size = 202108, upload-time = "2026-02-02T12:37:01.718Z" }, + { url = "https://files.pythonhosted.org/packages/8d/96/cdcf54dd0b0341db7d25413229888a346c7130bd20820530905fdb65727b/jiter-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:f22ef501c3f87ede88f23f9b11e608581c14f04db59b6a801f354397ae13739f", size = 204027, upload-time = "2026-02-02T12:37:03.075Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f9/724bcaaab7a3cd727031fe4f6995cb86c4bd344909177c186699c8dec51a/jiter-0.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:07b75fe09a4ee8e0c606200622e571e44943f47254f95e2436c8bdcaceb36d7d", size = 187199, upload-time = "2026-02-02T12:37:04.414Z" }, + { url = "https://files.pythonhosted.org/packages/62/92/1661d8b9fd6a3d7a2d89831db26fe3c1509a287d83ad7838831c7b7a5c7e/jiter-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:964538479359059a35fb400e769295d4b315ae61e4105396d355a12f7fef09f0", size = 318423, upload-time = "2026-02-02T12:37:05.806Z" }, + { url = "https://files.pythonhosted.org/packages/4f/3b/f77d342a54d4ebcd128e520fc58ec2f5b30a423b0fd26acdfc0c6fef8e26/jiter-0.13.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e104da1db1c0991b3eaed391ccd650ae8d947eab1480c733e5a3fb28d4313e40", size = 351438, upload-time = "2026-02-02T12:37:07.189Z" }, + { url = "https://files.pythonhosted.org/packages/76/b3/ba9a69f0e4209bd3331470c723c2f5509e6f0482e416b612431a5061ed71/jiter-0.13.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e3a5f0cde8ff433b8e88e41aa40131455420fb3649a3c7abdda6145f8cb7202", size = 364774, upload-time = "2026-02-02T12:37:08.579Z" }, + { url = "https://files.pythonhosted.org/packages/b3/16/6cdb31fa342932602458dbb631bfbd47f601e03d2e4950740e0b2100b570/jiter-0.13.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:57aab48f40be1db920a582b30b116fe2435d184f77f0e4226f546794cedd9cf0", size = 487238, upload-time = "2026-02-02T12:37:10.066Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b1/956cc7abaca8d95c13aa8d6c9b3f3797241c246cd6e792934cc4c8b250d2/jiter-0.13.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7772115877c53f62beeb8fd853cab692dbc04374ef623b30f997959a4c0e7e95", size = 372892, upload-time = "2026-02-02T12:37:11.656Z" }, + { url = "https://files.pythonhosted.org/packages/26/c4/97ecde8b1e74f67b8598c57c6fccf6df86ea7861ed29da84629cdbba76c4/jiter-0.13.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1211427574b17b633cfceba5040de8081e5abf114f7a7602f73d2e16f9fdaa59", size = 360309, upload-time = "2026-02-02T12:37:13.244Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d7/eabe3cf46715854ccc80be2cd78dd4c36aedeb30751dbf85a1d08c14373c/jiter-0.13.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7beae3a3d3b5212d3a55d2961db3c292e02e302feb43fce6a3f7a31b90ea6dfe", size = 389607, upload-time = "2026-02-02T12:37:14.881Z" }, + { url = "https://files.pythonhosted.org/packages/df/2d/03963fc0804e6109b82decfb9974eb92df3797fe7222428cae12f8ccaa0c/jiter-0.13.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:e5562a0f0e90a6223b704163ea28e831bd3a9faa3512a711f031611e6b06c939", size = 514986, upload-time = "2026-02-02T12:37:16.326Z" }, + { url = "https://files.pythonhosted.org/packages/f6/6c/8c83b45eb3eb1c1e18d841fe30b4b5bc5619d781267ca9bc03e005d8fd0a/jiter-0.13.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:6c26a424569a59140fb51160a56df13f438a2b0967365e987889186d5fc2f6f9", size = 548756, upload-time = "2026-02-02T12:37:17.736Z" }, + { url = "https://files.pythonhosted.org/packages/47/66/eea81dfff765ed66c68fd2ed8c96245109e13c896c2a5015c7839c92367e/jiter-0.13.0-cp314-cp314t-win32.whl", hash = "sha256:24dc96eca9f84da4131cdf87a95e6ce36765c3b156fc9ae33280873b1c32d5f6", size = 201196, upload-time = "2026-02-02T12:37:19.101Z" }, + { url = "https://files.pythonhosted.org/packages/ff/32/4ac9c7a76402f8f00d00842a7f6b83b284d0cf7c1e9d4227bc95aa6d17fa/jiter-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0a8d76c7524087272c8ae913f5d9d608bd839154b62c4322ef65723d2e5bb0b8", size = 204215, upload-time = "2026-02-02T12:37:20.495Z" }, + { url = "https://files.pythonhosted.org/packages/f9/8e/7def204fea9f9be8b3c21a6f2dd6c020cf56c7d5ff753e0e23ed7f9ea57e/jiter-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2c26cf47e2cad140fa23b6d58d435a7c0161f5c514284802f25e87fddfe11024", size = 187152, upload-time = "2026-02-02T12:37:22.124Z" }, + { url = "https://files.pythonhosted.org/packages/79/b3/3c29819a27178d0e461a8571fb63c6ae38be6dc36b78b3ec2876bbd6a910/jiter-0.13.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b1cbfa133241d0e6bdab48dcdc2604e8ba81512f6bbd68ec3e8e1357dd3c316c", size = 307016, upload-time = "2026-02-02T12:37:42.755Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ae/60993e4b07b1ac5ebe46da7aa99fdbb802eb986c38d26e3883ac0125c4e0/jiter-0.13.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:db367d8be9fad6e8ebbac4a7578b7af562e506211036cba2c06c3b998603c3d2", size = 305024, upload-time = "2026-02-02T12:37:44.774Z" }, + { url = "https://files.pythonhosted.org/packages/77/fa/2227e590e9cf98803db2811f172b2d6460a21539ab73006f251c66f44b14/jiter-0.13.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45f6f8efb2f3b0603092401dc2df79fa89ccbc027aaba4174d2d4133ed661434", size = 339337, upload-time = "2026-02-02T12:37:46.668Z" }, + { url = "https://files.pythonhosted.org/packages/2d/92/015173281f7eb96c0ef580c997da8ef50870d4f7f4c9e03c845a1d62ae04/jiter-0.13.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:597245258e6ad085d064780abfb23a284d418d3e61c57362d9449c6c7317ee2d", size = 346395, upload-time = "2026-02-02T12:37:48.09Z" }, + { url = "https://files.pythonhosted.org/packages/80/60/e50fa45dd7e2eae049f0ce964663849e897300433921198aef94b6ffa23a/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:3d744a6061afba08dd7ae375dcde870cffb14429b7477e10f67e9e6d68772a0a", size = 305169, upload-time = "2026-02-02T12:37:50.376Z" }, + { url = "https://files.pythonhosted.org/packages/d2/73/a009f41c5eed71c49bec53036c4b33555afcdee70682a18c6f66e396c039/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:ff732bd0a0e778f43d5009840f20b935e79087b4dc65bd36f1cd0f9b04b8ff7f", size = 303808, upload-time = "2026-02-02T12:37:52.092Z" }, + { url = "https://files.pythonhosted.org/packages/c4/10/528b439290763bff3d939268085d03382471b442f212dca4ff5f12802d43/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab44b178f7981fcaea7e0a5df20e773c663d06ffda0198f1a524e91b2fde7e59", size = 337384, upload-time = "2026-02-02T12:37:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/67/8a/a342b2f0251f3dac4ca17618265d93bf244a2a4d089126e81e4c1056ac50/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19", size = 343768, upload-time = "2026-02-02T12:37:55.055Z" }, +] + [[package]] name = "jmespath" version = "1.0.1" @@ -2255,6 +2397,8 @@ dev = [ { name = "fastapi" }, { name = "flash-linear-attention" }, { name = "flashinfer-python" }, + { name = "flask", extra = ["async"] }, + { name = "hypercorn" }, { name = "mamba-ssm" }, { name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-dev'" }, { name = "multi-storage-client" }, @@ -2263,6 +2407,7 @@ dev = [ { name = "nvidia-resiliency-ext" }, { name = "nvtx" }, { name = "onnxscript" }, + { name = "openai" }, { name = "opentelemetry-api" }, { name = "tensorstore", version = "0.1.78", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "tensorstore", version = "0.1.80", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, @@ -2369,7 +2514,9 @@ requires-dist = [ { name = "flash-linear-attention", marker = "extra == 'dev'", specifier = "~=0.4.0" }, { name = "flashinfer-python", marker = "extra == 'dev'", specifier = "~=0.5.0" }, { name = "flashinfer-python", marker = "extra == 'lts'", specifier = "~=0.5.0" }, + { name = "flask", extras = ["async"], marker = "extra == 'dev'" }, { name = "flask-restful", marker = "extra == 'mlm'" }, + { name = "hypercorn", marker = "extra == 'dev'" }, { name = "mamba-ssm", marker = "extra == 'dev'", specifier = "~=2.2" }, { name = "mamba-ssm", marker = "extra == 'lts'", specifier = "~=2.2" }, { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'dev'", specifier = "~=6.0" }, @@ -2385,6 +2532,7 @@ requires-dist = [ { name = "nvtx", marker = "extra == 'lts'", specifier = "~=0.2" }, { name = "onnxscript", marker = "extra == 'dev'" }, { name = "onnxscript", marker = "extra == 'lts'" }, + { name = "openai", marker = "extra == 'dev'" }, { name = "opentelemetry-api", marker = "extra == 'dev'", specifier = "~=1.33.1" }, { name = "opentelemetry-api", marker = "extra == 'lts'", specifier = "~=1.33.1" }, { name = "packaging", specifier = ">=24.2" }, @@ -3358,6 +3506,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f6/ec/1656ea93be1e50baf429c20603dce249fa3571f3a180407cee79b1afa013/onnxscript-0.5.7-py3-none-any.whl", hash = "sha256:f94a66059c56d13b44908e9b7fd9dae4b4faa6681c784f3fd4c29cfa863e454e", size = 693353, upload-time = "2025-12-16T20:47:17.897Z" }, ] +[[package]] +name = "openai" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/5a/f495777c02625bfa18212b6e3b73f1893094f2bf660976eb4bc6f43a1ca2/openai-2.20.0.tar.gz", hash = "sha256:2654a689208cd0bf1098bb9462e8d722af5cbe961e6bba54e6f19fb843d88db1", size = 642355, upload-time = "2026-02-10T19:02:54.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/a0/cf4297aa51bbc21e83ef0ac018947fa06aea8f2364aad7c96cbf148590e6/openai-2.20.0-py3-none-any.whl", hash = "sha256:38d989c4b1075cd1f76abc68364059d822327cf1a932531d429795f4fc18be99", size = 1098479, upload-time = "2026-02-10T19:02:52.157Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.33.1" @@ -3612,6 +3779,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/8c/83087ebc47ab0396ce092363001fa37c17153119ee282700c0713a195853/prettytable-3.17.0-py3-none-any.whl", hash = "sha256:aad69b294ddbe3e1f95ef8886a060ed1666a0b83018bbf56295f6f226c43d287", size = 34433, upload-time = "2025-11-14T17:33:19.093Z" }, ] +[[package]] +name = "priority" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/3c/eb7c35f4dcede96fca1842dac5f4f5d15511aa4b52f3a961219e68ae9204/priority-2.0.0.tar.gz", hash = "sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0", size = 24792, upload-time = "2021-06-27T10:15:05.487867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/5f/82c8074f7e84978129347c2c6ec8b6c59f3584ff1a20bc3c940a3e061790/priority-2.0.0-py3-none-any.whl", hash = "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa", size = 8946, upload-time = "2021-06-27T10:15:03.856590Z" }, +] + [[package]] name = "prometheus-client" version = "0.24.0" @@ -5347,6 +5523,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, ] +[[package]] +name = "taskgroup" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/8d/e218e0160cc1b692e6e0e5ba34e8865dbb171efeb5fc9a704544b3020605/taskgroup-0.2.2.tar.gz", hash = "sha256:078483ac3e78f2e3f973e2edbf6941374fbea81b9c5d0a96f51d297717f4752d", size = 11504, upload-time = "2025-01-03T09:24:13.761490Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/b1/74babcc824a57904e919f3af16d86c08b524c0691504baf038ef2d7f655c/taskgroup-0.2.2-py2.py3-none-any.whl", hash = "sha256:e2c53121609f4ae97303e9ea1524304b4de6faf9eb2c9280c7f87976479a52fb", size = 14237, upload-time = "2025-01-03T09:24:11.410239Z" }, +] + [[package]] name = "tensorboard" version = "2.20.0" @@ -6237,6 +6426,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, ] +[[package]] +name = "wsproto" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454632Z" }, +] + [[package]] name = "xattr" version = "1.3.0"