Repository navigation
Move to using the Inference OpenAI API server #3107
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 21 commits
cbd57c1
2b92df8
bd32db8
054942d
0cc1b08
fc668b6
e9d1a52
31bdce4
1aec041
39fd47a
49da256
590ce52
15395b8
39d36cc
14f1347
4b80457
36e850a
c3e7854
0a72c80
d62038d
351c00e
c195e24
6489600
09b83eb
f32d215
055964d
076fac5
3f0bb15
ebf542c
9a5207b
fc0307d
d321cc5
d69730f
d5edaaa
2412044
dc4fcbc
441e49a
81c78f9
fc27fa3
2216694
652431c
338f36d
ad40331
ee45dd6
0666c2e
9109ca1
7a36e5e
7feee53
473a652
23ceea5
0bd3202
3912fd7
120c22f
452acb2
20760a5
9b8843a
1ca3a82
96dc390
bf27426
26002fc
75d31e1
db37b3b
e0d3dd5
601b77d
a13eafb
dda0b6a
045b07b
48bb9b9
b632151
5072d5d
e3c6cab
061228c
aeb73d1
e084310
c05da3b
a99ddee
8657146
2594c39
11a0b46
8fd7f70
6baad1e
593f2f1
14be3e1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,8 +3,10 @@ | |
| import asyncio | ||
| import logging | ||
| import time | ||
| import traceback | ||
|
|
||
| from megatron.core.inference.sampling_params import SamplingParams | ||
| from megatron.core.tokenizers.text.parsers import PARSER_MAPPING | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
@@ -19,6 +21,7 @@ async def chat_completions(): | |
| """Handles async POST requests for chat completions.""" | ||
| client = current_app.config['client'] | ||
| tokenizer = current_app.config['tokenizer'] | ||
| parsers = current_app.config['parsers'] | ||
|
|
||
| req = request.get_json() | ||
|
|
||
|
|
@@ -31,14 +34,17 @@ async def chat_completions(): | |
|
|
||
| try: | ||
| prompt_tokens = tokenizer.apply_chat_template( | ||
| messages, tokenize=True, add_generation_prompt=True | ||
| messages, tokenize=True, add_generation_prompt=True, tools=req.get("tools", None) | ||
| ) | ||
| except (AttributeError, AssertionError): | ||
| logger.warning( | ||
| "Tokenizer does not support 'apply_chat_template'. Using tokenize instead." | ||
| ) | ||
| prompt_tokens = tokenizer.tokenize( | ||
| "\n".join([message["content"] for message in messages]) | ||
| ) | ||
| except AttributeError: | ||
| return ( | ||
| "Tokenizer does not support 'apply_chat_template'. " | ||
| "Chat completions requires a tokenizer with a configured chat template." | ||
| ), 500 | ||
| except Exception as e: | ||
| logger.error(f"{traceback.format_exc()}") | ||
| return f"Error processing 'messages': {e}", 500 | ||
|
|
||
| # --- 2. Parse Sampling Params --- | ||
|
|
@@ -62,7 +68,12 @@ async def chat_completions(): | |
| top_p=top_p, | ||
| return_log_probs=return_log_probs, | ||
| top_n_logprobs=top_n_logprobs, | ||
| num_tokens_to_generate=int(req.get("max_tokens", 16)), | ||
| num_tokens_to_generate=( | ||
| int(max_tokens) | ||
| if ((max_tokens := req.get("max_tokens", None)) is not None) | ||
| else None | ||
| ), | ||
| skip_prompt_log_probs=True, | ||
| ) | ||
| except ValueError as e: | ||
| return f"Invalid sampling parameter: {e}", 400 | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's add
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd rather not have add_BOS over the API, really we should do it on a tokenizer basis.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is; I think the stop conditions should be both stop string and termination ID. Stop string is less reliable and uses more compute.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Over the OpenAI API you don't know the ID. |
||
|
|
@@ -71,20 +82,13 @@ async def chat_completions(): | |
| # For chat, we run the *same* prompt 'n' times. | ||
| tasks = [] | ||
| for _ in range(n): | ||
| per_req_params = SamplingParams( | ||
| temperature=sampling_params.temperature, | ||
| top_k=sampling_params.top_k, | ||
| top_p=sampling_params.top_p, | ||
| return_log_probs=sampling_params.return_log_probs, | ||
| top_n_logprobs=sampling_params.top_n_logprobs, | ||
| num_tokens_to_generate=sampling_params.num_tokens_to_generate, | ||
| ) | ||
| tasks.append(client.add_request(prompt_tokens, per_req_params)) | ||
| tasks.append(client.add_request(prompt_tokens, sampling_params)) | ||
|
|
||
| start_time = time.perf_counter() | ||
| try: | ||
| batch_results = await asyncio.gather(*tasks) | ||
| except Exception as e: | ||
| logger.error(f"Error during inference: {e}") | ||
| return f"Error during inference: {e}", 500 | ||
|
|
||
| logger.info( | ||
|
|
@@ -133,12 +137,37 @@ async def chat_completions(): | |
| } | ||
| logprobs_content.append(entry) | ||
|
|
||
| metadata = {} | ||
| message_text = text_output | ||
| if parsers: | ||
| for parser in parsers: | ||
| if parser not in PARSER_MAPPING: | ||
| raise ValueError(f"Parser {parser} not found in PARSER_MAPPING") | ||
| message_text, new_info = PARSER_MAPPING[parser].parse( | ||
| message_text, tools=req.get("tools", None) | ||
| ) | ||
| assert not ( | ||
| metadata.keys() & new_info.keys() | ||
| ), "Multiple parsers found the same information." | ||
| metadata.update(new_info) | ||
| message = {"role": "assistant", "content": message_text} | ||
| if "tool_calls" in metadata: | ||
| message["tool_calls"] = metadata["tool_calls"] | ||
| if "reasoning" in metadata: | ||
| message["reasoning"] = metadata["reasoning"] | ||
|
|
||
| choice_data = { | ||
| "index": 0, | ||
| "message": {"role": "assistant", "content": text_output}, | ||
| "index": request_idx, | ||
| "message": message, | ||
| "prompt_token_ids": prompt_tokens, | ||
| "generation_token_ids": result.generated_tokens, | ||
| "generation_log_probs": result.generated_log_probs, | ||
| "raw_text": result.prompt + result.generated_text, | ||
| # 'logprobs' in chat API is an object containing 'content' | ||
| "logprobs": {"content": logprobs_content} if logprobs_content else None, | ||
| "finish_reason": "length", # Original code hardcoded this. | ||
| "finish_reason": ( | ||
| "tool_calls" if metadata.get("tool_calls", []) else "stop" | ||
| ), # Original code hardcoded this. | ||
| } | ||
| choices.append(choice_data) | ||
| total_completion_tokens += len(result.generated_tokens) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| from megatron.core.tokenizers.text.parsers.deepseek_r1_reasoning_parser import ( | ||
| DeepSeekR1ReasoningParser, | ||
| ) | ||
| from megatron.core.tokenizers.text.parsers.qwen3_coder_tool_parser import Qwen3CoderToolParser | ||
|
|
||
| PARSER_MAPPING = { | ||
| "deepseek-r1-reasoning": DeepSeekR1ReasoningParser, | ||
| "qwen3-coder-tool": Qwen3CoderToolParser, | ||
| } | ||
|
|
||
| __all__ = ["PARSER_MAPPING"] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| from typing import Any | ||
|
|
||
|
|
||
| class BaseParser: | ||
| """Base class for text parsers.""" | ||
|
|
||
| @staticmethod | ||
| def parse(text: str, **kwargs) -> tuple[str, dict[str, Any]]: | ||
| """ | ||
| Parses the text into a tuple containing extracted content | ||
| and a dictionary of additional information. | ||
|
|
||
| Args: | ||
| text (str): The text to parse. | ||
|
|
||
| Returns: | ||
| tuple[str, dict[str, Any]]: A tuple containing the unprocessed text | ||
| and a dictionary with the extracted information. | ||
| """ | ||
| return text, {} |
Uh oh!
There was an error while loading. Please reload this page.