From d8cfa254d7fcb16131df64bead22a8f133118081 Mon Sep 17 00:00:00 2001 From: surajm20061998 Date: Sat, 17 Oct 2026 12:46:53 -0400 Subject: [PATCH 1/3] [Bugfix][Frontend] Honor --enable-log-outputs on /v1/completions --enable-log-outputs is server-wide, but /v1/completions never logged generated text: OpenAIServingCompletion did not accept the flag, and init_generate_state did not pass it. Chat, Responses, Anthropic, Cohere and /inference/v1/generate all log outputs. Log completion outputs the way chat does: one line per output for non-streaming requests; for streaming requests, one line per delta (unless --enable-log-deltas is off) and the full text once the stream ends. Lines use the same per-prompt request id as the input log. Co-Authored-By: Claude Opus 5.5 Signed-off-by: surajm20061998 --- .../completion/test_completion_error.py | 76 ++++++++++++++++++- .../completion/test_completion_text_offset.py | 1 + vllm/entrypoints/generate/api_router.py | 2 + vllm/entrypoints/openai/completion/serving.py | 40 ++++++++++ 4 files changed, 117 insertions(+), 2 deletions(-) diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 0217d8a73351..feacd82c37a8 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -16,6 +16,7 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.scale_out.render.serving import ServingRender from vllm.entrypoints.serve.engine.protocol import ErrorResponse +from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.exceptions import GenerationError, VLLMValidationError from vllm.logprobs import Logprob from vllm.outputs import CompletionOutput, RequestOutput, WeightVersionSpan @@ -91,7 +92,11 @@ class MockVllmConfig: parallel_config: MockParallelConfig -def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion: +def _build_serving_completion( + engine: AsyncLLM, + request_logger: RequestLogger | None = None, + **kwargs: Any, +) -> OpenAIServingCompletion: models = OpenAIServingModels( engine_client=engine, base_model_paths=BASE_MODEL_PATHS, @@ -107,7 +112,8 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion: engine, models, online_renderer=online_renderer, - request_logger=None, + request_logger=request_logger, + **kwargs, ) @@ -694,6 +700,72 @@ async def test_renderer_only_completion_request_skips_mm_cache(): ) +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_completion_logs_outputs_when_enabled(stream: bool): + """`--enable-log-outputs` covers /v1/completions like the chat endpoint.""" + mock_engine = MagicMock(spec=AsyncLLM) + mock_engine.errored = False + mock_engine.model_config = MockModelConfig() + mock_engine.input_processor = MagicMock() + mock_engine.renderer = _build_renderer(mock_engine.model_config) + request_logger = MagicMock(spec=RequestLogger) + serving_completion = _build_serving_completion( + mock_engine, request_logger=request_logger, enable_log_outputs=True + ) + + def request_output(text: str, token_ids: list[int], finish_reason: str | None): + return RequestOutput( + request_id="test-id", + prompt="Test prompt", + prompt_token_ids=[1, 2, 3], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text=text, + token_ids=token_ids, + cumulative_logprob=None, + logprobs=None, + finish_reason=finish_reason, + ) + ], + finished=finish_reason is not None, + ) + + async def mock_generate(*args, **kwargs): + if stream: + yield request_output("Hello", [100], None) + yield request_output(" world", [101], "stop") + else: + yield request_output("Hello world", [100, 101], "stop") + + mock_engine.generate = MagicMock(side_effect=mock_generate) + request = CompletionRequest( + model=MODEL_NAME, prompt="Test prompt", max_tokens=10, stream=stream + ) + + response = await serving_completion.create_completion(request) + if stream: + async for _ in response: + pass + + input_request_id = request_logger.log_inputs.call_args.args[0] + logged = [call.kwargs for call in request_logger.log_outputs.call_args_list] + assert {log["request_id"] for log in logged} == {input_request_id} + if stream: + assert [(log["outputs"], log["delta"]) for log in logged] == [ + ("Hello", True), + (" world", True), + ("Hello world", False), + ] + else: + assert len(logged) == 1 + assert logged[0]["outputs"] == "Hello world" + assert logged[0]["output_token_ids"] == [100, 101] + assert logged[0]["finish_reason"] == "stop" + + @pytest.mark.asyncio @pytest.mark.parametrize("include_usage", [False, True]) async def test_completion_error_stream(include_usage: bool): diff --git a/tests/entrypoints/openai/completion/test_completion_text_offset.py b/tests/entrypoints/openai/completion/test_completion_text_offset.py index c8ba7df62f8a..841847dae3e9 100644 --- a/tests/entrypoints/openai/completion/test_completion_text_offset.py +++ b/tests/entrypoints/openai/completion/test_completion_text_offset.py @@ -46,6 +46,7 @@ def _serving() -> OpenAIServingCompletion: serving = OpenAIServingCompletion.__new__(OpenAIServingCompletion) serving.enable_prompt_tokens_details = False serving.enable_force_include_usage = False + serving.enable_log_outputs = False serving.enable_per_request_metrics = False serving.system_fingerprint = None serving.return_tokens_as_token_ids = False diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 942a1c3e78c2..c1e501d44764 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -163,6 +163,8 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, + enable_log_outputs=args.enable_log_outputs, + enable_log_deltas=args.enable_log_deltas, enable_per_request_metrics=args.enable_per_request_metrics, ) if "generate" in supported_tasks diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index 944da27530a3..a1620217b903 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -65,6 +65,8 @@ def __init__( return_tokens_as_token_ids: bool = False, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, + enable_log_outputs: bool = False, + enable_log_deltas: bool = True, enable_per_request_metrics: bool = False, ): super().__init__( @@ -77,6 +79,8 @@ def __init__( self.online_renderer = online_renderer self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage + self.enable_log_outputs = enable_log_outputs + self.enable_log_deltas = enable_log_deltas self.enable_per_request_metrics = enable_per_request_metrics self.default_sampling_params = self.model_config.get_diff_sampling_param() @@ -262,6 +266,18 @@ async def _create_completion( tokenizer, request_metadata, ) + + if self.enable_log_outputs and self.request_logger: + for i, final_res in enumerate(final_res_batch_checked): + for output in final_res.outputs: + self.request_logger.log_outputs( + request_id=f"{request_id}-{i}", + outputs=output.text, + output_token_ids=output.token_ids, + finish_reason=output.finish_reason, + is_streaming=False, + delta=False, + ) except asyncio.CancelledError: return self.create_error_response("Client disconnected") @@ -293,6 +309,7 @@ async def completion_stream_generator( num_choices = 1 if request.n is None else request.n next_text_offsets = [0] * num_choices * num_prompts previous_num_tokens = [0] * num_choices * num_prompts + previous_texts = [""] * num_choices * num_prompts has_echoed = [False] * num_choices * num_prompts num_prompt_tokens = [0] * num_prompts num_cached_tokens = None @@ -404,6 +421,18 @@ async def completion_stream_generator( previous_num_tokens[i] += len(output.token_ids) stop_reason = output.stop_reason + if self.enable_log_outputs and self.request_logger: + previous_texts[i] += output.text + if output.text and self.enable_log_deltas: + self.request_logger.log_outputs( + request_id=f"{request_id}-{prompt_idx}", + outputs=output.text, + output_token_ids=as_list(output.token_ids), + finish_reason=finish_reason, + is_streaming=True, + delta=True, + ) + chunk = CompletionStreamResponse( id=request_id, object="text_completion", @@ -450,6 +479,17 @@ async def completion_stream_generator( response_json = chunk.model_dump_json(exclude_unset=True) yield f"data: {response_json}\n\n" + if self.enable_log_outputs and self.request_logger: + for i, full_text in enumerate(previous_texts): + self.request_logger.log_outputs( + request_id=f"{request_id}-{i // num_choices}", + outputs=full_text, + output_token_ids=None, + finish_reason="streaming_complete", + is_streaming=True, + delta=False, + ) + total_prompt_tokens = sum(num_prompt_tokens) total_completion_tokens = sum(previous_num_tokens) final_usage_info = UsageInfo( From c5106cd536e2d6de7c37581b225b2d88a61956fa Mon Sep 17 00:00:00 2001 From: surajm20061998 Date: Sun, 18 Oct 2026 10:09:25 -0400 Subject: [PATCH 2/3] Review comment based fixes Signed-off-by: surajm20061998 --- .../openai/completion/test_completion_error.py | 14 ++++++++------ vllm/entrypoints/openai/completion/serving.py | 17 ++++++++++------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index feacd82c37a8..c8a99eb06bfb 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -714,9 +714,11 @@ async def test_completion_logs_outputs_when_enabled(stream: bool): mock_engine, request_logger=request_logger, enable_log_outputs=True ) - def request_output(text: str, token_ids: list[int], finish_reason: str | None): + def request_output( + request_id: str, text: str, token_ids: list[int], finish_reason: str | None + ): return RequestOutput( - request_id="test-id", + request_id=request_id, prompt="Test prompt", prompt_token_ids=[1, 2, 3], prompt_logprobs=None, @@ -733,12 +735,12 @@ def request_output(text: str, token_ids: list[int], finish_reason: str | None): finished=finish_reason is not None, ) - async def mock_generate(*args, **kwargs): + async def mock_generate(prompt, params, request_id, **kwargs): if stream: - yield request_output("Hello", [100], None) - yield request_output(" world", [101], "stop") + yield request_output(request_id, "Hello", [100], None) + yield request_output(request_id, " world", [101], "stop") else: - yield request_output("Hello world", [100, 101], "stop") + yield request_output(request_id, "Hello world", [100, 101], "stop") mock_engine.generate = MagicMock(side_effect=mock_generate) request = CompletionRequest( diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index a1620217b903..3b0eda85f730 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -268,10 +268,10 @@ async def _create_completion( ) if self.enable_log_outputs and self.request_logger: - for i, final_res in enumerate(final_res_batch_checked): + for final_res in final_res_batch_checked: for output in final_res.outputs: self.request_logger.log_outputs( - request_id=f"{request_id}-{i}", + request_id=final_res.request_id, outputs=output.text, output_token_ids=output.token_ids, finish_reason=output.finish_reason, @@ -310,6 +310,7 @@ async def completion_stream_generator( next_text_offsets = [0] * num_choices * num_prompts previous_num_tokens = [0] * num_choices * num_prompts previous_texts = [""] * num_choices * num_prompts + prompt_request_ids = [request_id] * num_prompts has_echoed = [False] * num_choices * num_prompts num_prompt_tokens = [0] * num_prompts num_cached_tokens = None @@ -422,11 +423,13 @@ async def completion_stream_generator( stop_reason = output.stop_reason if self.enable_log_outputs and self.request_logger: - previous_texts[i] += output.text - if output.text and self.enable_log_deltas: + delta_content = output.text + previous_texts[i] += delta_content + prompt_request_ids[prompt_idx] = res.request_id + if delta_content and self.enable_log_deltas: self.request_logger.log_outputs( - request_id=f"{request_id}-{prompt_idx}", - outputs=output.text, + request_id=res.request_id, + outputs=delta_content, output_token_ids=as_list(output.token_ids), finish_reason=finish_reason, is_streaming=True, @@ -482,7 +485,7 @@ async def completion_stream_generator( if self.enable_log_outputs and self.request_logger: for i, full_text in enumerate(previous_texts): self.request_logger.log_outputs( - request_id=f"{request_id}-{i // num_choices}", + request_id=prompt_request_ids[i // num_choices], outputs=full_text, output_token_ids=None, finish_reason="streaming_complete", From 169f5a715388fe0de1680e434a88b43b85c396f3 Mon Sep 17 00:00:00 2001 From: surajm20061998 Date: Sun, 18 Oct 2026 12:50:10 -0400 Subject: [PATCH 3/3] More fixes Signed-off-by: surajm20061998 --- .../openai/completion/test_completion_error.py | 9 ++++++--- vllm/entrypoints/openai/completion/serving.py | 10 ++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index c8a99eb06bfb..0075254af024 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -744,7 +744,11 @@ async def mock_generate(prompt, params, request_id, **kwargs): mock_engine.generate = MagicMock(side_effect=mock_generate) request = CompletionRequest( - model=MODEL_NAME, prompt="Test prompt", max_tokens=10, stream=stream + model=MODEL_NAME, + prompt="Test prompt", + max_tokens=10, + stream=stream, + request_id="test-req", ) response = await serving_completion.create_completion(request) @@ -752,9 +756,8 @@ async def mock_generate(prompt, params, request_id, **kwargs): async for _ in response: pass - input_request_id = request_logger.log_inputs.call_args.args[0] logged = [call.kwargs for call in request_logger.log_outputs.call_args_list] - assert {log["request_id"] for log in logged} == {input_request_id} + assert {log["request_id"] for log in logged} == {"cmpl-test-req"} if stream: assert [(log["outputs"], log["delta"]) for log in logged] == [ ("Hello", True), diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index 3b0eda85f730..bb4db91b494f 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -271,7 +271,7 @@ async def _create_completion( for final_res in final_res_batch_checked: for output in final_res.outputs: self.request_logger.log_outputs( - request_id=final_res.request_id, + request_id=request_id, outputs=output.text, output_token_ids=output.token_ids, finish_reason=output.finish_reason, @@ -310,7 +310,6 @@ async def completion_stream_generator( next_text_offsets = [0] * num_choices * num_prompts previous_num_tokens = [0] * num_choices * num_prompts previous_texts = [""] * num_choices * num_prompts - prompt_request_ids = [request_id] * num_prompts has_echoed = [False] * num_choices * num_prompts num_prompt_tokens = [0] * num_prompts num_cached_tokens = None @@ -425,10 +424,9 @@ async def completion_stream_generator( if self.enable_log_outputs and self.request_logger: delta_content = output.text previous_texts[i] += delta_content - prompt_request_ids[prompt_idx] = res.request_id if delta_content and self.enable_log_deltas: self.request_logger.log_outputs( - request_id=res.request_id, + request_id=request_id, outputs=delta_content, output_token_ids=as_list(output.token_ids), finish_reason=finish_reason, @@ -483,9 +481,9 @@ async def completion_stream_generator( yield f"data: {response_json}\n\n" if self.enable_log_outputs and self.request_logger: - for i, full_text in enumerate(previous_texts): + for full_text in previous_texts: self.request_logger.log_outputs( - request_id=prompt_request_ids[i // num_choices], + request_id=request_id, outputs=full_text, output_token_ids=None, finish_reason="streaming_complete",