Skip to content

fix(anthropic): prevent streaming tool calling responses when internal execution is enabled #3679

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

fix(anthropic): prevent streaming tool calling responses when internal execution is enabled #3679

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 26 additions & 19 deletions ...pring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,26 +260,33 @@ public Flux<ChatResponse> internalStream(Prompt prompt, ChatResponse previousCha
Usage accumulatedUsage = UsageCalculator.getCumulativeUsage(currentChatResponseUsage, previousChatResponse);
ChatResponse chatResponse = toChatResponse(chatCompletionResponse, accumulatedUsage);

if (this.toolExecutionEligibilityPredicate.isToolExecutionRequired(prompt.getOptions(), chatResponse) && chatResponse.hasFinishReasons(Set.of("tool_use"))) {
// FIXME: bounded elastic needs to be used since tool calling
// is currently only synchronous
return Flux.defer(() -> {
var toolExecutionResult = this.toolCallingManager.executeToolCalls(prompt, chatResponse);
if (toolExecutionResult.returnDirect()) {
// Return tool execution result directly to the client.
return Flux.just(ChatResponse.builder().from(chatResponse)
.generations(ToolExecutionResult.buildGenerations(toolExecutionResult))
.build());
}
else {
// Send the tool execution result back to the model.
return this.internalStream(new Prompt(toolExecutionResult.conversationHistory(), prompt.getOptions()),
chatResponse);
}
}).subscribeOn(Schedulers.boundedElastic());
}
if (this.toolExecutionEligibilityPredicate.isToolExecutionRequired(prompt.getOptions(), chatResponse)) {

if (chatResponse.hasFinishReasons(Set.of("tool_use"))) {
// FIXME: bounded elastic needs to be used since tool calling
// is currently only synchronous
return Flux.defer(() -> {
var toolExecutionResult = this.toolCallingManager.executeToolCalls(prompt, chatResponse);
if (toolExecutionResult.returnDirect()) {
// Return tool execution result directly to the client.
return Flux.just(ChatResponse.builder().from(chatResponse)
.generations(ToolExecutionResult.buildGenerations(toolExecutionResult))
.build());
}
else {
// Send the tool execution result back to the model.
return this.internalStream(new Prompt(toolExecutionResult.conversationHistory(), prompt.getOptions()),
chatResponse);
}
}).subscribeOn(Schedulers.boundedElastic());
} else {
return Mono.empty();
}

return Mono.just(chatResponse);
} else {
// If internal tool execution is not required, just return the chat response.
return Mono.just(chatResponse);
}
})
.doOnError(observation::error)
.doFinally(s -> observation.stop())
Expand Down
26 changes: 17 additions & 9 deletions .../spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/api/AnthropicApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,8 @@
import java.util.function.Consumer;
import java.util.function.Predicate;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.anthropic.api.StreamHelper.ChatCompletionResponseBuilder;
import org.springframework.ai.model.ApiKey;
import org.springframework.ai.model.ChatModelDescription;
Expand All @@ -52,6 +45,16 @@
import org.springframework.web.client.RestClient;
import org.springframework.web.reactive.function.client.WebClient;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;

import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;

/**
* The Anthropic API client.
*
Expand All @@ -67,6 +70,8 @@
*/
public final class AnthropicApi {

private static final Logger logger = LoggerFactory.getLogger(AnthropicApi.class);

public static Builder builder() {
return new Builder();
}
Expand Down Expand Up @@ -222,6 +227,9 @@ public Flux<ChatCompletionResponse> chatCompletionStream(ChatCompletionRequest c
.filter(event -> event.type() != EventType.PING)
// Detect if the chunk is part of a streaming function call.
.map(event -> {

logger.debug("Received event: {}", event);

if (this.streamHelper.isToolUseStart(event)) {
isInsideTool.set(true);
}
Expand Down
43 changes: 40 additions & 3 deletions ...nthropic/src/test/java/org/springframework/ai/anthropic/client/AnthropicChatClientIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

package org.springframework.ai.anthropic.client;

import static org.assertj.core.api.Assertions.assertThat;

import java.io.IOException;
import java.net.URL;
import java.util.Arrays;
Expand All @@ -29,8 +31,6 @@
import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.publisher.Flux;

import org.springframework.ai.anthropic.AnthropicChatOptions;
import org.springframework.ai.anthropic.AnthropicTestConfiguration;
import org.springframework.ai.anthropic.api.AnthropicApi;
Expand All @@ -41,7 +41,9 @@
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.ai.converter.ListOutputConverter;
import org.springframework.ai.model.tool.ToolCallingChatOptions;
import org.springframework.ai.test.CurlyBracketEscaper;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.ai.tool.function.FunctionToolCallback;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
Expand All @@ -53,7 +55,7 @@
import org.springframework.test.context.ActiveProfiles;
import org.springframework.util.MimeTypeUtils;

import static org.assertj.core.api.Assertions.assertThat;
import reactor.core.publisher.Flux;

@SpringBootTest(classes = AnthropicTestConfiguration.class, properties = "spring.ai.retry.on-http-codes=429")
@EnabledIfEnvironmentVariable(named = "ANTHROPIC_API_KEY", matches = ".+")
Expand Down Expand Up @@ -343,4 +345,39 @@ record ActorsFilms(String actor, List<String> movies) {

}

@ParameterizedTest(name = "{0} : {displayName} ")
@ValueSource(strings = { "claude-3-7-sonnet-latest", "claude-sonnet-4-0" })
void streamToolCallingResponseShouldNotContainToolCallMessages(String modelName) {

ChatClient chatClient = ChatClient.builder(this.chatModel).build();

Flux<ChatResponse> responses = chatClient.prompt()
.options(ToolCallingChatOptions.builder().model(modelName).build())
.tools(new MyTools())
.user("Get current weather in Amsterdam and Paris")
// .user("Get current weather in Amsterdam. Please don't explain that you will
// call tools.")
.stream()
.chatResponse();

List<ChatResponse> chatResponses = responses.collectList().block();

assertThat(chatResponses).isNotEmpty();

// Verify that none of the ChatResponse objects have tool calls
chatResponses.forEach(chatResponse -> {
logger.info("ChatResponse Results: {}", chatResponse.getResults());
assertThat(chatResponse.hasToolCalls()).isFalse();
});
}

public static class MyTools {

@Tool(description = "Get the current weather forecast by city name")
String getCurrentDateTime(String cityName) {
return "For " + cityName + " Weather is hot and sunny with a temperature of 20 degrees";
}

}

}
2 changes: 1 addition & 1 deletion ...ingframework/ai/anthropic/client/AnthropicChatClientMethodInvokingFunctionCallbackIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ void streamingParameterLessTool(String modelName) {
.map(cr -> cr.getResult().getOutput().getText())
.collect(Collectors.joining());

assertThat(content).contains("20 degrees");
assertThat(content).contains("20");
}

public static class ParameterLessTools {
Expand Down
2 changes: 2 additions & 0 deletions models/spring-ai-anthropic/src/test/resources/application-logging-test.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,5 @@
#

logging.level.org.springframework.ai.chat.client.advisor=DEBUG

logging.level.org.springframework.ai.anthropic.api.AnthropicApi=DEBUG