From 14b91dc91de6988380e7e456d41beb0edd63a4db Mon Sep 17 00:00:00 2001 From: Minu Kim Date: Sun, 31 Aug 2025 14:26:06 +0900 Subject: [PATCH 1/2] Fix: handle JSON parse error in streaming tool calls Signed-off-by: Minu Kim --- .../ai/model/tool/ToolOnFinishPredicate.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 spring-ai-model/src/main/java/org/springframework/ai/model/tool/ToolOnFinishPredicate.java diff --git a/spring-ai-model/src/main/java/org/springframework/ai/model/tool/ToolOnFinishPredicate.java b/spring-ai-model/src/main/java/org/springframework/ai/model/tool/ToolOnFinishPredicate.java new file mode 100644 index 00000000000..be49f962084 --- /dev/null +++ b/spring-ai-model/src/main/java/org/springframework/ai/model/tool/ToolOnFinishPredicate.java @@ -0,0 +1,73 @@ +/* + * Copyright 2023-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.model.tool; + +import java.util.List; +import java.util.function.BiPredicate; + +import org.jetbrains.annotations.NotNull; + +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.ChatOptions; + +/** + * Executes tools only when the assistant signals completion (finishReason = "tool_calls" + * or "stop"). + */ +public final class ToolOnFinishPredicate implements ToolExecutionEligibilityPredicate { + + @Override + public boolean isToolExecutionRequired(ChatOptions opts, ChatResponse resp) { + List gens = resp.getResults(); + if (gens.isEmpty()) { + return false; + } + + Generation gen = gens.get(0); + boolean hasToolCalls = !gen.getOutput().getToolCalls().isEmpty(); + String finish = String.valueOf(gen.getMetadata().get("finishReason")); + + return hasToolCalls && ("tool_calls".equalsIgnoreCase(finish) || "stop".equalsIgnoreCase(finish)); + } + + @Override + public boolean test(ChatOptions chatOptions, ChatResponse chatResponse) { + return false; + } + + @NotNull + @Override + public BiPredicate and( + @NotNull BiPredicate other) { + return ToolExecutionEligibilityPredicate.super.and(other); + } + + @NotNull + @Override + public BiPredicate negate() { + return ToolExecutionEligibilityPredicate.super.negate(); + } + + @NotNull + @Override + public BiPredicate or( + @NotNull BiPredicate other) { + return ToolExecutionEligibilityPredicate.super.or(other); + } + +} From 601f0e2092b9f6b4e50ecc7860d0693d27595134 Mon Sep 17 00:00:00 2001 From: Minu Kim Date: Sun, 28 Sep 2025 16:01:08 +0900 Subject: [PATCH 2/2] fix: handle invalid JSON chunk in OpenAiChatModel Signed-off-by: Minu Kim --- .../ai/openai/OpenAiChatModel.java | 23 ++++-- .../openai/OpenAiChatModelStreamingTest.java | 49 +++++++++++++ .../ai/model/tool/ToolOnFinishPredicate.java | 73 ------------------- 3 files changed, 64 insertions(+), 81 deletions(-) create mode 100644 models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelStreamingTest.java delete mode 100644 spring-ai-model/src/main/java/org/springframework/ai/model/tool/ToolOnFinishPredicate.java diff --git a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java index cb0fed3e549..7c82db1eabf 100644 --- a/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java +++ b/models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiChatModel.java @@ -329,6 +329,7 @@ public Flux internalStream(Prompt prompt, ChatResponse previousCha previousChatResponse); return new ChatResponse(generations, from(chatCompletion2, null, accumulatedUsage)); } + catch (Exception e) { logger.error("Error processing chat completion", e); return new ChatResponse(List.of()); @@ -492,14 +493,20 @@ private ChatResponseMetadata from(ChatResponseMetadata chatResponseMetadata, Usa * @return the ChatCompletion */ private OpenAiApi.ChatCompletion chunkToChatCompletion(OpenAiApi.ChatCompletionChunk chunk) { - List choices = chunk.choices() - .stream() - .map(chunkChoice -> new Choice(chunkChoice.finishReason(), chunkChoice.index(), chunkChoice.delta(), - chunkChoice.logprobs())) - .toList(); - - return new OpenAiApi.ChatCompletion(chunk.id(), choices, chunk.created(), chunk.model(), chunk.serviceTier(), - chunk.systemFingerprint(), "chat.completion", chunk.usage()); + try { + List choices = chunk.choices() + .stream() + .map(chunkChoice -> new Choice(chunkChoice.finishReason(), chunkChoice.index(), chunkChoice.delta(), + chunkChoice.logprobs())) + .toList(); + + return new OpenAiApi.ChatCompletion(chunk.id(), choices, chunk.created(), chunk.model(), + chunk.serviceTier(), chunk.systemFingerprint(), "chat.completion", chunk.usage()); + } + catch (Exception e) { + logger.warn("Invalid JSON chunk received, skipping. Raw chunk: {}", chunk, e); + throw new RuntimeException("Failed to parse ChatCompletionChunk", e); + } } private DefaultUsage getDefaultUsage(OpenAiApi.Usage usage) { diff --git a/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelStreamingTest.java b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelStreamingTest.java new file mode 100644 index 00000000000..491df21d5cc --- /dev/null +++ b/models/spring-ai-openai/src/test/java/org/springframework/ai/openai/OpenAiChatModelStreamingTest.java @@ -0,0 +1,49 @@ +/* + * Copyright 2025-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.openai; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import reactor.core.publisher.Flux; + +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.openai.api.OpenAiApi; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +public class OpenAiChatModelStreamingTest { + + @Test + void shouldThrowExceptionOnInvalidJsonChunk() { + OpenAiApi mockApi = Mockito.mock(OpenAiApi.class); + + OpenAiApi.ChatCompletionChunk invalidChunk = new OpenAiApi.ChatCompletionChunk("invalid-id", null, + System.currentTimeMillis() / 1000L, "gpt-test-model", null, null, null, null); + + when(mockApi.chatCompletionStream(any(), any())).thenReturn(Flux.just(invalidChunk)); + + OpenAiChatOptions options = OpenAiChatOptions.builder().model("gpt-test-model").build(); + OpenAiChatModel model = OpenAiChatModel.builder().openAiApi(mockApi).defaultOptions(options).build(); + + assertThatThrownBy(() -> model.stream(new Prompt("Hello")).collectList().block()) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Failed to parse ChatCompletionChunk"); + } + +} diff --git a/spring-ai-model/src/main/java/org/springframework/ai/model/tool/ToolOnFinishPredicate.java b/spring-ai-model/src/main/java/org/springframework/ai/model/tool/ToolOnFinishPredicate.java deleted file mode 100644 index be49f962084..00000000000 --- a/spring-ai-model/src/main/java/org/springframework/ai/model/tool/ToolOnFinishPredicate.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2023-2025 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.ai.model.tool; - -import java.util.List; -import java.util.function.BiPredicate; - -import org.jetbrains.annotations.NotNull; - -import org.springframework.ai.chat.model.ChatResponse; -import org.springframework.ai.chat.model.Generation; -import org.springframework.ai.chat.prompt.ChatOptions; - -/** - * Executes tools only when the assistant signals completion (finishReason = "tool_calls" - * or "stop"). - */ -public final class ToolOnFinishPredicate implements ToolExecutionEligibilityPredicate { - - @Override - public boolean isToolExecutionRequired(ChatOptions opts, ChatResponse resp) { - List gens = resp.getResults(); - if (gens.isEmpty()) { - return false; - } - - Generation gen = gens.get(0); - boolean hasToolCalls = !gen.getOutput().getToolCalls().isEmpty(); - String finish = String.valueOf(gen.getMetadata().get("finishReason")); - - return hasToolCalls && ("tool_calls".equalsIgnoreCase(finish) || "stop".equalsIgnoreCase(finish)); - } - - @Override - public boolean test(ChatOptions chatOptions, ChatResponse chatResponse) { - return false; - } - - @NotNull - @Override - public BiPredicate and( - @NotNull BiPredicate other) { - return ToolExecutionEligibilityPredicate.super.and(other); - } - - @NotNull - @Override - public BiPredicate negate() { - return ToolExecutionEligibilityPredicate.super.negate(); - } - - @NotNull - @Override - public BiPredicate or( - @NotNull BiPredicate other) { - return ToolExecutionEligibilityPredicate.super.or(other); - } - -}