From 28f85c1516020d4b30305386128a11f0fb01a4ed Mon Sep 17 00:00:00 2001 From: James Brown Date: Thu, 30 Jan 2025 21:06:17 +0800 Subject: [PATCH 1/8] Update launch.json --- .vscode/launch.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 977eeda..2dcb99c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,6 +10,7 @@ "request": "launch", "django": true, "module": "mcp_bridge.main", + "pythonArgs": ["-Xutf8"] } ] -} \ No newline at end of file +} From 8823a93e4a518114c8f4a56d1beda8b8c1f8c8e6 Mon Sep 17 00:00:00 2001 From: James Brown Date: Thu, 30 Jan 2025 21:09:00 +0800 Subject: [PATCH 2/8] Update utils.py --- mcp_bridge/openai_clients/utils.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mcp_bridge/openai_clients/utils.py b/mcp_bridge/openai_clients/utils.py index 58c269b..57c1be7 100644 --- a/mcp_bridge/openai_clients/utils.py +++ b/mcp_bridge/openai_clients/utils.py @@ -3,6 +3,7 @@ from lmos_openai_types import CreateChatCompletionRequest import mcp.types import json +import traceback from mcp_bridge.mcp_clients.McpClientManager import ClientManager from mcp_bridge.tool_mappers import mcp2openai @@ -10,17 +11,22 @@ async def chat_completion_add_tools(request: CreateChatCompletionRequest): request.tools = [] + logger.info("adding tools to request") for _, session in ClientManager.get_clients(): # if session is None, then the client is not running if session.session is None: - logger.error(f"session is `None` for {session.name}") + logger.error(f"session is `None` for {session.name}") # Date:2025/01/25 why not running? continue - + logger.debug(f"session ready for {session.name}") tools = await session.session.list_tools() for tool in tools.tools: request.tools.append(mcp2openai(tool)) - + + if request.tools == []: + logger.info("this request loads no tools") + # raise Exception("no tools found. unable to initiate chat completion.") + request.tools = None return request @@ -42,9 +48,10 @@ async def call_tool( return None try: - tool_call_args = json.loads(tool_call_json) + tool_call_args = json.loads(tool_call_json) # Date: 2025/01/26 cannot load this tool call json? except json.JSONDecodeError: logger.error(f"failed to decode json for {tool_call_name}") + traceback.print_exc() return None return await session.call_tool(tool_call_name, tool_call_args, timeout) From a0cf387058af746bb95e4fd1306fb5ce20e86186 Mon Sep 17 00:00:00 2001 From: James Brown Date: Thu, 30 Jan 2025 21:14:53 +0800 Subject: [PATCH 3/8] Update utils.py --- mcp_bridge/openai_clients/utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mcp_bridge/openai_clients/utils.py b/mcp_bridge/openai_clients/utils.py index 57c1be7..73e318d 100644 --- a/mcp_bridge/openai_clients/utils.py +++ b/mcp_bridge/openai_clients/utils.py @@ -9,6 +9,21 @@ from mcp_bridge.tool_mappers import mcp2openai +def validate_if_json_object_parsable(content: str): + try: + json.loads(content) + return True + except ValueError: + return False + + +def salvage_parsable_json_object(content: str): + content = content.strip() + for i in range(0, len(content)): + snippet = content[: len(content) - i] + if validate_if_json_object_parsable(snippet): + return snippet + async def chat_completion_add_tools(request: CreateChatCompletionRequest): request.tools = [] logger.info("adding tools to request") From 1fb17d7a5a5e4362227f9ff1e223076e8d6b0946 Mon Sep 17 00:00:00 2001 From: James Brown Date: Thu, 30 Jan 2025 21:16:32 +0800 Subject: [PATCH 4/8] Delete mcp_bridge/openai_clients/streamCompletion.py --- mcp_bridge/openai_clients/streamCompletion.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 mcp_bridge/openai_clients/streamCompletion.py diff --git a/mcp_bridge/openai_clients/streamCompletion.py b/mcp_bridge/openai_clients/streamCompletion.py deleted file mode 100644 index e69de29..0000000 From 96e91b825cb79b14a28db3ecba1d05e4d995a335 Mon Sep 17 00:00:00 2001 From: James Brown Date: Thu, 30 Jan 2025 21:31:32 +0800 Subject: [PATCH 5/8] Update streamChatCompletion.py --- .../openai_clients/streamChatCompletion.py | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/mcp_bridge/openai_clients/streamChatCompletion.py b/mcp_bridge/openai_clients/streamChatCompletion.py index c87c2cc..7c27b88 100644 --- a/mcp_bridge/openai_clients/streamChatCompletion.py +++ b/mcp_bridge/openai_clients/streamChatCompletion.py @@ -1,5 +1,8 @@ +import datetime import json +import os import time +import traceback from typing import Optional from secrets import token_hex from lmos_openai_types import ( @@ -8,13 +11,19 @@ CreateChatCompletionRequest, CreateChatCompletionStreamResponse, Function1, + FinishReason1, ) from mcp_bridge.inference_engine_mappers.chat.requester import chat_completion_requester from mcp_bridge.inference_engine_mappers.chat.stream_responder import ( chat_completion_stream_responder, ) -from .utils import call_tool, chat_completion_add_tools +from .utils import ( + call_tool, + chat_completion_add_tools, + validate_if_json_object_parsable, + salvage_parsable_json_object, +) from mcp_bridge.models import SSEData, upstream_error from mcp_bridge.http_clients import get_client from loguru import logger @@ -68,9 +77,9 @@ async def chat_completions(request: CreateChatCompletionRequest): # exclude_defaults=True, exclude_none=True, exclude_unset=True # ) - json_data = json.dumps(chat_completion_requester(request)) + json_data = json.dumps(chat_completion_requester(request), indent=4, ensure_ascii=False) - # logger.debug(json_data) + logger.debug("Request JSON:\n%s" % json_data) last: Optional[CreateChatCompletionStreamResponse] = None # last message @@ -211,6 +220,29 @@ async def chat_completions(request: CreateChatCompletionRequest): # save the last message last = parsed_data + # perform early stopping on parsable tool_call_json + if tool_call_json: + if tool_call_json.strip().startswith("{"): + if validate_if_json_object_parsable(tool_call_json): + logger.debug( + f"tool call json '{tool_call_json}' is parsable now." + ) + logger.debug("exiting message receive loop") + last.choices[0].finish_reason = FinishReason1.tool_calls + break + salvaged_json_object = salvage_parsable_json_object( + tool_call_json + ) + if salvaged_json_object: + tool_call_json = salvaged_json_object + logger.debug( + f"tool call json '{tool_call_json}' is salvagable now." + ) + logger.debug("salvaged json content:", tool_call_json) + logger.debug("exiting message receive loop") + last.choices[0].finish_reason = FinishReason1.tool_calls + break + # ideally we should check this properly assert last is not None @@ -229,6 +261,9 @@ async def chat_completions(request: CreateChatCompletionRequest): f"{tool_call_name=} {tool_call_json=}" ) # this should not be error but its easier to debug + logger.debug("clearing tool contexts to prevent tool call loops") + request.tools = None + # add received message to the history msg = ChatCompletionRequestMessage( role="assistant", From e4a83e08a357de0e613f48cfeba51894eb54c6e9 Mon Sep 17 00:00:00 2001 From: James Brown Date: Thu, 30 Jan 2025 21:37:42 +0800 Subject: [PATCH 6/8] Update chatCompletion.py --- mcp_bridge/openai_clients/chatCompletion.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/mcp_bridge/openai_clients/chatCompletion.py b/mcp_bridge/openai_clients/chatCompletion.py index 8bf21a7..3241c91 100644 --- a/mcp_bridge/openai_clients/chatCompletion.py +++ b/mcp_bridge/openai_clients/chatCompletion.py @@ -10,7 +10,7 @@ FinishReason1, ) -from .utils import call_tool, chat_completion_add_tools +from .utils import call_tool, chat_completion_add_tools, validate_if_json_object_parsable, json_pretty_print from mcp_bridge.http_clients import get_client from mcp_bridge.inference_engine_mappers.chat.requester import chat_completion_requester from mcp_bridge.inference_engine_mappers.chat.responder import chat_completion_responder @@ -86,11 +86,22 @@ async def chat_completions( return response logger.debug("tool calls found") + + logger.debug("clearing tool contexts to prevent tool call loops") + request.tools = None + for tool_call in response.choices[0].message.tool_calls.root: logger.debug( - f"tool call: {tool_call.function.name} arguments: {json.loads(tool_call.function.arguments)}" + f"tool call: {tool_call.function.name}" ) + if validate_if_json_object_parsable(tool): + logger.debug(f"arguments:\n{json_pretty_print(tool_call.function.arguments)}") + else: + logger.debug("non-json arguments given: %s" % tool_call.function.arguments) + logger.debug("unable to parse tool call argument as json. skipping...") + continue + # FIXME: this can probably be done in parallel using asyncio gather tool_call_result = await call_tool( tool_call.function.name, tool_call.function.arguments From e21cff532f176a15744b1bfd17a531d0bbc83508 Mon Sep 17 00:00:00 2001 From: James Brown Date: Thu, 30 Jan 2025 21:40:29 +0800 Subject: [PATCH 7/8] Update utils.py --- mcp_bridge/openai_clients/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mcp_bridge/openai_clients/utils.py b/mcp_bridge/openai_clients/utils.py index 73e318d..010974c 100644 --- a/mcp_bridge/openai_clients/utils.py +++ b/mcp_bridge/openai_clients/utils.py @@ -9,6 +9,14 @@ from mcp_bridge.tool_mappers import mcp2openai +def json_pretty_print(obj) -> str: + if type(obj) == bytes: + obj = obj.decode() + if type(obj) == str: + obj = json.loads(obj) + ret = json.dumps(obj, indent=4, ensure_ascii=False) + return ret + def validate_if_json_object_parsable(content: str): try: json.loads(content) From 1a322e5144389f1304af58a635080cbf6e986464 Mon Sep 17 00:00:00 2001 From: James Brown Date: Thu, 30 Jan 2025 21:41:41 +0800 Subject: [PATCH 8/8] Update streamChatCompletion.py --- mcp_bridge/openai_clients/streamChatCompletion.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mcp_bridge/openai_clients/streamChatCompletion.py b/mcp_bridge/openai_clients/streamChatCompletion.py index 7c27b88..494e1d2 100644 --- a/mcp_bridge/openai_clients/streamChatCompletion.py +++ b/mcp_bridge/openai_clients/streamChatCompletion.py @@ -21,8 +21,9 @@ from .utils import ( call_tool, chat_completion_add_tools, - validate_if_json_object_parsable, + json_pretty_print, salvage_parsable_json_object, + validate_if_json_object_parsable, ) from mcp_bridge.models import SSEData, upstream_error from mcp_bridge.http_clients import get_client @@ -77,7 +78,7 @@ async def chat_completions(request: CreateChatCompletionRequest): # exclude_defaults=True, exclude_none=True, exclude_unset=True # ) - json_data = json.dumps(chat_completion_requester(request), indent=4, ensure_ascii=False) + json_data = json_pretty_print(chat_completion_requester(request)) logger.debug("Request JSON:\n%s" % json_data)