diff --git a/.dockerignore b/.dockerignore index 62316abc186..f8b9278001b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,29 @@ -.BC.* -tmp* -*~ -OLD* -*.pyc -.DS_Store -.env -.venv -.aider.* -venv/* -build +# Ignore everything +* + +# But descend into directories +!*/ + +# Recursively allow files under subtree +!/.github/** +!/aider/** +!/benchmark/** +!/docker/** +!/requirements/** +!/scripts/** +!/tests/** + +# Specific Files +!/.dockerignore +!/.flake8 +!/.gitignore +!/.pre-commit-config.yaml +!/CNAME +!/CONTRIBUTING.metadata +!/HISTORY.md +!/LICENSE.txt +!/MANIFEST.in +!/pyproject.toml +!/pytest.ini +!/README.md +!/requirements.txt \ No newline at end of file diff --git a/.gitignore b/.gitignore index a15c4e88d2c..bcc1ecf0595 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ !/requirements/** !/scripts/** !/tests/** + # Specific Files !/.dockerignore !/.flake8 @@ -28,24 +29,6 @@ !/requirements.txt # Ignore specific files -.DS_Store -.vscode/ -aider.code-workspace -*.pyc -.aider* -aider_chat.egg-info/ -build -dist/ -Gemfile.lock -_site -.jekyll-cache/ -.jekyll-metadata aider/__version__.py aider/_version.py -.venv/ -.#* -.gitattributes -tmp.benchmarks/ -.docker_bash_history -/venv -/venv/* +*.pyc \ No newline at end of file diff --git a/aider/__init__.py b/aider/__init__.py index dbbf6f23871..00b338056c3 100644 --- a/aider/__init__.py +++ b/aider/__init__.py @@ -1,6 +1,6 @@ from packaging import version -__version__ = "0.86.1.dev" +__version__ = "0.86.2.dev" safe_version = __version__ try: diff --git a/aider/coders/base_coder.py b/aider/coders/base_coder.py index c1a30834680..74c37915686 100755 --- a/aider/coders/base_coder.py +++ b/aider/coders/base_coder.py @@ -38,6 +38,7 @@ from aider.io import ConfirmGroup, InputOutput from aider.linter import Linter from aider.llm import litellm +from aider.mcp.server import LocalServer from aider.models import RETRY_TIMEOUT from aider.reasoning_tags import ( REASONING_TAG, @@ -1762,7 +1763,48 @@ def process_tool_calls(self, tool_call_response): if tool_call_response is None: return False - tool_calls = tool_call_response.choices[0].message.tool_calls + original_tool_calls = tool_call_response.choices[0].message.tool_calls + if not original_tool_calls: + return False + + # Expand any tool calls that have concatenated JSON in their arguments. + # This is necessary because some models (like Gemini) will serialize + # multiple tool calls in this way. + expanded_tool_calls = [] + for tool_call in original_tool_calls: + args_string = tool_call.function.arguments.strip() + + # If there are no arguments, or it's not a string that looks like it could + # be concatenated JSON, just add it and continue. + if not args_string or not ( + args_string.startswith("{") or args_string.startswith("[") + ): + expanded_tool_calls.append(tool_call) + continue + + json_chunks = utils.split_concatenated_json(args_string) + + # If it's just a single JSON object, there's nothing to expand. + if len(json_chunks) <= 1: + expanded_tool_calls.append(tool_call) + continue + + # We have concatenated JSON, so expand it into multiple tool calls. + for i, chunk in enumerate(json_chunks): + if not chunk.strip(): + continue + + # Create a new tool call for each JSON chunk, with a unique ID. + new_function = tool_call.function.model_copy(update={"arguments": chunk}) + new_tool_call = tool_call.model_copy( + update={"id": f"{tool_call.id}-{i}", "function": new_function} + ) + expanded_tool_calls.append(new_tool_call) + + # Replace the original tool_calls in the response object with the expanded list. + tool_call_response.choices[0].message.tool_calls = expanded_tool_calls + tool_calls = expanded_tool_calls + # Collect all tool calls grouped by server server_tool_calls = self._gather_server_tool_calls(tool_calls) @@ -1772,10 +1814,9 @@ def process_tool_calls(self, tool_call_response): if self.io.confirm_ask("Run tools?"): tool_responses = self._execute_tool_calls(server_tool_calls) - # Add the assistant message with tool calls - # Converting to a dict so it can be safely dumped to json - self.cur_messages.append( - tool_call_response.choices[0].message.to_dict()) + # Add the assistant message with the modified (expanded) tool calls. + # This ensures that what's stored in history is valid. + self.cur_messages.append(tool_call_response.choices[0].message.to_dict()) # Add all tool responses for tool_response in tool_responses: @@ -1784,8 +1825,10 @@ def process_tool_calls(self, tool_call_response): return True elif self.num_tool_calls >= self.max_tool_calls: self.io.tool_warning( - f"Only {self.max_tool_calls} tool calls allowed, stopping.") - return False + f"Only {self.max_tool_calls} tool calls allowed, stopping." + ) + + return False def _print_tool_call_info(self, server_tool_calls): """Print information about an MCP tool call.""" @@ -1838,6 +1881,22 @@ def _execute_tool_calls(self, tool_calls): # Define the coroutine to execute all tool calls for a single server async def _exec_server_tools(server, tool_calls_list): + if isinstance(server, LocalServer): + if hasattr(self, "_execute_local_tool_calls"): + return await self._execute_local_tool_calls(tool_calls_list) + else: + # This coder doesn't support local tools, return errors for all calls + error_responses = [] + for tool_call in tool_calls_list: + error_responses.append( + { + "role": "tool", + "tool_call_id": tool_call.id, + "content": f"Coder does not support local tool: {tool_call.function.name}", + } + ) + return error_responses + tool_responses = [] try: # Connect to the server once @@ -1847,24 +1906,19 @@ async def _exec_server_tools(server, tool_calls_list): try: # Arguments can be a stream of JSON objects. # We need to parse them and run a tool call for each. - decoder = json.JSONDecoder() args_string = tool_call.function.arguments.strip() - pos = 0 parsed_args_list = [] if args_string: - while pos < len(args_string): + json_chunks = utils.split_concatenated_json(args_string) + for chunk in json_chunks: try: - obj, end_pos = decoder.raw_decode(args_string, pos) - parsed_args_list.append(obj) - pos = end_pos - # skip whitespace - while pos < len(args_string) and args_string[pos].isspace(): - pos += 1 + parsed_args_list.append(json.loads(chunk)) except json.JSONDecodeError: - # If we fail to decode, just break and process what we have. - # If we haven't parsed anything, it will be an error below - # if the arg string was not empty. - break + self.io.tool_warning( + "Could not parse JSON chunk for tool" + f" {tool_call.function.name}: {chunk}" + ) + continue if not parsed_args_list and not args_string: parsed_args_list.append({}) # For tool calls with no arguments diff --git a/aider/coders/navigator_coder.py b/aider/coders/navigator_coder.py index 8fbcb72845e..e566777cadc 100644 --- a/aider/coders/navigator_coder.py +++ b/aider/coders/navigator_coder.py @@ -1,4 +1,5 @@ import ast +import json import re import fnmatch import os @@ -21,6 +22,7 @@ from .navigator_legacy_prompts import NavigatorLegacyPrompts from aider.repo import ANY_GIT_ERROR from aider import urls +from aider import utils # Import run_cmd for potentially interactive execution and run_cmd_subprocess for guaranteed non-interactive from aider.run_cmd import run_cmd, run_cmd_subprocess # Import the change tracker @@ -49,6 +51,8 @@ from aider.tools.list_changes import _execute_list_changes from aider.tools.extract_lines import _execute_extract_lines from aider.tools.show_numbered_context import execute_show_numbered_context +from aider.tools.grep import _execute_grep +from aider.mcp.server import LocalServer class NavigatorCoder(Coder): @@ -100,6 +104,540 @@ def __init__(self, *args, **kwargs): self.tokens_calculated = False super().__init__(*args, **kwargs) + self.initialize_local_tools() + + def initialize_local_tools(self): + if not self.use_granular_editing: + return + + local_tools = self.get_local_tool_schemas() + if not local_tools: + return + + local_server_config = {"name": "local_tools"} + local_server = LocalServer(local_server_config) + + if not self.mcp_servers: + self.mcp_servers = [] + if not any(isinstance(s, LocalServer) for s in self.mcp_servers): + self.mcp_servers.append(local_server) + + if not self.mcp_tools: + self.mcp_tools = [] + + if "local_tools" not in [name for name, _ in self.mcp_tools]: + self.mcp_tools.append((local_server.name, local_tools)) + self.functions = self.get_tool_list() + + def get_local_tool_schemas(self): + """Returns the JSON schemas for all local tools.""" + return [ + { + "type": "function", + "function": { + "name": "ViewFilesAtGlob", + "description": "View files matching a glob pattern.", + "parameters": { + "type": "object", + "properties": { + "pattern": {"type": "string", "description": "The glob pattern to match files."}, + }, + "required": ["pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "ViewFilesMatching", + "description": "View files containing a specific pattern.", + "parameters": { + "type": "object", + "properties": { + "pattern": {"type": "string", "description": "The pattern to search for in file contents."}, + "file_pattern": {"type": "string", "description": "An optional glob pattern to filter which files are searched."}, + "regex": {"type": "boolean", "description": "Whether the pattern is a regular expression. Defaults to False."}, + }, + "required": ["pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "Ls", + "description": "List files in a directory.", + "parameters": { + "type": "object", + "properties": { + "directory": {"type": "string", "description": "The directory to list."}, + }, + "required": ["directory"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "View", + "description": "View a specific file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string", "description": "The path to the file to view."}, + }, + "required": ["file_path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "Remove", + "description": "Remove a file from the chat context.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string", "description": "The path to the file to remove."}, + }, + "required": ["file_path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "MakeEditable", + "description": "Make a read-only file editable.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string", "description": "The path to the file to make editable."}, + }, + "required": ["file_path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "MakeReadonly", + "description": "Make an editable file read-only.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string", "description": "The path to the file to make read-only."}, + }, + "required": ["file_path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "ViewFilesWithSymbol", + "description": "View files that contain a specific symbol (e.g., class, function).", + "parameters": { + "type": "object", + "properties": { + "symbol": {"type": "string", "description": "The symbol to search for."}, + }, + "required": ["symbol"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "Command", + "description": "Execute a shell command.", + "parameters": { + "type": "object", + "properties": { + "command_string": {"type": "string", "description": "The shell command to execute."}, + }, + "required": ["command_string"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "CommandInteractive", + "description": "Execute a shell command interactively.", + "parameters": { + "type": "object", + "properties": { + "command_string": {"type": "string", "description": "The interactive shell command to execute."}, + }, + "required": ["command_string"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "Grep", + "description": "Search for a pattern in files.", + "parameters": { + "type": "object", + "properties": { + "pattern": {"type": "string", "description": "The pattern to search for."}, + "file_pattern": {"type": "string", "description": "Glob pattern for files to search. Defaults to '*'."}, + "directory": {"type": "string", "description": "Directory to search in. Defaults to '.'."}, + "use_regex": {"type": "boolean", "description": "Whether to use regex. Defaults to False."}, + "case_insensitive": {"type": "boolean", "description": "Whether to perform a case-insensitive search. Defaults to False."}, + "context_before": {"type": "integer", "description": "Number of lines to show before a match. Defaults to 5."}, + "context_after": {"type": "integer", "description": "Number of lines to show after a match. Defaults to 5."}, + }, + "required": ["pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "ReplaceText", + "description": "Replace text in a file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "find_text": {"type": "string"}, + "replace_text": {"type": "string"}, + "near_context": {"type": "string"}, + "occurrence": {"type": "integer", "default": 1}, + "change_id": {"type": "string"}, + "dry_run": {"type": "boolean", "default": False}, + }, + "required": ["file_path", "find_text", "replace_text"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "ReplaceAll", + "description": "Replace all occurrences of text in a file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "find_text": {"type": "string"}, + "replace_text": {"type": "string"}, + "change_id": {"type": "string"}, + "dry_run": {"type": "boolean", "default": False}, + }, + "required": ["file_path", "find_text", "replace_text"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "InsertBlock", + "description": "Insert a block of content into a file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "content": {"type": "string"}, + "after_pattern": {"type": "string"}, + "before_pattern": {"type": "string"}, + "occurrence": {"type": "integer", "default": 1}, + "change_id": {"type": "string"}, + "dry_run": {"type": "boolean", "default": False}, + "position": {"type": "string", "enum": ["top", "bottom"]}, + "auto_indent": {"type": "boolean", "default": True}, + "use_regex": {"type": "boolean", "default": False}, + }, + "required": ["file_path", "content"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "DeleteBlock", + "description": "Delete a block of lines from a file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "start_pattern": {"type": "string"}, + "end_pattern": {"type": "string"}, + "line_count": {"type": "integer"}, + "near_context": {"type": "string"}, + "occurrence": {"type": "integer", "default": 1}, + "change_id": {"type": "string"}, + "dry_run": {"type": "boolean", "default": False}, + }, + "required": ["file_path", "start_pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "ReplaceLine", + "description": "Replace a single line in a file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "line_number": {"type": "integer"}, + "new_content": {"type": "string"}, + "change_id": {"type": "string"}, + "dry_run": {"type": "boolean", "default": False}, + }, + "required": ["file_path", "line_number", "new_content"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "ReplaceLines", + "description": "Replace a range of lines in a file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "start_line": {"type": "integer"}, + "end_line": {"type": "integer"}, + "new_content": {"type": "string"}, + "change_id": {"type": "string"}, + "dry_run": {"type": "boolean", "default": False}, + }, + "required": ["file_path", "start_line", "end_line", "new_content"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "IndentLines", + "description": "Indent a block of lines in a file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "start_pattern": {"type": "string"}, + "end_pattern": {"type": "string"}, + "line_count": {"type": "integer"}, + "indent_levels": {"type": "integer", "default": 1}, + "near_context": {"type": "string"}, + "occurrence": {"type": "integer", "default": 1}, + "change_id": {"type": "string"}, + "dry_run": {"type": "boolean", "default": False}, + }, + "required": ["file_path", "start_pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "DeleteLine", + "description": "Delete a single line from a file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "line_number": {"type": "integer"}, + "change_id": {"type": "string"}, + "dry_run": {"type": "boolean", "default": False}, + }, + "required": ["file_path", "line_number"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "DeleteLines", + "description": "Delete a range of lines from a file.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "start_line": {"type": "integer"}, + "end_line": {"type": "integer"}, + "change_id": {"type": "string"}, + "dry_run": {"type": "boolean", "default": False}, + }, + "required": ["file_path", "start_line", "end_line"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "UndoChange", + "description": "Undo a previously applied change.", + "parameters": { + "type": "object", + "properties": { + "change_id": {"type": "string"}, + "file_path": {"type": "string"}, + }, + }, + }, + }, + { + "type": "function", + "function": { + "name": "ListChanges", + "description": "List recent changes made.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "limit": {"type": "integer", "default": 10}, + }, + }, + }, + }, + { + "type": "function", + "function": { + "name": "ExtractLines", + "description": "Extract lines from a source file and append them to a target file.", + "parameters": { + "type": "object", + "properties": { + "source_file_path": {"type": "string"}, + "target_file_path": {"type": "string"}, + "start_pattern": {"type": "string"}, + "end_pattern": {"type": "string"}, + "line_count": {"type": "integer"}, + "near_context": {"type": "string"}, + "occurrence": {"type": "integer", "default": 1}, + "dry_run": {"type": "boolean", "default": False}, + }, + "required": ["source_file_path", "target_file_path", "start_pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "ShowNumberedContext", + "description": "Show numbered lines of context around a pattern or line number.", + "parameters": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "pattern": {"type": "string"}, + "line_number": {"type": "integer"}, + "context_lines": {"type": "integer", "default": 3}, + }, + "required": ["file_path"], + }, + }, + }, + ] + + async def _execute_local_tool_calls(self, tool_calls_list): + tool_responses = [] + for tool_call in tool_calls_list: + tool_name = tool_call.function.name + result_message = "" + try: + # Arguments can be a stream of JSON objects. + # We need to parse them and run a tool call for each. + args_string = tool_call.function.arguments.strip() + parsed_args_list = [] + if args_string: + json_chunks = utils.split_concatenated_json(args_string) + for chunk in json_chunks: + try: + parsed_args_list.append(json.loads(chunk)) + except json.JSONDecodeError: + self.io.tool_warning( + f"Could not parse JSON chunk for tool {tool_name}: {chunk}" + ) + continue + + if not parsed_args_list and not args_string: + parsed_args_list.append({}) # For tool calls with no arguments + + all_results_content = [] + norm_tool_name = tool_name.lower() + + for params in parsed_args_list: + single_result = "" + # Dispatch to the correct tool execution function + if norm_tool_name == "viewfilesatglob": + single_result = execute_view_files_at_glob(self, **params) + elif norm_tool_name == "viewfilesmatching": + single_result = execute_view_files_matching(self, **params) + elif norm_tool_name == "ls": + single_result = execute_ls(self, **params) + elif norm_tool_name == "view": + single_result = execute_view(self, **params) + elif norm_tool_name == "remove": + single_result = _execute_remove(self, **params) + elif norm_tool_name == "makeeditable": + single_result = _execute_make_editable(self, **params) + elif norm_tool_name == "makereadonly": + single_result = _execute_make_readonly(self, **params) + elif norm_tool_name == "viewfileswithsymbol": + single_result = _execute_view_files_with_symbol(self, **params) + elif norm_tool_name == "command": + single_result = _execute_command(self, **params) + elif norm_tool_name == "commandinteractive": + single_result = _execute_command_interactive(self, **params) + elif norm_tool_name == "grep": + single_result = _execute_grep(self, **params) + elif norm_tool_name == "replacetext": + single_result = _execute_replace_text(self, **params) + elif norm_tool_name == "replaceall": + single_result = _execute_replace_all(self, **params) + elif norm_tool_name == "insertblock": + single_result = _execute_insert_block(self, **params) + elif norm_tool_name == "deleteblock": + single_result = _execute_delete_block(self, **params) + elif norm_tool_name == "replaceline": + single_result = _execute_replace_line(self, **params) + elif norm_tool_name == "replacelines": + single_result = _execute_replace_lines(self, **params) + elif norm_tool_name == "indentlines": + single_result = _execute_indent_lines(self, **params) + elif norm_tool_name == "deleteline": + single_result = _execute_delete_line(self, **params) + elif norm_tool_name == "deletelines": + single_result = _execute_delete_lines(self, **params) + elif norm_tool_name == "undochange": + single_result = _execute_undo_change(self, **params) + elif norm_tool_name == "listchanges": + single_result = _execute_list_changes(self, **params) + elif norm_tool_name == "extractlines": + single_result = _execute_extract_lines(self, **params) + elif norm_tool_name == "shownumberedcontext": + single_result = execute_show_numbered_context(self, **params) + else: + single_result = f"Error: Unknown local tool name '{tool_name}'" + + all_results_content.append(str(single_result)) + + result_message = "\n\n".join(all_results_content) + + except Exception as e: + result_message = f"Error executing {tool_name}: {e}" + self.io.tool_error( + f"Error during {tool_name} execution: {e}\n{traceback.format_exc()}" + ) + + tool_responses.append( + { + "role": "tool", + "tool_call_id": tool_call.id, + "name": tool_name, + "content": result_message, + } + ) + return tool_responses def _calculate_context_block_tokens(self, force=False): """ @@ -604,6 +1142,34 @@ def reply_completed(self): iteratively discover and analyze relevant files before providing a final answer to the user's question. """ + # In granular editing mode, tool calls are handled by BaseCoder's process_tool_calls. + # This method is now only for legacy tool call format and search/replace blocks. + if self.use_granular_editing: + # Handle SEARCH/REPLACE blocks + content = self.partial_response_content + if not content or not content.strip(): + return True + + # Check for search/replace blocks + has_search = "<<<<<<< SEARCH" in content + has_divider = "=======" in content + has_replace = ">>>>>>> REPLACE" in content + if has_search and has_divider and has_replace: + self.io.tool_output("Detected edit blocks, applying changes...") + edited_files = self._apply_edits_from_response() + if self.reflected_message: + return False # Trigger reflection if edits failed + + # If edits were successful, we might want to reflect. + # For now, let's consider the turn complete. + + # Since tool calls are handled earlier, we finalize the turn. + self.tool_call_count = 0 + self.files_added_in_exploration = set() + self.move_back_cur_messages(None) + return True + + # Legacy tool call processing for use_granular_editing=False content = self.partial_response_content if not content or not content.strip(): return True @@ -744,7 +1310,7 @@ def _process_tool_commands(self, content): max_calls = self.max_tool_calls # Check if there's a '---' separator and only process tool calls after the LAST one - separator_marker = "\n---\n" + separator_marker = "---" content_parts = content.split(separator_marker) # If there's no separator, treat the entire content as before the separator @@ -760,23 +1326,34 @@ def _process_tool_commands(self, content): # Find tool calls using a more robust method, but only in the content after separator processed_content = content_before_separator + separator_marker last_index = 0 - start_marker = "[tool_call(" + + # Support any [tool_...(...)] format + tool_call_pattern = re.compile(r"\[tool_.*?\(", re.DOTALL) end_marker = "]" # The parenthesis balancing finds the ')', we just need the final ']' while True: - start_pos = content_after_separator.find(start_marker, last_index) - if start_pos == -1: + match = tool_call_pattern.search(content_after_separator, last_index) + if not match: processed_content += content_after_separator[last_index:] break - # Check for escaped tool call: \[tool_call( - if start_pos > 0 and content_after_separator[start_pos - 1] == '\\': - # Append the content including the escaped marker - # We append up to start_pos + len(start_marker) to include the marker itself. + start_pos = match.start() + start_marker = match.group(0) + + # Check for escaped tool call: \[tool_... + # Count preceding backslashes to handle \\ + backslashes = 0 + p = start_pos - 1 + while p >= 0 and content_after_separator[p] == '\\': + backslashes += 1 + p -= 1 + + if backslashes % 2 == 1: + # Odd number of backslashes means it's escaped. Treat as text. + # We append up to the end of the marker and continue searching. processed_content += content_after_separator[last_index : start_pos + len(start_marker)] - # Update last_index to search after this escaped marker last_index = start_pos + len(start_marker) - continue # Continue searching for the next potential marker + continue # Append content before the (non-escaped) tool call processed_content += content_after_separator[last_index:start_pos] @@ -870,6 +1447,25 @@ def _process_tool_commands(self, content): tool_calls_found = True try: + # Pre-process inner_content to handle non-identifier tool names by quoting them. + # This allows ast.parse to succeed on names like 'resolve-library-id'. + if inner_content: + parts = inner_content.split(",", 1) + potential_tool_name = parts[0].strip() + + is_string = (potential_tool_name.startswith("'") and potential_tool_name.endswith( + "'" + )) or (potential_tool_name.startswith('"') and potential_tool_name.endswith('"')) + + if not potential_tool_name.isidentifier() and not is_string: + # It's not a valid identifier and not a string, so quote it. + # Use json.dumps to handle escaping correctly. + quoted_tool_name = json.dumps(potential_tool_name) + if len(parts) > 1: + inner_content = quoted_tool_name + ", " + parts[1] + else: + inner_content = quoted_tool_name + # Wrap the inner content to make it parseable as a function call # Example: ToolName, key="value" becomes f(ToolName, key="value") parse_str = f"f({inner_content})" @@ -883,9 +1479,16 @@ def _process_tool_commands(self, content): raise ValueError("Expected a Call node") # Extract tool name (should be the first positional argument) - if not call_node.args or not isinstance(call_node.args[0], ast.Name): + if not call_node.args: raise ValueError("Tool name not found or invalid") - tool_name = call_node.args[0].id + + tool_name_node = call_node.args[0] + if isinstance(tool_name_node, ast.Name): + tool_name = tool_name_node.id + elif isinstance(tool_name_node, ast.Constant) and isinstance(tool_name_node.value, str): + tool_name = tool_name_node.value + else: + raise ValueError("Tool name must be an identifier or a string literal") # Extract keyword arguments for keyword in call_node.keywords: @@ -905,8 +1508,16 @@ def _process_tool_commands(self, content): value = value[1:] if value.endswith('\n'): value = value[:-1] - elif isinstance(value_node, ast.Name): # Handle unquoted values like True/False/None or variables (though variables are unlikely here) - value = value_node.id + elif isinstance(value_node, ast.Name): # Handle unquoted values like True/False/None or variables + id_val = value_node.id.lower() + if id_val == 'true': + value = True + elif id_val == 'false': + value = False + elif id_val == 'none': + value = None + else: + value = value_node.id # Keep as string if it's something else # Add more types if needed (e.g., ast.List, ast.Dict) else: # Attempt to reconstruct the source for complex types, or raise error diff --git a/aider/commands.py b/aider/commands.py index 1169f7588d4..49f67358800 100644 --- a/aider/commands.py +++ b/aider/commands.py @@ -452,6 +452,11 @@ def cmd_tokens(self, args): self.coder.choose_fence() + # Show progress indicator + total_files = len(self.coder.abs_fnames) + len(self.coder.abs_read_only_fnames) + if total_files > 20: + self.io.tool_output(f"Calculating tokens for {total_files} files...") + # system messages main_sys = self.coder.fmt_system_prompt(self.coder.gpt_prompts.main_system) main_sys += "\n" + self.coder.fmt_system_prompt(self.coder.gpt_prompts.system_reminder) @@ -497,27 +502,50 @@ def cmd_tokens(self, args): fence = "`" * 3 file_res = [] - # files - for fname in self.coder.abs_fnames: - relative_fname = self.coder.get_rel_fname(fname) - content = self.io.read_text(fname) - if is_image_file(relative_fname): - tokens = self.coder.main_model.token_count_for_image(fname) - else: - # approximate - content = f"{relative_fname}\n{fence}\n" + content + "{fence}\n" - tokens = self.coder.main_model.token_count(content) - file_res.append((tokens, f"{relative_fname}", "/drop to remove")) - - # read-only files - for fname in self.coder.abs_read_only_fnames: - relative_fname = self.coder.get_rel_fname(fname) - content = self.io.read_text(fname) - if content is not None and not is_image_file(relative_fname): - # approximate - content = f"{relative_fname}\n{fence}\n" + content + "{fence}\n" - tokens = self.coder.main_model.token_count(content) - file_res.append((tokens, f"{relative_fname} (read-only)", "/drop to remove")) + # Process files with progress indication + total_editable_files = len(self.coder.abs_fnames) + total_readonly_files = len(self.coder.abs_read_only_fnames) + + # Display progress for editable files + if total_editable_files > 0: + if total_editable_files > 20: + self.io.tool_output(f"Calculating tokens for {total_editable_files} editable files...") + + # Calculate tokens for editable files + for i, fname in enumerate(self.coder.abs_fnames): + if i > 0 and i % 20 == 0 and total_editable_files > 20: + self.io.tool_output(f"Processed {i}/{total_editable_files} editable files...") + + relative_fname = self.coder.get_rel_fname(fname) + content = self.io.read_text(fname) + if is_image_file(relative_fname): + tokens = self.coder.main_model.token_count_for_image(fname) + else: + # approximate + content = f"{relative_fname}\n{fence}\n" + content + "{fence}\n" + tokens = self.coder.main_model.token_count(content) + file_res.append((tokens, f"{relative_fname}", "/drop to remove")) + + # Display progress for read-only files + if total_readonly_files > 0: + if total_readonly_files > 20: + self.io.tool_output(f"Calculating tokens for {total_readonly_files} read-only files...") + + # Calculate tokens for read-only files + for i, fname in enumerate(self.coder.abs_read_only_fnames): + if i > 0 and i % 20 == 0 and total_readonly_files > 20: + self.io.tool_output(f"Processed {i}/{total_readonly_files} read-only files...") + + relative_fname = self.coder.get_rel_fname(fname) + content = self.io.read_text(fname) + if content is not None and not is_image_file(relative_fname): + # approximate + content = f"{relative_fname}\n{fence}\n" + content + "{fence}\n" + tokens = self.coder.main_model.token_count(content) + file_res.append((tokens, f"{relative_fname} (read-only)", "/drop to remove")) + + if total_files > 20: + self.io.tool_output("Token calculation complete. Generating report...") file_res.sort() res.extend(file_res) @@ -533,7 +561,7 @@ def cmd_tokens(self, args): def fmt(v): return format(int(v), ",").rjust(width) - col_width = max(len(row[1]) for row in res) + col_width = max(len(row[1]) for row in res) if res else 0 cost_pad = " " * cost_width total = 0 diff --git a/aider/io.py b/aider/io.py index 80489919fc3..e087123a243 100644 --- a/aider/io.py +++ b/aider/io.py @@ -128,8 +128,20 @@ def tokenize(self): if self.tokenized: return self.tokenized = True - - for fname in self.all_fnames: + + # Performance optimization for large file sets + if len(self.all_fnames) > 100: + # Skip tokenization for very large numbers of files to avoid input lag + self.tokenized = True + return + + # Limit number of files to process to avoid excessive tokenization time + process_fnames = self.all_fnames + if len(process_fnames) > 50: + # Only process a subset of files to maintain responsiveness + process_fnames = process_fnames[:50] + + for fname in process_fnames: try: with open(fname, "r", encoding=self.encoding) as f: content = f.read() @@ -1173,6 +1185,21 @@ def append_chat_history(self, text, linebreak=False, blockquote=False, strip=Tru self.chat_history_file = None # Disable further attempts to write def format_files_for_input(self, rel_fnames, rel_read_only_fnames): + # Optimization for large number of files + total_files = len(rel_fnames) + len(rel_read_only_fnames or []) + + # For very large numbers of files, use a summary display + if total_files > 50: + read_only_count = len(rel_read_only_fnames or []) + editable_count = len([f for f in rel_fnames if f not in (rel_read_only_fnames or [])]) + + summary = f"{editable_count} editable file(s)" + if read_only_count > 0: + summary += f", {read_only_count} read-only file(s)" + summary += " (use /ls to list all files)\n" + return summary + + # Original implementation for reasonable number of files if not self.pretty: read_only_files = [] for full_path in sorted(rel_read_only_fnames or []): diff --git a/aider/mcp/server.py b/aider/mcp/server.py index be9afe85ac7..c21a709950f 100644 --- a/aider/mcp/server.py +++ b/aider/mcp/server.py @@ -97,3 +97,22 @@ async def connect(self): await self.disconnect() raise +class LocalServer(McpServer): + """ + A dummy McpServer for executing local, in-process tools + that are not provided by an external MCP server. + """ + + async def connect(self): + """Local tools don't need a connection.""" + if self.session is not None: + logging.info(f"Using existing session for local tools: {self.name}") + return self.session + + self.session = object() # Dummy session object + return self.session + + async def disconnect(self): + """Disconnect from the MCP server and clean up resources.""" + self.session = None + diff --git a/aider/models.py b/aider/models.py index 947a1bff390..9f78fa3e9da 100644 --- a/aider/models.py +++ b/aider/models.py @@ -950,7 +950,7 @@ def send_completion(self, messages, functions, stream, temperature=None, tools=N if self.is_deepseek_r1(): messages = ensure_alternating_roles(messages) - kwargs = dict(model=self.name, stream=stream, tools=[]) + kwargs = dict(model=self.name, stream=stream) if self.use_temperature is not False: if temperature is None: @@ -961,19 +961,37 @@ def send_completion(self, messages, functions, stream, temperature=None, tools=N kwargs["temperature"] = temperature - if functions is not None: + # `tools` is for modern tool usage. `functions` is for legacy/forced calls. + # If `tools` is provided, it's the canonical list. If not, use `functions`. + # This handles `base_coder` sending both with same content for `navigator_coder`. + effective_tools = tools if tools is not None else functions + + if effective_tools: + # Check if we have legacy format functions (which lack a 'type' key) and convert them. + # This is a simplifying assumption that works for aider's use cases. + is_legacy = any("type" not in tool for tool in effective_tools) + if is_legacy: + kwargs["tools"] = [dict(type="function", function=tool) for tool in effective_tools] + else: + kwargs["tools"] = effective_tools + + # Forcing a function call is for legacy style `functions` with a single function. + # This is used by ArchitectCoder and not intended for NavigatorCoder's tools. + if functions and len(functions) == 1: function = functions[0] - kwargs["tools"] = [dict(type="function", function=function)] - kwargs["tool_choice"] = {"type": "function", "function": {"name": function["name"]}} + is_legacy = "type" not in function + + if is_legacy and "name" in function: + tool_name = function.get("name") + if tool_name: + kwargs["tool_choice"] = {"type": "function", "function": {"name": tool_name}} + if self.extra_params: kwargs.update(self.extra_params) if self.is_ollama() and "num_ctx" not in kwargs: num_ctx = int(self.token_count(messages) * 1.25) + 8192 kwargs["num_ctx"] = num_ctx - if tools: - kwargs["tools"] = kwargs["tools"] + tools - key = json.dumps(kwargs, sort_keys=True).encode() # dump(kwargs) diff --git a/aider/utils.py b/aider/utils.py index d6c279bff99..938030c58a5 100644 --- a/aider/utils.py +++ b/aider/utils.py @@ -159,7 +159,18 @@ def format_messages(messages, title=None): else: output.append(f"{role} {item}") elif isinstance(content, str): # Handle string content - output.append(format_content(role, content)) + # For large content, especially with many files, use a truncated display approach + if len(content) > 5000: + # Count the number of code blocks (approximation) + fence_count = content.count("```") // 2 + if fence_count > 5: + # Show truncated content with file count for large files to improve performance + first_line = content.split("\n", 1)[0] + output.append(f"{role} {first_line} [content with ~{fence_count} files truncated]") + else: + output.append(format_content(role, content)) + else: + output.append(format_content(role, content)) function_call = msg.get("function_call") if function_call: output.append(f"{role} Function Call: {function_call}") @@ -376,3 +387,66 @@ def printable_shell_command(cmd_list): str: Shell-escaped command string. """ return oslex.join(cmd_list) + + +def split_concatenated_json(s: str) -> list[str]: + """ + Splits a string containing one or more concatenated JSON objects. + """ + res = [] + i = 0 + s_len = len(s) + while i < s_len: + # skip leading whitespace + while i < s_len and s[i].isspace(): + i += 1 + if i >= s_len: + break + + start_char = s[i] + if start_char == "{": + end_char = "}" + elif start_char == "[": + end_char = "]" + else: + # Doesn't start with a JSON object/array, so we can't parse it as a stream. + # Return the rest of the string as a single chunk. + res.append(s[i:]) + break + + start_index = i + stack_depth = 0 + in_string = False + escape = False + + for j in range(start_index, s_len): + char = s[j] + + if escape: + escape = False + continue + + if char == "\\": + escape = True + continue + + if char == '"': + in_string = not in_string + + if in_string: + continue + + if char == start_char: + stack_depth += 1 + elif char == end_char: + stack_depth -= 1 + if stack_depth == 0: + res.append(s[start_index : j + 1]) + i = j + 1 + break + else: + # Unclosed object, add the remainder as the last chunk + res.append(s[start_index:]) + break + + return res