From 54e60ce2c085dc4b10622d9cd3e71c1d46a614dc Mon Sep 17 00:00:00 2001 From: Olivier Lacroix Date: Sat, 4 Apr 2026 15:34:47 +1100 Subject: [PATCH 01/10] feat(google): implement native grounding support and update documentation This change enables native Google Search and URL Context grounding when function calling is active, eliminating the need for the external google_search_tool filter. Documentation has been updated to reflect this new mechanism. --- .github/copilot-instructions.md | 2 +- README.md | 2 +- docs/google-gemini-integration.md | 16 +++++++----- filters/google_search_tool.py | 33 ------------------------- pipelines/google/google_gemini.py | 41 +++++++++++++------------------ 5 files changed, 29 insertions(+), 65 deletions(-) delete mode 100644 filters/google_search_tool.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index e15084f..1930999 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -19,7 +19,7 @@ This repo extends Open WebUI with pluggable Python pipelines and filters. If you - Azure and N8N stream with SSE (`StreamingResponse`); always close `aiohttp` `ClientSession`/response in `finally` (see `cleanup_response`). - Gemini disables streaming for image-generation models; thinking is wrapped in `
` and emitted incrementally. - Cross-component integration: - - `filters/google_search_tool.py` converts `features.web_search` → `metadata.features.google_search_tool`; Gemini reads this to enable grounding tools. + - Gemini reads `search_web` and `fetch_url` tools from `__metadata__.get("tools", {})` to enable grounding tools natively when function calling is enabled. - N8N non-streaming responses can append a tool-calls section built by `_format_tool_calls_section` with verbosity and truncation valves. ## Dev workflow (local) diff --git a/README.md b/README.md index c01fd65..d16336f 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ The functions include a built-in encryption mechanism for sensitive information: - **Video generation (Veo)**: Generate videos with Google Veo models (3.1, 3, 2). Configurable aspect ratio, resolution, duration, negative prompt, and person generation controls. Supports text-to-video and image-to-video for all supported Veo models. Videos are automatically uploaded and embedded with playback controls. - **Token usage tracking**: Returns prompt, completion, and total token counts to Open WebUI for automatic persistence in the database. - **Model whitelist & additional models**: Restrict the visible model list via `GOOGLE_MODEL_WHITELIST` and add SDK-unsupported models via `GOOGLE_MODEL_ADDITIONAL`. -- Grounding with Google Search via the [google_search_tool.py filter](./filters/google_search_tool.py) +- Grounding with Google Search and URL Context (natively supported via native tool calling) - Grounding with Vertex AI Search via the [vertex_ai_search_tool.py filter](./filters/vertex_ai_search_tool.py) - Native tool calling support - Configurable API version support diff --git a/docs/google-gemini-integration.md b/docs/google-gemini-integration.md index c378b50..4a1a38f 100644 --- a/docs/google-gemini-integration.md +++ b/docs/google-gemini-integration.md @@ -55,8 +55,8 @@ This integration enables **Open WebUI** to interact with **Google Gemini** model - **Customizable Generation Settings** Use environment variables to configure token limits, temperature, etc. -- **Grounding with Google search** - Improve the accuracy and recency of Gemini responses with Google search grounding. +- **Grounding with Google search and URL Context** + Improve the accuracy and recency of Gemini responses with Google search grounding and the URL Context tool, automatically enabled when native tool calling is active and appropriate tools (`search_web`, `fetch_url`) are provided. - **Ability to forward User Headers and change gemini base url** Forward user information headers (like Name, Id, Email and Role) to Google API or LiteLLM for better context and analytics. Also, change the base URL for the Google Generative AI API if needed. @@ -558,11 +558,15 @@ GOOGLE_MODEL_WHITELIST="gemini-exp-1206,gemini-2.0-flash-exp,gemini-1.5-pro" ## Web search and access -[Grounding with Google search](https://ai.google.dev/gemini-api/docs/google-search) together with the [URL context tool](https://ai.google.dev/gemini-api/docs/url-context) are enabled/disabled together via the `google_search_tool` feature, which can be switched on/off in a Filter. +[Grounding with Google search](https://ai.google.dev/gemini-api/docs/google-search) and the [URL context tool](https://ai.google.dev/gemini-api/docs/url-context) are natively integrated into the Gemini pipeline. They are automatically enabled when **Native tool calling** is toggled on in Open WebUI, and the corresponding tools are available to the pipeline: -For instance, the following [Filter (google_search_tool.py)](../filters/google_search_tool.py) will replace Open Web UI default web search function with Google search grounding + the URL context tool. +- **`search_web`**: Maps to Google Search grounding (or Enterprise Search if configured). +- **`fetch_url`**: Maps to the URL Context tool for accessing webpage content. -When enabled, sources and google queries from the search used by Gemini will be displayed with the response. +When these tools are used, sources and Google search queries used by Gemini will be displayed with the response. + +> [!NOTE] +> The separate `google_search_tool` filter is no longer required for Gemini grounding, as the pipeline now handles these tools natively when function calling is enabled. ### Enterprise Search @@ -573,7 +577,7 @@ To enable Enterprise Search: 1. Set `GOOGLE_USE_ENTERPRISE_SEARCH=true` (or toggle the Valve in the UI). 2. Ensure `GOOGLE_GENAI_USE_VERTEXAI=true` (Enterprise Search is a Vertex AI feature). -When enabled, the pipeline will use the `enterprise_web_search` tool instead of the standard `google_search` tool whenever grounding is requested. +When enabled, the pipeline will use the `enterprise_web_search` tool instead of the standard `google_search` tool whenever the `search_web` tool is called. ## Grounding with Vertex AI Search diff --git a/filters/google_search_tool.py b/filters/google_search_tool.py deleted file mode 100644 index 9d15d05..0000000 --- a/filters/google_search_tool.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -title: Google Search Tool Filter for https://github.com/owndev/Open-WebUI-Functions/blob/main/pipelines/google/google_gemini.py -author: owndev, olivier-lacroix -author_url: https://github.com/owndev/ -project_url: https://github.com/owndev/Open-WebUI-Functions -funding_url: https://github.com/sponsors/owndev -version: 1.0.0 -license: Apache License 2.0 -requirements: - - https://github.com/owndev/Open-WebUI-Functions/blob/main/pipelines/google/google_gemini.py -description: Replacing web_search tool with google search grounding -""" - -import logging -from open_webui.env import SRC_LOG_LEVELS - - -class Filter: - def __init__(self): - self.log = logging.getLogger("google_ai.pipe") - self.log.setLevel(SRC_LOG_LEVELS.get("OPENAI", logging.INFO)) - - def inlet(self, body: dict) -> dict: - features = body.get("features", {}) - - # Ensure metadata structure exists and add new feature - metadata = body.setdefault("metadata", {}) - metadata_features = metadata.setdefault("features", {}) - - if features.pop("web_search"): - self.log.debug("Replacing web_search tool with google search grounding") - metadata_features["google_search_tool"] = True - return body diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 41e63a0..128c7a6 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -2187,7 +2187,6 @@ def _configure_generation( body: Dict[str, Any], system_instruction: Optional[str], __metadata__: Dict[str, Any], - __tools__: dict[str, Any] | None = None, __user__: Optional[dict] = None, enable_image_generation: bool = False, model_id: str = "", @@ -2402,18 +2401,6 @@ def _configure_generation( params = __metadata__.get("params", {}) tools = [] - if features.get("google_search_tool", False): - if self.valves.USE_ENTERPRISE_WEB_SEARCH: - self.log.debug("Enabling Enterprise Web Search grounding") - tools.append( - types.Tool(enterprise_web_search=types.EnterpriseWebSearch()) - ) - else: - self.log.debug("Enabling Google search grounding") - tools.append(types.Tool(google_search=types.GoogleSearch())) - self.log.debug("Enabling URL context grounding") - tools.append(types.Tool(url_context=types.UrlContext())) - if features.get("vertex_ai_search", False) or ( self.valves.USE_VERTEX_AI and (self.valves.VERTEX_AI_RAG_STORE or os.getenv("VERTEX_AI_RAG_STORE")) @@ -2441,14 +2428,23 @@ def _configure_generation( "Vertex AI Search requested but vertex_rag_store not provided in params, valves, or env" ) - if __tools__ is not None and params.get("function_calling") == "native": - for name, tool_def in __tools__.items(): - if not name.startswith("_"): - tool = tool_def["callable"] - self.log.debug( - f"Adding tool '{name}' with signature {tool.__signature__}" - ) - tools.append(tool) + # metadata['tools'] is populated only in native tool calling mode, + # and contains all tools, not only user-defined tools, contrarily to __tools__ + for name, tool_def in __metadata__.get("tools", {}).items(): + if name == "search_web": + if self.valves.USE_ENTERPRISE_WEB_SEARCH: + self.log.debug("Enabling Enterprise Web Search grounding") + tool = types.Tool(enterprise_web_search=types.EnterpriseWebSearch()) + else: + self.log.debug("Enabling Google search grounding") + tool = types.Tool(google_search=types.GoogleSearch()) + elif name == "fetch_url": + self.log.debug("Enabling URL context grounding") + tool = types.Tool(url_context=types.UrlContext()) + else: + tool = tool_def["callable"] + self.log.debug(f"Adding tool '{name}' with signature {tool.__signature__}") + tools.append(tool) if tools: gen_config_params["tools"] = tools @@ -3081,7 +3077,6 @@ async def pipe( body: Dict[str, Any], __metadata__: dict[str, Any], __event_emitter__: Callable, - __tools__: dict[str, Any] | None, __request__: Optional[Request] = None, __user__: Optional[dict] = None, ) -> Union[str, Dict[str, Any], AsyncIterator[Union[str, Dict[str, Any]]]]: @@ -3092,7 +3087,6 @@ async def pipe( body: The request body containing messages and other parameters. __metadata__: Request metadata __event_emitter__: Event emitter for status updates - __tools__: Available tools __request__: FastAPI request object (for image upload) __user__: User information (for image upload) @@ -3167,7 +3161,6 @@ async def pipe( body, system_instruction, __metadata__, - __tools__, __user__, supports_image_generation, model_id, From 3177002b2ee811cc945c9777891eca0799c543e1 Mon Sep 17 00:00:00 2001 From: Olivier Lacroix Date: Sat, 4 Apr 2026 16:04:54 +1100 Subject: [PATCH 02/10] don't access __signature__ --- pipelines/google/google_gemini.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 128c7a6..83da564 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -2433,17 +2433,21 @@ def _configure_generation( for name, tool_def in __metadata__.get("tools", {}).items(): if name == "search_web": if self.valves.USE_ENTERPRISE_WEB_SEARCH: - self.log.debug("Enabling Enterprise Web Search grounding") + self.log.debug( + "Replacing 'search_web' with Enterprise Web Search grounding" + ) tool = types.Tool(enterprise_web_search=types.EnterpriseWebSearch()) else: - self.log.debug("Enabling Google search grounding") + self.log.debug( + "Replacing 'search_web' with Google search grounding" + ) tool = types.Tool(google_search=types.GoogleSearch()) elif name == "fetch_url": - self.log.debug("Enabling URL context grounding") + self.log.debug("Replacing 'fetch_url' with URL context grounding") tool = types.Tool(url_context=types.UrlContext()) else: tool = tool_def["callable"] - self.log.debug(f"Adding tool '{name}' with signature {tool.__signature__}") + self.log.debug(f"Adding tool '{name}'") tools.append(tool) if tools: From d30b6899ec60b51f2f7fa916c511be1fe31d9652 Mon Sep 17 00:00:00 2001 From: Olivier Lacroix Date: Sun, 5 Apr 2026 12:33:07 +1000 Subject: [PATCH 03/10] Handle MCP tools --- docs/google-gemini-integration.md | 14 +++++++++++++- pipelines/google/google_gemini.py | 8 ++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/google-gemini-integration.md b/docs/google-gemini-integration.md index 4a1a38f..ab4d430 100644 --- a/docs/google-gemini-integration.md +++ b/docs/google-gemini-integration.md @@ -62,7 +62,11 @@ This integration enables **Open WebUI** to interact with **Google Gemini** model Forward user information headers (like Name, Id, Email and Role) to Google API or LiteLLM for better context and analytics. Also, change the base URL for the Google Generative AI API if needed. - **Native tool calling support** - Leverage Google genai native function calling to orchestrate the use of tools + Leverage Google genai native function calling to orchestrate the use of tools. + +- **Native MCP Tool Support** + Directly integrate Model Context Protocol (MCP) tool sessions with Gemini's native tool calling capabilities. + ## Environment Variables @@ -623,6 +627,14 @@ To use this filter, ensure it's enabled in your Open WebUI configuration. Then, Native tool calling is enabled/disabled via the standard 'Function calling' Open Web UI toggle. +## Native MCP Tool Support + +The Google Gemini pipeline supports **Native MCP Tool Support**, allowing Gemini models to directly use tools from any connected MCP (Model Context Protocol) servers when **Native tool calling** is enabled. + +When using this feature: +- The pipeline automatically detects connected MCP clients and includes their entire sessions in the Gemini tool list. +- **Important**: The standard Open Web UI **MCP function whitelist does not apply** when using native tool calling with Gemini. All tools provided by the connected MCP servers will be available to the model. + ## Default System Prompt The Google Gemini pipeline supports a configurable default system prompt that is applied to all chats. This is useful when you want to consistently apply certain behaviors or instructions to all Gemini models without having to configure each model individually. diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 83da564..b330d1c 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -2445,11 +2445,19 @@ def _configure_generation( elif name == "fetch_url": self.log.debug("Replacing 'fetch_url' with URL context grounding") tool = types.Tool(url_context=types.UrlContext()) + elif tool_def.get("type") == "mcp": + # Don't add mcp tools one by one, add the mcp session directly + pass else: tool = tool_def["callable"] self.log.debug(f"Adding tool '{name}'") tools.append(tool) + # Add MCP server sessions + for name, mcp_client in __metadata__.get("mcp_clients", {}).items(): + self.log.debug(f"Adding MCP server '{name}'") + tools.append(mcp_client.session) + if tools: gen_config_params["tools"] = tools From 340b5459f0e134b6280cb2cf8074d4f65422a725 Mon Sep 17 00:00:00 2001 From: Olivier Lacroix Date: Sun, 5 Apr 2026 13:13:24 +1000 Subject: [PATCH 04/10] Refactor RAG store as native tool --- docs/google-gemini-integration.md | 45 ++++++------------ filters/vertex_ai_search_tool.py | 43 ----------------- pipelines/google/google_gemini.py | 76 ++++++++++++------------------- tools/vertex_ai_search.py | 28 ++++++++++++ 4 files changed, 70 insertions(+), 122 deletions(-) delete mode 100644 filters/vertex_ai_search_tool.py create mode 100644 tools/vertex_ai_search.py diff --git a/docs/google-gemini-integration.md b/docs/google-gemini-integration.md index ab4d430..db841c7 100644 --- a/docs/google-gemini-integration.md +++ b/docs/google-gemini-integration.md @@ -58,15 +58,18 @@ This integration enables **Open WebUI** to interact with **Google Gemini** model - **Grounding with Google search and URL Context** Improve the accuracy and recency of Gemini responses with Google search grounding and the URL Context tool, automatically enabled when native tool calling is active and appropriate tools (`search_web`, `fetch_url`) are provided. -- **Ability to forward User Headers and change gemini base url** - Forward user information headers (like Name, Id, Email and Role) to Google API or LiteLLM for better context and analytics. Also, change the base URL for the Google Generative AI API if needed. - - **Native tool calling support** Leverage Google genai native function calling to orchestrate the use of tools. - **Native MCP Tool Support** Directly integrate Model Context Protocol (MCP) tool sessions with Gemini's native tool calling capabilities. +- **Extensible Native Gemini Tools** + The pipeline automatically detects and executes Open WebUI tools that return a `google.genai.types.Tool` object. This allows for seamless integration of Gemini-specific features like Vertex AI Search grounding as standalone tools. + +- **Ability to forward User Headers and change gemini base url** + Forward user information headers (like Name, Id, Email and Role) to Google API or LiteLLM for better context and analytics. Also, change the base URL for the Google Generative AI API if needed. + ## Environment Variables @@ -585,43 +588,23 @@ When enabled, the pipeline will use the `enterprise_web_search` tool instead of ## Grounding with Vertex AI Search -Improve the accuracy and recency of Gemini responses by grounding them with your own data in Vertex AI Search. +Improve the accuracy and recency of Gemini responses by grounding them with your own data in Vertex AI Search. This feature is implemented as a [native Gemini tool](../tools/vertex_ai_search.py). ### Configuration To enable Vertex AI Search grounding, you need to: 1. **Set up a Vertex AI Search Data Store**: Follow the [Google Cloud documentation](https://cloud.google.com/vertex-ai/docs/search/overview) to create a Data Store in Discovery Engine and ingest your documents. -2. **Provide the RAG Store Path**: The path should be in the format `projects/PROJECT/locations/LOCATION/ragCorpora/DATA_STORE_ID` or `projects/PROJECT/locations/global/collections/default_collection/dataStores/DATA_STORE_ID`. - - Set the `VERTEX_AI_RAG_STORE` environment variable, or - - Use the [Filter (vertex_ai_search_tool.py)](../filters/vertex_ai_search_tool.py) to enable the feature and optionally pass the store ID via chat metadata. -3. **Enable Vertex AI**: Set `GOOGLE_GENAI_USE_VERTEXAI=true` to use Vertex AI (required for Vertex AI Search grounding). +2. **Configure the Tool**: Enable the **Vertex AI Search** tool in Open WebUI. +3. **Set the RAG Store Path**: The path should be in the format `projects/PROJECT/locations/LOCATION/ragCorpora/DATA_STORE_ID` or `projects/PROJECT/locations/global/collections/default_collection/dataStores/DATA_STORE_ID`. + - This is configured via the `VERTEX_AI_RAG_STORE` valve on the tool itself. +4. **Enable Vertex AI**: Set `GOOGLE_GENAI_USE_VERTEXAI="true"` in the Gemini pipeline settings (required for Vertex AI Search grounding). -When `USE_VERTEX_AI` is `true` and `VERTEX_AI_RAG_STORE` is configured, Vertex AI Search grounding will be automatically enabled. You can also explicitly enable it via the `vertex_ai_search` feature flag. +### Usage -When enabled, Gemini will use the specified Vertex AI Search Data Store to retrieve relevant information and ground its responses, providing citations to the source documents. +Once the tool is configured, you can enable it for any chat by selecting the **Vertex AI Search** tool. -### Example Filter Usage - -The [vertex_ai_search_tool.py](../filters/vertex_ai_search_tool.py) filter enables Vertex AI Search grounding when the `vertex_ai_search` feature is requested: - -```python -# filters/vertex_ai_search_tool.py -# ... (filter code) ... -``` - -To use this filter, ensure it's enabled in your Open WebUI configuration. Then, in your chat settings or via metadata, you can enable the `vertex_ai_search` feature: - -```json -{ - "features": { - "vertex_ai_search": true - }, - "params": { - "vertex_rag_store": "projects/your-project/locations/global/collections/default_collection/dataStores/your-data-store-id" - } -} -``` +When enabled, Gemini will use the specified Vertex AI Search Data Store to retrieve relevant information and ground its responses, providing citations to the source documents. ## Native tool calling support diff --git a/filters/vertex_ai_search_tool.py b/filters/vertex_ai_search_tool.py deleted file mode 100644 index 8be1534..0000000 --- a/filters/vertex_ai_search_tool.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -title: Vertex AI Search Tool Filter for https://github.com/owndev/Open-WebUI-Functions/blob/main/pipelines/google/google_gemini.py -author: owndev, eun2ce -author_url: https://github.com/owndev/ -project_url: https://github.com/owndev/Open-WebUI-Functions -funding_url: https://github.com/sponsors/owndev -version: 1.0.0 -license: Apache License 2.0 -requirements: - - https://github.com/owndev/Open-WebUI-Functions/blob/main/pipelines/google/google_gemini.py -description: Enable Vertex AI Search grounding for RAG -""" - -import logging -import os -from open_webui.env import SRC_LOG_LEVELS - - -class Filter: - def __init__(self): - self.log = logging.getLogger("google_ai.pipe") - self.log.setLevel(SRC_LOG_LEVELS.get("OPENAI", logging.INFO)) - - def inlet(self, body: dict) -> dict: - features = body.get("features", {}) - - metadata = body.setdefault("metadata", {}) - metadata_features = metadata.setdefault("features", {}) - metadata_params = metadata.setdefault("params", {}) - - if features.pop("vertex_ai_search", False): - self.log.debug("Enabling Vertex AI Search grounding") - metadata_features["vertex_ai_search"] = True - - if "vertex_rag_store" not in metadata_params: - vertex_rag_store = os.getenv("VERTEX_AI_RAG_STORE") - if vertex_rag_store: - metadata_params["vertex_rag_store"] = vertex_rag_store - else: - self.log.warning( - "vertex_ai_search enabled but vertex_rag_store not provided in params or VERTEX_AI_RAG_STORE env var" - ) - return body diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index b330d1c..ce97014 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -53,6 +53,7 @@ import base64 import hashlib import logging +import inspect import io import uuid import aiofiles @@ -279,10 +280,6 @@ class Valves(BaseModel): default=os.getenv("GOOGLE_CLOUD_LOCATION", "global"), description="The Google Cloud region to use with Vertex AI.", ) - VERTEX_AI_RAG_STORE: str | None = Field( - default=os.getenv("GOOGLE_VERTEX_AI_RAG_STORE"), - description="Vertex AI RAG Store path for grounding (e.g., projects/PROJECT/locations/LOCATION/ragCorpora/DATA_STORE_ID). Only used when USE_VERTEX_AI is true.", - ) USE_PERMISSIVE_SAFETY: bool = Field( default=os.getenv("GOOGLE_USE_PERMISSIVE_SAFETY", "false").lower() == "true", @@ -2397,61 +2394,44 @@ def _configure_generation( gen_config_params |= {"safety_settings": safety_settings} # Add various tools to Gemini as required - features = __metadata__.get("features", {}) - params = __metadata__.get("params", {}) tools = [] - if features.get("vertex_ai_search", False) or ( - self.valves.USE_VERTEX_AI - and (self.valves.VERTEX_AI_RAG_STORE or os.getenv("VERTEX_AI_RAG_STORE")) - ): - vertex_rag_store = ( - params.get("vertex_rag_store") - or self.valves.VERTEX_AI_RAG_STORE - or os.getenv("VERTEX_AI_RAG_STORE") - ) - if vertex_rag_store: + # metadata['tools'] is populated only in native tool calling mode, + # and contains all tools, not only user-defined tools, contrarily to __tools__ + for name, tool_def in __metadata__.get("tools", {}).items(): + if name == "search_web" and self.valves.USE_ENTERPRISE_WEB_SEARCH: self.log.debug( - f"Enabling Vertex AI Search grounding: {vertex_rag_store}" + "Replacing 'search_web' with Enterprise Web Search grounding" ) tools.append( - types.Tool( - retrieval=types.Retrieval( - vertex_ai_search=types.VertexAISearch( - datastore=vertex_rag_store - ) - ) - ) + types.Tool(enterprise_web_search=types.EnterpriseWebSearch()) ) - else: - self.log.warning( - "Vertex AI Search requested but vertex_rag_store not provided in params, valves, or env" - ) - - # metadata['tools'] is populated only in native tool calling mode, - # and contains all tools, not only user-defined tools, contrarily to __tools__ - for name, tool_def in __metadata__.get("tools", {}).items(): - if name == "search_web": - if self.valves.USE_ENTERPRISE_WEB_SEARCH: - self.log.debug( - "Replacing 'search_web' with Enterprise Web Search grounding" - ) - tool = types.Tool(enterprise_web_search=types.EnterpriseWebSearch()) - else: - self.log.debug( - "Replacing 'search_web' with Google search grounding" - ) - tool = types.Tool(google_search=types.GoogleSearch()) + elif name == "search_web": + self.log.debug("Replacing 'search_web' with Google search grounding") + tools.append(types.Tool(google_search=types.GoogleSearch())) elif name == "fetch_url": self.log.debug("Replacing 'fetch_url' with URL context grounding") - tool = types.Tool(url_context=types.UrlContext()) + tools.append(types.Tool(url_context=types.UrlContext())) elif tool_def.get("type") == "mcp": - # Don't add mcp tools one by one, add the mcp session directly - pass + # Don't add mcp tools one by one, add the mcp session directly later + self.log.debug(f"Skipping MCP tool {name}") + continue + elif ( + inspect.signature(tool_def["callable"]).return_annotation is types.Tool + ): + try: + self.log.debug(f"Getting native Gemini tool: {name}") + native_tool = tool_def["callable"]() + if isinstance(native_tool, types.Tool): + tools.append(native_tool) + else: + self.log.warning(f"'{name}' is not a 'types.Tool'. Skipping.") + continue + except Exception as e: + self.log.warning(f"Failed to check/execute native tool {name}: {e}") else: - tool = tool_def["callable"] self.log.debug(f"Adding tool '{name}'") - tools.append(tool) + tools.append(tool_def["callable"]) # Add MCP server sessions for name, mcp_client in __metadata__.get("mcp_clients", {}).items(): diff --git a/tools/vertex_ai_search.py b/tools/vertex_ai_search.py new file mode 100644 index 0000000..1f11a57 --- /dev/null +++ b/tools/vertex_ai_search.py @@ -0,0 +1,28 @@ +import os +from google.genai import types +from pydantic import BaseModel, Field + +class Tools: + class Valves(BaseModel): + VERTEX_AI_RAG_STORE: str = Field( + default=os.getenv("GOOGLE_VERTEX_AI_RAG_STORE", ""), + description="Vertex AI RAG Store path for grounding (e.g., projects/PROJECT/locations/LOCATION/ragCorpora/DATA_STORE_ID).", + ) + + def __init__(self): + self.valves = self.Valves() + + def vertex_ai_search(self) -> types.Tool: + """ + Enable Vertex AI Search grounding for RAG. + """ + if not self.valves.VERTEX_AI_RAG_STORE: + raise ValueError("VERTEX_AI_RAG_STORE valve is not set.") + + return types.Tool( + retrieval=types.Retrieval( + vertex_ai_search=types.VertexAISearch( + datastore=self.valves.VERTEX_AI_RAG_STORE + ) + ) + ) From a022480417e178ff7eac1ab6fb5f01ea2e63cd7f Mon Sep 17 00:00:00 2001 From: Olivier Lacroix Date: Sun, 5 Apr 2026 13:22:48 +1000 Subject: [PATCH 05/10] await --- pipelines/google/google_gemini.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index ce97014..0d6bc3e 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -2179,7 +2179,7 @@ def _get_user_valve_value( return value return None - def _configure_generation( + async def _configure_generation( self, body: Dict[str, Any], system_instruction: Optional[str], @@ -2421,8 +2421,9 @@ def _configure_generation( ): try: self.log.debug(f"Getting native Gemini tool: {name}") - native_tool = tool_def["callable"]() + native_tool = await tool_def["callable"]() if isinstance(native_tool, types.Tool): + self.log.debug(f"Adding tool '{name}'") tools.append(native_tool) else: self.log.warning(f"'{name}' is not a 'types.Tool'. Skipping.") @@ -3149,7 +3150,7 @@ async def pipe( # Configure generation parameters and safety settings self.log.debug(f"Supports image generation: {supports_image_generation}") - generation_config = self._configure_generation( + generation_config = await self._configure_generation( body, system_instruction, __metadata__, From c0ef901c25ced9a0603f14998006078bdd24495d Mon Sep 17 00:00:00 2001 From: Olivier Lacroix Date: Sun, 5 Apr 2026 13:30:34 +1000 Subject: [PATCH 06/10] improve doc --- docs/google-gemini-integration.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/google-gemini-integration.md b/docs/google-gemini-integration.md index db841c7..632b8cd 100644 --- a/docs/google-gemini-integration.md +++ b/docs/google-gemini-integration.md @@ -573,7 +573,7 @@ GOOGLE_MODEL_WHITELIST="gemini-exp-1206,gemini-2.0-flash-exp,gemini-1.5-pro" When these tools are used, sources and Google search queries used by Gemini will be displayed with the response. > [!NOTE] -> The separate `google_search_tool` filter is no longer required for Gemini grounding, as the pipeline now handles these tools natively when function calling is enabled. +> The separate `google_search_tool` filter is no longer required for Gemini grounding, as the pipeline now handles these tools natively. ### Enterprise Search @@ -606,6 +606,9 @@ Once the tool is configured, you can enable it for any chat by selecting the **V When enabled, Gemini will use the specified Vertex AI Search Data Store to retrieve relevant information and ground its responses, providing citations to the source documents. +> [!NOTE] +> The separate `vertex_ai_search_tool` filter is no longer required for Vertex AI grounding, as the pipeline now handles these tools natively. + ## Native tool calling support Native tool calling is enabled/disabled via the standard 'Function calling' Open Web UI toggle. From 4fc7b7f2a3af5098dce4044bcf5835878a594fac Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Tue, 7 Apr 2026 08:27:02 +0200 Subject: [PATCH 07/10] Update version to 1.15.0 in google_gemini.py --- pipelines/google/google_gemini.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 0d6bc3e..5eb06f1 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -4,7 +4,7 @@ author_url: https://github.com/owndev/ project_url: https://github.com/owndev/Open-WebUI-Functions funding_url: https://github.com/sponsors/owndev -version: 1.14.1 +version: 1.15.0 required_open_webui_version: 0.8.0 license: Apache License 2.0 description: Highly optimized Google Gemini pipeline with advanced image and video generation capabilities, intelligent compression, and streamlined processing workflows. From cc24604d4027b7e191bb2d13c2c762a0ada973d1 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Tue, 7 Apr 2026 10:52:58 +0200 Subject: [PATCH 08/10] Update version to 2.0.0 in google_gemini.py --- pipelines/google/google_gemini.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 5eb06f1..295fd9f 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -4,7 +4,7 @@ author_url: https://github.com/owndev/ project_url: https://github.com/owndev/Open-WebUI-Functions funding_url: https://github.com/sponsors/owndev -version: 1.15.0 +version: 2.0.0 required_open_webui_version: 0.8.0 license: Apache License 2.0 description: Highly optimized Google Gemini pipeline with advanced image and video generation capabilities, intelligent compression, and streamlined processing workflows. From 5e9ffd9b7bc316cfc837df1cbc7cd80c07954532 Mon Sep 17 00:00:00 2001 From: Olivier Lacroix Date: Sat, 11 Apr 2026 14:36:30 +1000 Subject: [PATCH 09/10] Generate MCP tool signatures --- pipelines/google/google_gemini.py | 103 +++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 9 deletions(-) diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index 295fd9f..c231e97 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -61,7 +61,17 @@ from google import genai from google.genai import types from google.genai.errors import ClientError, ServerError, APIError -from typing import List, Union, Optional, Dict, Any, Tuple, AsyncIterator, Callable +from typing import ( + List, + Union, + Optional, + Dict, + Any, + Tuple, + AsyncIterator, + Callable, + Awaitable, +) from pydantic_core import core_schema from pydantic import BaseModel, Field, GetCoreSchemaHandler from cryptography.fernet import Fernet, InvalidToken @@ -2413,9 +2423,11 @@ async def _configure_generation( self.log.debug("Replacing 'fetch_url' with URL context grounding") tools.append(types.Tool(url_context=types.UrlContext())) elif tool_def.get("type") == "mcp": - # Don't add mcp tools one by one, add the mcp session directly later - self.log.debug(f"Skipping MCP tool {name}") - continue + self.log.debug(f"Adding MCP tool '{name}'") + mcp_tool = self._create_callable_from_spec( + name, tool_def["spec"], tool_def["callable"] + ) + tools.append(mcp_tool) elif ( inspect.signature(tool_def["callable"]).return_annotation is types.Tool ): @@ -2434,11 +2446,6 @@ async def _configure_generation( self.log.debug(f"Adding tool '{name}'") tools.append(tool_def["callable"]) - # Add MCP server sessions - for name, mcp_client in __metadata__.get("mcp_clients", {}).items(): - self.log.debug(f"Adding MCP server '{name}'") - tools.append(mcp_client.session) - if tools: gen_config_params["tools"] = tools @@ -2446,6 +2453,84 @@ async def _configure_generation( filtered_params = {k: v for k, v in gen_config_params.items() if v is not None} return types.GenerateContentConfig(**filtered_params) + @staticmethod + def _create_callable_from_spec( + name: str, spec: dict, callable_func: Callable[..., Awaitable[Any]] + ) -> Callable[..., Awaitable[Any]]: + """ + Dynamically creates a well-typed async function from an MCP-style tool specification. + This satisfies inspection-based SDKs (like Gemini) by providing proper + signatures, docstrings, and unique function names. + """ + import inspect + + description = spec.get("description", "") + parameters_spec = spec.get("parameters", spec.get("inputSchema", {})) + properties = parameters_spec.get("properties", {}) + required_params = parameters_spec.get("required", []) + + # Type mapping from JSON schema to Python + type_map = { + "string": str, + "number": float, + "integer": int, + "boolean": bool, + "object": dict, + "array": list, + } + + params = [] + doc_params = [] + + # Sort properties so required parameters come first to avoid "non-default argument follows default argument" + sorted_properties = sorted( + properties.items(), + key=lambda item: item[0] not in required_params, + ) + + for param_name, param_info in sorted_properties: + if param_name.startswith("__"): + continue + + param_type = type_map.get(param_info.get("type"), Any) + param_desc = param_info.get("description", "") + + default = inspect.Parameter.empty + if param_name not in required_params: + # If not required, default to None or a provided default + default = param_info.get("default", None) + + params.append( + inspect.Parameter( + name=param_name, + kind=inspect.Parameter.KEYWORD_ONLY, + default=default, + annotation=param_type, + ) + ) + + if param_desc: + doc_params.append(f":param {param_name}: {param_desc}") + + # Build the docstring + docstring = description + if doc_params: + docstring += "\n\n" + "\n".join(doc_params) + + # The actual wrapper function + async def wrapped_func(*args, **kwargs): + return await callable_func(*args, **kwargs) + + # Set metadata to satisfy SDK inspection + wrapped_func.__name__ = name + wrapped_func.__qualname__ = name + wrapped_func.__doc__ = docstring + wrapped_func.__signature__ = inspect.Signature( + parameters=params, return_annotation=Any + ) + + return wrapped_func + @staticmethod def _format_grounding_chunks_as_sources( grounding_chunks: list[types.GroundingChunk], From 611d08f269ad4d38f04de3fb30c224419fada6e9 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Thu, 30 Apr 2026 10:58:49 +0200 Subject: [PATCH 10/10] Update required Open WebUI version to 0.9.0 --- pipelines/google/google_gemini.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index fb91766..fdaedc9 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -5,7 +5,7 @@ project_url: https://github.com/owndev/Open-WebUI-Functions funding_url: https://github.com/sponsors/owndev version: 2.0.0 -required_open_webui_version: 0.8.0 +required_open_webui_version: 0.9.0 license: Apache License 2.0 description: Highly optimized Google Gemini pipeline with advanced image and video generation capabilities, intelligent compression, and streamlined processing workflows. features: