From 346e2059c968e13dab421a895847a3e31c5d778e Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Mon, 17 Nov 2025 11:51:48 -0500 Subject: [PATCH 01/14] feat: Add custom exceptions --- nemoguardrails/actions/llm/utils.py | 82 ++++++++++++- nemoguardrails/exceptions.py | 170 +++++++++++++++++++++++++++ nemoguardrails/rails/llm/config.py | 105 ++++++++++++++--- nemoguardrails/rails/llm/llmrails.py | 50 +++++--- 4 files changed, 368 insertions(+), 39 deletions(-) create mode 100644 nemoguardrails/exceptions.py diff --git a/nemoguardrails/actions/llm/utils.py b/nemoguardrails/actions/llm/utils.py index 12f0c0c64..3c2ba13da 100644 --- a/nemoguardrails/actions/llm/utils.py +++ b/nemoguardrails/actions/llm/utils.py @@ -36,6 +36,18 @@ logger = logging.getLogger(__name__) +# Since different providers have different attributes for the base URL we'll use this list as a best-effort way +# to extract the base URL from a `BaseLanguageModel` instance. +BASE_URL_ATTRIBUTES = [ + "api_base", + "api_host", + "azure_endpoint", + "base_url", + "endpoint", + "endpoint_url", + "openai_api_base", +] + class LLMCallException(Exception): """A wrapper around the LLM call invocation exception. @@ -44,9 +56,18 @@ class LLMCallException(Exception): catch it and return an "Internal server error." message. """ - def __init__(self, inner_exception: Any): - super().__init__(f"LLM Call Exception: {str(inner_exception)}") + def __init__(self, inner_exception: Any, context_message: Optional[str] = None): + """Initialize LLMCallException. + + Args: + inner_exception: The original exception that occurred + context_message: Optional context to prepend (for example, the model name or endpoint) + """ + message = f"{context_message or 'LLM Call Exception'}: {str(inner_exception)}" + super().__init__(message) + self.inner_exception = inner_exception + self.context_message = context_message def _infer_provider_from_module(llm: BaseLanguageModel) -> Optional[str]: @@ -202,6 +223,59 @@ def _prepare_callbacks( return logging_callbacks +def _raise_llm_call_exception( + exception: Exception, + llm: Union[BaseLanguageModel, Runnable], +) -> None: + """Raise an LLMCallException with enriched context about the failed invocation. + + Args: + exception: The original exception that occurred + llm: The LLM instance that was being invoked + + Raises: + LLMCallException with context message including model name and endpoint + """ + # Extract model info from context (set by _setup_llm_call_info) + llm_call_info = llm_call_info_var.get() + logger.info(llm_call_info) + model_name = ( + llm_call_info.llm_model_name + if llm_call_info + else _infer_model_name(llm) + if isinstance(llm, BaseLanguageModel) + else "" + ) + + # Extract endpoint URL from the LLM instance + endpoint_url = None + for attr in BASE_URL_ATTRIBUTES: + if hasattr(llm, attr): + value = getattr(llm, attr, None) + if value: + endpoint_url = str(value) + break + + # If we didn't find endpoint URL, check the nested client object + if not endpoint_url and hasattr(llm, "client"): + client = getattr(llm, "client", None) + if client and hasattr(client, "base_url"): + endpoint_url = str(client.base_url) + + # Build context message with model and endpoint info + context_parts = [] + if model_name: + context_parts.append(f"model={model_name}") + if endpoint_url: + context_parts.append(f"endpoint={endpoint_url}") + + if context_parts: + context_message = f"Error invoking LLM ({', '.join(context_parts)})" + raise LLMCallException(exception, context_message=context_message) + else: + raise LLMCallException(exception) + + async def _invoke_with_string_prompt( llm: Union[BaseLanguageModel, Runnable], prompt: str, @@ -211,7 +285,7 @@ async def _invoke_with_string_prompt( try: return await llm.ainvoke(prompt, config=RunnableConfig(callbacks=callbacks)) except Exception as e: - raise LLMCallException(e) + _raise_llm_call_exception(e, llm) async def _invoke_with_message_list( @@ -225,7 +299,7 @@ async def _invoke_with_message_list( try: return await llm.ainvoke(messages, config=RunnableConfig(callbacks=callbacks)) except Exception as e: - raise LLMCallException(e) + _raise_llm_call_exception(e, llm) def _convert_messages_to_langchain_format(prompt: List[dict]) -> List: diff --git a/nemoguardrails/exceptions.py b/nemoguardrails/exceptions.py new file mode 100644 index 000000000..857906bea --- /dev/null +++ b/nemoguardrails/exceptions.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Custom exceptions for NeMo Guardrails. + +These exceptions represent errors that the SDK detects and raises. +Provider errors (authentication, network, rate limits, etc.) are NOT wrapped +and should bubble up unchanged to the consuming application. +""" + +from typing import Any, Dict, Optional + + +class NemoGuardrailsError(Exception): + """Base exception for all NeMo Guardrails SDK errors. + + Attributes: + message: Human-readable error message + context: Additional context about the error (model names, config details, etc.) + """ + + def __init__( + self, + message: str, + context: Optional[Dict[str, Any]] = None, + ): + """Initialize the exception. + + Args: + message: Error message describing what went wrong + context: Additional context (e.g., model name, provider, file path, etc.) + """ + super().__init__(message) + self.context = context or {} + + def __str__(self) -> str: + """Return a formatted error message with context.""" + parts = [super().__str__()] + + if self.context: + context_str = ", ".join(f"{k}={v!r}" for k, v in self.context.items()) + parts.append(f"Context: {context_str}") + + return "\n".join(parts) + + +# ============================================================================ +# Configuration Errors (Creation-Time) +# ============================================================================ + + +class ConfigurationError(NemoGuardrailsError): + """Base class for configuration-related errors. + + These errors occur when loading or validating guardrail configurations, + typically during RailsConfig loading or LLMRails initialization. + """ + + pass + + +class InvalidModelConfigurationError(ConfigurationError): + """Raised when model configuration is invalid. + + Examples: + - model.model is empty or missing + - model.engine is invalid + - Invalid model parameters + - Model type not recognized + - Base URL is malformed + """ + + pass + + +class InvalidConfigurationFileError(ConfigurationError): + """Raised when configuration files cannot be successfullyparsed. + + Examples: + - Invalid YAML syntax in config.yml + - Invalid Colang syntax in .co files + - Circular import dependencies + - File not found + """ + + pass + + +class InvalidRailsConfigurationError(ConfigurationError): + """Raised when rails configuration is invalid. + + Examples: + - Input/output rail references a model that doesn't exist in config + - Rail references a flow that doesn't exist + - Missing required prompt template + - Invalid rail parameters + """ + + pass + + +# ============================================================================ +# Runtime Errors (Inference-Time) +# ============================================================================ + + +class ModelAuthenticationError(NemoGuardrailsError): + """Raised at inference-time when an authentication error occurs when calling the model.""" + + pass + + +class LLMInvocationError(NemoGuardrailsError): + """Base class for runtime errors that occur during inference. + + These errors occur after configuration has been loaded and validated, + during the actual execution of guardrails. + """ + + pass + + +class MalformedLLMResponseError(LLMInvocationError): + """Raised when the SDK cannot parse an LLM response. + + Examples: + - NemoGuard JSON parsing failures + - Structured output doesn't match expected format + - LLM returns invalid JSON when JSON is required + - Response missing required fields + """ + + pass + + +class InvalidConfigurationError(LLMInvocationError): + """Raised when an invalid configuration is detected at runtime. + + Examples: + - Dynamic configuration issues discovered during inference + - Rail references a flow or action that doesn't exist (only discovered at runtime) + - Configuration state becomes invalid during execution + """ + + pass + + +class ActionExecutionError(LLMInvocationError): + """Raised when a custom action fails during execution. + + Examples: + - User-defined custom action raised an exception + - Action returned invalid format + - Action parameter validation failed at runtime + - Action depends on external resource that's unavailable + """ + + pass diff --git a/nemoguardrails/rails/llm/config.py b/nemoguardrails/rails/llm/config.py index 6e463f963..2394b9c79 100644 --- a/nemoguardrails/rails/llm/config.py +++ b/nemoguardrails/rails/llm/config.py @@ -37,6 +37,11 @@ from nemoguardrails.colang.v1_0.runtime.flows import _normalize_flow_id from nemoguardrails.colang.v2_x.lang.utils import format_colang_parsing_error_message from nemoguardrails.colang.v2_x.runtime.errors import ColangParsingError +from nemoguardrails.exceptions import ( + InvalidModelConfigurationError, + InvalidRailsConfigurationError, +) +from nemoguardrails.llm.types import Task log = logging.getLogger(__name__) @@ -136,8 +141,12 @@ def set_and_validate_model(cls, data: Any) -> Any: model_from_params = parameters.get("model_name") or parameters.get("model") if model_field and model_from_params: - raise ValueError( - "Model name must be specified in exactly one place: either in the 'model' field or in parameters, not both." + raise InvalidModelConfigurationError( + message=f"Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`).", + context={ + "model": model_field, + "parameters": parameters, + }, ) if not model_field and model_from_params: data["model"] = model_from_params @@ -151,8 +160,8 @@ def set_and_validate_model(cls, data: Any) -> Any: def model_must_be_none_empty(self) -> "Model": """Validate that a model name is present either directly or in parameters.""" if not self.model or not self.model.strip(): - raise ValueError( - "Model name must be specified either directly in the 'model' field or through 'model_name'/'model' in parameters" + raise InvalidModelConfigurationError( + message=f"Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`).", ) return self @@ -334,10 +343,15 @@ class TaskPrompt(BaseModel): @root_validator(pre=True, allow_reuse=True) def check_fields(cls, values): if not values.get("content") and not values.get("messages"): - raise ValueError("One of `content` or `messages` must be provided.") + raise InvalidRailsConfigurationError( + message="One of `content` or `messages` must be provided." + ) + # raise ValueError("One of `content` or `messages` must be provided.") if values.get("content") and values.get("messages"): - raise ValueError("Only one of `content` or `messages` must be provided.") + raise InvalidRailsConfigurationError( + message="Only one of `content` or `messages` must be provided." + ) return values @@ -1414,7 +1428,15 @@ def check_model_exists_for_input_rails(cls, values): if not flow_model: continue if flow_model not in model_types: - raise ValueError(f"No `{flow_model}` model provided for input flow `{_normalize_flow_id(flow)}`") + flow_id = _normalize_flow_id(flow) + available_types = ( + ", ".join(f"'{str(t)}'" for t in sorted(model_types)) + if model_types + else "none" + ) + raise InvalidRailsConfigurationError( + message=f"Input flow '{flow_id}' references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}.", + ) return values @root_validator(pre=True) @@ -1436,7 +1458,15 @@ def check_model_exists_for_output_rails(cls, values): if not flow_model: continue if flow_model not in model_types: - raise ValueError(f"No `{flow_model}` model provided for output flow `{_normalize_flow_id(flow)}`") + flow_id = _normalize_flow_id(flow) + available_types = ( + ", ".join(f"'{str(t)}'" for t in sorted(model_types)) + if model_types + else "none" + ) + raise InvalidRailsConfigurationError( + message=f"Output flow {flow_id} references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}.", + ) return values @root_validator(pre=True) @@ -1449,10 +1479,23 @@ def check_prompt_exist_for_self_check_rails(cls, values): provided_task_prompts = [prompt.task if hasattr(prompt, "task") else prompt.get("task") for prompt in prompts] # Input moderation prompt verification - if "self check input" in enabled_input_rails and "self_check_input" not in provided_task_prompts: - raise ValueError("You must provide a `self_check_input` prompt template.") - if "llama guard check input" in enabled_input_rails and "llama_guard_check_input" not in provided_task_prompts: - raise ValueError("You must provide a `llama_guard_check_input` prompt template.") + if ( + "self check input" in enabled_input_rails + and "self_check_input" not in provided_task_prompts + ): + raise InvalidRailsConfigurationError( + f"Missing a `self_check_input` prompt, which is required for the `self check input` rail." + ) + if ( + "llama guard check input" in enabled_input_rails + and "llama_guard_check_input" not in provided_task_prompts + ): + raise InvalidRailsConfigurationError( + f"Missing a `llama_guard_check_input` prompt, which is required for the `llama guard check input` rail." + ) + # raise ValueError( + # "You must provide a `llama_guard_check_input` prompt template." + # ) # Only content-safety and topic-safety include a $model reference in the rail flow text # Need to match rails with flow_id (excluding $model reference) and match prompts @@ -1461,21 +1504,43 @@ def check_prompt_exist_for_self_check_rails(cls, values): _validate_rail_prompts(enabled_input_rails, provided_task_prompts, "topic safety check input") # Output moderation prompt verification - if "self check output" in enabled_output_rails and "self_check_output" not in provided_task_prompts: - raise ValueError("You must provide a `self_check_output` prompt template.") + if ( + "self check output" in enabled_output_rails + and "self_check_output" not in provided_task_prompts + ): + raise InvalidRailsConfigurationError( + f"Missing a `self_check_output` prompt, which is required for the `self check output` rail." + ) + # raise ValueError("You must provide a `self_check_output` prompt template.") if ( "llama guard check output" in enabled_output_rails and "llama_guard_check_output" not in provided_task_prompts ): - raise ValueError("You must provide a `llama_guard_check_output` prompt template.") + raise InvalidRailsConfigurationError( + f"Missing a `llama_guard_check_output` prompt, which is required for the `llama guard check output` rail." + ) + # raise ValueError( + # "You must provide a `llama_guard_check_output` prompt template." + # ) if ( "patronus lynx check output hallucination" in enabled_output_rails and "patronus_lynx_check_output_hallucination" not in provided_task_prompts ): - raise ValueError("You must provide a `patronus_lynx_check_output_hallucination` prompt template.") + raise InvalidRailsConfigurationError( + f"Missing a `patronus_lynx_check_output_hallucination` prompt, which is required for the `patronus lynx check output hallucination` rail." + ) + # raise ValueError( + # "You must provide a `patronus_lynx_check_output_hallucination` prompt template." + # ) - if "self check facts" in enabled_output_rails and "self_check_facts" not in provided_task_prompts: - raise ValueError("You must provide a `self_check_facts` prompt template.") + if ( + "self check facts" in enabled_output_rails + and "self_check_facts" not in provided_task_prompts + ): + raise InvalidRailsConfigurationError( + f"Missing a `self_check_facts` prompt, which is required for the `self check facts` rail." + ) + # raise ValueError("You must provide a `self_check_facts` prompt template.") # Only content-safety and topic-safety include a $model reference in the rail flow text # Need to match rails with flow_id (excluding $model reference) and match prompts @@ -1801,4 +1866,6 @@ def _validate_rail_prompts(rails: list[str], prompts: list[Any], validation_rail prompt_flow_id = flow_id.replace(" ", "_") expected_prompt = f"{prompt_flow_id} $model={flow_model}" if expected_prompt not in prompts: - raise ValueError(f"You must provide a `{expected_prompt}` prompt template.") + raise InvalidRailsConfigurationError( + f"Missing a `{expected_prompt}` prompt, which is required for the `{validation_rail}` rail." + ) diff --git a/nemoguardrails/rails/llm/llmrails.py b/nemoguardrails/rails/llm/llmrails.py index c4d33f83d..e8eed2da0 100644 --- a/nemoguardrails/rails/llm/llmrails.py +++ b/nemoguardrails/rails/llm/llmrails.py @@ -70,6 +70,10 @@ from nemoguardrails.embeddings.index import EmbeddingsIndex from nemoguardrails.embeddings.providers import register_embedding_provider from nemoguardrails.embeddings.providers.base import EmbeddingModel +from nemoguardrails.exceptions import ( + InvalidModelConfigurationError, + InvalidRailsConfigurationError, +) from nemoguardrails.kb.kb import KnowledgeBase from nemoguardrails.llm.cache import CacheInterface, LFUCache from nemoguardrails.llm.models.initializer import ( @@ -225,13 +229,19 @@ def __init__( spec.loader.exec_module(config_module) config_modules.append(config_module) + colang_version_to_runtime: Dict[str, Type[Runtime]] = { + "1.0": RuntimeV1_0, + "2.x": RuntimeV2_x, + } + if config.colang_version not in colang_version_to_runtime: + raise InvalidRailsConfigurationError( + f"Unsupported colang version: {config.colang_version}. Supported versions: {list(colang_version_to_runtime.keys())}" + ) + # First, we initialize the runtime. - if config.colang_version == "1.0": - self.runtime = RuntimeV1_0(config=config, verbose=verbose) - elif config.colang_version == "2.x": - self.runtime = RuntimeV2_x(config=config, verbose=verbose) - else: - raise ValueError(f"Unsupported colang version: {config.colang_version}.") + self.runtime = colang_version_to_runtime[config.colang_version]( + config=config, verbose=verbose + ) # If we have a config_modules with an `init` function, we call it. # We need to call this here because the `init` might register additional @@ -317,20 +327,26 @@ def _validate_config(self): # content safety check input/output flows are special as they have parameters flow_name = _normalize_flow_id(flow_name) if flow_name not in existing_flows_names: - raise ValueError(f"The provided input rail flow `{flow_name}` does not exist") + raise InvalidRailsConfigurationError( + f"The provided input rail flow `{flow_name}` does not exist" + ) for flow_name in self.config.rails.output.flows: flow_name = _normalize_flow_id(flow_name) if flow_name not in existing_flows_names: - raise ValueError(f"The provided output rail flow `{flow_name}` does not exist") + raise InvalidRailsConfigurationError( + f"The provided output rail flow `{flow_name}` does not exist" + ) for flow_name in self.config.rails.retrieval.flows: if flow_name not in existing_flows_names: - raise ValueError(f"The provided retrieval rail flow `{flow_name}` does not exist") + raise InvalidRailsConfigurationError( + f"The provided retrieval rail flow `{flow_name}` does not exist" + ) # If both passthrough mode and single call mode are specified, we raise an exception. if self.config.passthrough and self.config.rails.dialog.single_call.enabled: - raise ValueError( + raise InvalidRailsConfigurationError( "The passthrough mode and the single call dialog rails mode can't be used at the same time. " "The single call mode needs to use an altered prompt when prompting the LLM. " ) @@ -470,7 +486,9 @@ def _init_llms(self): try: model_name = llm_config.model if not model_name: - raise ValueError("LLM Config model field not set") + raise InvalidModelConfigurationError( + message=f"`model` field must be set in model configuration: {llm_config.model_dump_json()}" + ) provider_name = llm_config.engine kwargs = self._prepare_model_kwargs(llm_config) @@ -1179,11 +1197,11 @@ def _validate_streaming_with_output_rails(self) -> None: if len(self.config.rails.output.flows) > 0 and ( not self.config.rails.output.streaming or not self.config.rails.output.streaming.enabled ): - raise ValueError( - "stream_async() cannot be used when output rails are configured but " - "rails.output.streaming.enabled is False. Either set " - "rails.output.streaming.enabled to True in your configuration, or use " - "generate_async() instead of stream_async()." + raise InvalidRailsConfigurationError( + message=f"stream_async() cannot be used when output rails are configured but " + f"rails.output.streaming.enabled is False. Either set " + f"rails.output.streaming.enabled to True in your configuration, or use " + f"generate_async() instead of stream_async()." ) @overload From 22facb50c94aa248bdf0bcce82e579094e2acf7e Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Tue, 18 Nov 2025 15:52:06 -0500 Subject: [PATCH 02/14] fix: Propagate custom provider error messages; consolidate custom errors --- nemoguardrails/actions/action_dispatcher.py | 3 +- nemoguardrails/actions/llm/utils.py | 24 +-- nemoguardrails/exceptions.py | 141 +++--------------- .../llm/models/langchain_initializer.py | 13 +- nemoguardrails/rails/llm/config.py | 41 ++--- nemoguardrails/rails/llm/llmrails.py | 2 +- 6 files changed, 43 insertions(+), 181 deletions(-) diff --git a/nemoguardrails/actions/action_dispatcher.py b/nemoguardrails/actions/action_dispatcher.py index 11cc6e420..6dc5df77f 100644 --- a/nemoguardrails/actions/action_dispatcher.py +++ b/nemoguardrails/actions/action_dispatcher.py @@ -26,7 +26,8 @@ from langchain_core.runnables import Runnable from nemoguardrails import utils -from nemoguardrails.actions.llm.utils import LLMCallException +from nemoguardrails.exceptions import LLMCallException +from nemoguardrails.logging.callbacks import logging_callbacks log = logging.getLogger(__name__) diff --git a/nemoguardrails/actions/llm/utils.py b/nemoguardrails/actions/llm/utils.py index 3c2ba13da..4f80e458c 100644 --- a/nemoguardrails/actions/llm/utils.py +++ b/nemoguardrails/actions/llm/utils.py @@ -30,13 +30,14 @@ reasoning_trace_var, tool_calls_var, ) +from nemoguardrails.exceptions import LLMCallException from nemoguardrails.integrations.langchain.message_utils import dicts_to_messages from nemoguardrails.logging.callbacks import logging_callbacks from nemoguardrails.logging.explain import LLMCallInfo logger = logging.getLogger(__name__) -# Since different providers have different attributes for the base URL we'll use this list as a best-effort way +# Since different providers have different attributes for the base URL, we'll use this list as a best-effort way # to extract the base URL from a `BaseLanguageModel` instance. BASE_URL_ATTRIBUTES = [ "api_base", @@ -49,27 +50,6 @@ ] -class LLMCallException(Exception): - """A wrapper around the LLM call invocation exception. - - This is used to propagate the exception out of the `generate_async` call (the default behavior is to - catch it and return an "Internal server error." message. - """ - - def __init__(self, inner_exception: Any, context_message: Optional[str] = None): - """Initialize LLMCallException. - - Args: - inner_exception: The original exception that occurred - context_message: Optional context to prepend (for example, the model name or endpoint) - """ - message = f"{context_message or 'LLM Call Exception'}: {str(inner_exception)}" - super().__init__(message) - - self.inner_exception = inner_exception - self.context_message = context_message - - def _infer_provider_from_module(llm: BaseLanguageModel) -> Optional[str]: """Infer provider name from the LLM's module path. diff --git a/nemoguardrails/exceptions.py b/nemoguardrails/exceptions.py index 857906bea..4eeabed7e 100644 --- a/nemoguardrails/exceptions.py +++ b/nemoguardrails/exceptions.py @@ -12,88 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""Custom exceptions for NeMo Guardrails. - -These exceptions represent errors that the SDK detects and raises. -Provider errors (authentication, network, rate limits, etc.) are NOT wrapped -and should bubble up unchanged to the consuming application. -""" - from typing import Any, Dict, Optional -class NemoGuardrailsError(Exception): - """Base exception for all NeMo Guardrails SDK errors. - - Attributes: - message: Human-readable error message - context: Additional context about the error (model names, config details, etc.) +class ConfigurationError(ValueError): """ - - def __init__( - self, - message: str, - context: Optional[Dict[str, Any]] = None, - ): - """Initialize the exception. - - Args: - message: Error message describing what went wrong - context: Additional context (e.g., model name, provider, file path, etc.) - """ - super().__init__(message) - self.context = context or {} - - def __str__(self) -> str: - """Return a formatted error message with context.""" - parts = [super().__str__()] - - if self.context: - context_str = ", ".join(f"{k}={v!r}" for k, v in self.context.items()) - parts.append(f"Context: {context_str}") - - return "\n".join(parts) - - -# ============================================================================ -# Configuration Errors (Creation-Time) -# ============================================================================ - - -class ConfigurationError(NemoGuardrailsError): - """Base class for configuration-related errors. - - These errors occur when loading or validating guardrail configurations, - typically during RailsConfig loading or LLMRails initialization. + Base class for Guardrails Configuration validation errors. """ pass class InvalidModelConfigurationError(ConfigurationError): - """Raised when model configuration is invalid. - - Examples: - - model.model is empty or missing - - model.engine is invalid - - Invalid model parameters - - Model type not recognized - - Base URL is malformed - """ - - pass - - -class InvalidConfigurationFileError(ConfigurationError): - """Raised when configuration files cannot be successfullyparsed. - - Examples: - - Invalid YAML syntax in config.yml - - Invalid Colang syntax in .co files - - Circular import dependencies - - File not found - """ + """Raised when a guardrail configuration's model is invalid.""" pass @@ -111,60 +42,22 @@ class InvalidRailsConfigurationError(ConfigurationError): pass -# ============================================================================ -# Runtime Errors (Inference-Time) -# ============================================================================ - - -class ModelAuthenticationError(NemoGuardrailsError): - """Raised at inference-time when an authentication error occurs when calling the model.""" - - pass - - -class LLMInvocationError(NemoGuardrailsError): - """Base class for runtime errors that occur during inference. - - These errors occur after configuration has been loaded and validated, - during the actual execution of guardrails. - """ - - pass - - -class MalformedLLMResponseError(LLMInvocationError): - """Raised when the SDK cannot parse an LLM response. - - Examples: - - NemoGuard JSON parsing failures - - Structured output doesn't match expected format - - LLM returns invalid JSON when JSON is required - - Response missing required fields - """ - - pass - - -class InvalidConfigurationError(LLMInvocationError): - """Raised when an invalid configuration is detected at runtime. +class LLMCallException(Exception): + """A wrapper around the LLM call invocation exception. - Examples: - - Dynamic configuration issues discovered during inference - - Rail references a flow or action that doesn't exist (only discovered at runtime) - - Configuration state becomes invalid during execution + This is used to propagate the exception out of the `generate_async` call. The default behavior is to + catch it and return an "Internal server error." message. """ - pass - + def __init__(self, inner_exception: Any, context_message: Optional[str] = None): + """Initialize LLMCallException. -class ActionExecutionError(LLMInvocationError): - """Raised when a custom action fails during execution. - - Examples: - - User-defined custom action raised an exception - - Action returned invalid format - - Action parameter validation failed at runtime - - Action depends on external resource that's unavailable - """ + Args: + inner_exception: The original exception that occurred + context_message: Optional context to prepend (for example, the model name or endpoint) + """ + message = f"{context_message or 'LLM Call Exception'}: {str(inner_exception)}" + super().__init__(message) - pass + self.inner_exception = inner_exception + self.context_message = context_message diff --git a/nemoguardrails/llm/models/langchain_initializer.py b/nemoguardrails/llm/models/langchain_initializer.py index 6cb937d33..24e67dd24 100644 --- a/nemoguardrails/llm/models/langchain_initializer.py +++ b/nemoguardrails/llm/models/langchain_initializer.py @@ -138,13 +138,13 @@ def init_langchain_model( initializers: list[ModelInitializer] = [ # Try special case handlers first (handles both chat and text) ModelInitializer(_handle_model_special_cases, ["chat", "text"]), + # FIXME: is text and chat a good idea? + # For text mode, use text completion, we are using both text and chat as the last resort + ModelInitializer(_init_text_completion_model, ["text", "chat"]), # For chat mode, first try the standard chat completion API ModelInitializer(_init_chat_completion_model, ["chat"]), # For chat mode, fall back to community chat models ModelInitializer(_init_community_chat_models, ["chat"]), - # FIXME: is text and chat a good idea? - # For text mode, use text completion, we are using both text and chat as the last resort - ModelInitializer(_init_text_completion_model, ["text", "chat"]), ] # Track the last exception for better error reporting @@ -168,10 +168,10 @@ def init_langchain_model( if first_import_error is None: first_import_error = e last_exception = e - log.debug(f"Initialization import‐failure in {initializer}: {e}") + log.error(f"Initialization import‐failure in {initializer}: {e}") except Exception as e: last_exception = e - log.debug(f"Initialization failed with {initializer}: {e}") + log.error(f"Initialization failed with {initializer}: {e}") # build the final message, preferring that first ImportError if we saw one base = f"Failed to initialize model {model_name!r} with provider {provider_name!r} in {mode!r} mode" @@ -223,6 +223,9 @@ def _init_chat_completion_model(model_name: str, provider_name: str, kwargs: Dic ) except ValueError: raise + except Exception as e: + log.error(e, exc_info=True) + raise def _init_text_completion_model(model_name: str, provider_name: str, kwargs: Dict[str, Any]) -> BaseLLM: diff --git a/nemoguardrails/rails/llm/config.py b/nemoguardrails/rails/llm/config.py index 2394b9c79..7f8c1ad32 100644 --- a/nemoguardrails/rails/llm/config.py +++ b/nemoguardrails/rails/llm/config.py @@ -142,11 +142,7 @@ def set_and_validate_model(cls, data: Any) -> Any: if model_field and model_from_params: raise InvalidModelConfigurationError( - message=f"Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`).", - context={ - "model": model_field, - "parameters": parameters, - }, + f"Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`).", ) if not model_field and model_from_params: data["model"] = model_from_params @@ -161,7 +157,7 @@ def model_must_be_none_empty(self) -> "Model": """Validate that a model name is present either directly or in parameters.""" if not self.model or not self.model.strip(): raise InvalidModelConfigurationError( - message=f"Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`).", + f"Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`)." ) return self @@ -344,13 +340,13 @@ class TaskPrompt(BaseModel): def check_fields(cls, values): if not values.get("content") and not values.get("messages"): raise InvalidRailsConfigurationError( - message="One of `content` or `messages` must be provided." + "One of `content` or `messages` must be provided." ) # raise ValueError("One of `content` or `messages` must be provided.") if values.get("content") and values.get("messages"): raise InvalidRailsConfigurationError( - message="Only one of `content` or `messages` must be provided." + "Only one of `content` or `messages` must be provided." ) return values @@ -1435,7 +1431,7 @@ def check_model_exists_for_input_rails(cls, values): else "none" ) raise InvalidRailsConfigurationError( - message=f"Input flow '{flow_id}' references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}.", + f"Input flow '{flow_id}' references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}." ) return values @@ -1465,7 +1461,7 @@ def check_model_exists_for_output_rails(cls, values): else "none" ) raise InvalidRailsConfigurationError( - message=f"Output flow {flow_id} references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}.", + "Output flow {flow_id} references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}." ) return values @@ -1484,18 +1480,15 @@ def check_prompt_exist_for_self_check_rails(cls, values): and "self_check_input" not in provided_task_prompts ): raise InvalidRailsConfigurationError( - f"Missing a `self_check_input` prompt, which is required for the `self check input` rail." + f"Missing a `self_check_input` prompt template, which is required for the `self check input` rail." ) if ( "llama guard check input" in enabled_input_rails and "llama_guard_check_input" not in provided_task_prompts ): raise InvalidRailsConfigurationError( - f"Missing a `llama_guard_check_input` prompt, which is required for the `llama guard check input` rail." + f"Missing a `llama_guard_check_input` prompt template, which is required for the `llama guard check input` rail." ) - # raise ValueError( - # "You must provide a `llama_guard_check_input` prompt template." - # ) # Only content-safety and topic-safety include a $model reference in the rail flow text # Need to match rails with flow_id (excluding $model reference) and match prompts @@ -1509,38 +1502,30 @@ def check_prompt_exist_for_self_check_rails(cls, values): and "self_check_output" not in provided_task_prompts ): raise InvalidRailsConfigurationError( - f"Missing a `self_check_output` prompt, which is required for the `self check output` rail." + f"Missing a `self_check_output` prompt template, which is required for the `self check output` rail." ) - # raise ValueError("You must provide a `self_check_output` prompt template.") if ( "llama guard check output" in enabled_output_rails and "llama_guard_check_output" not in provided_task_prompts ): raise InvalidRailsConfigurationError( - f"Missing a `llama_guard_check_output` prompt, which is required for the `llama guard check output` rail." + f"Missing a `llama_guard_check_output` prompt template, which is required for the `llama guard check output` rail." ) - # raise ValueError( - # "You must provide a `llama_guard_check_output` prompt template." - # ) if ( "patronus lynx check output hallucination" in enabled_output_rails and "patronus_lynx_check_output_hallucination" not in provided_task_prompts ): raise InvalidRailsConfigurationError( - f"Missing a `patronus_lynx_check_output_hallucination` prompt, which is required for the `patronus lynx check output hallucination` rail." + f"Missing a `patronus_lynx_check_output_hallucination` prompt template, which is required for the `patronus lynx check output hallucination` rail." ) - # raise ValueError( - # "You must provide a `patronus_lynx_check_output_hallucination` prompt template." - # ) if ( "self check facts" in enabled_output_rails and "self_check_facts" not in provided_task_prompts ): raise InvalidRailsConfigurationError( - f"Missing a `self_check_facts` prompt, which is required for the `self check facts` rail." + f"Missing a `self_check_facts` prompt template, which is required for the `self check facts` rail." ) - # raise ValueError("You must provide a `self_check_facts` prompt template.") # Only content-safety and topic-safety include a $model reference in the rail flow text # Need to match rails with flow_id (excluding $model reference) and match prompts @@ -1867,5 +1852,5 @@ def _validate_rail_prompts(rails: list[str], prompts: list[Any], validation_rail expected_prompt = f"{prompt_flow_id} $model={flow_model}" if expected_prompt not in prompts: raise InvalidRailsConfigurationError( - f"Missing a `{expected_prompt}` prompt, which is required for the `{validation_rail}` rail." + f"Missing a `{expected_prompt}` prompt template, which is required for the `{validation_rail}` rail." ) diff --git a/nemoguardrails/rails/llm/llmrails.py b/nemoguardrails/rails/llm/llmrails.py index e8eed2da0..1d5be59fe 100644 --- a/nemoguardrails/rails/llm/llmrails.py +++ b/nemoguardrails/rails/llm/llmrails.py @@ -487,7 +487,7 @@ def _init_llms(self): model_name = llm_config.model if not model_name: raise InvalidModelConfigurationError( - message=f"`model` field must be set in model configuration: {llm_config.model_dump_json()}" + f"`model` field must be set in model configuration: {llm_config.model_dump_json()}" ) provider_name = llm_config.engine From 77b0f711c3b6beb7aaea9c55345ae6f71a3e902b Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 19 Nov 2025 10:11:09 -0500 Subject: [PATCH 03/14] Tests and cleanup --- nemoguardrails/actions/llm/utils.py | 11 +- nemoguardrails/exceptions.py | 9 +- .../llm/models/langchain_initializer.py | 7 +- nemoguardrails/rails/llm/config.py | 5 +- tests/test_actions_llm_utils.py | 106 ++++++++++++++++++ 5 files changed, 123 insertions(+), 15 deletions(-) diff --git a/nemoguardrails/actions/llm/utils.py b/nemoguardrails/actions/llm/utils.py index 4f80e458c..117b4870d 100644 --- a/nemoguardrails/actions/llm/utils.py +++ b/nemoguardrails/actions/llm/utils.py @@ -35,10 +35,8 @@ from nemoguardrails.logging.callbacks import logging_callbacks from nemoguardrails.logging.explain import LLMCallInfo -logger = logging.getLogger(__name__) - -# Since different providers have different attributes for the base URL, we'll use this list as a best-effort way -# to extract the base URL from a `BaseLanguageModel` instance. +# Since different providers have different attributes for the base URL, we'll use this list +# to attempt to extract the base URL from a `BaseLanguageModel` instance. BASE_URL_ATTRIBUTES = [ "api_base", "api_host", @@ -216,9 +214,8 @@ def _raise_llm_call_exception( Raises: LLMCallException with context message including model name and endpoint """ - # Extract model info from context (set by _setup_llm_call_info) + # Extract model name from context llm_call_info = llm_call_info_var.get() - logger.info(llm_call_info) model_name = ( llm_call_info.llm_model_name if llm_call_info @@ -236,7 +233,7 @@ def _raise_llm_call_exception( endpoint_url = str(value) break - # If we didn't find endpoint URL, check the nested client object + # If we didn't find endpoint URL, check the nested client object. if not endpoint_url and hasattr(llm, "client"): client = getattr(llm, "client", None) if client and hasattr(client, "base_url"): diff --git a/nemoguardrails/exceptions.py b/nemoguardrails/exceptions.py index 4eeabed7e..abe5b143e 100644 --- a/nemoguardrails/exceptions.py +++ b/nemoguardrails/exceptions.py @@ -12,7 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, Optional +from typing import Any, Optional + +__all__ = [ + "ConfigurationError", + "InvalidModelConfigurationError", + "InvalidRailsConfigurationError", + "LLMCallException", +] class ConfigurationError(ValueError): diff --git a/nemoguardrails/llm/models/langchain_initializer.py b/nemoguardrails/llm/models/langchain_initializer.py index 24e67dd24..66c8f4d2d 100644 --- a/nemoguardrails/llm/models/langchain_initializer.py +++ b/nemoguardrails/llm/models/langchain_initializer.py @@ -168,10 +168,10 @@ def init_langchain_model( if first_import_error is None: first_import_error = e last_exception = e - log.error(f"Initialization import‐failure in {initializer}: {e}") + log.debug(f"Initialization import‐failure in {initializer}: {e}") except Exception as e: last_exception = e - log.error(f"Initialization failed with {initializer}: {e}") + log.debug(f"Initialization failed with {initializer}: {e}") # build the final message, preferring that first ImportError if we saw one base = f"Failed to initialize model {model_name!r} with provider {provider_name!r} in {mode!r} mode" @@ -223,9 +223,6 @@ def _init_chat_completion_model(model_name: str, provider_name: str, kwargs: Dic ) except ValueError: raise - except Exception as e: - log.error(e, exc_info=True) - raise def _init_text_completion_model(model_name: str, provider_name: str, kwargs: Dict[str, Any]) -> BaseLLM: diff --git a/nemoguardrails/rails/llm/config.py b/nemoguardrails/rails/llm/config.py index 7f8c1ad32..6957a069a 100644 --- a/nemoguardrails/rails/llm/config.py +++ b/nemoguardrails/rails/llm/config.py @@ -342,7 +342,6 @@ def check_fields(cls, values): raise InvalidRailsConfigurationError( "One of `content` or `messages` must be provided." ) - # raise ValueError("One of `content` or `messages` must be provided.") if values.get("content") and values.get("messages"): raise InvalidRailsConfigurationError( @@ -1578,7 +1577,9 @@ def validate_models_api_key_env_var(cls, models): api_keys = [m.api_key_env_var for m in models] for api_key in api_keys: if api_key and not os.environ.get(api_key): - raise ValueError(f"Model API Key environment variable '{api_key}' not set.") + raise InvalidRailsConfigurationError( + f"Model API Key environment variable '{api_key}' not set." + ) return models raw_llm_call_action: Optional[str] = Field( diff --git a/tests/test_actions_llm_utils.py b/tests/test_actions_llm_utils.py index ed3bcfce5..0639dd830 100644 --- a/tests/test_actions_llm_utils.py +++ b/tests/test_actions_llm_utils.py @@ -14,7 +14,10 @@ # limitations under the License. import pytest +from typing import cast +from langchain_core.language_models import BaseLanguageModel from langchain_core.messages import AIMessage +from unittest.mock import AsyncMock from nemoguardrails.actions.llm.utils import ( _extract_reasoning_from_additional_kwargs, @@ -26,6 +29,7 @@ _store_tool_calls, ) from nemoguardrails.context import reasoning_trace_var, tool_calls_var +from nemoguardrails.exceptions import LLMCallException @pytest.fixture(autouse=True) @@ -63,6 +67,24 @@ class MockNVIDIAOriginal: __module__ = "langchain_nvidia_ai_endpoints.chat_models" +class MockTRTLLM: + __module__ = "nemoguardrails.llm.providers.trtllm.llm" + + +class MockAzureLLM: + __module__ = "langchain_openai.chat_models" + + +class MockLLMWithClient: + __module__ = "langchain_openai.chat_models" + + class _MockClient: + base_url = "https://custom.endpoint.com/v1" + + def __init__(self): + self.client = self._MockClient() + + class MockPatchedNVIDIA(MockNVIDIAOriginal): __module__ = "nemoguardrails.llm.providers._langchain_nvidia_ai_endpoints_patch" @@ -532,3 +554,87 @@ def test_store_tool_calls_with_real_aimessage_multiple_tool_calls(): assert len(tool_calls) == 2 assert tool_calls[0]["name"] == "foo" assert tool_calls[1]["name"] == "bar" + +@pytest.mark.asyncio +async def test_llm_call_exception_enrichment_with_model_and_endpoint(): + """Test that LLM invocation errors include model and endpoint context.""" + mock_llm = MockOpenAILLM() + mock_llm.model_name = "gpt-4" + mock_llm.base_url = "https://api.openai.com/v1" + mock_llm.ainvoke = AsyncMock(side_effect=ConnectionError("Connection refused")) + + with pytest.raises(LLMCallException) as exc_info: + await llm_call(cast(BaseLanguageModel, mock_llm), "test prompt") + + exc_str = str(exc_info.value) + assert "gpt-4" in exc_str + assert "https://api.openai.com/v1" in exc_str + assert "Connection refused" in exc_str + assert isinstance(exc_info.value.inner_exception, ConnectionError) + + +@pytest.mark.asyncio +async def test_llm_call_exception_without_endpoint(): + """Test exception enrichment when endpoint URL is not available.""" + mock_llm = AsyncMock() + mock_llm.__module__ = "langchain_openai.chat_models" + mock_llm.model_name = "custom-model" + # No base_url attribute + mock_llm.ainvoke = AsyncMock(side_effect=ValueError("Invalid request")) + + with pytest.raises(LLMCallException) as exc_info: + await llm_call(mock_llm, "test prompt") + + # Should still have model name but no endpoint + assert "custom-model" in str(exc_info.value) + assert "Invalid request" in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_llm_call_exception_extracts_azure_endpoint(): + """Test that Azure-style endpoint URLs are extracted.""" + mock_llm = MockAzureLLM() + mock_llm.model_name = "gpt-4" + mock_llm.azure_endpoint = "https://example.openai.azure.com" + mock_llm.ainvoke = AsyncMock(side_effect=Exception("Azure error")) + + with pytest.raises(LLMCallException) as exc_info: + await llm_call(cast(BaseLanguageModel, mock_llm), "test prompt") + + exc_str = str(exc_info.value) + assert "https://example.openai.azure.com" in exc_str + assert "gpt-4" in exc_str + assert "Azure error" in exc_str + + +@pytest.mark.asyncio +async def test_llm_call_exception_extracts_server_url(): + """Test that TRT-style server_url is extracted.""" + mock_llm = MockTRTLLM() + mock_llm.model_name = "llama-2-70b" + mock_llm.server_url = "https://triton.example.com:8000" + mock_llm.ainvoke = AsyncMock(side_effect=Exception("Triton server error")) + + with pytest.raises(LLMCallException) as exc_info: + await llm_call(cast(BaseLanguageModel, mock_llm), "test prompt") + + exc_str = str(exc_info.value) + assert "https://triton.example.com:8000" in exc_str + assert "llama-2-70b" in exc_str + assert "Triton server error" in exc_str + + +@pytest.mark.asyncio +async def test_llm_call_exception_extracts_nested_client_base_url(): + """Test that nested client.base_url is extracted.""" + mock_llm = MockLLMWithClient() + mock_llm.model_name = "gpt-4-turbo" + mock_llm.ainvoke = AsyncMock(side_effect=Exception("Client error")) + + with pytest.raises(LLMCallException) as exc_info: + await llm_call(cast(BaseLanguageModel, mock_llm), "test prompt") + + exc_str = str(exc_info.value) + assert "https://custom.endpoint.com/v1" in exc_str + assert "gpt-4-turbo" in exc_str + assert "Client error" in exc_str \ No newline at end of file From a5b903c263484d5738051228311c6f1e7788565e Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 19 Nov 2025 12:10:05 -0500 Subject: [PATCH 04/14] Fix build issue --- nemoguardrails/rails/llm/llmrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemoguardrails/rails/llm/llmrails.py b/nemoguardrails/rails/llm/llmrails.py index 1d5be59fe..7267a00c4 100644 --- a/nemoguardrails/rails/llm/llmrails.py +++ b/nemoguardrails/rails/llm/llmrails.py @@ -1198,7 +1198,7 @@ def _validate_streaming_with_output_rails(self) -> None: not self.config.rails.output.streaming or not self.config.rails.output.streaming.enabled ): raise InvalidRailsConfigurationError( - message=f"stream_async() cannot be used when output rails are configured but " + f"stream_async() cannot be used when output rails are configured but " f"rails.output.streaming.enabled is False. Either set " f"rails.output.streaming.enabled to True in your configuration, or use " f"generate_async() instead of stream_async()." From ec818cda868fc261152071d44c0de86aef20993a Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 19 Nov 2025 12:24:58 -0500 Subject: [PATCH 05/14] Fix tests --- nemoguardrails/actions/llm/utils.py | 1 + nemoguardrails/rails/llm/config.py | 2 +- tests/test_actions_llm_utils.py | 18 ++++++++++++++++++ tests/test_config_validation.py | 4 ++-- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/nemoguardrails/actions/llm/utils.py b/nemoguardrails/actions/llm/utils.py index 117b4870d..ef65a8c4e 100644 --- a/nemoguardrails/actions/llm/utils.py +++ b/nemoguardrails/actions/llm/utils.py @@ -45,6 +45,7 @@ "endpoint", "endpoint_url", "openai_api_base", + "server_url", ] diff --git a/nemoguardrails/rails/llm/config.py b/nemoguardrails/rails/llm/config.py index 6957a069a..d9da7ceb1 100644 --- a/nemoguardrails/rails/llm/config.py +++ b/nemoguardrails/rails/llm/config.py @@ -1460,7 +1460,7 @@ def check_model_exists_for_output_rails(cls, values): else "none" ) raise InvalidRailsConfigurationError( - "Output flow {flow_id} references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}." + f"Output flow '{flow_id}' references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}." ) return values diff --git a/tests/test_actions_llm_utils.py b/tests/test_actions_llm_utils.py index 0639dd830..f33a09f1c 100644 --- a/tests/test_actions_llm_utils.py +++ b/tests/test_actions_llm_utils.py @@ -89,6 +89,24 @@ class MockPatchedNVIDIA(MockNVIDIAOriginal): __module__ = "nemoguardrails.llm.providers._langchain_nvidia_ai_endpoints_patch" +class MockTRTLLM: + __module__ = "nemoguardrails.llm.providers.trtllm.llm" + + +class MockAzureLLM: + __module__ = "langchain_openai.chat_models" + + +class MockLLMWithClient: + __module__ = "langchain_openai.chat_models" + + class _MockClient: + base_url = "https://custom.endpoint.com/v1" + + def __init__(self): + self.client = self._MockClient() + + def test_infer_provider_openai(): llm = MockOpenAILLM() provider = _infer_provider_from_module(llm) diff --git a/tests/test_config_validation.py b/tests/test_config_validation.py index 3e0bf62d7..a73216695 100644 --- a/tests/test_config_validation.py +++ b/tests/test_config_validation.py @@ -75,7 +75,7 @@ def test_self_check_input_prompt_exception(): ) LLMRails(config=config) - assert "You must provide a `self_check_input` prompt" in str(exc_info.value) + assert "Missing a `self_check_input` prompt template" in str(exc_info.value) def test_self_check_output_prompt_exception(): @@ -90,7 +90,7 @@ def test_self_check_output_prompt_exception(): ) LLMRails(config=config) - assert "You must provide a `self_check_output` prompt" in str(exc_info.value) + assert "Missing a `self_check_output` prompt template" in str(exc_info.value) def test_passthrough_and_single_call_incompatibility(): From 87de16e55adf27de4a73a0cfc3e4bbc0b1c8ec88 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 19 Nov 2025 13:12:56 -0500 Subject: [PATCH 06/14] Fix more tests --- .../test_langchain_initializer.py | 22 ++++++----- tests/test_rails_config.py | 38 ++++++++++--------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/tests/llm_providers/test_langchain_initializer.py b/tests/llm_providers/test_langchain_initializer.py index a8ff2df34..c9432067d 100644 --- a/tests/llm_providers/test_langchain_initializer.py +++ b/tests/llm_providers/test_langchain_initializer.py @@ -58,25 +58,27 @@ def test_special_case_called_first(mock_initializers): def test_chat_completion_called(mock_initializers): mock_initializers["special"].return_value = None + mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = "chat_model" result = init_langchain_model("chat-model", "provider", "chat", {}) assert result == "chat_model" mock_initializers["special"].assert_called_once() + mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_not_called() - mock_initializers["text"].assert_not_called() def test_community_chat_called(mock_initializers): mock_initializers["special"].return_value = None + mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = None mock_initializers["community"].return_value = "community_model" result = init_langchain_model("community-chat", "provider", "chat", {}) assert result == "community_model" mock_initializers["special"].assert_called_once() + mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_called_once() - mock_initializers["text"].assert_not_called() def test_text_completion_called(mock_initializers): @@ -138,36 +140,39 @@ def test_all_initializers_raise_exceptions(mock_initializers): def test_duplicate_modes_in_initializer(mock_initializers): mock_initializers["special"].return_value = None + mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = "chat_model" result = init_langchain_model("chat-model", "provider", "chat", {}) assert result == "chat_model" mock_initializers["special"].assert_called_once() + mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_not_called() - mock_initializers["text"].assert_not_called() def test_chat_completion_called_when_special_returns_none(mock_initializers): mock_initializers["special"].return_value = None + mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = "chat_model" result = init_langchain_model("chat-model", "provider", "chat", {}) assert result == "chat_model" mock_initializers["special"].assert_called_once() + mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_not_called() - mock_initializers["text"].assert_not_called() def test_community_chat_called_when_previous_fail(mock_initializers): mock_initializers["special"].return_value = None + mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = None mock_initializers["community"].return_value = "community_model" result = init_langchain_model("community-chat", "provider", "chat", {}) assert result == "community_model" mock_initializers["special"].assert_called_once() + mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_called_once() - mock_initializers["text"].assert_not_called() def test_text_completion_called_when_previous_fail(mock_initializers): @@ -185,12 +190,11 @@ def test_text_completion_called_when_previous_fail(mock_initializers): def test_text_completion_supports_chat_mode(mock_initializers): mock_initializers["special"].return_value = None - mock_initializers["chat"].return_value = None - mock_initializers["community"].return_value = None mock_initializers["text"].return_value = "text_model" result = init_langchain_model("text-model", "provider", "chat", {}) assert result == "text_model" mock_initializers["special"].assert_called_once() - mock_initializers["chat"].assert_called_once() - mock_initializers["community"].assert_called_once() mock_initializers["text"].assert_called_once() + # Since text returns a value, chat and community are not called + mock_initializers["chat"].assert_not_called() + mock_initializers["community"].assert_not_called() diff --git a/tests/test_rails_config.py b/tests/test_rails_config.py index 4896d0014..e05e5855f 100644 --- a/tests/test_rails_config.py +++ b/tests/test_rails_config.py @@ -91,7 +91,9 @@ def test_check_prompt_exist_for_self_check_rails(): # missings self_check_output prompt ], } - with pytest.raises(ValueError, match="You must provide a `self_check_output` prompt template"): + with pytest.raises( + ValueError, match="Missing a `self_check_output` prompt template" + ): RailsConfig.check_prompt_exist_for_self_check_rails(values) @@ -340,7 +342,7 @@ def test_validate_rail_prompts_wrong_flow_id_raises(self): with pytest.raises( ValueError, - match="You must provide a `content_safety_check_input \$model=content_safety` prompt template.", + match="Missing a `content_safety_check_input \$model=content_safety` prompt template", ): _validate_rail_prompts( ["content safety check input $model=content_safety"], @@ -353,7 +355,7 @@ def test_validate_rail_prompts_wrong_model_raises(self): with pytest.raises( ValueError, - match="You must provide a `content_safety_check_input \$model=content_safety` prompt template.", + match="Missing a `content_safety_check_input \$model=content_safety` prompt template", ): _validate_rail_prompts( ["content safety check input $model=content_safety"], @@ -366,7 +368,7 @@ def test_validate_rail_prompts_no_prompt_raises(self): with pytest.raises( ValueError, - match="You must provide a `content_safety_check_input \$model=content_safety` prompt template.", + match="Missing a `content_safety_check_input \$model=content_safety` prompt template", ): _validate_rail_prompts( ["content safety check input $model=content_safety"], @@ -382,7 +384,7 @@ def test_content_safety_input_missing_prompt_raises(self): """Check Content Safety output rail raises ValueError if we don't have a prompt""" with pytest.raises( ValueError, - match="You must provide a `content_safety_check_input \$model=content_safety` prompt template.", + match="Missing a `content_safety_check_input \$model=content_safety` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" @@ -402,7 +404,7 @@ def test_content_safety_output_missing_prompt_raises(self): """Check Content Safety output rail raises ValueError if we don't have a prompt""" with pytest.raises( ValueError, - match="You must provide a `content_safety_check_output \$model=content_safety` prompt template.", + match="Missing a `content_safety_check_output \$model=content_safety` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" @@ -506,7 +508,7 @@ def test_input_content_safety_no_model_raises(self): with pytest.raises( ValueError, - match="No `content_safety` model provided for input flow `content safety check input`", + match="Input flow 'content safety check input' references model type 'content_safety' that is not defined", ): _ = RailsConfig.from_content( yaml_content=""" @@ -531,7 +533,7 @@ def test_input_content_safety_wrong_model_raises(self): with pytest.raises( ValueError, - match="No `content_safety` model provided for input flow `content safety check input", + match="Input flow 'content safety check input' references model type 'content_safety' that is not defined", ): _ = RailsConfig.from_content( yaml_content=""" @@ -556,7 +558,7 @@ def test_output_content_safety_no_model_raises(self): with pytest.raises( ValueError, - match="No `content_safety` model provided for output flow `content safety check output`", + match="Output flow 'content safety check output' references model type 'content_safety' that is not defined", ): _ = RailsConfig.from_content( yaml_content=""" @@ -581,7 +583,7 @@ def test_output_content_safety_wrong_model_raises(self): with pytest.raises( ValueError, - match="You must provide a `content_safety_check_output \$model=content_safety` prompt template", + match="Missing a `content_safety_check_output \$model=content_safety` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" @@ -636,7 +638,7 @@ def test_topic_safety_no_prompt_raises(self): with pytest.raises( ValueError, - match="You must provide a `topic_safety_check_input \$model=topic_control` prompt template", + match="Missing a `topic_safety_check_input \$model=topic_control` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" @@ -660,7 +662,7 @@ def test_topic_safety_no_model_raises(self): """Check if we don't provide a topic-safety model we raise a ValueError""" with pytest.raises( ValueError, - match="No `topic_control` model provided for input flow `topic safety check input`", + match="Input flow 'topic safety check input' references model type 'topic_control' that is not defined", ): _ = RailsConfig.from_content( yaml_content=""" @@ -684,7 +686,7 @@ def test_topic_safety_no_model_no_prompt_raises(self): """Check a missing model and prompt raises ValueError""" with pytest.raises( ValueError, - match="You must provide a `topic_safety_check_input \$model=topic_control` prompt template", + match="Missing a `topic_safety_check_input \$model=topic_control` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" @@ -713,7 +715,7 @@ def test_hero_separate_models_no_prompts_raises(self): with pytest.raises( ValueError, - match="You must provide a `content_safety_check_input \$model=my_content_safety` prompt template", + match="Missing a `content_safety_check_input \$model=my_content_safety` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" @@ -846,7 +848,7 @@ def test_hero_no_prompts_raises(self): """Create hero workflow with no prompts. Expect Content Safety input prompt check to fail""" with pytest.raises( ValueError, - match="You must provide a `content_safety_check_input \$model=content_safety` prompt template", + match="Missing a `content_safety_check_input \$model=content_safety` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" @@ -886,7 +888,7 @@ def test_hero_no_output_content_safety_prompt_raises(self): """Create hero workflow with no prompts. Expect Content Safety input prompt check to fail""" with pytest.raises( ValueError, - match="You must provide a `topic_safety_check_input \$model=your_topic_control` prompt template", + match="Missing a `topic_safety_check_input \$model=your_topic_control` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" @@ -930,7 +932,7 @@ def test_hero_no_topic_safety_prompt_raises(self): """Create hero workflow with no prompts. Expect Content Safety input prompt check to fail""" with pytest.raises( ValueError, - match="You must provide a `topic_safety_check_input \$model=your_topic_control` prompt template", + match="Missing a `topic_safety_check_input \$model=your_topic_control` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" @@ -976,7 +978,7 @@ def test_hero_topic_safety_prompt_raises(self): """Create hero workflow with no prompts. Expect Content Safety input prompt check to fail""" with pytest.raises( ValueError, - match="You must provide a `content_safety_check_input \$model=content_safety` prompt template", + match="Missing a `content_safety_check_input \$model=content_safety` prompt template", ): _ = RailsConfig.from_content( yaml_content=""" From 87c7b4ced709730745f86f499fd371e773bd3d67 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 19 Nov 2025 13:23:30 -0500 Subject: [PATCH 07/14] Fix imports in tests --- tests/test_embeddings_only_user_messages.py | 2 +- tests/test_tool_calling_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_embeddings_only_user_messages.py b/tests/test_embeddings_only_user_messages.py index c1dc69f05..6794e01ce 100644 --- a/tests/test_embeddings_only_user_messages.py +++ b/tests/test_embeddings_only_user_messages.py @@ -17,7 +17,7 @@ import pytest from nemoguardrails import LLMRails, RailsConfig -from nemoguardrails.actions.llm.utils import LLMCallException +from nemoguardrails.exceptions import LLMCallException from tests.utils import TestChat diff --git a/tests/test_tool_calling_utils.py b/tests/test_tool_calling_utils.py index aafc9f937..906521de4 100644 --- a/tests/test_tool_calling_utils.py +++ b/tests/test_tool_calling_utils.py @@ -19,7 +19,6 @@ from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage from nemoguardrails.actions.llm.utils import ( - LLMCallException, _convert_messages_to_langchain_format, _extract_content, _store_tool_calls, @@ -27,6 +26,7 @@ llm_call, ) from nemoguardrails.context import tool_calls_var +from nemoguardrails.exceptions import LLMCallException from nemoguardrails.rails.llm.llmrails import GenerationResponse From 0a7a742b8c3576f429c575d5b4858bf2b8269708 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Tue, 25 Nov 2025 15:38:19 -0500 Subject: [PATCH 08/14] Revert model initializer order, but only try _init_text_completion_model for text models --- .../llm/models/langchain_initializer.py | 5 ++-- .../test_langchain_initializer.py | 25 ++++++++----------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/nemoguardrails/llm/models/langchain_initializer.py b/nemoguardrails/llm/models/langchain_initializer.py index 66c8f4d2d..08ddd909d 100644 --- a/nemoguardrails/llm/models/langchain_initializer.py +++ b/nemoguardrails/llm/models/langchain_initializer.py @@ -138,13 +138,12 @@ def init_langchain_model( initializers: list[ModelInitializer] = [ # Try special case handlers first (handles both chat and text) ModelInitializer(_handle_model_special_cases, ["chat", "text"]), - # FIXME: is text and chat a good idea? - # For text mode, use text completion, we are using both text and chat as the last resort - ModelInitializer(_init_text_completion_model, ["text", "chat"]), # For chat mode, first try the standard chat completion API ModelInitializer(_init_chat_completion_model, ["chat"]), # For chat mode, fall back to community chat models ModelInitializer(_init_community_chat_models, ["chat"]), + # For text mode, use text completion + ModelInitializer(_init_text_completion_model, ["text"]), ] # Track the last exception for better error reporting diff --git a/tests/llm_providers/test_langchain_initializer.py b/tests/llm_providers/test_langchain_initializer.py index c9432067d..9c0bf830e 100644 --- a/tests/llm_providers/test_langchain_initializer.py +++ b/tests/llm_providers/test_langchain_initializer.py @@ -58,27 +58,25 @@ def test_special_case_called_first(mock_initializers): def test_chat_completion_called(mock_initializers): mock_initializers["special"].return_value = None - mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = "chat_model" result = init_langchain_model("chat-model", "provider", "chat", {}) assert result == "chat_model" mock_initializers["special"].assert_called_once() - mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_not_called() + mock_initializers["text"].assert_not_called() def test_community_chat_called(mock_initializers): mock_initializers["special"].return_value = None - mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = None mock_initializers["community"].return_value = "community_model" result = init_langchain_model("community-chat", "provider", "chat", {}) assert result == "community_model" mock_initializers["special"].assert_called_once() - mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_called_once() + mock_initializers["text"].assert_not_called() def test_text_completion_called(mock_initializers): @@ -98,13 +96,12 @@ def test_all_initializers_fail(mock_initializers): mock_initializers["special"].return_value = None mock_initializers["chat"].return_value = None mock_initializers["community"].return_value = None - mock_initializers["text"].return_value = None with pytest.raises(ModelInitializationError): init_langchain_model("unknown-model", "provider", "chat", {}) mock_initializers["special"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_called_once() - mock_initializers["text"].assert_called_once() + mock_initializers["text"].assert_not_called() def test_unsupported_mode(mock_initializers): @@ -135,44 +132,41 @@ def test_all_initializers_raise_exceptions(mock_initializers): mock_initializers["special"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_called_once() - mock_initializers["text"].assert_called_once() + mock_initializers["text"].assert_not_called() def test_duplicate_modes_in_initializer(mock_initializers): mock_initializers["special"].return_value = None - mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = "chat_model" result = init_langchain_model("chat-model", "provider", "chat", {}) assert result == "chat_model" mock_initializers["special"].assert_called_once() - mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_not_called() + mock_initializers["text"].assert_not_called() def test_chat_completion_called_when_special_returns_none(mock_initializers): mock_initializers["special"].return_value = None - mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = "chat_model" result = init_langchain_model("chat-model", "provider", "chat", {}) assert result == "chat_model" mock_initializers["special"].assert_called_once() - mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_not_called() + mock_initializers["text"].assert_not_called() def test_community_chat_called_when_previous_fail(mock_initializers): mock_initializers["special"].return_value = None - mock_initializers["text"].return_value = None mock_initializers["chat"].return_value = None mock_initializers["community"].return_value = "community_model" result = init_langchain_model("community-chat", "provider", "chat", {}) assert result == "community_model" mock_initializers["special"].assert_called_once() - mock_initializers["text"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_called_once() + mock_initializers["text"].assert_not_called() def test_text_completion_called_when_previous_fail(mock_initializers): @@ -188,10 +182,11 @@ def test_text_completion_called_when_previous_fail(mock_initializers): mock_initializers["text"].assert_called_once() -def test_text_completion_supports_chat_mode(mock_initializers): +def test_text_mode_only_calls_text_initializers(mock_initializers): + """Test that text mode only tries initializers that support text mode.""" mock_initializers["special"].return_value = None mock_initializers["text"].return_value = "text_model" - result = init_langchain_model("text-model", "provider", "chat", {}) + result = init_langchain_model("text-model", "provider", "text", {}) assert result == "text_model" mock_initializers["special"].assert_called_once() mock_initializers["text"].assert_called_once() From ec7d8f2aa956ed476e51c6e2885ac5cf44b8030e Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Tue, 25 Nov 2025 15:52:07 -0500 Subject: [PATCH 09/14] Update tests --- tests/test_actions_llm_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_actions_llm_utils.py b/tests/test_actions_llm_utils.py index f33a09f1c..28f767a39 100644 --- a/tests/test_actions_llm_utils.py +++ b/tests/test_actions_llm_utils.py @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest from typing import cast +from unittest.mock import AsyncMock + +import pytest from langchain_core.language_models import BaseLanguageModel from langchain_core.messages import AIMessage -from unittest.mock import AsyncMock from nemoguardrails.actions.llm.utils import ( _extract_reasoning_from_additional_kwargs, @@ -27,6 +28,7 @@ _infer_provider_from_module, _store_reasoning_traces, _store_tool_calls, + llm_call, ) from nemoguardrails.context import reasoning_trace_var, tool_calls_var from nemoguardrails.exceptions import LLMCallException @@ -573,6 +575,7 @@ def test_store_tool_calls_with_real_aimessage_multiple_tool_calls(): assert tool_calls[0]["name"] == "foo" assert tool_calls[1]["name"] == "bar" + @pytest.mark.asyncio async def test_llm_call_exception_enrichment_with_model_and_endpoint(): """Test that LLM invocation errors include model and endpoint context.""" @@ -655,4 +658,4 @@ async def test_llm_call_exception_extracts_nested_client_base_url(): exc_str = str(exc_info.value) assert "https://custom.endpoint.com/v1" in exc_str assert "gpt-4-turbo" in exc_str - assert "Client error" in exc_str \ No newline at end of file + assert "Client error" in exc_str From de1b8f8feac07fa74aced48c219044b75f69fdee Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Tue, 25 Nov 2025 15:55:19 -0500 Subject: [PATCH 10/14] Add back removed code --- nemoguardrails/actions/llm/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nemoguardrails/actions/llm/utils.py b/nemoguardrails/actions/llm/utils.py index ef65a8c4e..e111eb6c7 100644 --- a/nemoguardrails/actions/llm/utils.py +++ b/nemoguardrails/actions/llm/utils.py @@ -35,6 +35,8 @@ from nemoguardrails.logging.callbacks import logging_callbacks from nemoguardrails.logging.explain import LLMCallInfo +logger = logging.getLogger(__name__) + # Since different providers have different attributes for the base URL, we'll use this list # to attempt to extract the base URL from a `BaseLanguageModel` instance. BASE_URL_ATTRIBUTES = [ From a9d77e45031c0ffeab72440bffc224ccfea3235f Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Tue, 25 Nov 2025 22:39:27 -0500 Subject: [PATCH 11/14] Update model initializers to return None if initializer not found --- .../llm/models/langchain_initializer.py | 25 +++++++++++++------ .../test_langchain_initialization_methods.py | 8 +++--- .../test_langchain_initializer.py | 5 ++-- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/nemoguardrails/llm/models/langchain_initializer.py b/nemoguardrails/llm/models/langchain_initializer.py index 08ddd909d..cf1fa4245 100644 --- a/nemoguardrails/llm/models/langchain_initializer.py +++ b/nemoguardrails/llm/models/langchain_initializer.py @@ -143,7 +143,7 @@ def init_langchain_model( # For chat mode, fall back to community chat models ModelInitializer(_init_community_chat_models, ["chat"]), # For text mode, use text completion - ModelInitializer(_init_text_completion_model, ["text"]), + ModelInitializer(_init_text_completion_model, ["text", "chat"]), ] # Track the last exception for better error reporting @@ -224,7 +224,9 @@ def _init_chat_completion_model(model_name: str, provider_name: str, kwargs: Dic raise -def _init_text_completion_model(model_name: str, provider_name: str, kwargs: Dict[str, Any]) -> BaseLLM: +def _init_text_completion_model( + model_name: str, provider_name: str, kwargs: Dict[str, Any] +) -> BaseLLM | None: """Initialize a text completion model. Args: @@ -238,9 +240,14 @@ def _init_text_completion_model(model_name: str, provider_name: str, kwargs: Dic Raises: RuntimeError: If the provider is not found """ - provider_cls = _get_text_completion_provider(provider_name) + try: + provider_cls = _get_text_completion_provider(provider_name) + except RuntimeError as e: + return None + if provider_cls is None: - raise ValueError() + return None + kwargs = _update_model_kwargs(provider_cls, model_name, kwargs) # remove stream_usage parameter as it's not supported by text completion APIs # (e.g., OpenAI's AsyncCompletions.create() doesn't accept this parameter) @@ -248,7 +255,9 @@ def _init_text_completion_model(model_name: str, provider_name: str, kwargs: Dic return provider_cls(**kwargs) -def _init_community_chat_models(model_name: str, provider_name: str, kwargs: Dict[str, Any]) -> BaseChatModel: +def _init_community_chat_models( + model_name: str, provider_name: str, kwargs: Dict[str, Any] +) -> BaseChatModel | None: """Initialize community chat models. Args: @@ -265,12 +274,14 @@ def _init_community_chat_models(model_name: str, provider_name: str, kwargs: Dic """ provider_cls = _get_chat_completion_provider(provider_name) if provider_cls is None: - raise ValueError() + return None kwargs = _update_model_kwargs(provider_cls, model_name, kwargs) return provider_cls(**kwargs) -def _init_gpt35_turbo_instruct(model_name: str, provider_name: str, kwargs: Dict[str, Any]) -> BaseLLM: +def _init_gpt35_turbo_instruct( + model_name: str, provider_name: str, kwargs: Dict[str, Any] +) -> BaseLLM | None: """Initialize GPT-3.5 Turbo Instruct model. Currently init_chat_model from langchain infers this as a chat model. diff --git a/tests/llm_providers/test_langchain_initialization_methods.py b/tests/llm_providers/test_langchain_initialization_methods.py index 14b6f0e0f..7bd5ff63c 100644 --- a/tests/llm_providers/test_langchain_initialization_methods.py +++ b/tests/llm_providers/test_langchain_initialization_methods.py @@ -116,8 +116,9 @@ def test_init_community_chat_models_no_provider(self): "nemoguardrails.llm.models.langchain_initializer._get_chat_completion_provider" ) as mock_get_provider: mock_get_provider.return_value = None - with pytest.raises(ValueError): - _init_community_chat_models("community-model", "provider", {}) + assert ( + _init_community_chat_models("community-model", "provider", {}) is None + ) class TestTextCompletionInitializer: @@ -156,8 +157,7 @@ def test_init_text_completion_model_no_provider(self): "nemoguardrails.llm.models.langchain_initializer._get_text_completion_provider" ) as mock_get_provider: mock_get_provider.return_value = None - with pytest.raises(ValueError): - _init_text_completion_model("text-model", "provider", {}) + assert _init_text_completion_model("text-model", "provider", {}) is None class TestUpdateModelKwargs: diff --git a/tests/llm_providers/test_langchain_initializer.py b/tests/llm_providers/test_langchain_initializer.py index 9c0bf830e..070d3bb58 100644 --- a/tests/llm_providers/test_langchain_initializer.py +++ b/tests/llm_providers/test_langchain_initializer.py @@ -96,12 +96,13 @@ def test_all_initializers_fail(mock_initializers): mock_initializers["special"].return_value = None mock_initializers["chat"].return_value = None mock_initializers["community"].return_value = None + mock_initializers["text"].return_value = None with pytest.raises(ModelInitializationError): init_langchain_model("unknown-model", "provider", "chat", {}) mock_initializers["special"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_called_once() - mock_initializers["text"].assert_not_called() + mock_initializers["text"].assert_called_once() def test_unsupported_mode(mock_initializers): @@ -132,7 +133,7 @@ def test_all_initializers_raise_exceptions(mock_initializers): mock_initializers["special"].assert_called_once() mock_initializers["chat"].assert_called_once() mock_initializers["community"].assert_called_once() - mock_initializers["text"].assert_not_called() + mock_initializers["text"].assert_called_once() def test_duplicate_modes_in_initializer(mock_initializers): From 9dacd9c105d8ef950cdd4878e938c3f5bfcc03aa Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Tue, 25 Nov 2025 22:52:19 -0500 Subject: [PATCH 12/14] Add comment back --- nemoguardrails/llm/models/langchain_initializer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemoguardrails/llm/models/langchain_initializer.py b/nemoguardrails/llm/models/langchain_initializer.py index cf1fa4245..da0dad57b 100644 --- a/nemoguardrails/llm/models/langchain_initializer.py +++ b/nemoguardrails/llm/models/langchain_initializer.py @@ -142,7 +142,8 @@ def init_langchain_model( ModelInitializer(_init_chat_completion_model, ["chat"]), # For chat mode, fall back to community chat models ModelInitializer(_init_community_chat_models, ["chat"]), - # For text mode, use text completion + # FIXME: is text and chat a good idea? + # For text mode, use text completion, we are using both text and chat as the last resort ModelInitializer(_init_text_completion_model, ["text", "chat"]), ] From fb0cc19fd0ad260b050468ff14afe5f94722a3ba Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 26 Nov 2025 08:36:16 -0500 Subject: [PATCH 13/14] Revert langchain_initializer changes --- .../llm/models/langchain_initializer.py | 17 ++++++----------- .../test_langchain_initialization_methods.py | 8 ++++---- .../test_langchain_initializer.py | 12 ++++++------ tests/test_actions_llm_utils.py | 18 ------------------ 4 files changed, 16 insertions(+), 39 deletions(-) diff --git a/nemoguardrails/llm/models/langchain_initializer.py b/nemoguardrails/llm/models/langchain_initializer.py index da0dad57b..ef4d2b057 100644 --- a/nemoguardrails/llm/models/langchain_initializer.py +++ b/nemoguardrails/llm/models/langchain_initializer.py @@ -227,7 +227,7 @@ def _init_chat_completion_model(model_name: str, provider_name: str, kwargs: Dic def _init_text_completion_model( model_name: str, provider_name: str, kwargs: Dict[str, Any] -) -> BaseLLM | None: +) -> BaseLLM: """Initialize a text completion model. Args: @@ -241,14 +241,9 @@ def _init_text_completion_model( Raises: RuntimeError: If the provider is not found """ - try: - provider_cls = _get_text_completion_provider(provider_name) - except RuntimeError as e: - return None - + provider_cls = _get_text_completion_provider(provider_name) if provider_cls is None: - return None - + raise ValueError() kwargs = _update_model_kwargs(provider_cls, model_name, kwargs) # remove stream_usage parameter as it's not supported by text completion APIs # (e.g., OpenAI's AsyncCompletions.create() doesn't accept this parameter) @@ -258,7 +253,7 @@ def _init_text_completion_model( def _init_community_chat_models( model_name: str, provider_name: str, kwargs: Dict[str, Any] -) -> BaseChatModel | None: +) -> BaseChatModel: """Initialize community chat models. Args: @@ -275,14 +270,14 @@ def _init_community_chat_models( """ provider_cls = _get_chat_completion_provider(provider_name) if provider_cls is None: - return None + raise ValueError() kwargs = _update_model_kwargs(provider_cls, model_name, kwargs) return provider_cls(**kwargs) def _init_gpt35_turbo_instruct( model_name: str, provider_name: str, kwargs: Dict[str, Any] -) -> BaseLLM | None: +) -> BaseLLM: """Initialize GPT-3.5 Turbo Instruct model. Currently init_chat_model from langchain infers this as a chat model. diff --git a/tests/llm_providers/test_langchain_initialization_methods.py b/tests/llm_providers/test_langchain_initialization_methods.py index 7bd5ff63c..14b6f0e0f 100644 --- a/tests/llm_providers/test_langchain_initialization_methods.py +++ b/tests/llm_providers/test_langchain_initialization_methods.py @@ -116,9 +116,8 @@ def test_init_community_chat_models_no_provider(self): "nemoguardrails.llm.models.langchain_initializer._get_chat_completion_provider" ) as mock_get_provider: mock_get_provider.return_value = None - assert ( - _init_community_chat_models("community-model", "provider", {}) is None - ) + with pytest.raises(ValueError): + _init_community_chat_models("community-model", "provider", {}) class TestTextCompletionInitializer: @@ -157,7 +156,8 @@ def test_init_text_completion_model_no_provider(self): "nemoguardrails.llm.models.langchain_initializer._get_text_completion_provider" ) as mock_get_provider: mock_get_provider.return_value = None - assert _init_text_completion_model("text-model", "provider", {}) is None + with pytest.raises(ValueError): + _init_text_completion_model("text-model", "provider", {}) class TestUpdateModelKwargs: diff --git a/tests/llm_providers/test_langchain_initializer.py b/tests/llm_providers/test_langchain_initializer.py index 070d3bb58..a8ff2df34 100644 --- a/tests/llm_providers/test_langchain_initializer.py +++ b/tests/llm_providers/test_langchain_initializer.py @@ -183,14 +183,14 @@ def test_text_completion_called_when_previous_fail(mock_initializers): mock_initializers["text"].assert_called_once() -def test_text_mode_only_calls_text_initializers(mock_initializers): - """Test that text mode only tries initializers that support text mode.""" +def test_text_completion_supports_chat_mode(mock_initializers): mock_initializers["special"].return_value = None + mock_initializers["chat"].return_value = None + mock_initializers["community"].return_value = None mock_initializers["text"].return_value = "text_model" - result = init_langchain_model("text-model", "provider", "text", {}) + result = init_langchain_model("text-model", "provider", "chat", {}) assert result == "text_model" mock_initializers["special"].assert_called_once() + mock_initializers["chat"].assert_called_once() + mock_initializers["community"].assert_called_once() mock_initializers["text"].assert_called_once() - # Since text returns a value, chat and community are not called - mock_initializers["chat"].assert_not_called() - mock_initializers["community"].assert_not_called() diff --git a/tests/test_actions_llm_utils.py b/tests/test_actions_llm_utils.py index 28f767a39..643f83644 100644 --- a/tests/test_actions_llm_utils.py +++ b/tests/test_actions_llm_utils.py @@ -91,24 +91,6 @@ class MockPatchedNVIDIA(MockNVIDIAOriginal): __module__ = "nemoguardrails.llm.providers._langchain_nvidia_ai_endpoints_patch" -class MockTRTLLM: - __module__ = "nemoguardrails.llm.providers.trtllm.llm" - - -class MockAzureLLM: - __module__ = "langchain_openai.chat_models" - - -class MockLLMWithClient: - __module__ = "langchain_openai.chat_models" - - class _MockClient: - base_url = "https://custom.endpoint.com/v1" - - def __init__(self): - self.client = self._MockClient() - - def test_infer_provider_openai(): llm = MockOpenAILLM() provider = _infer_provider_from_module(llm) From 5300be12e735f1aac2e2a7a2722acd820fab5447 Mon Sep 17 00:00:00 2001 From: Jash Gulabrai Date: Wed, 26 Nov 2025 09:02:47 -0500 Subject: [PATCH 14/14] Fix formatting --- nemoguardrails/actions/action_dispatcher.py | 1 - nemoguardrails/actions/llm/utils.py | 2 +- .../llm/models/langchain_initializer.py | 12 +--- nemoguardrails/rails/llm/config.py | 61 ++++++------------- nemoguardrails/rails/llm/llmrails.py | 24 +++----- tests/test_rails_config.py | 4 +- 6 files changed, 30 insertions(+), 74 deletions(-) diff --git a/nemoguardrails/actions/action_dispatcher.py b/nemoguardrails/actions/action_dispatcher.py index 6dc5df77f..e237f95e1 100644 --- a/nemoguardrails/actions/action_dispatcher.py +++ b/nemoguardrails/actions/action_dispatcher.py @@ -27,7 +27,6 @@ from nemoguardrails import utils from nemoguardrails.exceptions import LLMCallException -from nemoguardrails.logging.callbacks import logging_callbacks log = logging.getLogger(__name__) diff --git a/nemoguardrails/actions/llm/utils.py b/nemoguardrails/actions/llm/utils.py index e111eb6c7..747c0ec1c 100644 --- a/nemoguardrails/actions/llm/utils.py +++ b/nemoguardrails/actions/llm/utils.py @@ -15,7 +15,7 @@ import logging import re -from typing import Any, Dict, List, Optional, Sequence, Union +from typing import Dict, List, Optional, Sequence, Union from langchain_core.callbacks.base import AsyncCallbackHandler, BaseCallbackManager from langchain_core.language_models import BaseLanguageModel diff --git a/nemoguardrails/llm/models/langchain_initializer.py b/nemoguardrails/llm/models/langchain_initializer.py index ef4d2b057..6cb937d33 100644 --- a/nemoguardrails/llm/models/langchain_initializer.py +++ b/nemoguardrails/llm/models/langchain_initializer.py @@ -225,9 +225,7 @@ def _init_chat_completion_model(model_name: str, provider_name: str, kwargs: Dic raise -def _init_text_completion_model( - model_name: str, provider_name: str, kwargs: Dict[str, Any] -) -> BaseLLM: +def _init_text_completion_model(model_name: str, provider_name: str, kwargs: Dict[str, Any]) -> BaseLLM: """Initialize a text completion model. Args: @@ -251,9 +249,7 @@ def _init_text_completion_model( return provider_cls(**kwargs) -def _init_community_chat_models( - model_name: str, provider_name: str, kwargs: Dict[str, Any] -) -> BaseChatModel: +def _init_community_chat_models(model_name: str, provider_name: str, kwargs: Dict[str, Any]) -> BaseChatModel: """Initialize community chat models. Args: @@ -275,9 +271,7 @@ def _init_community_chat_models( return provider_cls(**kwargs) -def _init_gpt35_turbo_instruct( - model_name: str, provider_name: str, kwargs: Dict[str, Any] -) -> BaseLLM: +def _init_gpt35_turbo_instruct(model_name: str, provider_name: str, kwargs: Dict[str, Any]) -> BaseLLM: """Initialize GPT-3.5 Turbo Instruct model. Currently init_chat_model from langchain infers this as a chat model. diff --git a/nemoguardrails/rails/llm/config.py b/nemoguardrails/rails/llm/config.py index d9da7ceb1..c3909fafa 100644 --- a/nemoguardrails/rails/llm/config.py +++ b/nemoguardrails/rails/llm/config.py @@ -41,7 +41,6 @@ InvalidModelConfigurationError, InvalidRailsConfigurationError, ) -from nemoguardrails.llm.types import Task log = logging.getLogger(__name__) @@ -142,7 +141,7 @@ def set_and_validate_model(cls, data: Any) -> Any: if model_field and model_from_params: raise InvalidModelConfigurationError( - f"Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`).", + "Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`).", ) if not model_field and model_from_params: data["model"] = model_from_params @@ -157,7 +156,7 @@ def model_must_be_none_empty(self) -> "Model": """Validate that a model name is present either directly or in parameters.""" if not self.model or not self.model.strip(): raise InvalidModelConfigurationError( - f"Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`)." + "Model name must be specified in exactly one place: either the `model` field, or in `parameters` (`parameters.model` or `parameters.model_name`)." ) return self @@ -339,14 +338,10 @@ class TaskPrompt(BaseModel): @root_validator(pre=True, allow_reuse=True) def check_fields(cls, values): if not values.get("content") and not values.get("messages"): - raise InvalidRailsConfigurationError( - "One of `content` or `messages` must be provided." - ) + raise InvalidRailsConfigurationError("One of `content` or `messages` must be provided.") if values.get("content") and values.get("messages"): - raise InvalidRailsConfigurationError( - "Only one of `content` or `messages` must be provided." - ) + raise InvalidRailsConfigurationError("Only one of `content` or `messages` must be provided.") return values @@ -1424,11 +1419,7 @@ def check_model_exists_for_input_rails(cls, values): continue if flow_model not in model_types: flow_id = _normalize_flow_id(flow) - available_types = ( - ", ".join(f"'{str(t)}'" for t in sorted(model_types)) - if model_types - else "none" - ) + available_types = ", ".join(f"'{str(t)}'" for t in sorted(model_types)) if model_types else "none" raise InvalidRailsConfigurationError( f"Input flow '{flow_id}' references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}." ) @@ -1454,11 +1445,7 @@ def check_model_exists_for_output_rails(cls, values): continue if flow_model not in model_types: flow_id = _normalize_flow_id(flow) - available_types = ( - ", ".join(f"'{str(t)}'" for t in sorted(model_types)) - if model_types - else "none" - ) + available_types = ", ".join(f"'{str(t)}'" for t in sorted(model_types)) if model_types else "none" raise InvalidRailsConfigurationError( f"Output flow '{flow_id}' references model type '{flow_model}' that is not defined in the configuration. Detected model types: {available_types}." ) @@ -1474,19 +1461,13 @@ def check_prompt_exist_for_self_check_rails(cls, values): provided_task_prompts = [prompt.task if hasattr(prompt, "task") else prompt.get("task") for prompt in prompts] # Input moderation prompt verification - if ( - "self check input" in enabled_input_rails - and "self_check_input" not in provided_task_prompts - ): + if "self check input" in enabled_input_rails and "self_check_input" not in provided_task_prompts: raise InvalidRailsConfigurationError( - f"Missing a `self_check_input` prompt template, which is required for the `self check input` rail." + "Missing a `self_check_input` prompt template, which is required for the `self check input` rail." ) - if ( - "llama guard check input" in enabled_input_rails - and "llama_guard_check_input" not in provided_task_prompts - ): + if "llama guard check input" in enabled_input_rails and "llama_guard_check_input" not in provided_task_prompts: raise InvalidRailsConfigurationError( - f"Missing a `llama_guard_check_input` prompt template, which is required for the `llama guard check input` rail." + "Missing a `llama_guard_check_input` prompt template, which is required for the `llama guard check input` rail." ) # Only content-safety and topic-safety include a $model reference in the rail flow text @@ -1496,34 +1477,28 @@ def check_prompt_exist_for_self_check_rails(cls, values): _validate_rail_prompts(enabled_input_rails, provided_task_prompts, "topic safety check input") # Output moderation prompt verification - if ( - "self check output" in enabled_output_rails - and "self_check_output" not in provided_task_prompts - ): + if "self check output" in enabled_output_rails and "self_check_output" not in provided_task_prompts: raise InvalidRailsConfigurationError( - f"Missing a `self_check_output` prompt template, which is required for the `self check output` rail." + "Missing a `self_check_output` prompt template, which is required for the `self check output` rail." ) if ( "llama guard check output" in enabled_output_rails and "llama_guard_check_output" not in provided_task_prompts ): raise InvalidRailsConfigurationError( - f"Missing a `llama_guard_check_output` prompt template, which is required for the `llama guard check output` rail." + "Missing a `llama_guard_check_output` prompt template, which is required for the `llama guard check output` rail." ) if ( "patronus lynx check output hallucination" in enabled_output_rails and "patronus_lynx_check_output_hallucination" not in provided_task_prompts ): raise InvalidRailsConfigurationError( - f"Missing a `patronus_lynx_check_output_hallucination` prompt template, which is required for the `patronus lynx check output hallucination` rail." + "Missing a `patronus_lynx_check_output_hallucination` prompt template, which is required for the `patronus lynx check output hallucination` rail." ) - if ( - "self check facts" in enabled_output_rails - and "self_check_facts" not in provided_task_prompts - ): + if "self check facts" in enabled_output_rails and "self_check_facts" not in provided_task_prompts: raise InvalidRailsConfigurationError( - f"Missing a `self_check_facts` prompt template, which is required for the `self check facts` rail." + "Missing a `self_check_facts` prompt template, which is required for the `self check facts` rail." ) # Only content-safety and topic-safety include a $model reference in the rail flow text @@ -1577,9 +1552,7 @@ def validate_models_api_key_env_var(cls, models): api_keys = [m.api_key_env_var for m in models] for api_key in api_keys: if api_key and not os.environ.get(api_key): - raise InvalidRailsConfigurationError( - f"Model API Key environment variable '{api_key}' not set." - ) + raise InvalidRailsConfigurationError(f"Model API Key environment variable '{api_key}' not set.") return models raw_llm_call_action: Optional[str] = Field( diff --git a/nemoguardrails/rails/llm/llmrails.py b/nemoguardrails/rails/llm/llmrails.py index 7267a00c4..0833710fa 100644 --- a/nemoguardrails/rails/llm/llmrails.py +++ b/nemoguardrails/rails/llm/llmrails.py @@ -239,9 +239,7 @@ def __init__( ) # First, we initialize the runtime. - self.runtime = colang_version_to_runtime[config.colang_version]( - config=config, verbose=verbose - ) + self.runtime = colang_version_to_runtime[config.colang_version](config=config, verbose=verbose) # If we have a config_modules with an `init` function, we call it. # We need to call this here because the `init` might register additional @@ -327,22 +325,16 @@ def _validate_config(self): # content safety check input/output flows are special as they have parameters flow_name = _normalize_flow_id(flow_name) if flow_name not in existing_flows_names: - raise InvalidRailsConfigurationError( - f"The provided input rail flow `{flow_name}` does not exist" - ) + raise InvalidRailsConfigurationError(f"The provided input rail flow `{flow_name}` does not exist") for flow_name in self.config.rails.output.flows: flow_name = _normalize_flow_id(flow_name) if flow_name not in existing_flows_names: - raise InvalidRailsConfigurationError( - f"The provided output rail flow `{flow_name}` does not exist" - ) + raise InvalidRailsConfigurationError(f"The provided output rail flow `{flow_name}` does not exist") for flow_name in self.config.rails.retrieval.flows: if flow_name not in existing_flows_names: - raise InvalidRailsConfigurationError( - f"The provided retrieval rail flow `{flow_name}` does not exist" - ) + raise InvalidRailsConfigurationError(f"The provided retrieval rail flow `{flow_name}` does not exist") # If both passthrough mode and single call mode are specified, we raise an exception. if self.config.passthrough and self.config.rails.dialog.single_call.enabled: @@ -1198,10 +1190,10 @@ def _validate_streaming_with_output_rails(self) -> None: not self.config.rails.output.streaming or not self.config.rails.output.streaming.enabled ): raise InvalidRailsConfigurationError( - f"stream_async() cannot be used when output rails are configured but " - f"rails.output.streaming.enabled is False. Either set " - f"rails.output.streaming.enabled to True in your configuration, or use " - f"generate_async() instead of stream_async()." + "stream_async() cannot be used when output rails are configured but " + "rails.output.streaming.enabled is False. Either set " + "rails.output.streaming.enabled to True in your configuration, or use " + "generate_async() instead of stream_async()." ) @overload diff --git a/tests/test_rails_config.py b/tests/test_rails_config.py index e05e5855f..796011d82 100644 --- a/tests/test_rails_config.py +++ b/tests/test_rails_config.py @@ -91,9 +91,7 @@ def test_check_prompt_exist_for_self_check_rails(): # missings self_check_output prompt ], } - with pytest.raises( - ValueError, match="Missing a `self_check_output` prompt template" - ): + with pytest.raises(ValueError, match="Missing a `self_check_output` prompt template"): RailsConfig.check_prompt_exist_for_self_check_rails(values)