From 71d00f083fb59bda34c82b82eea85602c1710265 Mon Sep 17 00:00:00 2001 From: tgasser-nv <200644301+tgasser-nv@users.noreply.github.com> Date: Tue, 2 Sep 2025 11:17:40 -0500 Subject: [PATCH 1/4] Dummy commit to set up the chore/type-clean-guardrails PR and branch --- nemoguardrails/actions/llm/generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemoguardrails/actions/llm/generation.py b/nemoguardrails/actions/llm/generation.py index 2a57e1c26..cd11e70a7 100644 --- a/nemoguardrails/actions/llm/generation.py +++ b/nemoguardrails/actions/llm/generation.py @@ -137,7 +137,7 @@ async def init(self): self._init_flows_index(), ) - def _extract_user_message_example(self, flow: Flow): + def _extract_user_message_example(self, flow: Flow) -> None: """Heuristic to extract user message examples from a flow.""" elements = [ item From 0a9bcd7ce0a9a13a7077e0826420f641497bce19 Mon Sep 17 00:00:00 2001 From: tgasser-nv <200644301+tgasser-nv@users.noreply.github.com> Date: Wed, 10 Sep 2025 15:52:09 -0500 Subject: [PATCH 2/4] Checkin agent-generated type-fixes before reviewing and manual fixes --- nemoguardrails/context.py | 9 ++++- nemoguardrails/library/attention/actions.py | 28 ++++++++------ nemoguardrails/library/autoalign/actions.py | 38 +++++++++++++++---- nemoguardrails/library/clavata/actions.py | 7 ++-- nemoguardrails/library/cleanlab/actions.py | 6 ++- .../factchecking/align_score/actions.py | 3 ++ .../factchecking/align_score/server.py | 20 ++++++++-- nemoguardrails/library/fiddler/actions.py | 12 ++++++ .../library/gcp_moderate_text/actions.py | 9 +++-- .../library/guardrails_ai/actions.py | 28 +++++++++++--- .../library/guardrails_ai/errors.py | 2 +- .../library/guardrails_ai/registry.py | 4 +- .../library/hallucination/actions.py | 25 ++++++++++-- .../library/injection_detection/actions.py | 22 +++++++---- .../injection_detection/yara_config.py | 6 +-- .../library/jailbreak_detection/actions.py | 12 +++++- .../jailbreak_detection/heuristics/checks.py | 24 ++++++++---- .../jailbreak_detection/model_based/checks.py | 10 ++++- .../jailbreak_detection/model_based/models.py | 4 +- .../library/jailbreak_detection/request.py | 6 ++- .../library/jailbreak_detection/server.py | 8 ++-- nemoguardrails/library/llama_guard/actions.py | 10 +++++ nemoguardrails/library/patronusai/actions.py | 15 +++++++- .../library/self_check/facts/actions.py | 12 +++++- .../library/self_check/input_check/actions.py | 10 ++++- .../self_check/output_check/actions.py | 10 ++++- .../sensitive_data_detection/actions.py | 32 ++++++++++++---- .../library/topic_safety/actions.py | 15 +++++++- nemoguardrails/llm/taskmanager.py | 9 +++-- 29 files changed, 310 insertions(+), 86 deletions(-) diff --git a/nemoguardrails/context.py b/nemoguardrails/context.py index e66f1a0d5..a8bd625f6 100644 --- a/nemoguardrails/context.py +++ b/nemoguardrails/context.py @@ -14,7 +14,10 @@ # limitations under the License. import contextvars -from typing import Optional +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from nemoguardrails.logging.explain import LLMCallInfo streaming_handler_var = contextvars.ContextVar("streaming_handler", default=None) @@ -22,7 +25,9 @@ explain_info_var = contextvars.ContextVar("explain_info", default=None) # The current LLM call. -llm_call_info_var = contextvars.ContextVar("llm_call_info", default=None) +llm_call_info_var: contextvars.ContextVar[ + Optional["LLMCallInfo"] +] = contextvars.ContextVar("llm_call_info", default=None) # All the generation options applicable to the current context. generation_options_var = contextvars.ContextVar("generation_options", default=None) diff --git a/nemoguardrails/library/attention/actions.py b/nemoguardrails/library/attention/actions.py index 06ef2c304..89b079928 100644 --- a/nemoguardrails/library/attention/actions.py +++ b/nemoguardrails/library/attention/actions.py @@ -60,8 +60,10 @@ def _get_action_timestamp(action_event_name: str, event_args) -> Optional[dateti return None try: return read_isoformat(event_args[_mapping[action_event_name]]) - except Exception: - log_p(f"Could not parse timestamp {event_args[_mapping[action_event_name]]}") + except (ValueError, KeyError, TypeError) as e: + log_p( + f"Could not parse timestamp {event_args[_mapping[action_event_name]]}: {e}" + ) return None @@ -98,8 +100,8 @@ def __init__(self) -> None: self.user_is_talking = False self.sentence_distribution = {UNKNOWN_ATTENTION_STATE: 0.0} self.attention_events: list[ActionEvent] = [] - self.utterance_started_event = None - self.utterance_last_event = None + self.utterance_started_event: Optional[ActionEvent] = None + self.utterance_last_event: Optional[ActionEvent] = None def reset_view(self) -> None: """Reset the view. Removing all attention events except for the most recent one""" @@ -111,16 +113,17 @@ def update(self, event: ActionEvent, offsets: dict[str, float]) -> None: Args: event (ActionEvent): Action event to use for updating the view - offsets (dict[str, float]): You can provide static offsets in seconds for every event type to correct for known latencies of these events. + offsets (dict[str, float]): You can provide static offsets in seconds for every event type to + correct for known latencies of these events. """ # print(f"attention_events: {self.attention_events}") timestamp = _get_action_timestamp(event.name, event.arguments) if not timestamp: return - event.corrected_datetime = timestamp + timedelta( - seconds=offsets.get(event.name, 0.0) - ) + # Dynamically add corrected_datetime attribute to the event + corrected_time = timestamp + timedelta(seconds=offsets.get(event.name, 0.0)) + setattr(event, "corrected_datetime", corrected_time) if event.name == "UtteranceUserActionStarted": self.reset_view() @@ -144,7 +147,8 @@ def get_time_spent_percentage(self, attention_levels: list[str]) -> float: attention_levels (list[str]): List of attention level names to consider `attentive` Returns: - float: The percentage the user was in the attention levels provided. Returns 1.0 if no attention events have been registered. + float: The percentage the user was in the attention levels provided. Returns 1.0 if no + attention events have been registered. """ log_p(f"attention_events={self.attention_events}") @@ -194,7 +198,8 @@ def get_time_spent_percentage(self, attention_levels: list[str]) -> float: ) durations = compute_time_spent_in_states(state_changes) - # If the only state we observed during the duration of the utterance is UNKNOWN_ATTENTION_STATE we treat it as 1.0 + # If the only state we observed during the duration of the utterance is UNKNOWN_ATTENTION_STATE + # we treat it as 1.0 if len(durations) == 1 and UNKNOWN_ATTENTION_STATE in durations: return 1.0 @@ -238,6 +243,7 @@ async def get_attention_percentage_action(attention_levels: list[str]) -> float: attention_levels : Name of attention levels for which the user is considered to be `attentive` Returns: - float: The percentage the user was in the attention levels provided. Returns 1.0 if no attention events have been registered. + float: The percentage the user was in the attention levels provided. Returns 1.0 if no + attention events have been registered. """ return _attention_view.get_time_spent_percentage(attention_levels) diff --git a/nemoguardrails/library/autoalign/actions.py b/nemoguardrails/library/autoalign/actions.py index 57dd79e2a..f901b8d1b 100644 --- a/nemoguardrails/library/autoalign/actions.py +++ b/nemoguardrails/library/autoalign/actions.py @@ -151,7 +151,8 @@ def process_autoalign_output(responses: List[Any], show_toxic_phrases: bool = Fa response_dict["combined_response"] = ", ".join(prefixes) + " detected." if ( "toxicity_detection" in response_dict.keys() - and response_dict["toxicity_detection"]["guarded"] + and isinstance(response_dict["toxicity_detection"], dict) + and response_dict["toxicity_detection"].get("guarded", False) and show_toxic_phrases ): response_dict["combined_response"] += suffix @@ -173,11 +174,12 @@ async def autoalign_infer( headers = {"x-api-key": api_key} config = copy.deepcopy(DEFAULT_CONFIG) # enable the select guardrail - for task in task_config.keys(): - if task != "factcheck": - config[task]["mode"] = "DETECT" - if task_config[task]: - config[task].update(task_config[task]) + if task_config is not None: + for task in task_config.keys(): + if task != "factcheck" and isinstance(config.get(task), dict): + config[task]["mode"] = "DETECT" + if task_config[task] and isinstance(config.get(task), dict): + config[task].update(task_config[task]) request_body = {"prompt": text, "config": config, "multi_language": multi_language} guardrails_configured = [] @@ -287,7 +289,12 @@ async def autoalign_input_api( **kwargs, ): """Calls AutoAlign API for the user message and guardrail configuration provided""" + if context is None: + raise ValueError("Context is required") user_message = context.get("user_message") + if user_message is None: + raise ValueError("user_message is required in context") + autoalign_config = llm_task_manager.config.rails.config.autoalign autoalign_api_url = autoalign_config.parameters.get("endpoint") multi_language = autoalign_config.parameters.get("multi_language", False) @@ -327,7 +334,12 @@ async def autoalign_output_api( **kwargs, ): """Calls AutoAlign API for the bot message and guardrail configuration provided""" + if context is None: + raise ValueError("Context is required") bot_message = context.get("bot_message") + if bot_message is None: + raise ValueError("bot_message is required in context") + autoalign_config = llm_task_manager.config.rails.config.autoalign autoalign_api_url = autoalign_config.parameters.get("endpoint") multi_language = autoalign_config.parameters.get("multi_language", False) @@ -366,8 +378,12 @@ async def autoalign_groundedness_output_api( ): """Calls AutoAlign groundedness check API and checks whether the bot message is factually grounded according to given documents""" - + if context is None: + raise ValueError("Context is required") bot_message = context.get("bot_message") + if bot_message is None: + raise ValueError("bot_message is required in context") + documents = context.get("relevant_chunks_sep", []) autoalign_config = llm_task_manager.config.rails.config.autoalign @@ -404,9 +420,15 @@ async def autoalign_factcheck_output_api( show_autoalign_message: bool = True, ): """Calls Autoalign Factchecker API and checks if the user message is factually answered by the bot message""" - + if context is None: + raise ValueError("Context is required") user_message = context.get("user_message") + if user_message is None: + raise ValueError("user_message is required in context") bot_message = context.get("bot_message") + if bot_message is None: + raise ValueError("bot_message is required in context") + autoalign_config = llm_task_manager.config.rails.config.autoalign autoalign_factcheck_api_url = autoalign_config.parameters.get("fact_check_endpoint") multi_language = autoalign_config.parameters.get("multi_language", False) diff --git a/nemoguardrails/library/clavata/actions.py b/nemoguardrails/library/clavata/actions.py index 3f5aec6c6..f51adc055 100644 --- a/nemoguardrails/library/clavata/actions.py +++ b/nemoguardrails/library/clavata/actions.py @@ -152,10 +152,11 @@ def get_policy_id( pass # Not a valid UUID, try to match the provided alias to a policy ID and return that + policy_id = config.policies.get(policy) + if policy_id is None: + raise ClavataPluginValueError(f"Policy with alias '{policy}' not found.") + try: - policy_id = config.policies.get(policy) - if policy_id is None: - raise ClavataPluginValueError(f"Policy with alias '{policy}' not found.") return uuid.UUID(policy_id) except ValueError as e: # Specifically catch the ValueError for badly formed UUIDs so we can provide a more helpful error message diff --git a/nemoguardrails/library/cleanlab/actions.py b/nemoguardrails/library/cleanlab/actions.py index a7f95cb8d..5d61f5242 100644 --- a/nemoguardrails/library/cleanlab/actions.py +++ b/nemoguardrails/library/cleanlab/actions.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import json import logging import os from typing import Dict, Optional, Union @@ -38,13 +37,16 @@ async def call_cleanlab_api( context: Optional[dict] = None, **kwargs, ) -> Union[ValueError, ImportError, Dict]: + if context is None: + raise ValueError("Context is required") + api_key = os.environ.get("CLEANLAB_API_KEY") if api_key is None: raise ValueError("CLEANLAB_API_KEY environment variable not set.") try: - from cleanlab_studio import Studio + from cleanlab_studio import Studio # type: ignore except ImportError: raise ImportError( "Please install cleanlab-studio using 'pip install --upgrade cleanlab-studio' command" diff --git a/nemoguardrails/library/factchecking/align_score/actions.py b/nemoguardrails/library/factchecking/align_score/actions.py index b2650cc9a..ec0216f83 100644 --- a/nemoguardrails/library/factchecking/align_score/actions.py +++ b/nemoguardrails/library/factchecking/align_score/actions.py @@ -47,6 +47,9 @@ async def alignscore_check_facts( **kwargs, ): """Checks the facts for the bot response using an information alignment score.""" + if context is None: + raise ValueError("Context is required") + fact_checking_config = llm_task_manager.config.rails.config.fact_checking fallback_to_self_check = fact_checking_config.fallback_to_self_check diff --git a/nemoguardrails/library/factchecking/align_score/server.py b/nemoguardrails/library/factchecking/align_score/server.py index 18aacbed0..f7961b93c 100644 --- a/nemoguardrails/library/factchecking/align_score/server.py +++ b/nemoguardrails/library/factchecking/align_score/server.py @@ -17,15 +17,24 @@ from functools import lru_cache from typing import List -import nltk +try: + import nltk # type: ignore +except ImportError: + nltk = None + +try: + from alignscore import AlignScore # type: ignore +except ImportError: + AlignScore = None + import typer import uvicorn -from alignscore import AlignScore from fastapi import FastAPI from pydantic import BaseModel # Make sure we have the punkt tokenizer downloaded. -nltk.download("punkt") +if nltk is not None: + nltk.download("punkt") models_path = os.environ.get("ALIGN_SCORE_PATH") @@ -47,6 +56,11 @@ def get_model(model: str): Args model: The type of the model to be loaded, i.e. "base", "large". """ + if models_path is None: + raise ValueError("ALIGN_SCORE_PATH environment variable not set") + if AlignScore is None: + raise ImportError("alignscore package not available") + return AlignScore( model="roberta-base", batch_size=32, diff --git a/nemoguardrails/library/fiddler/actions.py b/nemoguardrails/library/fiddler/actions.py index 728a3748d..67c87dd1e 100644 --- a/nemoguardrails/library/fiddler/actions.py +++ b/nemoguardrails/library/fiddler/actions.py @@ -86,6 +86,10 @@ async def call_fiddler_guardrail( @action(name="call fiddler safety on user message", is_system_action=True) async def call_fiddler_safety_user(config: RailsConfig, context: Optional[dict] = None): + if context is None: + log.error("Context is required for Fiddler Jailbreak Guardrails") + return False + fiddler_config: FiddlerGuardrails = getattr(config.rails.config, "fiddler") base_url = fiddler_config.fiddler_endpoint @@ -114,6 +118,10 @@ async def call_fiddler_safety_user(config: RailsConfig, context: Optional[dict] @action(name="call fiddler safety on bot message", is_system_action=True) async def call_fiddler_safety_bot(config: RailsConfig, context: Optional[dict] = None): + if context is None: + log.error("Context is required for Fiddler Safety Guardrails") + return False + fiddler_config: FiddlerGuardrails = getattr(config.rails.config, "fiddler") base_url = fiddler_config.fiddler_endpoint @@ -144,6 +152,10 @@ async def call_fiddler_safety_bot(config: RailsConfig, context: Optional[dict] = async def call_fiddler_faithfulness( config: RailsConfig, context: Optional[dict] = None ): + if context is None: + log.error("Context is required for Fiddler Faithfulness Guardrails") + return False + fiddler_config: FiddlerGuardrails = getattr(config.rails.config, "fiddler") base_url = fiddler_config.fiddler_endpoint diff --git a/nemoguardrails/library/gcp_moderate_text/actions.py b/nemoguardrails/library/gcp_moderate_text/actions.py index afb7004f0..350d40c68 100644 --- a/nemoguardrails/library/gcp_moderate_text/actions.py +++ b/nemoguardrails/library/gcp_moderate_text/actions.py @@ -17,10 +17,10 @@ from typing import Optional try: - from google.cloud import language_v2 + from google.cloud import language_v2 # type: ignore except ImportError: # The exception about installing google-cloud-language will be on the first call to the moderation api - pass + language_v2 = None from nemoguardrails.actions import action @@ -115,8 +115,11 @@ async def call_gcp_text_moderation_api( For more information check https://cloud.google.com/docs/authentication/application-default-credentials """ + if context is None: + raise ValueError("Context is required") + try: - from google.cloud import language_v2 + from google.cloud import language_v2 # type: ignore except ImportError: raise ImportError( diff --git a/nemoguardrails/library/guardrails_ai/actions.py b/nemoguardrails/library/guardrails_ai/actions.py index 12fe0b93d..638da40bc 100644 --- a/nemoguardrails/library/guardrails_ai/actions.py +++ b/nemoguardrails/library/guardrails_ai/actions.py @@ -21,10 +21,10 @@ from typing import Any, Dict, Optional, Type try: - from guardrails import Guard + from guardrails import Guard # type: ignore except ImportError: # Mock Guard class for when guardrails is not available - class Guard: + class Guard: # type: ignore def __init__(self): pass @@ -110,11 +110,18 @@ def validate_guardrails_ai_input( Dict with validation_result """ - text = text or context.get("user_message", "") + text = text or (context.get("user_message", "") if context else "") if not text: raise ValueError("Either 'text' or 'context' must be provided.") - validator_config = config.rails.config.guardrails_ai.get_validator_config(validator) + guardrails_ai_config = config.rails.config.guardrails_ai + if guardrails_ai_config is None: + raise ValueError("Guardrails AI config is not configured") + + validator_config = guardrails_ai_config.get_validator_config(validator) + if validator_config is None: + raise ValueError(f"Validator config for '{validator}' not found") + parameters = validator_config.parameters or {} metadata = validator_config.metadata or {} @@ -149,11 +156,20 @@ def validate_guardrails_ai_output( Dict with validation_result """ - text = text or context.get("bot_message", "") + text = text or (context.get("bot_message", "") if context else "") if not text: raise ValueError("Either 'text' or 'context' must be provided.") + if config is None: + raise ValueError("Config is required") + + guardrails_ai_config = config.rails.config.guardrails_ai + if guardrails_ai_config is None: + raise ValueError("Guardrails AI config is not configured") + + validator_config = guardrails_ai_config.get_validator_config(validator) + if validator_config is None: + raise ValueError(f"Validator config for '{validator}' not found") - validator_config = config.rails.config.guardrails_ai.get_validator_config(validator) parameters = validator_config.parameters or {} metadata = validator_config.metadata or {} diff --git a/nemoguardrails/library/guardrails_ai/errors.py b/nemoguardrails/library/guardrails_ai/errors.py index 4615814ec..3b192b2cd 100644 --- a/nemoguardrails/library/guardrails_ai/errors.py +++ b/nemoguardrails/library/guardrails_ai/errors.py @@ -14,7 +14,7 @@ # limitations under the License. try: - from guardrails.errors import ValidationError + from guardrails.errors import ValidationError # type: ignore GuardrailsAIValidationError = ValidationError except ImportError: diff --git a/nemoguardrails/library/guardrails_ai/registry.py b/nemoguardrails/library/guardrails_ai/registry.py index 0aef9fcd0..a943eaeac 100644 --- a/nemoguardrails/library/guardrails_ai/registry.py +++ b/nemoguardrails/library/guardrails_ai/registry.py @@ -104,7 +104,9 @@ def get_validator_info(validator_path: str) -> Dict[str, str]: # not in registry, try to fetch from hub try: try: - from guardrails.hub.validator_package_service import get_validator_manifest + from guardrails.hub.validator_package_service import ( # type: ignore + get_validator_manifest, + ) except ImportError: raise GuardrailsAIConfigError( "Could not import get_validator_manifest. " diff --git a/nemoguardrails/library/hallucination/actions.py b/nemoguardrails/library/hallucination/actions.py index 778f43f51..de00e546a 100644 --- a/nemoguardrails/library/hallucination/actions.py +++ b/nemoguardrails/library/hallucination/actions.py @@ -16,7 +16,15 @@ import logging from typing import Optional -from langchain.chains import LLMChain +try: + from langchain.chains import LLMChain # type: ignore +except ImportError: + try: + from langchain_core.chains import LLMChain # type: ignore + except ImportError: + # Fallback if LLMChain is not available + LLMChain = None + from langchain.prompts import PromptTemplate from langchain_core.language_models.llms import BaseLLM @@ -52,8 +60,11 @@ async def self_check_hallucination( :return: True if hallucination is detected, False otherwise. """ + if context is None: + raise ValueError("Context is required") + try: - from langchain_openai import OpenAI + from langchain_openai import OpenAI # type: ignore except ImportError: log.warning( "The langchain_openai module is not installed. Please install it using pip: pip install langchain_openai" @@ -81,6 +92,12 @@ async def self_check_hallucination( return False # Use the "generate" call from langchain to get all completions in the same response. + if LLMChain is None: + log.warning( + "LLMChain is not available. Cannot perform hallucination check." + ) + return False + last_bot_prompt = PromptTemplate(template="{text}", input_variables=["text"]) chain = LLMChain(prompt=last_bot_prompt, llm=llm) @@ -131,7 +148,9 @@ async def self_check_hallucination( llm_call_info_var.set(LLMCallInfo(task=Task.SELF_CHECK_HALLUCINATION.value)) stop = llm_task_manager.get_stop_tokens(task=Task.SELF_CHECK_HALLUCINATION) - with llm_params(llm, temperature=config.lowest_temperature): + with llm_params( + llm, temperature=config.lowest_temperature if config else 0.1 + ): agreement = await llm_call(llm, prompt, stop=stop) agreement = agreement.lower().strip() diff --git a/nemoguardrails/library/injection_detection/actions.py b/nemoguardrails/library/injection_detection/actions.py index 7a85e2993..03a8f337e 100644 --- a/nemoguardrails/library/injection_detection/actions.py +++ b/nemoguardrails/library/injection_detection/actions.py @@ -32,13 +32,16 @@ import re from functools import lru_cache from pathlib import Path -from typing import Dict, List, Optional, Tuple, TypedDict, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, TypedDict, Union -yara = None -try: +if TYPE_CHECKING: import yara -except ImportError: - pass +else: + yara = None + try: + import yara + except ImportError: + pass from nemoguardrails import RailsConfig from nemoguardrails.actions import action @@ -115,7 +118,7 @@ def _validate_injection_config(config: RailsConfig) -> None: def _extract_injection_config( config: RailsConfig, -) -> Tuple[str, Path, Tuple[str], Optional[Dict[str, str]]]: +) -> Tuple[str, Path, Tuple[str, ...], Optional[Dict[str, str]]]: """ Extracts and processes the injection detection configuration values. @@ -130,6 +133,8 @@ def _extract_injection_config( ValueError: If the injection rules contain invalid elements. """ command_injection_config = config.rails.config.injection_detection + if command_injection_config is None: + raise ValueError("Injection detection config is not configured") yara_rules = command_injection_config.yara_rules # Set yara_path @@ -188,6 +193,9 @@ def _load_rules( ) return None + if yara is None: + return None + try: if yara_rules: rules_source = { @@ -202,7 +210,7 @@ def _load_rules( for rule_name in rule_names } rules = yara.compile(filepaths=rules_to_load) - except yara.SyntaxError as e: + except Exception as e: # yara.SyntaxError when yara is available msg = f"Failed to initialize injection detection due to configuration or YARA rule error: YARA compilation failed: {e}" log.error(msg) return None diff --git a/nemoguardrails/library/injection_detection/yara_config.py b/nemoguardrails/library/injection_detection/yara_config.py index 9e9dfd2d3..5b6e7abcd 100644 --- a/nemoguardrails/library/injection_detection/yara_config.py +++ b/nemoguardrails/library/injection_detection/yara_config.py @@ -41,13 +41,13 @@ def __contains__(cls, item): return True def __repr__(cls): - return ", ".join([member.value for member in list(cls)]) + return ", ".join([str(member) for member in list(cls)]) def __le__(cls, other): if isinstance(other, list): other = set(other) if isinstance(other, set): - values = {member.value for member in list(cls)} + values = {str(member) for member in list(cls)} return values <= other else: raise TypeError( @@ -58,7 +58,7 @@ def __ge__(cls, other): if isinstance(other, list): other = set(other) if isinstance(other, set): - values = {member.value for member in list(cls)} + values = {str(member) for member in list(cls)} return values >= other else: raise TypeError( diff --git a/nemoguardrails/library/jailbreak_detection/actions.py b/nemoguardrails/library/jailbreak_detection/actions.py index 223226b72..15b4989cd 100644 --- a/nemoguardrails/library/jailbreak_detection/actions.py +++ b/nemoguardrails/library/jailbreak_detection/actions.py @@ -50,13 +50,20 @@ async def jailbreak_detection_heuristics( **kwargs, ) -> bool: """Checks the user's prompt to determine if it is attempt to jailbreak the model.""" + if context is None: + return False + jailbreak_config = llm_task_manager.config.rails.config.jailbreak_detection + if jailbreak_config is None: + return False jailbreak_api_url = jailbreak_config.server_endpoint lp_threshold = jailbreak_config.length_per_perplexity_threshold ps_ppl_threshold = jailbreak_config.prefix_suffix_perplexity_threshold prompt = context.get("user_message") + if prompt is None: + return False if not jailbreak_api_url: from nemoguardrails.library.jailbreak_detection.heuristics.checks import ( @@ -93,6 +100,8 @@ async def jailbreak_detection_model( """Uses a trained classifier to determine if a user input is a jailbreak attempt""" prompt: str = "" jailbreak_config = llm_task_manager.config.rails.config.jailbreak_detection + if jailbreak_config is None: + return False jailbreak_api_url = jailbreak_config.server_endpoint nim_base_url = jailbreak_config.nim_base_url @@ -125,7 +134,8 @@ async def jailbreak_detection_model( ) return False - if nim_base_url: + jailbreak = None + if nim_base_url and nim_classification_path: jailbreak = await jailbreak_nim_request( prompt=prompt, nim_url=nim_base_url, diff --git a/nemoguardrails/library/jailbreak_detection/heuristics/checks.py b/nemoguardrails/library/jailbreak_detection/heuristics/checks.py index 188b89387..146dc3cbd 100644 --- a/nemoguardrails/library/jailbreak_detection/heuristics/checks.py +++ b/nemoguardrails/library/jailbreak_detection/heuristics/checks.py @@ -15,13 +15,20 @@ import os -import torch -from transformers import GPT2LMHeadModel, GPT2TokenizerFast - -device = os.environ.get("JAILBREAK_CHECK_DEVICE", "cpu") -model_id = "gpt2-large" -model = GPT2LMHeadModel.from_pretrained(model_id).to(device) -tokenizer = GPT2TokenizerFast.from_pretrained(model_id) +try: + import torch # type: ignore + from transformers import GPT2LMHeadModel, GPT2TokenizerFast # type: ignore + + device = os.environ.get("JAILBREAK_CHECK_DEVICE", "cpu") + model_id = "gpt2-large" + model = GPT2LMHeadModel.from_pretrained(model_id).to(device) + tokenizer = GPT2TokenizerFast.from_pretrained(model_id) +except ImportError: + torch = None + GPT2LMHeadModel = None + GPT2TokenizerFast = None + model = None + tokenizer = None def get_perplexity(input_string: str) -> bool: @@ -31,6 +38,9 @@ def get_perplexity(input_string: str) -> bool: Args input_string: The prompt to be sent to the model """ + if tokenizer is None or model is None or torch is None: + return False + encodings = tokenizer(input_string, return_tensors="pt") max_length = model.config.n_positions diff --git a/nemoguardrails/library/jailbreak_detection/model_based/checks.py b/nemoguardrails/library/jailbreak_detection/model_based/checks.py index b59bfa1e1..36597ef3e 100644 --- a/nemoguardrails/library/jailbreak_detection/model_based/checks.py +++ b/nemoguardrails/library/jailbreak_detection/model_based/checks.py @@ -17,7 +17,15 @@ import os from functools import lru_cache from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING, Union + +if TYPE_CHECKING: + from .models import JailbreakClassifier +else: + try: + from .models import JailbreakClassifier + except ImportError: + JailbreakClassifier = None logger = logging.getLogger(__name__) diff --git a/nemoguardrails/library/jailbreak_detection/model_based/models.py b/nemoguardrails/library/jailbreak_detection/model_based/models.py index 80dc23a5c..5ea85fa85 100644 --- a/nemoguardrails/library/jailbreak_detection/model_based/models.py +++ b/nemoguardrails/library/jailbreak_detection/model_based/models.py @@ -20,8 +20,8 @@ class SnowflakeEmbed: def __init__(self): - import torch - from transformers import AutoModel, AutoTokenizer + import torch # type: ignore + from transformers import AutoModel, AutoTokenizer # type: ignore self.device = "cuda:0" if torch.cuda.is_available() else "cpu" self.tokenizer = AutoTokenizer.from_pretrained( diff --git a/nemoguardrails/library/jailbreak_detection/request.py b/nemoguardrails/library/jailbreak_detection/request.py index 64d5a0b1a..4ebf19739 100644 --- a/nemoguardrails/library/jailbreak_detection/request.py +++ b/nemoguardrails/library/jailbreak_detection/request.py @@ -33,6 +33,7 @@ from typing import Optional import aiohttp +from aiohttp import ClientTimeout log = logging.getLogger(__name__) @@ -115,7 +116,10 @@ async def jailbreak_nim_request( if nim_auth_token is not None: headers["Authorization"] = f"Bearer {nim_auth_token}" async with session.post( - endpoint, json=payload, headers=headers, timeout=30 + endpoint, + json=payload, + headers=headers, + timeout=ClientTimeout(total=30), ) as resp: if resp.status != 200: log.error( diff --git a/nemoguardrails/library/jailbreak_detection/server.py b/nemoguardrails/library/jailbreak_detection/server.py index e956c0deb..86a652afc 100644 --- a/nemoguardrails/library/jailbreak_detection/server.py +++ b/nemoguardrails/library/jailbreak_detection/server.py @@ -80,14 +80,14 @@ def hello_world(): @app.post("/jailbreak_lp_heuristic") def lp_heuristic_check(request: JailbreakHeuristicRequest): return hc.check_jailbreak_length_per_perplexity( - request.prompt, request.lp_threshold + request.prompt, request.lp_threshold or 89.79 ) @app.post("/jailbreak_ps_heuristic") def ps_ppl_heuristic_check(request: JailbreakHeuristicRequest): return hc.check_jailbreak_prefix_suffix_perplexity( - request.prompt, request.ps_ppl_threshold + request.prompt, request.ps_ppl_threshold or 1845.65 ) @@ -95,10 +95,10 @@ def ps_ppl_heuristic_check(request: JailbreakHeuristicRequest): def run_all_heuristics(request: JailbreakHeuristicRequest): # Will add other heuristics as they become available lp_check = hc.check_jailbreak_length_per_perplexity( - request.prompt, request.lp_threshold + request.prompt, request.lp_threshold or 89.79 ) ps_ppl_check = hc.check_jailbreak_prefix_suffix_perplexity( - request.prompt, request.ps_ppl_threshold + request.prompt, request.ps_ppl_threshold or 1845.65 ) jailbreak = any([lp_check["jailbreak"], ps_ppl_check["jailbreak"]]) heuristic_checks = { diff --git a/nemoguardrails/library/llama_guard/actions.py b/nemoguardrails/library/llama_guard/actions.py index 23502be05..07bc49d62 100644 --- a/nemoguardrails/library/llama_guard/actions.py +++ b/nemoguardrails/library/llama_guard/actions.py @@ -64,6 +64,11 @@ async def llama_guard_check_input( Checks user messages using the configured Llama Guard model and the configured prompt containing the safety guidelines. """ + if context is None: + raise ValueError("Context is required") + if llama_guard_llm is None: + raise ValueError("llama_guard_llm is required") + user_input = context.get("user_message") check_input_prompt = llm_task_manager.render_task_prompt( task=Task.LLAMA_GUARD_CHECK_INPUT, @@ -109,6 +114,11 @@ async def llama_guard_check_output( Check the bot response using the configured Llama Guard model and the configured prompt containing the safety guidelines. """ + if context is None: + raise ValueError("Context is required") + if llama_guard_llm is None: + raise ValueError("llama_guard_llm is required") + user_input = context.get("user_message") bot_response = context.get("bot_message") diff --git a/nemoguardrails/library/patronusai/actions.py b/nemoguardrails/library/patronusai/actions.py index 19fe128ea..85d18e32a 100644 --- a/nemoguardrails/library/patronusai/actions.py +++ b/nemoguardrails/library/patronusai/actions.py @@ -83,6 +83,11 @@ async def patronus_lynx_check_output_hallucination( Check the bot response for hallucinations based on the given chunks using the configured Patronus Lynx model. """ + if context is None: + raise ValueError("Context is required") + if patronus_lynx_llm is None: + raise ValueError("patronus_lynx_llm is required") + user_input = context.get("user_message") bot_response = context.get("bot_message") provided_context = context.get("relevant_chunks") @@ -253,11 +258,19 @@ async def patronus_api_check_output( Check the user message, bot response, and/or provided context for issues based on the Patronus Evaluate API """ + if context is None: + raise ValueError("Context is required") + user_input = context.get("user_message") bot_response = context.get("bot_message") provided_context = context.get("relevant_chunks") - patronus_config = llm_task_manager.config.rails.config.patronus.output + patronus = llm_task_manager.config.rails.config.patronus + if patronus is None: + raise ValueError("Patronus config is not configured") + patronus_config = patronus.output + if patronus_config is None: + raise ValueError("Patronus output config is not configured") evaluate_config = getattr(patronus_config, "evaluate_config", {}) success_strategy: Literal["all_pass", "any_pass"] = getattr( evaluate_config, "success_strategy", "all_pass" diff --git a/nemoguardrails/library/self_check/facts/actions.py b/nemoguardrails/library/self_check/facts/actions.py index 91e1ad08b..1d57ea53a 100644 --- a/nemoguardrails/library/self_check/facts/actions.py +++ b/nemoguardrails/library/self_check/facts/actions.py @@ -50,6 +50,9 @@ async def self_check_facts( **kwargs, ): """Checks the facts for the bot response by appropriately prompting the base llm.""" + if context is None: + return False + _MAX_TOKENS = 3 evidence = context.get("relevant_chunks", []) response = context.get("bot_message") @@ -72,7 +75,14 @@ async def self_check_facts( # Initialize the LLMCallInfo object llm_call_info_var.set(LLMCallInfo(task=task.value)) - with llm_params(llm, temperature=config.lowest_temperature, max_tokens=max_tokens): + if llm is None: + return False + + with llm_params( + llm, + temperature=config.lowest_temperature if config else 0.1, + max_tokens=max_tokens, + ): response = await llm_call(llm, prompt, stop=stop) if llm_task_manager.has_output_parser(task): diff --git a/nemoguardrails/library/self_check/input_check/actions.py b/nemoguardrails/library/self_check/input_check/actions.py index 95dc36d67..144de24ec 100644 --- a/nemoguardrails/library/self_check/input_check/actions.py +++ b/nemoguardrails/library/self_check/input_check/actions.py @@ -48,6 +48,9 @@ async def self_check_input( True if the input should be allowed, False otherwise. """ + if context is None: + return False + _MAX_TOKENS = 3 user_input = context.get("user_message") task = Task.SELF_CHECK_INPUT @@ -66,8 +69,13 @@ async def self_check_input( # Initialize the LLMCallInfo object llm_call_info_var.set(LLMCallInfo(task=task.value)) + if llm is None: + return False + with llm_params( - llm, temperature=config.lowest_temperature, max_tokens=max_tokens + llm, + temperature=config.lowest_temperature if config else 0.1, + max_tokens=max_tokens, ): response = await llm_call(llm, prompt, stop=stop) diff --git a/nemoguardrails/library/self_check/output_check/actions.py b/nemoguardrails/library/self_check/output_check/actions.py index 20318b036..c6d34787c 100644 --- a/nemoguardrails/library/self_check/output_check/actions.py +++ b/nemoguardrails/library/self_check/output_check/actions.py @@ -50,6 +50,9 @@ async def self_check_output( True if the output should be allowed, False otherwise. """ + if context is None: + return False + _MAX_TOKENS = 3 bot_response = context.get("bot_message") user_input = context.get("user_message") @@ -71,8 +74,13 @@ async def self_check_output( # Initialize the LLMCallInfo object llm_call_info_var.set(LLMCallInfo(task=task.value)) + if llm is None: + return False + with llm_params( - llm, temperature=config.lowest_temperature, max_tokens=max_tokens + llm, + temperature=config.lowest_temperature if config else 0.1, + max_tokens=max_tokens, ): response = await llm_call(llm, prompt, stop=stop) diff --git a/nemoguardrails/library/sensitive_data_detection/actions.py b/nemoguardrails/library/sensitive_data_detection/actions.py index 8bd6748da..040a52b97 100644 --- a/nemoguardrails/library/sensitive_data_detection/actions.py +++ b/nemoguardrails/library/sensitive_data_detection/actions.py @@ -17,13 +17,15 @@ from functools import lru_cache try: - from presidio_analyzer import PatternRecognizer - from presidio_analyzer.nlp_engine import NlpEngineProvider - from presidio_anonymizer import AnonymizerEngine - from presidio_anonymizer.entities import OperatorConfig + from presidio_analyzer import PatternRecognizer # type: ignore + from presidio_analyzer.nlp_engine import NlpEngineProvider # type: ignore + from presidio_anonymizer import AnonymizerEngine # type: ignore + from presidio_anonymizer.entities import OperatorConfig # type: ignore except ImportError: - # The exception about installing presidio will be on the first call to the analyzer - pass + PatternRecognizer = None + NlpEngineProvider = None + AnonymizerEngine = None + OperatorConfig = None from nemoguardrails import RailsConfig from nemoguardrails.actions import action @@ -40,7 +42,7 @@ def _get_analyzer(score_threshold: float = 0.4): if not 0.0 <= score_threshold <= 1.0: raise ValueError("score_threshold must be a float between 0 and 1 (inclusive).") try: - from presidio_analyzer import AnalyzerEngine + from presidio_analyzer import AnalyzerEngine # type: ignore except ImportError: raise ImportError( @@ -49,7 +51,7 @@ def _get_analyzer(score_threshold: float = 0.4): ) try: - import spacy + import spacy # type: ignore except ImportError: raise RuntimeError( "The spacy module is not installed. Please install it using pip: pip install spacy." @@ -68,6 +70,8 @@ def _get_analyzer(score_threshold: float = 0.4): } # Create NLP engine based on configuration + if NlpEngineProvider is None: + raise ImportError("NlpEngineProvider not available") provider = NlpEngineProvider(nlp_configuration=configuration) nlp_engine = provider.create_engine() @@ -79,6 +83,8 @@ def _get_analyzer(score_threshold: float = 0.4): def _get_ad_hoc_recognizers(sdd_config: SensitiveDataDetection): """Helper to compute the ad hoc recognizers for a config.""" + if PatternRecognizer is None: + return [] ad_hoc_recognizers = [] for recognizer in sdd_config.recognizers: ad_hoc_recognizers.append(PatternRecognizer.from_dict(recognizer)) @@ -114,6 +120,8 @@ async def detect_sensitive_data( """ # Based on the source of the data, we use the right options sdd_config = config.rails.config.sensitive_data_detection + if sdd_config is None: + return False if source not in ["input", "output", "retrieval"]: raise ValueError("source must be one of 'input', 'output', or 'retrieval'") options: SensitiveDataDetectionOptions = getattr(sdd_config, source) @@ -152,6 +160,8 @@ async def mask_sensitive_data(source: str, text: str, config: RailsConfig): """ # Based on the source of the data, we use the right options sdd_config = config.rails.config.sensitive_data_detection + if sdd_config is None: + return text assert source in ["input", "output", "retrieval"] options: SensitiveDataDetectionOptions = getattr(sdd_config, source) @@ -159,6 +169,9 @@ async def mask_sensitive_data(source: str, text: str, config: RailsConfig): if len(options.entities) == 0: return text + if OperatorConfig is None: + return text + analyzer = _get_analyzer() operators = {} for entity in options.entities: @@ -170,6 +183,9 @@ async def mask_sensitive_data(source: str, text: str, config: RailsConfig): entities=options.entities, ad_hoc_recognizers=_get_ad_hoc_recognizers(sdd_config), ) + if AnonymizerEngine is None: + return text + anonymizer = AnonymizerEngine() masked_results = anonymizer.anonymize( text=text, analyzer_results=results, operators=operators diff --git a/nemoguardrails/library/topic_safety/actions.py b/nemoguardrails/library/topic_safety/actions.py index 7e2fb6dc2..599c72db9 100644 --- a/nemoguardrails/library/topic_safety/actions.py +++ b/nemoguardrails/library/topic_safety/actions.py @@ -40,6 +40,7 @@ async def topic_safety_check_input( ) -> dict: _MAX_TOKENS = 10 user_input: str = "" + conversation_history = [] if context is not None: user_input = context.get("user_message", "") @@ -49,7 +50,11 @@ async def topic_safety_check_input( # convert InternalEvent objects to dictionary format for compatibility with to_chat_messages dict_events = [] for event in events: - if hasattr(event, "name") and hasattr(event, "arguments"): + if ( + not isinstance(event, dict) + and hasattr(event, "name") + and hasattr(event, "arguments") + ): dict_event = {"type": event.name} dict_event.update(event.arguments) dict_events.append(dict_event) @@ -77,10 +82,16 @@ async def topic_safety_check_input( task = f"topic_safety_check_input $model={model_name}" - system_prompt = llm_task_manager.render_task_prompt( + system_prompt_result = llm_task_manager.render_task_prompt( task=task, ) + # Ensure we have a string + if isinstance(system_prompt_result, list): + system_prompt = "\n".join(str(item) for item in system_prompt_result) + else: + system_prompt = str(system_prompt_result) + TOPIC_SAFETY_OUTPUT_RESTRICTION = ( 'If any of the above conditions are violated, please respond with "off-topic". ' 'Otherwise, respond with "on-topic". ' diff --git a/nemoguardrails/llm/taskmanager.py b/nemoguardrails/llm/taskmanager.py index 3651676db..f393456e2 100644 --- a/nemoguardrails/llm/taskmanager.py +++ b/nemoguardrails/llm/taskmanager.py @@ -425,12 +425,15 @@ def render_task_prompt( return task_messages def parse_task_output( - self, task: Task, output: str, forced_output_parser: Optional[str] = None + self, + task: Union[str, Task], + output: str, + forced_output_parser: Optional[str] = None, ) -> ParsedTaskOutput: """Parses the output of a task, optionally extracting reasoning traces. Args: - task (Task): The task for which the output is being parsed. + task (Union[str, Task]): The task for which the output is being parsed. output (str): The output string to be parsed. forced_output_parser (Optional[str]): An optional parser name to force @@ -471,7 +474,7 @@ def parse_task_output( return ParsedTaskOutput(text=parsed_text, reasoning_trace=reasoning_trace) - def has_output_parser(self, task: Task): + def has_output_parser(self, task: Union[str, Task]): prompt = get_prompt(self.config, task) return prompt.output_parser is not None From f708b6faec41c2605b192203b7a94f87bdf0820d Mon Sep 17 00:00:00 2001 From: tgasser-nv <200644301+tgasser-nv@users.noreply.github.com> Date: Wed, 10 Sep 2025 17:09:59 -0500 Subject: [PATCH 3/4] Cleaned up some type-fixes --- nemoguardrails/library/attention/actions.py | 9 ++++----- nemoguardrails/library/autoalign/actions.py | 4 ++-- .../library/jailbreak_detection/server.py | 13 +++++++++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/nemoguardrails/library/attention/actions.py b/nemoguardrails/library/attention/actions.py index 89b079928..74bc18c2c 100644 --- a/nemoguardrails/library/attention/actions.py +++ b/nemoguardrails/library/attention/actions.py @@ -60,10 +60,8 @@ def _get_action_timestamp(action_event_name: str, event_args) -> Optional[dateti return None try: return read_isoformat(event_args[_mapping[action_event_name]]) - except (ValueError, KeyError, TypeError) as e: - log_p( - f"Could not parse timestamp {event_args[_mapping[action_event_name]]}: {e}" - ) + except Exception: + log_p(f"Could not parse timestamp {event_args[_mapping[action_event_name]]}") return None @@ -121,7 +119,8 @@ def update(self, event: ActionEvent, offsets: dict[str, float]) -> None: if not timestamp: return - # Dynamically add corrected_datetime attribute to the event + # Neither ActionEvent nor base class Event have `corrected_time` attribute + # so add it dynamically corrected_time = timestamp + timedelta(seconds=offsets.get(event.name, 0.0)) setattr(event, "corrected_datetime", corrected_time) diff --git a/nemoguardrails/library/autoalign/actions.py b/nemoguardrails/library/autoalign/actions.py index f901b8d1b..f2f4a3c36 100644 --- a/nemoguardrails/library/autoalign/actions.py +++ b/nemoguardrails/library/autoalign/actions.py @@ -289,10 +289,10 @@ async def autoalign_input_api( **kwargs, ): """Calls AutoAlign API for the user message and guardrail configuration provided""" - if context is None: + if not context: raise ValueError("Context is required") user_message = context.get("user_message") - if user_message is None: + if not user_message: raise ValueError("user_message is required in context") autoalign_config = llm_task_manager.config.rails.config.autoalign diff --git a/nemoguardrails/library/jailbreak_detection/server.py b/nemoguardrails/library/jailbreak_detection/server.py index 86a652afc..48016f14d 100644 --- a/nemoguardrails/library/jailbreak_detection/server.py +++ b/nemoguardrails/library/jailbreak_detection/server.py @@ -37,6 +37,7 @@ import uvicorn from fastapi import FastAPI from pydantic import BaseModel +from pydantic.fields import Field app = FastAPI() cli_app = typer.Typer() @@ -52,8 +53,12 @@ class JailbreakHeuristicRequest(BaseModel): """ prompt: str - lp_threshold: Optional[float] = 89.79 - ps_ppl_threshold: Optional[float] = 1845.65 + lp_threshold: float = Field( + default=89.79, description="The length/perplexity threshold." + ) + ps_ppl_threshold: float = Field( + default=1845.65, description="The prefix/suffix perplexity threshold." + ) class JailbreakModelRequest(BaseModel): @@ -95,10 +100,10 @@ def ps_ppl_heuristic_check(request: JailbreakHeuristicRequest): def run_all_heuristics(request: JailbreakHeuristicRequest): # Will add other heuristics as they become available lp_check = hc.check_jailbreak_length_per_perplexity( - request.prompt, request.lp_threshold or 89.79 + request.prompt, request.lp_threshold ) ps_ppl_check = hc.check_jailbreak_prefix_suffix_perplexity( - request.prompt, request.ps_ppl_threshold or 1845.65 + request.prompt, request.ps_ppl_threshold ) jailbreak = any([lp_check["jailbreak"], ps_ppl_check["jailbreak"]]) heuristic_checks = { From e1d2e32ecbcd2d82a2fb5b854f63f11a7a0f5a47 Mon Sep 17 00:00:00 2001 From: tgasser-nv <200644301+tgasser-nv@users.noreply.github.com> Date: Mon, 22 Sep 2025 13:41:27 -0500 Subject: [PATCH 4/4] Revert "Dummy commit to set up the chore/type-clean-guardrails PR and branch" This reverts commit 71d00f083fb59bda34c82b82eea85602c1710265. --- nemoguardrails/actions/llm/generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemoguardrails/actions/llm/generation.py b/nemoguardrails/actions/llm/generation.py index cd11e70a7..2a57e1c26 100644 --- a/nemoguardrails/actions/llm/generation.py +++ b/nemoguardrails/actions/llm/generation.py @@ -137,7 +137,7 @@ async def init(self): self._init_flows_index(), ) - def _extract_user_message_example(self, flow: Flow) -> None: + def _extract_user_message_example(self, flow: Flow): """Heuristic to extract user message examples from a flow.""" elements = [ item