From d7efb95c06c3b014a020136e94249b50f83d5d85 Mon Sep 17 00:00:00 2001 From: Emre Turan Date: Wed, 5 Nov 2025 09:37:11 -0800 Subject: [PATCH 1/2] Add fallback handling for global LLM failures (nonresponsive scenarios) --- .../initialize_listener_event.py | 2 + whatsapp_bot/app/handlers/ListenerMode.py | 95 +++++++++++++++++-- 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/tools/2ndRoundDeliberation/initialize_listener_event.py b/tools/2ndRoundDeliberation/initialize_listener_event.py index dee6c4c..ba44341 100644 --- a/tools/2ndRoundDeliberation/initialize_listener_event.py +++ b/tools/2ndRoundDeliberation/initialize_listener_event.py @@ -64,6 +64,8 @@ def initialize_event_collection(event_id, event_name, event_location, event_back 'extra_questions': extra_questions, 'mode': 'listener', # or "followup" / "survey" 'interaction_limit': 450, # Default; can be customized per event later + 'default_model': 'gpt-4o-mini', # ✅ added here + 'second_round_prompts': { diff --git a/whatsapp_bot/app/handlers/ListenerMode.py b/whatsapp_bot/app/handlers/ListenerMode.py index 389f7e9..7bd6a8e 100644 --- a/whatsapp_bot/app/handlers/ListenerMode.py +++ b/whatsapp_bot/app/handlers/ListenerMode.py @@ -12,7 +12,7 @@ from fastapi import Response from app.deliberation.second_round_agent import run_second_round_for_user from app.utils.blacklist_helpers import get_interaction_limit, is_blocked_number - +import random from config.config import ( db, logger, client, twilio_client, @@ -772,20 +772,97 @@ def process_second_round(transaction, ref, user_msg, sr_reply=None): 'interactions': firestore.ArrayUnion([{'message': Body}]) }) - run = client.beta.threads.runs.create_and_poll( - thread_id=thread.id, - assistant_id=assistant_id, - instructions=event_instructions - ) - if run.status == 'completed': + +# --- PRIMARY + FALLBACK LOGIC (DYNAMIC MODEL) --- + + try: + event_info_ref = db.collection(normalize_event_path(current_event_id)).document("info") + event_info_doc = event_info_ref.get() + if event_info_doc.exists: + event_info_data = event_info_doc.to_dict() + default_model = event_info_data.get("default_model", "gpt-4o-mini") + else: + default_model = "gpt-4o-mini" + logger.info(f"[LLM Config] Using model from Firestore: {default_model}") + except Exception as e: + logger.error(f"[LLM Config] Failed to fetch model from Firestore, defaulting to gpt-4o-mini: {e}") + default_model = "gpt-4o-mini" + + + try: + # First attempt using the model fetched from Firestore + logger.info(f"[LLM Run] Starting primary run with model: {default_model}") + + run = client.beta.threads.runs.create_and_poll( + thread_id=thread.id, + assistant_id=assistant_id, + instructions=event_instructions, + model=default_model + ) + + logger.info(f"[LLM Debug] Primary run status: {getattr(run, 'status', 'N/A')}") + + # Fallback if it fails + if run.status != "completed": + logger.warning(f"[LLM Fallback] Model {default_model} failed, retrying with gpt-4.1-mini") + + if hasattr(run, 'last_error'): + logger.error(f"[LLM Debug] last_error (primary): {run.last_error}") + if hasattr(run, 'incomplete_details'): + logger.error(f"[LLM Debug] incomplete_details (primary): {run.incomplete_details}") + + run = client.beta.threads.runs.create_and_poll( + thread_id=thread.id, + assistant_id=assistant_id, + instructions=event_instructions, + model="gpt-4.1-mini" + ) + + logger.info(f"[LLM Debug] Fallback run status: {getattr(run, 'status', 'N/A')}") + + except Exception as e: + logger.exception(f"[LLM Exception] Error while creating run: {e}") + run = None + + + # --- RESPONSE HANDLING --- + if run and run.status == "completed": + final_model = getattr(run, 'model', default_model) + logger.info(f"[LLM Success] Final model used: {final_model}") + messages = client.beta.threads.messages.list(thread_id=thread.id) assistant_response = extract_text_from_messages(messages) + send_message(From, assistant_response) event_doc_ref.update({ - 'interactions': firestore.ArrayUnion([{'response': assistant_response}]) + 'interactions': firestore.ArrayUnion([ + {'response': assistant_response, 'model': final_model} + ]) }) + else: - send_message(From, "There was an issue processing your request.") + logger.warning("[LLM Fallback] Both models failed or returned incomplete response.") + + if run and hasattr(run, 'last_error'): + logger.error(f"[LLM Debug] last_error (final): {run.last_error}") + if run and hasattr(run, 'incomplete_details'): + logger.error(f"[LLM Debug] incomplete_details (final): {run.incomplete_details}") + + fallback_responses = [ + "Agreed.", + "Please continue.", + "That’s an interesting point, tell me more.", + "I understand.", + "Go on, I’m listening." + ] + fallback_message = random.choice(fallback_responses) + + send_message(From, fallback_message) + event_doc_ref.update({ + 'interactions': firestore.ArrayUnion([ + {'response': fallback_message, 'fallback': True} + ]) + }) return Response(status_code=200) \ No newline at end of file From 25875f9288a6f2788d7e5460285719afe7e0c27a Mon Sep 17 00:00:00 2001 From: Emre Turan Date: Thu, 6 Nov 2025 21:34:47 -0800 Subject: [PATCH 2/2] minor add --- .../initialize_listener_event.py | 2 +- whatsapp_bot/app/handlers/ListenerMode.py | 33 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/tools/2ndRoundDeliberation/initialize_listener_event.py b/tools/2ndRoundDeliberation/initialize_listener_event.py index ba44341..bb3f08d 100644 --- a/tools/2ndRoundDeliberation/initialize_listener_event.py +++ b/tools/2ndRoundDeliberation/initialize_listener_event.py @@ -64,7 +64,7 @@ def initialize_event_collection(event_id, event_name, event_location, event_back 'extra_questions': extra_questions, 'mode': 'listener', # or "followup" / "survey" 'interaction_limit': 450, # Default; can be customized per event later - 'default_model': 'gpt-4o-mini', # ✅ added here + 'default_model': 'gpt-4o-mini', diff --git a/whatsapp_bot/app/handlers/ListenerMode.py b/whatsapp_bot/app/handlers/ListenerMode.py index 7bd6a8e..72656aa 100644 --- a/whatsapp_bot/app/handlers/ListenerMode.py +++ b/whatsapp_bot/app/handlers/ListenerMode.py @@ -36,7 +36,8 @@ from app.utils.validators import _norm from app.utils.validators import normalize_event_path - +DEFAULT_MODEL = os.getenv("DEFAULT_MODEL", "gpt-4o-mini") +FALLBACK_MODEL = os.getenv("FALLBACK_MODEL", "gpt-4.1-mini") def is_second_round_enabled(event_id: str) -> bool: """Return True iff info.second_round_claims_source.enabled is truthy.""" @@ -772,26 +773,26 @@ def process_second_round(transaction, ref, user_msg, sr_reply=None): 'interactions': firestore.ArrayUnion([{'message': Body}]) }) - - -# --- PRIMARY + FALLBACK LOGIC (DYNAMIC MODEL) --- - try: + # Attempt to fetch model configuration from Firestore event_info_ref = db.collection(normalize_event_path(current_event_id)).document("info") event_info_doc = event_info_ref.get() + + # Pre-initialize with the environment or constant default + default_model = DEFAULT_MODEL if event_info_doc.exists: event_info_data = event_info_doc.to_dict() - default_model = event_info_data.get("default_model", "gpt-4o-mini") - else: - default_model = "gpt-4o-mini" + default_model = event_info_data.get("default_model", default_model) + logger.info(f"[LLM Config] Using model from Firestore: {default_model}") + except Exception as e: - logger.error(f"[LLM Config] Failed to fetch model from Firestore, defaulting to gpt-4o-mini: {e}") - default_model = "gpt-4o-mini" + logger.error(f"[LLM Config] Failed to fetch model from Firestore, defaulting to {DEFAULT_MODEL}: {e}") + default_model = DEFAULT_MODEL try: - # First attempt using the model fetched from Firestore + # Primary model attempt logger.info(f"[LLM Run] Starting primary run with model: {default_model}") run = client.beta.threads.runs.create_and_poll( @@ -803,9 +804,9 @@ def process_second_round(transaction, ref, user_msg, sr_reply=None): logger.info(f"[LLM Debug] Primary run status: {getattr(run, 'status', 'N/A')}") - # Fallback if it fails + # Fallback if the primary model failed or didn’t complete if run.status != "completed": - logger.warning(f"[LLM Fallback] Model {default_model} failed, retrying with gpt-4.1-mini") + logger.warning(f"[LLM Fallback] Model {default_model} failed, retrying with {FALLBACK_MODEL}") if hasattr(run, 'last_error'): logger.error(f"[LLM Debug] last_error (primary): {run.last_error}") @@ -816,7 +817,7 @@ def process_second_round(transaction, ref, user_msg, sr_reply=None): thread_id=thread.id, assistant_id=assistant_id, instructions=event_instructions, - model="gpt-4.1-mini" + model=FALLBACK_MODEL ) logger.info(f"[LLM Debug] Fallback run status: {getattr(run, 'status', 'N/A')}") @@ -837,7 +838,7 @@ def process_second_round(transaction, ref, user_msg, sr_reply=None): send_message(From, assistant_response) event_doc_ref.update({ 'interactions': firestore.ArrayUnion([ - {'response': assistant_response, 'model': final_model} + {'response': assistant_response, 'model': final_model, 'fallback': False} ]) }) @@ -861,7 +862,7 @@ def process_second_round(transaction, ref, user_msg, sr_reply=None): send_message(From, fallback_message) event_doc_ref.update({ 'interactions': firestore.ArrayUnion([ - {'response': fallback_message, 'fallback': True} + {'response': fallback_message, 'model': None, 'fallback': True} ]) })