From 9accc1ef598f708e5ec41214e89434d4aacb0d07 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Fri, 26 Dec 2025 13:49:23 +0000 Subject: [PATCH 01/14] Enhance configuration management and add new setup scripts - Updated .gitignore to include config.yml and its template. - Added config.yml.template for default configuration settings. - Introduced restart.sh script for service management. - Enhanced services.py to load config.yml and check for Obsidian/Neo4j integration. - Updated wizard.py to prompt for Obsidian/Neo4j configuration during setup and create config.yml from template if it doesn't exist. --- .gitignore | 2 + config.yml => config.yml.template | 8 ++- restart.sh | 2 + services.py | 39 +++++++++++++- wizard.py | 90 ++++++++++++++++++++++++++++--- 5 files changed, 132 insertions(+), 9 deletions(-) rename config.yml => config.yml.template (96%) create mode 100755 restart.sh diff --git a/.gitignore b/.gitignore index b2b052b3..38cb0c88 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ !**/.env.template **/memory_config.yaml !**/memory_config.yaml.template +config.yml +!config.yml.template example/* **/node_modules/* **/ollama-data/* diff --git a/config.yml b/config.yml.template similarity index 96% rename from config.yml rename to config.yml.template index ac412d1e..37209d4b 100644 --- a/config.yml +++ b/config.yml.template @@ -128,7 +128,7 @@ models: language: multi smart_format: 'true' punctuate: 'true' - diarize: false + diarize: 'true' encoding: linear16 sample_rate: 16000 channels: '1' @@ -191,7 +191,7 @@ memory: enabled: true prompt: 'Extract important information from this conversation and return a JSON object with an array named "facts". Include personal preferences, plans, names, - dates, locations, numbers, and key details hehehe. Keep items concise and useful. + dates, locations, numbers, and key details. Keep items concise and useful. ' openmemory_mcp: @@ -202,3 +202,7 @@ memory: mycelia: api_url: http://localhost:5173 timeout: 30 + obsidian: + enabled: false + neo4j_host: neo4j-mem0 + timeout: 30 diff --git a/restart.sh b/restart.sh new file mode 100755 index 00000000..019518c4 --- /dev/null +++ b/restart.sh @@ -0,0 +1,2 @@ +#!/bin/bash +uv run --with-requirements setup-requirements.txt python services.py restart --all diff --git a/services.py b/services.py index 716e437e..0deeff8a 100755 --- a/services.py +++ b/services.py @@ -8,12 +8,26 @@ import subprocess from pathlib import Path +import yaml from rich.console import Console from rich.table import Table from dotenv import dotenv_values console = Console() +def load_config_yml(): + """Load config.yml from repository root""" + config_path = Path(__file__).parent / 'config.yml' + if not config_path.exists(): + return None + + try: + with open(config_path, 'r') as f: + return yaml.safe_load(f) + except Exception as e: + console.print(f"[yellow]โš ๏ธ Warning: Could not load config.yml: {e}[/yellow]") + return None + SERVICES = { 'backend': { 'path': 'backends/advanced', @@ -74,7 +88,30 @@ def run_compose_command(service_name, command, build=False): if caddyfile_path.exists() and caddyfile_path.is_file(): # Enable HTTPS profile to start Caddy service cmd.extend(['--profile', 'https']) - + + # Check if Obsidian/Neo4j is enabled + obsidian_enabled = False + + # Method 1: Check config.yml (preferred) + config_data = load_config_yml() + if config_data: + memory_config = config_data.get('memory', {}) + obsidian_config = memory_config.get('obsidian', {}) + if obsidian_config.get('enabled', False): + obsidian_enabled = True + + # Method 2: Fallback to .env for backward compatibility + if not obsidian_enabled: + env_file = service_path / '.env' + if env_file.exists(): + env_values = dotenv_values(env_file) + if env_values.get('OBSIDIAN_ENABLED', 'false').lower() == 'true': + obsidian_enabled = True + + if obsidian_enabled: + cmd.extend(['--profile', 'obsidian']) + console.print("[blue]โ„น๏ธ Starting with Obsidian/Neo4j support[/blue]") + # Handle speaker-recognition service specially if service_name == 'speaker-recognition' and command in ['up', 'down']: # Read configuration to determine profile diff --git a/wizard.py b/wizard.py index 8e3fa041..05e97e59 100755 --- a/wizard.py +++ b/wizard.py @@ -150,7 +150,8 @@ def cleanup_unselected_services(selected_services): env_file.rename(backup_file) console.print(f"๐Ÿงน [dim]Backed up {service_name} configuration to {backup_file.name} (service not selected)[/dim]") -def run_service_setup(service_name, selected_services, https_enabled=False, server_ip=None): +def run_service_setup(service_name, selected_services, https_enabled=False, server_ip=None, + obsidian_enabled=False, neo4j_password=None): """Execute individual service setup script""" if service_name == 'advanced': service = SERVICES['backend'][service_name] @@ -165,7 +166,11 @@ def run_service_setup(service_name, selected_services, https_enabled=False, serv # Add HTTPS configuration if https_enabled and server_ip: cmd.extend(['--enable-https', '--server-ip', server_ip]) - + + # Add Obsidian configuration + if obsidian_enabled and neo4j_password: + cmd.extend(['--enable-obsidian', '--neo4j-password', neo4j_password]) + else: service = SERVICES['extras'][service_name] cmd = service['cmd'].copy() @@ -308,10 +313,28 @@ def setup_git_hooks(): except Exception as e: console.print(f"โš ๏ธ [yellow]Could not setup git hooks: {e} (optional)[/yellow]") +def setup_config_file(): + """Setup config.yml from template if it doesn't exist""" + config_file = Path("config.yml") + config_template = Path("config.yml.template") + + if not config_file.exists(): + if config_template.exists(): + import shutil + shutil.copy(config_template, config_file) + console.print("โœ… [green]Created config.yml from template[/green]") + else: + console.print("โš ๏ธ [yellow]config.yml.template not found, skipping config setup[/yellow]") + else: + console.print("โ„น๏ธ [blue]config.yml already exists, keeping existing configuration[/blue]") + def main(): """Main orchestration logic""" console.print("๐ŸŽ‰ [bold green]Welcome to Chronicle![/bold green]\n") + # Setup config file from template + setup_config_file() + # Setup git hooks first setup_git_hooks() @@ -371,7 +394,43 @@ def main(): break console.print(f"[green]โœ…[/green] HTTPS configured for: {server_ip}") - + + # Obsidian/Neo4j Integration + obsidian_enabled = False + neo4j_password = None + + # Check if advanced backend is selected + if 'advanced' in selected_services: + console.print("\n๐Ÿ—‚๏ธ [bold cyan]Obsidian/Neo4j Integration[/bold cyan]") + console.print("Enable graph-based knowledge management for Obsidian vault notes") + console.print() + + try: + obsidian_enabled = Confirm.ask("Enable Obsidian/Neo4j integration?", default=False) + except EOFError: + console.print("Using default: No") + obsidian_enabled = False + + if obsidian_enabled: + console.print("[blue][INFO][/blue] Neo4j will be configured for graph-based memory storage") + console.print() + + # Prompt for Neo4j password + while True: + try: + neo4j_password = console.input("Neo4j password (min 8 chars) [default: neo4jpassword]: ").strip() + if not neo4j_password: + neo4j_password = "neo4jpassword" + if len(neo4j_password) >= 8: + break + console.print("[yellow][WARNING][/yellow] Password must be at least 8 characters") + except EOFError: + neo4j_password = "neo4jpassword" + console.print(f"Using default password") + break + + console.print("[green]โœ…[/green] Obsidian/Neo4j integration will be configured") + # Pure Delegation - Run Each Service Setup console.print(f"\n๐Ÿ“‹ [bold]Setting up {len(selected_services)} services...[/bold]") @@ -382,17 +441,36 @@ def main(): failed_services = [] for service in selected_services: - if run_service_setup(service, selected_services, https_enabled, server_ip): + if run_service_setup(service, selected_services, https_enabled, server_ip, + obsidian_enabled, neo4j_password): success_count += 1 else: failed_services.append(service) - + + # Check for Obsidian/Neo4j configuration + obsidian_enabled = False + if 'advanced' in selected_services and 'advanced' not in failed_services: + backend_env_path = Path('backends/advanced/.env') + if backend_env_path.exists(): + neo4j_host = read_env_value(str(backend_env_path), 'NEO4J_HOST') + obsidian_enabled_flag = read_env_value(str(backend_env_path), 'OBSIDIAN_ENABLED') + if neo4j_host and not is_placeholder(neo4j_host, 'your-neo4j-host-here', 'your_neo4j_host_here'): + obsidian_enabled = True + elif obsidian_enabled_flag == 'true': + obsidian_enabled = True + # Final Summary console.print(f"\n๐ŸŽŠ [bold green]Setup Complete![/bold green]") console.print(f"โœ… {success_count}/{len(selected_services)} services configured successfully") - + if failed_services: console.print(f"โŒ Failed services: {', '.join(failed_services)}") + + # Inform about Obsidian/Neo4j if configured + if obsidian_enabled: + console.print(f"\n๐Ÿ“š [bold cyan]Obsidian Integration Detected[/bold cyan]") + console.print(" Neo4j will be started with the 'obsidian' profile") + console.print(" when you start the backend service.") # Next Steps console.print("\n๐Ÿ“– [bold]Next Steps:[/bold]") From 892b1b2a23e7a3630377ef28d000a857b221b72b Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Tue, 30 Dec 2025 00:11:56 +0000 Subject: [PATCH 02/14] Refactor transcription providers and enhance configuration management - Updated Docker Compose files to include the new Neo4j service configuration. - Added support for Obsidian/Neo4j integration in the setup process. - Refactored transcription providers to utilize a registry-driven approach for Deepgram and Parakeet. - Enhanced error handling and logging in transcription processes. - Improved environment variable management in test scripts to prioritize command-line overrides. - Removed deprecated Parakeet provider implementation and streamlined audio stream workers. --- backends/advanced/docker-compose-test.yml | 3 +- backends/advanced/docker-compose.yml | 36 +- backends/advanced/init.py | 70 ++- backends/advanced/run-test.sh | 60 ++- .../services/transcription/__init__.py | 14 + .../services/transcription/deepgram.py | 429 +----------------- .../services/transcription/parakeet.py | 303 ------------- .../transcription/parakeet_stream_consumer.py | 19 +- .../workers/audio_stream_deepgram_worker.py | 11 +- .../workers/audio_stream_parakeet_worker.py | 11 +- .../workers/memory_jobs.py | 7 +- 11 files changed, 196 insertions(+), 767 deletions(-) delete mode 100644 backends/advanced/src/advanced_omi_backend/services/transcription/parakeet.py diff --git a/backends/advanced/docker-compose-test.yml b/backends/advanced/docker-compose-test.yml index 2498ea1a..20b4fd08 100644 --- a/backends/advanced/docker-compose-test.yml +++ b/backends/advanced/docker-compose-test.yml @@ -32,7 +32,7 @@ services: - ADMIN_EMAIL=test-admin@example.com # Transcription provider configuration - TRANSCRIPTION_PROVIDER=${TRANSCRIPTION_PROVIDER:-deepgram} - # - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} + - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} # Memory provider configuration - MEMORY_PROVIDER=${MEMORY_PROVIDER:-chronicle} - OPENMEMORY_MCP_URL=${OPENMEMORY_MCP_URL:-http://host.docker.internal:8765} @@ -144,6 +144,7 @@ services: - ADMIN_PASSWORD=test-admin-password-123 - ADMIN_EMAIL=test-admin@example.com - TRANSCRIPTION_PROVIDER=${TRANSCRIPTION_PROVIDER:-deepgram} + - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} - MEMORY_PROVIDER=${MEMORY_PROVIDER:-chronicle} - OPENMEMORY_MCP_URL=${OPENMEMORY_MCP_URL:-http://host.docker.internal:8765} - OPENMEMORY_USER_ID=${OPENMEMORY_USER_ID:-openmemory} diff --git a/backends/advanced/docker-compose.yml b/backends/advanced/docker-compose.yml index b84d2ebe..313c0f23 100644 --- a/backends/advanced/docker-compose.yml +++ b/backends/advanced/docker-compose.yml @@ -177,20 +177,28 @@ services: ## Additional - # neo4j-mem0: - # image: neo4j:5.15-community - # ports: - # - "7474:7474" # HTTP - # - "7687:7687" # Bolt - # environment: - # - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD:-password} - # - NEO4J_PLUGINS=["apoc"] - # - NEO4J_dbms_security_procedures_unrestricted=apoc.* - # - NEO4J_dbms_security_procedures_allowlist=apoc.* - # volumes: - # - ./data/neo4j_data:/data - # - ./data/neo4j_logs:/logs - # restart: unless-stopped + neo4j-mem0: + image: neo4j:5.15-community + hostname: neo4j-mem0 + ports: + - "7474:7474" # HTTP + - "7687:7687" # Bolt + environment: + - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD:-password} + - NEO4J_PLUGINS=["apoc"] + - NEO4J_dbms_security_procedures_unrestricted=apoc.* + - NEO4J_dbms_security_procedures_allowlist=apoc.* + - NEO4J_server_default__listen__address=0.0.0.0 + - NEO4J_server_bolt_listen__address=0.0.0.0:7687 + - NEO4J_server_http_listen__address=0.0.0.0:7474 + - NEO4J_dbms_memory_heap_initial__size=512m + - NEO4J_dbms_memory_heap_max__size=2G + volumes: + - ./data/neo4j_data:/data + - ./data/neo4j_logs:/logs + restart: unless-stopped + profiles: + - obsidian # ollama: # image: ollama/ollama:latest diff --git a/backends/advanced/init.py b/backends/advanced/init.py index 25a614aa..851d56e1 100644 --- a/backends/advanced/init.py +++ b/backends/advanced/init.py @@ -410,11 +410,56 @@ def setup_optional_services(self): self.console.print("[green][SUCCESS][/green] Speaker Recognition configured") self.console.print("[blue][INFO][/blue] Start with: cd ../../extras/speaker-recognition && docker compose up -d") - # Check if ASR service URL provided via args + # Check if ASR service URL provided via args if hasattr(self.args, 'parakeet_asr_url') and self.args.parakeet_asr_url: self.config["PARAKEET_ASR_URL"] = self.args.parakeet_asr_url self.console.print(f"[green][SUCCESS][/green] Parakeet ASR configured via args: {self.args.parakeet_asr_url}") + def setup_obsidian(self): + """Configure Obsidian/Neo4j integration""" + # Check if enabled via command line + if hasattr(self.args, 'enable_obsidian') and self.args.enable_obsidian: + enable_obsidian = True + neo4j_password = getattr(self.args, 'neo4j_password', None) + + if not neo4j_password: + self.console.print("[yellow][WARNING][/yellow] --enable-obsidian provided but no password") + neo4j_password = self.prompt_password("Neo4j password (min 8 chars)") + else: + # Interactive prompt (fallback) + self.console.print() + self.console.print("[bold cyan]Obsidian/Neo4j Integration[/bold cyan]") + self.console.print("Enable graph-based knowledge management for Obsidian vault notes") + self.console.print() + + try: + enable_obsidian = Confirm.ask("Enable Obsidian/Neo4j integration?", default=False) + except EOFError: + self.console.print("Using default: No") + enable_obsidian = False + + if enable_obsidian: + neo4j_password = self.prompt_password("Neo4j password (min 8 chars)") + + if enable_obsidian: + # Update .env with credentials + self.config["NEO4J_HOST"] = "neo4j-mem0" + self.config["NEO4J_USER"] = "neo4j" + self.config["NEO4J_PASSWORD"] = neo4j_password + + # Update config.yml with feature flag + if "memory" not in self.config_yml_data: + self.config_yml_data["memory"] = {} + if "obsidian" not in self.config_yml_data["memory"]: + self.config_yml_data["memory"]["obsidian"] = {} + + self.config_yml_data["memory"]["obsidian"]["enabled"] = True + self.config_yml_data["memory"]["obsidian"]["neo4j_host"] = "neo4j-mem0" + self.config_yml_data["memory"]["obsidian"]["timeout"] = 30 + + self.console.print("[green][SUCCESS][/green] Obsidian/Neo4j configured") + self.console.print("[blue][INFO][/blue] Neo4j will start automatically with --profile obsidian") + def setup_network(self): """Configure network settings""" self.print_section("Network Configuration") @@ -589,6 +634,11 @@ def show_summary(self): memory_provider = self.config_yml_data.get("memory", {}).get("provider", "chronicle") self.console.print(f"โœ… Memory Provider: {memory_provider} (config.yml)") + # Show Obsidian/Neo4j status + if self.config.get('OBSIDIAN_ENABLED') == 'true': + neo4j_host = self.config.get('NEO4J_HOST', 'not set') + self.console.print(f"โœ… Obsidian/Neo4j: Enabled ({neo4j_host})") + # Auto-determine URLs based on HTTPS configuration if self.config.get('HTTPS_ENABLED') == 'true': server_ip = self.config.get('SERVER_IP', 'localhost') @@ -604,9 +654,14 @@ def show_next_steps(self): """Show next steps""" self.print_section("Next Steps") self.console.print() - + self.console.print("1. Start the main services:") - self.console.print(" [cyan]docker compose up --build -d[/cyan]") + # Include --profile obsidian if Obsidian is enabled + if self.config.get('OBSIDIAN_ENABLED') == 'true': + self.console.print(" [cyan]docker compose --profile obsidian up --build -d[/cyan]") + self.console.print(" [dim](Includes Neo4j for Obsidian integration)[/dim]") + else: + self.console.print(" [cyan]docker compose up --build -d[/cyan]") self.console.print() # Auto-determine URLs for next steps @@ -653,6 +708,7 @@ def run(self): self.setup_llm() self.setup_memory() self.setup_optional_services() + self.setup_obsidian() self.setup_network() self.setup_https() @@ -695,9 +751,13 @@ def main(): help="Parakeet ASR service URL (default: prompt user)") parser.add_argument("--enable-https", action="store_true", help="Enable HTTPS configuration (default: prompt user)") - parser.add_argument("--server-ip", + parser.add_argument("--server-ip", help="Server IP/domain for SSL certificate (default: prompt user)") - + parser.add_argument("--enable-obsidian", action="store_true", + help="Enable Obsidian/Neo4j integration (default: prompt user)") + parser.add_argument("--neo4j-password", + help="Neo4j password (default: prompt user)") + args = parser.parse_args() setup = ChronicleSetup(args) diff --git a/backends/advanced/run-test.sh b/backends/advanced/run-test.sh index 925e3615..4f944256 100755 --- a/backends/advanced/run-test.sh +++ b/backends/advanced/run-test.sh @@ -39,8 +39,16 @@ print_info "Advanced Backend Integration Test Runner" print_info "========================================" # Load environment variables (CI or local) -# Priority: CI environment > .env.test > .env -if [ -n "$DEEPGRAM_API_KEY" ]; then +# Priority: Command-line env vars > CI environment > .env.test > .env +# Save any pre-existing environment variables to preserve command-line overrides +_TRANSCRIPTION_PROVIDER_OVERRIDE=${TRANSCRIPTION_PROVIDER} +_PARAKEET_ASR_URL_OVERRIDE=${PARAKEET_ASR_URL} +_DEEPGRAM_API_KEY_OVERRIDE=${DEEPGRAM_API_KEY} +_OPENAI_API_KEY_OVERRIDE=${OPENAI_API_KEY} +_LLM_PROVIDER_OVERRIDE=${LLM_PROVIDER} +_MEMORY_PROVIDER_OVERRIDE=${MEMORY_PROVIDER} + +if [ -n "$DEEPGRAM_API_KEY" ] && [ -z "$_TRANSCRIPTION_PROVIDER_OVERRIDE" ]; then print_info "Using environment variables from CI/environment..." elif [ -f ".env.test" ]; then print_info "Loading environment variables from .env.test..." @@ -59,6 +67,30 @@ else exit 1 fi +# Restore command-line overrides (these take highest priority) +if [ -n "$_TRANSCRIPTION_PROVIDER_OVERRIDE" ]; then + export TRANSCRIPTION_PROVIDER=$_TRANSCRIPTION_PROVIDER_OVERRIDE + print_info "Using command-line override: TRANSCRIPTION_PROVIDER=$TRANSCRIPTION_PROVIDER" +fi +if [ -n "$_PARAKEET_ASR_URL_OVERRIDE" ]; then + export PARAKEET_ASR_URL=$_PARAKEET_ASR_URL_OVERRIDE + print_info "Using command-line override: PARAKEET_ASR_URL=$PARAKEET_ASR_URL" +fi +if [ -n "$_DEEPGRAM_API_KEY_OVERRIDE" ]; then + export DEEPGRAM_API_KEY=$_DEEPGRAM_API_KEY_OVERRIDE +fi +if [ -n "$_OPENAI_API_KEY_OVERRIDE" ]; then + export OPENAI_API_KEY=$_OPENAI_API_KEY_OVERRIDE +fi +if [ -n "$_LLM_PROVIDER_OVERRIDE" ]; then + export LLM_PROVIDER=$_LLM_PROVIDER_OVERRIDE + print_info "Using command-line override: LLM_PROVIDER=$LLM_PROVIDER" +fi +if [ -n "$_MEMORY_PROVIDER_OVERRIDE" ]; then + export MEMORY_PROVIDER=$_MEMORY_PROVIDER_OVERRIDE + print_info "Using command-line override: MEMORY_PROVIDER=$MEMORY_PROVIDER" +fi + # Verify required environment variables based on configured providers TRANSCRIPTION_PROVIDER=${TRANSCRIPTION_PROVIDER:-deepgram} LLM_PROVIDER=${LLM_PROVIDER:-openai} @@ -161,17 +193,25 @@ else TEST_EXIT_CODE=$? print_error "Integration tests FAILED with exit code: $TEST_EXIT_CODE" - # Clean up test containers before exiting - print_info "Cleaning up test containers after failure..." - docker compose -f docker-compose-test.yml down -v || true - docker system prune -f || true + # Clean up test containers before exiting (unless CLEANUP_CONTAINERS=false) + if [ "${CLEANUP_CONTAINERS:-true}" != "false" ]; then + print_info "Cleaning up test containers after failure..." + docker compose -f docker-compose-test.yml down -v || true + docker system prune -f || true + else + print_warning "Skipping cleanup (CLEANUP_CONTAINERS=false) - containers left running for debugging" + fi exit $TEST_EXIT_CODE fi -# Clean up test containers -print_info "Cleaning up test containers..." -docker compose -f docker-compose-test.yml down -v || true -docker system prune -f || true +# Clean up test containers (unless CLEANUP_CONTAINERS=false) +if [ "${CLEANUP_CONTAINERS:-true}" != "false" ]; then + print_info "Cleaning up test containers..." + docker compose -f docker-compose-test.yml down -v || true + docker system prune -f || true +else + print_warning "Skipping cleanup (CLEANUP_CONTAINERS=false) - containers left running" +fi print_success "Advanced Backend integration tests completed!" diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/__init__.py b/backends/advanced/src/advanced_omi_backend/services/transcription/__init__.py index 507df738..2e20171b 100644 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/__init__.py +++ b/backends/advanced/src/advanced_omi_backend/services/transcription/__init__.py @@ -126,6 +126,13 @@ async def transcribe(self, audio_data: bytes, sample_rate: int, diarize: bool = resp.raise_for_status() data = resp.json() + # DEBUG: Log Deepgram response structure + if "results" in data and "channels" in data.get("results", {}): + channels = data["results"]["channels"] + if channels and "alternatives" in channels[0]: + alt = channels[0]["alternatives"][0] + logger.info(f"DEBUG Registry: Deepgram alternative keys: {list(alt.keys())}") + # Extract normalized shape text, words, segments = "", [], [] extract = (op.get("response", {}) or {}).get("extract") or {} @@ -133,6 +140,13 @@ async def transcribe(self, audio_data: bytes, sample_rate: int, diarize: bool = text = _dotted_get(data, extract.get("text")) or "" words = _dotted_get(data, extract.get("words")) or [] segments = _dotted_get(data, extract.get("segments")) or [] + + # DEBUG: Log what we extracted + logger.info(f"DEBUG Registry: Extracted {len(segments)} segments from response") + if segments and len(segments) > 0: + logger.info(f"DEBUG Registry: First segment keys: {list(segments[0].keys()) if isinstance(segments[0], dict) else 'not a dict'}") + logger.info(f"DEBUG Registry: First segment: {segments[0]}") + return {"text": text, "words": words, "segments": segments} class RegistryStreamingTranscriptionProvider(StreamingTranscriptionProvider): diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/deepgram.py b/backends/advanced/src/advanced_omi_backend/services/transcription/deepgram.py index ee7e23fa..03b2936d 100644 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/deepgram.py +++ b/backends/advanced/src/advanced_omi_backend/services/transcription/deepgram.py @@ -1,408 +1,13 @@ """ -Deepgram transcription provider implementations. +Deepgram transcription consumer for Redis Streams architecture. -Provides both batch and streaming transcription using Deepgram's Nova-3 model. +Uses the registry-driven transcription provider for Deepgram batch transcription. """ -import asyncio -import json import logging -import uuid -from typing import Dict, Optional - -import httpx -import websockets - -from .base import ( - BatchTranscriptionProvider, - StreamingTranscriptionProvider, -) logger = logging.getLogger(__name__) -class DeepgramProvider(BatchTranscriptionProvider): - """Deepgram batch transcription provider using Nova-3 model.""" - - def __init__(self, api_key: str): - self.api_key = api_key - self.url = "https://api.deepgram.com/v1/listen" - - @property - def name(self) -> str: - return "deepgram" - - async def transcribe(self, audio_data: bytes, sample_rate: int, diarize: bool = False) -> dict: - """Transcribe audio using Deepgram's REST API. - - Args: - audio_data: Raw audio bytes - sample_rate: Audio sample rate - diarize: Whether to enable speaker diarization - """ - try: - params = { - "model": "nova-3", - "language": "multi", - "smart_format": "true", - "punctuate": "true", - "diarize": "true" if diarize else "false", - "encoding": "linear16", - "sample_rate": str(sample_rate), - "channels": "1", - } - - headers = {"Authorization": f"Token {self.api_key}", "Content-Type": "audio/raw"} - - logger.debug(f"Sending {len(audio_data)} bytes to Deepgram API") - - # Calculate dynamic timeout based on audio file size - estimated_duration = len(audio_data) / (sample_rate * 2 * 1) # 16-bit mono - processing_timeout = max( - 120, int(estimated_duration * 3) - ) # Min 2 minutes, 3x audio duration - - timeout_config = httpx.Timeout( - connect=30.0, - read=processing_timeout, - write=max( - 180.0, int(len(audio_data) / (sample_rate * 2)) - ), # bytes per second for 16-bit PCM - pool=10.0, - ) - - logger.info( - f"Estimated audio duration: {estimated_duration:.1f}s, timeout: {processing_timeout}s" - ) - - async with httpx.AsyncClient(timeout=timeout_config) as client: - response = await client.post( - self.url, params=params, headers=headers, content=audio_data - ) - - if response.status_code == 200: - result = response.json() - logger.debug(f"Deepgram response: {result}") - - # Extract transcript from response - if result.get("results", {}).get("channels", []) and result["results"][ - "channels" - ][0].get("alternatives", []): - - alternative = result["results"]["channels"][0]["alternatives"][0] - - # Extract segments from diarized utterances if available - segments = [] - if "paragraphs" in alternative and alternative["paragraphs"].get("paragraphs"): - transcript = alternative["paragraphs"]["transcript"].strip() - logger.info( - f"Deepgram diarized transcription successful: {len(transcript)} characters" - ) - - # Extract speaker segments, grouping consecutive sentences from same speaker - current_speaker = None - current_segment = None - - for paragraph in alternative["paragraphs"]["paragraphs"]: - speaker = f"Speaker {paragraph.get('speaker', 'unknown')}" - - for sentence in paragraph.get("sentences", []): - if speaker == current_speaker and current_segment: - # Extend current segment with same speaker - current_segment["text"] += " " + sentence.get("text", "").strip() - current_segment["end"] = sentence.get("end", 0) - else: - # Save previous segment and start new one - if current_segment: - segments.append(current_segment) - current_segment = { - "text": sentence.get("text", "").strip(), - "speaker": speaker, - "start": sentence.get("start", 0), - "end": sentence.get("end", 0), - "confidence": None # Deepgram doesn't provide segment-level confidence - } - current_speaker = speaker - - # Don't forget the last segment - if current_segment: - segments.append(current_segment) - else: - transcript = alternative.get("transcript", "").strip() - logger.debug( - f"Deepgram basic transcription successful: {len(transcript)} characters" - ) - - if transcript: - # Extract speech timing information for logging - words = alternative.get("words", []) - if words: - first_word_start = words[0].get("start", 0) - last_word_end = words[-1].get("end", 0) - speech_duration = last_word_end - first_word_start - - # Calculate audio duration from data size - audio_duration = len(audio_data) / ( - sample_rate * 2 * 1 - ) # 16-bit mono - speech_percentage = ( - (speech_duration / audio_duration) * 100 - if audio_duration > 0 - else 0 - ) - - logger.info( - f"Deepgram speech analysis: {speech_duration:.1f}s speech detected in {audio_duration:.1f}s audio ({speech_percentage:.1f}%)" - ) - - # Check confidence levels - confidences = [ - w.get("confidence", 0) for w in words if "confidence" in w - ] - if confidences: - avg_confidence = sum(confidences) / len(confidences) - low_confidence_count = sum(1 for c in confidences if c < 0.5) - logger.info( - f"Deepgram confidence: avg={avg_confidence:.2f}, {low_confidence_count}/{len(words)} words <0.5 confidence" - ) - - # Keep raw transcript and word data without formatting - logger.info( - f"Keeping raw transcript with word-level data: {len(transcript)} characters, {len(segments)} segments" - ) - return { - "text": transcript, - "words": words, - "segments": segments, - } - else: - # No word-level data, return basic transcript - logger.info( - "No word-level data available, returning basic transcript" - ) - return {"text": transcript, "words": [], "segments": []} - else: - logger.warning("Deepgram returned empty transcript") - return {"text": "", "words": [], "segments": []} - else: - error_msg = "Deepgram response missing expected transcript structure" - logger.error(error_msg) - raise RuntimeError(error_msg) - else: - error_msg = f"Deepgram API error: {response.status_code} - {response.text}" - logger.error(error_msg) - raise RuntimeError(error_msg) - - except httpx.TimeoutException as e: - timeout_type = "unknown" - if "connect" in str(e).lower(): - timeout_type = "connection" - elif "read" in str(e).lower(): - timeout_type = "read" - elif "write" in str(e).lower(): - timeout_type = "write (upload)" - elif "pool" in str(e).lower(): - timeout_type = "connection pool" - error_msg = f"HTTP {timeout_type} timeout during Deepgram API call for {len(audio_data)} bytes: {e}" - logger.error(error_msg) - raise RuntimeError(error_msg) from e - except RuntimeError: - # Re-raise RuntimeError from above (API errors, timeouts) - raise - except Exception as e: - error_msg = f"Unexpected error calling Deepgram API: {e}" - logger.error(error_msg) - raise RuntimeError(error_msg) from e - - -class DeepgramStreamingProvider(StreamingTranscriptionProvider): - """Deepgram streaming transcription provider using WebSocket connection.""" - - def __init__(self, api_key: str): - self.api_key = api_key - self.ws_url = "wss://api.deepgram.com/v1/listen" - self._streams: Dict[str, Dict] = {} # client_id -> stream data - - @property - def name(self) -> str: - return "deepgram" - - async def start_stream(self, client_id: str, sample_rate: int = 16000, diarize: bool = False): - """Start a WebSocket connection for streaming transcription. - - Args: - client_id: Unique client identifier - sample_rate: Audio sample rate - diarize: Whether to enable speaker diarization - """ - try: - logger.info(f"Starting Deepgram streaming for client {client_id} (diarize={diarize})") - - # WebSocket connection parameters - params = { - "model": "nova-3", - "language": "multi", - "smart_format": "true", - "punctuate": "true", - "diarize": "true" if diarize else "false", - "encoding": "linear16", - "sample_rate": str(sample_rate), - "channels": "1", - "interim_results": "true", - "endpointing": "300", # 300ms silence for endpoint detection - } - - # Build WebSocket URL with parameters - query_string = "&".join([f"{k}={v}" for k, v in params.items()]) - ws_url = f"{self.ws_url}?{query_string}" - - # Connect to WebSocket - websocket = await websockets.connect( - ws_url, - extra_headers={"Authorization": f"Token {self.api_key}"} - ) - - # Store stream data - self._streams[client_id] = { - "websocket": websocket, - "final_transcript": "", - "words": [], - "stream_id": str(uuid.uuid4()) - } - - logger.debug(f"Deepgram WebSocket connected for client {client_id}") - - except Exception as e: - logger.error(f"Failed to start Deepgram streaming for {client_id}: {e}") - raise - - async def process_audio_chunk(self, client_id: str, audio_chunk: bytes) -> Optional[dict]: - """Send audio chunk to WebSocket and process responses.""" - if client_id not in self._streams: - logger.error(f"No active stream for client {client_id}") - return None - - try: - stream_data = self._streams[client_id] - websocket = stream_data["websocket"] - - # Send audio chunk - await websocket.send(audio_chunk) - - # Check for responses (non-blocking) - try: - while True: - response = await asyncio.wait_for(websocket.recv(), timeout=0.01) - result = json.loads(response) - - if result.get("type") == "Results": - channel = result.get("channel", {}) - alternatives = channel.get("alternatives", []) - - if alternatives: - alt = alternatives[0] - is_final = channel.get("is_final", False) - - if is_final: - # Accumulate final transcript and words - transcript = alt.get("transcript", "") - words = alt.get("words", []) - - if transcript.strip(): - stream_data["final_transcript"] += transcript + " " - stream_data["words"].extend(words) - - logger.debug(f"Final transcript chunk: {transcript}") - - except asyncio.TimeoutError: - # No response available, continue - pass - - return None # Streaming, no final result yet - - except Exception as e: - logger.error(f"Error processing audio chunk for {client_id}: {e}") - return None - - async def end_stream(self, client_id: str) -> dict: - """Close WebSocket connection and return final transcription.""" - if client_id not in self._streams: - logger.error(f"No active stream for client {client_id}") - return {"text": "", "words": [], "segments": []} - - try: - stream_data = self._streams[client_id] - websocket = stream_data["websocket"] - - # Send close message - close_msg = json.dumps({"type": "CloseStream"}) - await websocket.send(close_msg) - - # Wait a bit for final responses - try: - end_time = asyncio.get_event_loop().time() + 2.0 # 2 second timeout - while asyncio.get_event_loop().time() < end_time: - response = await asyncio.wait_for(websocket.recv(), timeout=0.5) - result = json.loads(response) - - if result.get("type") == "Results": - channel = result.get("channel", {}) - alternatives = channel.get("alternatives", []) - - if alternatives and channel.get("is_final", False): - alt = alternatives[0] - transcript = alt.get("transcript", "") - words = alt.get("words", []) - - if transcript.strip(): - stream_data["final_transcript"] += transcript - stream_data["words"].extend(words) - - except asyncio.TimeoutError: - pass - - # Close WebSocket - await websocket.close() - - # Prepare final result - final_transcript = stream_data["final_transcript"].strip() - final_words = stream_data["words"] - - logger.info(f"Deepgram streaming completed for {client_id}: {len(final_transcript)} chars, {len(final_words)} words") - - # Clean up - del self._streams[client_id] - - return { - "text": final_transcript, - "words": final_words, - "segments": [] - } - - except Exception as e: - logger.error(f"Error ending stream for {client_id}: {e}") - # Clean up on error - if client_id in self._streams: - del self._streams[client_id] - return {"text": "", "words": [], "segments": []} - - async def transcribe(self, audio_data: bytes, sample_rate: int, **kwargs) -> dict: - """For streaming provider, this method is not typically used.""" - logger.warning("transcribe() called on streaming provider - use streaming methods instead") - return {"text": "", "words": [], "segments": []} - - async def disconnect(self): - """Close all active WebSocket connections.""" - for client_id in list(self._streams.keys()): - try: - websocket = self._streams[client_id]["websocket"] - await websocket.close() - except Exception as e: - logger.error(f"Error closing WebSocket for {client_id}: {e}") - finally: - del self._streams[client_id] - - logger.info("All Deepgram streaming connections closed") - class DeepgramStreamConsumer: """ @@ -411,40 +16,42 @@ class DeepgramStreamConsumer: Reads from: specified stream (client-specific or provider-specific) Writes to: transcription:results:{session_id} - This inherits from BaseAudioStreamConsumer and implements transcribe_audio(). + Uses RegistryBatchTranscriptionProvider configured via config.yml for + Deepgram transcription. This ensures consistent behavior with batch + transcription jobs. """ - def __init__(self, redis_client, api_key: str = None, buffer_chunks: int = 30): + def __init__(self, redis_client, buffer_chunks: int = 30): """ Initialize Deepgram consumer. Dynamically discovers all audio:stream:* streams and claims them using Redis locks. + Uses config.yml stt-deepgram configuration for transcription. Args: redis_client: Connected Redis client - api_key: Deepgram API key (defaults to DEEPGRAM_API_KEY env var) buffer_chunks: Number of chunks to buffer before transcribing (default: 30 = ~7.5s) """ - import os from advanced_omi_backend.services.audio_stream.consumer import BaseAudioStreamConsumer + from advanced_omi_backend.services.transcription import get_transcription_provider - self.api_key = api_key or os.getenv("DEEPGRAM_API_KEY") - if not self.api_key: - raise ValueError("DEEPGRAM_API_KEY is required") - - # Initialize Deepgram provider - self.provider = DeepgramProvider(api_key=self.api_key) + # Get registry-driven transcription provider + self.provider = get_transcription_provider(mode="batch") + if not self.provider: + raise RuntimeError( + "Failed to load transcription provider. Ensure config.yml has a default 'stt' model configured." + ) # Create a concrete subclass that implements transcribe_audio class _ConcreteConsumer(BaseAudioStreamConsumer): def __init__(inner_self, provider_name: str, redis_client, buffer_chunks: int): super().__init__(provider_name, redis_client, buffer_chunks) - inner_self._deepgram_provider = self.provider + inner_self._transcription_provider = self.provider async def transcribe_audio(inner_self, audio_data: bytes, sample_rate: int) -> dict: - """Transcribe using DeepgramProvider.""" + """Transcribe using registry-driven transcription provider.""" try: - result = await inner_self._deepgram_provider.transcribe( + result = await inner_self._transcription_provider.transcribe( audio_data=audio_data, sample_rate=sample_rate, diarize=True @@ -482,5 +89,3 @@ async def start_consuming(self): async def stop(self): """Delegate to base consumer.""" return await self._consumer.stop() - - diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet.py b/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet.py deleted file mode 100644 index 97b5b751..00000000 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet.py +++ /dev/null @@ -1,303 +0,0 @@ -""" -Parakeet (NeMo) transcription provider implementations. - -Provides both batch and streaming transcription using NeMo's Parakeet ASR models. -""" - -import asyncio -import json -import logging -import os -import tempfile -from typing import Dict, Optional - -import httpx -import numpy as np -import websockets -from easy_audio_interfaces.audio_interfaces import AudioChunk -from easy_audio_interfaces.filesystem import LocalFileSink - -from .base import ( - BatchTranscriptionProvider, - StreamingTranscriptionProvider, -) - -logger = logging.getLogger(__name__) - -class ParakeetProvider(BatchTranscriptionProvider): - """Parakeet HTTP batch transcription provider.""" - - def __init__(self, service_url: str): - self.service_url = service_url.rstrip('/') - self.transcribe_url = f"{self.service_url}/transcribe" - - @property - def name(self) -> str: - return "parakeet" - - async def transcribe(self, audio_data: bytes, sample_rate: int, **kwargs) -> dict: - """Transcribe audio using Parakeet HTTP service.""" - try: - - logger.info(f"Sending {len(audio_data)} bytes to Parakeet service at {self.transcribe_url}") - - # Convert PCM bytes to audio file for upload - if sample_rate != 16000: - logger.warning(f"Sample rate {sample_rate} != 16000, audio may not be optimal") - - # Assume 16-bit PCM - audio_array = np.frombuffer(audio_data, dtype=np.int16).astype(np.float32) - audio_array = audio_array / np.iinfo(np.int16).max # Normalize to [-1, 1] - - # Create temporary WAV file - with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_file: - # sf.write(tmp_file.name, audio_array, 16000) # Force 16kHz - async with LocalFileSink(tmp_file.name, sample_rate, 1) as sink: - await sink.write(AudioChunk( - rate=sample_rate, - width=2, - channels=1, - audio=audio_data, - )) - - tmp_filename = tmp_file.name - - try: - # Upload file to Parakeet service - async with httpx.AsyncClient(timeout=180.0) as client: - with open(tmp_filename, "rb") as f: - files = {"file": ("audio.wav", f, "audio/wav")} - response = await client.post(self.transcribe_url, files=files) - - if response.status_code == 200: - result = response.json() - logger.info(f"Parakeet transcription successful: {len(result.get('text', ''))} chars, {len(result.get('words', []))} words") - return result - else: - error_msg = f"Parakeet service error: {response.status_code} - {response.text}" - logger.error(error_msg) - - # For 5xx errors, raise exception to trigger retry/failure handling - if response.status_code >= 500: - raise RuntimeError(f"Parakeet service unavailable: HTTP {response.status_code}") - - # For 4xx errors, return empty result (client error, won't retry) - return {"text": "", "words": [], "segments": []} - - finally: - # Clean up temporary file - if os.path.exists(tmp_filename): - os.unlink(tmp_filename) - - except Exception as e: - logger.error(f"Error calling Parakeet service: {e}") - raise e - - -class ParakeetStreamingProvider(StreamingTranscriptionProvider): - """Parakeet WebSocket streaming transcription provider.""" - - def __init__(self, service_url: str): - self.service_url = service_url.rstrip('/') - self.ws_url = service_url.replace("http://", "ws://").replace("https://", "wss://") + "/stream" - self._streams: Dict[str, Dict] = {} # client_id -> stream data - - @property - def name(self) -> str: - return "parakeet" - - async def start_stream(self, client_id: str, sample_rate: int = 16000, diarize: bool = False): - """Start a WebSocket connection for streaming transcription. - - Args: - client_id: Unique client identifier - sample_rate: Audio sample rate - diarize: Whether to enable speaker diarization (ignored - Parakeet doesn't support diarization) - """ - if diarize: - logger.warning(f"Parakeet streaming provider does not support diarization, ignoring diarize=True for client {client_id}") - try: - logger.info(f"Starting Parakeet streaming for client {client_id}") - - # Connect to WebSocket - websocket = await websockets.connect(self.ws_url) - - # Send transcribe event to start session - session_config = { - "vad_enabled": True, - "vad_silence_ms": 1000, - "time_interval_seconds": 30, - "return_interim_results": True, - "min_audio_seconds": 0.5 - } - - start_message = { - "type": "transcribe", - "session_id": client_id, - "config": session_config - } - - await websocket.send(json.dumps(start_message)) - - # Wait for session_started confirmation - response = await websocket.recv() - response_data = json.loads(response) - - if response_data.get("type") != "session_started": - raise RuntimeError(f"Failed to start session: {response_data}") - - # Store stream data - self._streams[client_id] = { - "websocket": websocket, - "sample_rate": sample_rate, - "session_id": client_id, - "interim_results": [], - "final_result": None - } - - logger.info(f"Parakeet WebSocket connected for client {client_id}") - - except Exception as e: - logger.error(f"Failed to start Parakeet streaming for {client_id}: {e}") - raise - - async def process_audio_chunk(self, client_id: str, audio_chunk: bytes) -> Optional[dict]: - """Send audio chunk to WebSocket and process responses.""" - if client_id not in self._streams: - logger.error(f"No active stream for client {client_id}") - return None - - try: - stream_data = self._streams[client_id] - websocket = stream_data["websocket"] - sample_rate = stream_data["sample_rate"] - - # Send audio_chunk event - chunk_message = { - "type": "audio_chunk", - "session_id": client_id, - "rate": sample_rate, - "width": 2, # 16-bit - "channels": 1 - } - - await websocket.send(json.dumps(chunk_message)) - await websocket.send(audio_chunk) - - # Check for responses (non-blocking) - try: - while True: - response = await asyncio.wait_for(websocket.recv(), timeout=0.01) - result = json.loads(response) - - if result.get("type") == "interim_result": - # Store interim result but don't return it (handled by backend differently) - stream_data["interim_results"].append(result) - logger.debug(f"Received interim result: {result.get('text', '')[:50]}...") - elif result.get("type") == "final_result": - # This shouldn't happen during chunk processing, but store it - stream_data["final_result"] = result - logger.debug(f"Received final result during chunk processing: {result.get('text', '')[:50]}...") - - except asyncio.TimeoutError: - # No response available, continue - pass - - return None # Streaming, no final result yet - - except Exception as e: - logger.error(f"Error processing audio chunk for {client_id}: {e}") - return None - - async def end_stream(self, client_id: str) -> dict: - """Close WebSocket connection and return final transcription.""" - if client_id not in self._streams: - logger.error(f"No active stream for client {client_id}") - return {"text": "", "words": [], "segments": []} - - try: - stream_data = self._streams[client_id] - websocket = stream_data["websocket"] - - # Send finalize event - finalize_message = { - "type": "finalize", - "session_id": client_id - } - await websocket.send(json.dumps(finalize_message)) - - # Wait for final result - try: - end_time = asyncio.get_event_loop().time() + 5.0 # 5 second timeout - while asyncio.get_event_loop().time() < end_time: - response = await asyncio.wait_for(websocket.recv(), timeout=1.0) - result = json.loads(response) - - if result.get("type") == "final_result": - stream_data["final_result"] = result - break - - except asyncio.TimeoutError: - logger.warning(f"Timeout waiting for final result from {client_id}") - - # Close WebSocket - await websocket.close() - - # Prepare final result - final_result = stream_data.get("final_result") - if final_result: - result_data = { - "text": final_result.get("text", ""), - "words": final_result.get("words", []), - "segments": final_result.get("segments", []) - } - else: - # Fallback: aggregate interim results if no final result received - interim_texts = [r.get("text", "") for r in stream_data["interim_results"]] - all_words = [] - for r in stream_data["interim_results"]: - all_words.extend(r.get("words", [])) - - result_data = { - "text": " ".join(interim_texts), - "words": all_words, - "segments": [] - } - - logger.info(f"Parakeet streaming completed for {client_id}: {len(result_data.get('text', ''))} chars") - - # Clean up - del self._streams[client_id] - - return result_data - - except Exception as e: - logger.error(f"Error ending stream for {client_id}: {e}") - # Clean up on error - if client_id in self._streams: - try: - await self._streams[client_id]["websocket"].close() - except: - pass - del self._streams[client_id] - return {"text": "", "words": [], "segments": []} - - async def transcribe(self, audio_data: bytes, sample_rate: int, **kwargs) -> dict: - """For streaming provider, this method is not typically used.""" - logger.warning("transcribe() called on streaming provider - use streaming methods instead") - return {"text": "", "words": [], "segments": []} - - async def disconnect(self): - """Close all active WebSocket connections.""" - for client_id in list(self._streams.keys()): - try: - websocket = self._streams[client_id]["websocket"] - await websocket.close() - except Exception as e: - logger.error(f"Error closing WebSocket for {client_id}: {e}") - finally: - del self._streams[client_id] - - logger.info("All Parakeet streaming connections closed") - - diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet_stream_consumer.py b/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet_stream_consumer.py index 740a5f84..f629cefd 100644 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet_stream_consumer.py +++ b/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet_stream_consumer.py @@ -6,10 +6,9 @@ """ import logging -import os from advanced_omi_backend.services.audio_stream.consumer import BaseAudioStreamConsumer -from advanced_omi_backend.services.transcription.parakeet import ParakeetProvider +from advanced_omi_backend.services.transcription import get_transcription_provider logger = logging.getLogger(__name__) @@ -24,23 +23,23 @@ class ParakeetStreamConsumer: This inherits from BaseAudioStreamConsumer and implements transcribe_audio(). """ - def __init__(self, redis_client, service_url: str = None, buffer_chunks: int = 30): + def __init__(self, redis_client, buffer_chunks: int = 30): """ Initialize Parakeet consumer. Dynamically discovers all audio:stream:* streams and claims them using Redis locks. + Uses config.yml stt-parakeet-batch configuration for transcription. Args: redis_client: Connected Redis client - service_url: Parakeet service URL (defaults to PARAKEET_ASR_URL env var) buffer_chunks: Number of chunks to buffer before transcribing (default: 30 = ~7.5s) """ - self.service_url = service_url or os.getenv("PARAKEET_ASR_URL") - if not self.service_url: - raise ValueError("PARAKEET_ASR_URL is required") - - # Initialize Parakeet provider - self.provider = ParakeetProvider(service_url=self.service_url) + # Get registry-driven transcription provider + self.provider = get_transcription_provider(mode="batch") + if not self.provider: + raise RuntimeError( + "Failed to load transcription provider. Ensure config.yml has a default 'stt' model configured." + ) # Create a concrete subclass that implements transcribe_audio class _ConcreteConsumer(BaseAudioStreamConsumer): diff --git a/backends/advanced/src/advanced_omi_backend/workers/audio_stream_deepgram_worker.py b/backends/advanced/src/advanced_omi_backend/workers/audio_stream_deepgram_worker.py index c8866eed..a58682c1 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/audio_stream_deepgram_worker.py +++ b/backends/advanced/src/advanced_omi_backend/workers/audio_stream_deepgram_worker.py @@ -27,12 +27,13 @@ async def main(): """Main worker entry point.""" logger.info("๐Ÿš€ Starting Deepgram audio stream worker") - # Get configuration from environment + # Check that config.yml has Deepgram configured + # The registry provider will load configuration from config.yml api_key = os.getenv("DEEPGRAM_API_KEY") if not api_key: - logger.warning("DEEPGRAM_API_KEY environment variable not set - Deepgram audio stream worker will not start") - logger.warning("Audio transcription will use alternative providers if configured") - return + logger.warning("DEEPGRAM_API_KEY environment variable not set") + logger.warning("Ensure config.yml has a default 'stt' model configured for Deepgram") + logger.warning("Audio transcription will use alternative providers if configured in config.yml") redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0") @@ -47,9 +48,9 @@ async def main(): # Create consumer with balanced buffer size # 20 chunks = ~5 seconds of audio # Balance between transcription accuracy and latency + # Consumer uses registry-driven provider from config.yml consumer = DeepgramStreamConsumer( redis_client=redis_client, - api_key=api_key, buffer_chunks=20 # 5 seconds - good context without excessive delay ) diff --git a/backends/advanced/src/advanced_omi_backend/workers/audio_stream_parakeet_worker.py b/backends/advanced/src/advanced_omi_backend/workers/audio_stream_parakeet_worker.py index 0c368a2b..56f2f26b 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/audio_stream_parakeet_worker.py +++ b/backends/advanced/src/advanced_omi_backend/workers/audio_stream_parakeet_worker.py @@ -27,12 +27,13 @@ async def main(): """Main worker entry point.""" logger.info("๐Ÿš€ Starting Parakeet audio stream worker") - # Get configuration from environment + # Check that config.yml has Parakeet configured + # The registry provider will load configuration from config.yml service_url = os.getenv("PARAKEET_ASR_URL") if not service_url: - logger.warning("PARAKEET_ASR_URL environment variable not set - Parakeet audio stream worker will not start") - logger.warning("Audio transcription will use alternative providers if configured") - return + logger.warning("PARAKEET_ASR_URL environment variable not set") + logger.warning("Ensure config.yml has a default 'stt' model configured for Parakeet") + logger.warning("Audio transcription will use alternative providers if configured in config.yml") redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0") @@ -47,9 +48,9 @@ async def main(): # Create consumer with balanced buffer size # 20 chunks = ~5 seconds of audio # Balance between transcription accuracy and latency + # Consumer uses registry-driven provider from config.yml consumer = ParakeetStreamConsumer( redis_client=redis_client, - service_url=service_url, buffer_chunks=20 # 5 seconds - good context without excessive delay ) diff --git a/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py index 31dba573..8b64d690 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py @@ -89,8 +89,11 @@ async def process_memory_job(conversation_id: str, *, redis_client=None) -> Dict if text: dialogue_lines.append(f"{speaker}: {text}") full_conversation = "\n".join(dialogue_lines) - elif conversation_model.transcript and isinstance(conversation_model.transcript, str): - # Fallback: if segments are empty but transcript text exists + + # Fallback: if segments have no text content but transcript exists, use transcript + # This handles cases where speaker recognition fails/is disabled + if len(full_conversation) < 10 and conversation_model.transcript and isinstance(conversation_model.transcript, str): + logger.info(f"Segments empty or too short, falling back to transcript text for {conversation_id}") full_conversation = conversation_model.transcript if len(full_conversation) < 10: From ea58d6eae40ea0faf8c39f1601ea425368818ddc Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Tue, 30 Dec 2025 02:30:18 +0000 Subject: [PATCH 03/14] Update configuration management and enhance file structure - Refactored configuration file paths to use a dedicated `config/` directory, including updates to `config.yml` and its template. - Modified service scripts to load the new configuration path for `config.yml`. - Enhanced `.gitignore` to include the new configuration files and templates. - Updated documentation to reflect changes in configuration file locations and usage. - Improved setup scripts to ensure proper creation and management of configuration files. - Added new test configurations for various provider combinations to streamline testing processes. --- .gitignore | 12 +- Docs/getting-started.md | 10 +- backends/advanced/Docs/README.md | 20 +-- backends/advanced/Docs/contribution.md | 4 +- backends/advanced/Docs/memories.md | 4 +- .../Docs/memory-configuration-guide.md | 6 +- backends/advanced/Docs/quickstart.md | 10 +- backends/advanced/SETUP_SCRIPTS.md | 2 +- backends/advanced/docker-compose-test.yml | 4 +- backends/advanced/docker-compose.yml | 4 +- backends/advanced/init.py | 4 +- backends/advanced/run-test.sh | 13 ++ backends/advanced/start-workers.sh | 23 ++- config/README.md | 106 ++++++++++++++ .../config.yml.template | 0 extras/speaker-recognition/run-test.sh | 9 +- services.py | 4 +- tests/configs/README.md | 132 +++++++++++++++++ tests/configs/deepgram-openai.yml | 84 +++++++++++ tests/configs/full-local.yml | 1 + tests/configs/parakeet-ollama.yml | 73 ++++++++++ tests/configs/parakeet-openai.yml | 73 ++++++++++ tests/integration/integration_test.robot | 40 +++++ tests/resources/audio_keywords.robot | 27 ++++ tests/resources/memory_keywords.robot | 137 ++++++++++++++++++ tests/run-robot-tests.sh | 16 +- tests/setup/test_data.py | 14 ++ wizard.py | 14 +- 28 files changed, 792 insertions(+), 54 deletions(-) create mode 100644 config/README.md rename config.yml.template => config/config.yml.template (100%) create mode 100644 tests/configs/README.md create mode 100644 tests/configs/deepgram-openai.yml create mode 120000 tests/configs/full-local.yml create mode 100644 tests/configs/parakeet-ollama.yml create mode 100644 tests/configs/parakeet-openai.yml diff --git a/.gitignore b/.gitignore index 38cb0c88..933a1165 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,16 @@ !**/.env.template **/memory_config.yaml !**/memory_config.yaml.template -config.yml -!config.yml.template +tests/setup/.env.test + +# Main config (user-specific) +config/config.yml +!config/config.yml.template + +# Config backups +config/*.backup.* +config/*.backup* + example/* **/node_modules/* **/ollama-data/* diff --git a/Docs/getting-started.md b/Docs/getting-started.md index dfa3dabf..506dd2f6 100644 --- a/Docs/getting-started.md +++ b/Docs/getting-started.md @@ -342,7 +342,7 @@ curl -X POST "http://localhost:8000/api/process-audio-files" \ **Implementation**: - **Memory System**: `src/advanced_omi_backend/memory/memory_service.py` + `src/advanced_omi_backend/controllers/memory_controller.py` -- **Configuration**: memory settings in `config.yml` (memory section) +- **Configuration**: memory settings in `config/config.yml` (memory section) ### Authentication & Security - **Email Authentication**: Login with email and password @@ -541,10 +541,10 @@ OPENMEMORY_MCP_URL=http://host.docker.internal:8765 > ๐ŸŽฏ **New to memory configuration?** Read our [Memory Configuration Guide](./memory-configuration-guide.md) for a step-by-step setup guide with examples. -The system uses **centralized configuration** via `config.yml` for all models (LLM, embeddings, vector store) and memory extraction settings. +The system uses **centralized configuration** via `config/config.yml` for all models (LLM, embeddings, vector store) and memory extraction settings. ### Configuration File Location -- **Path**: repository `config.yml` (override with `CONFIG_FILE` env var) +- **Path**: repository `config/config.yml` (override with `CONFIG_FILE` env var) - **Hot-reload**: Changes are applied on next processing cycle (no restart required) - **Fallback**: If file is missing, system uses safe defaults with environment variables @@ -613,7 +613,7 @@ If you experience JSON parsing errors in fact extraction: 2. **Enable fact extraction** with reliable JSON output: ```yaml - # In config.yml (memory section) + # In config/config.yml (memory section) fact_extraction: enabled: true # Safe to enable with GPT-4o ``` @@ -727,5 +727,5 @@ curl -H "Authorization: Bearer $ADMIN_TOKEN" \ - **Connect audio clients** using the WebSocket API - **Explore the dashboard** to manage conversations and users - **Review the user data architecture** for understanding data organization -- **Customize memory extraction** by editing the `memory` section in `config.yml` +- **Customize memory extraction** by editing the `memory` section in `config/config.yml` - **Monitor processing performance** using debug API endpoints diff --git a/backends/advanced/Docs/README.md b/backends/advanced/Docs/README.md index abddef9b..11e683e8 100644 --- a/backends/advanced/Docs/README.md +++ b/backends/advanced/Docs/README.md @@ -13,7 +13,7 @@ Welcome to chronicle! This guide provides the optimal reading sequence to unders - What the system does (voice โ†’ memories) - Key features and capabilities - Basic setup and configuration -- **Code References**: `src/advanced_omi_backend/main.py`, `config.yml`, `docker-compose.yml` +- **Code References**: `src/advanced_omi_backend/main.py`, `config/config.yml`, `docker-compose.yml` ### 2. **[System Architecture](./architecture.md)** **Read second** - Complete technical architecture with diagrams @@ -70,7 +70,7 @@ Welcome to chronicle! This guide provides the optimal reading sequence to unders ## ๐Ÿ” **Configuration & Customization** -### 6. **Configuration File** โ†’ `../config.yml` +### 6. **Configuration File** โ†’ `../config/config.yml` **Central configuration for all extraction** - Memory extraction settings and prompts - Quality control and debug settings @@ -86,11 +86,11 @@ Welcome to chronicle! This guide provides the optimal reading sequence to unders 1. [quickstart.md](./quickstart.md) - System overview 2. [architecture.md](./architecture.md) - Technical architecture 3. `src/advanced_omi_backend/main.py` - Core imports and setup -4. `config.yml` - Configuration overview +4. `config/config.yml` - Configuration overview ### **"I want to work on memory extraction"** 1. [memories.md](./memories.md) - Memory system details -2. `../config.yml` - Models and memory configuration +2. `../config/config.yml` - Models and memory configuration 3. `src/advanced_omi_backend/memory/memory_service.py` - Implementation 4. `src/advanced_omi_backend/controllers/memory_controller.py` - Processing triggers @@ -130,7 +130,7 @@ backends/advanced-backend/ โ”‚ โ”‚ โ””โ”€โ”€ memory_service.py # Memory system (Mem0) โ”‚ โ””โ”€โ”€ model_registry.py # Configuration loading โ”‚ -โ”œโ”€โ”€ config.yml # ๐Ÿ“‹ Central configuration +โ”œโ”€โ”€ config/config.yml # ๐Ÿ“‹ Central configuration โ”œโ”€โ”€ MEMORY_DEBUG_IMPLEMENTATION.md # Debug system details ``` @@ -148,7 +148,7 @@ backends/advanced-backend/ ### **Configuration** - **Loading**: `src/advanced_omi_backend/model_registry.py` -- **File**: `config.yml` +- **File**: `config/config.yml` - **Usage**: `src/advanced_omi_backend/memory/memory_service.py` ### **Authentication** @@ -162,7 +162,7 @@ backends/advanced-backend/ 1. **Follow the references**: Each doc links to specific code files and line numbers 2. **Use the debug API**: `GET /api/debug/memory/stats` shows live system status -3. **Check configuration first**: Many behaviors are controlled by `config.yml` +3. **Check configuration first**: Many behaviors are controlled by `config/config.yml` 4. **Understand the memory pipeline**: Memories (end-of-conversation) 5. **Test with curl**: All API endpoints have curl examples in the docs @@ -175,20 +175,20 @@ backends/advanced-backend/ 1. **Set up the system**: Follow [quickstart.md](./quickstart.md) to get everything running 2. **Test the API**: Use the curl examples in the documentation to test endpoints 3. **Explore the debug system**: Check `GET /api/debug/memory/stats` to see live data -4. **Modify configuration**: Edit `config.yml` (memory section) to see how it affects extraction +4. **Modify configuration**: Edit `config/config.yml` (memory section) to see how it affects extraction 5. **Read the code**: Start with `src/advanced_omi_backend/main.py` and follow the references in each doc ### **Contributing Guidelines** - **Add code references**: When updating docs, include file paths and line numbers - **Test your changes**: Use the debug API to verify your modifications work -- **Update configuration**: Add new settings to `config.yml` when needed +- **Update configuration**: Add new settings to `config/config.yml` when needed - **Follow the architecture**: Keep memories in their respective services ### **Getting Help** - **Debug API**: `GET /api/debug/memory/*` endpoints show real-time system status -- **Configuration**: Check `config.yml` for behavior controls +- **Configuration**: Check `config/config.yml` for behavior controls - **Logs**: Check Docker logs with `docker compose logs chronicle-backend` - **Documentation**: Each doc file links to relevant code sections diff --git a/backends/advanced/Docs/contribution.md b/backends/advanced/Docs/contribution.md index a5766828..b78f4a5a 100644 --- a/backends/advanced/Docs/contribution.md +++ b/backends/advanced/Docs/contribution.md @@ -1,12 +1,12 @@ 1. Docs/quickstart.md (15 min) 2. Docs/architecture.md (20 min) 3. main.py - just the imports and WebSocket sections (15 min) - 4. config.yml (memory section) (10 min) + 4. config/config.yml (memory section) (10 min) ๐Ÿ”ง "I want to work on memory extraction" 1. Docs/quickstart.md โ†’ Docs/memories.md - 2. config.yml (memory.extraction section) + 2. config/config.yml (memory.extraction section) 3. main.py lines 1047-1065 (trigger) 4. main.py lines 1163-1195 (processing) 5. src/memory/memory_service.py diff --git a/backends/advanced/Docs/memories.md b/backends/advanced/Docs/memories.md index 38eed697..cae98383 100644 --- a/backends/advanced/Docs/memories.md +++ b/backends/advanced/Docs/memories.md @@ -10,7 +10,7 @@ This document explains how to configure and customize the memory service in the - **Repository Layer**: `src/advanced_omi_backend/conversation_repository.py` (clean data access) - **Processing Manager**: `src/advanced_omi_backend/processors.py` (MemoryProcessor class) - **Conversation Management**: `src/advanced_omi_backend/conversation_manager.py` (lifecycle coordination) -- **Configuration**: `config.yml` (memory section) + `src/model_registry.py` +- **Configuration**: `config/config.yml` (memory section) + `src/model_registry.py` ## Overview @@ -180,7 +180,7 @@ OPENAI_MODEL=gpt-5-mini # Recommended for reliable JSON output # OPENAI_MODEL=gpt-3.5-turbo # Budget option ``` -Or configure via `config.yml` (memory block): +Or configure via `config/config.yml` (memory block): ```yaml memory_extraction: diff --git a/backends/advanced/Docs/memory-configuration-guide.md b/backends/advanced/Docs/memory-configuration-guide.md index 9a694ac5..12796e13 100644 --- a/backends/advanced/Docs/memory-configuration-guide.md +++ b/backends/advanced/Docs/memory-configuration-guide.md @@ -6,10 +6,10 @@ This guide helps you set up and configure the memory system for the Friend Advan 1. **Copy the template configuration**: ```bash -Edit the `memory` section of `config.yml`. +Edit the `memory` section of `config/config.yml`. ``` -2. **Edit `config.yml`** with your preferred settings in the `memory` section: +2. **Edit `config/config.yml`** with your preferred settings in the `memory` section: ```yaml memory: provider: "mem0" # or "basic" for simpler setup @@ -127,6 +127,6 @@ memory: ## Next Steps -- Configure action items detection in `config.yml` (memory.extraction) +- Configure action items detection in `config/config.yml` (memory.extraction) - Set up custom prompt templates for your use case - Monitor memory processing in the debug dashboard diff --git a/backends/advanced/Docs/quickstart.md b/backends/advanced/Docs/quickstart.md index 6e7f03a2..922fe9b7 100644 --- a/backends/advanced/Docs/quickstart.md +++ b/backends/advanced/Docs/quickstart.md @@ -340,7 +340,7 @@ curl -X POST "http://localhost:8000/api/audio/upload" \ **Implementation**: - **Memory System**: `src/advanced_omi_backend/memory/memory_service.py` + `src/advanced_omi_backend/controllers/memory_controller.py` -- **Configuration**: `config.yml` (memory + models) in repo root +- **Configuration**: `config/config.yml` (memory + models) in repo root ### Authentication & Security - **Email Authentication**: Login with email and password @@ -539,10 +539,10 @@ OPENMEMORY_MCP_URL=http://host.docker.internal:8765 > ๐ŸŽฏ **New to memory configuration?** Read our [Memory Configuration Guide](./memory-configuration-guide.md) for a step-by-step setup guide with examples. -The system uses **centralized configuration** via `config.yml` for all memory extraction and model settings. +The system uses **centralized configuration** via `config/config.yml` for all memory extraction and model settings. ### Configuration File Location -- **Path**: `config.yml` in repo root +- **Path**: `config/config.yml` in repo root - **Hot-reload**: Changes are applied on next processing cycle (no restart required) - **Fallback**: If file is missing, system uses safe defaults with environment variables @@ -611,7 +611,7 @@ If you experience JSON parsing errors in fact extraction: 2. **Enable fact extraction** with reliable JSON output: ```yaml - # In config.yml (memory section) + # In config/config.yml (memory section) fact_extraction: enabled: true # Safe to enable with GPT-4o ``` @@ -725,5 +725,5 @@ curl -H "Authorization: Bearer $ADMIN_TOKEN" \ - **Connect audio clients** using the WebSocket API - **Explore the dashboard** to manage conversations and users - **Review the user data architecture** for understanding data organization -- **Customize memory extraction** by editing the `memory` section in `config.yml` +- **Customize memory extraction** by editing the `memory` section in `config/config.yml` - **Monitor processing performance** using debug API endpoints diff --git a/backends/advanced/SETUP_SCRIPTS.md b/backends/advanced/SETUP_SCRIPTS.md index 8fbc0ab2..b45c8910 100644 --- a/backends/advanced/SETUP_SCRIPTS.md +++ b/backends/advanced/SETUP_SCRIPTS.md @@ -6,7 +6,7 @@ This document explains the different setup scripts available in Friend-Lite and | Script | Purpose | When to Use | |--------|---------|-------------| -| `init.py` | **Main interactive setup wizard** | **Recommended for all users** - First time setup with guided configuration (located at repo root). Memory now configured in `config.yml`. | +| `init.py` | **Main interactive setup wizard** | **Recommended for all users** - First time setup with guided configuration (located at repo root). Memory now configured in `config/config.yml`. | | `setup-https.sh` | HTTPS certificate generation | **Optional** - When you need secure connections for microphone access | ## Main Setup Script: `init.py` diff --git a/backends/advanced/docker-compose-test.yml b/backends/advanced/docker-compose-test.yml index 20b4fd08..4d27c41e 100644 --- a/backends/advanced/docker-compose-test.yml +++ b/backends/advanced/docker-compose-test.yml @@ -14,7 +14,7 @@ services: - ./data/test_audio_chunks:/app/audio_chunks - ./data/test_debug_dir:/app/debug_dir - ./data/test_data:/app/data - - ../../config.yml:/app/config.yml:ro # Mount config.yml for model registry and memory settings + - ${CONFIG_FILE:-../../config/config.yml}:/app/config.yml:ro # Mount config.yml for model registry and memory settings environment: # Override with test-specific settings - MONGODB_URI=mongodb://mongo-test:27017/test_db @@ -129,7 +129,7 @@ services: - ./data/test_audio_chunks:/app/audio_chunks - ./data/test_debug_dir:/app/debug_dir - ./data/test_data:/app/data - - ../../config.yml:/app/config.yml:ro # Mount config.yml for model registry and memory settings + - ${CONFIG_FILE:-../../config/config.yml}:/app/config.yml:ro # Mount config.yml for model registry and memory settings environment: # Same environment as backend - MONGODB_URI=mongodb://mongo-test:27017/test_db diff --git a/backends/advanced/docker-compose.yml b/backends/advanced/docker-compose.yml index 313c0f23..80f27aae 100644 --- a/backends/advanced/docker-compose.yml +++ b/backends/advanced/docker-compose.yml @@ -12,7 +12,7 @@ services: - ./data/audio_chunks:/app/audio_chunks - ./data/debug_dir:/app/debug_dir - ./data:/app/data - - ../../config.yml:/app/config.yml # Removed :ro to allow UI config saving + - ../../config/config.yml:/app/config.yml # Removed :ro to allow UI config saving environment: - DEEPGRAM_API_KEY=${DEEPGRAM_API_KEY} - MISTRAL_API_KEY=${MISTRAL_API_KEY} @@ -65,7 +65,7 @@ services: - ./start-workers.sh:/app/start-workers.sh - ./data/audio_chunks:/app/audio_chunks - ./data:/app/data - - ../../config.yml:/app/config.yml # Removed :ro for consistency + - ../../config/config.yml:/app/config.yml # Removed :ro for consistency environment: - DEEPGRAM_API_KEY=${DEEPGRAM_API_KEY} - MISTRAL_API_KEY=${MISTRAL_API_KEY} diff --git a/backends/advanced/init.py b/backends/advanced/init.py index 851d56e1..11390ff1 100644 --- a/backends/advanced/init.py +++ b/backends/advanced/init.py @@ -28,7 +28,7 @@ def __init__(self, args=None): self.console = Console() self.config: Dict[str, Any] = {} self.args = args or argparse.Namespace() - self.config_yml_path = Path("../../config.yml") # Repo root config.yml + self.config_yml_path = Path("../../config/config.yml") # Main config at config/config.yml self.config_yml_data = None # Check if we're in the right directory @@ -726,7 +726,7 @@ def run(self): self.console.print() self.console.print("๐Ÿ“ [bold]Configuration files updated:[/bold]") self.console.print(f" โ€ข .env - API keys and environment variables") - self.console.print(f" โ€ข ../../config.yml - Model and memory provider configuration") + self.console.print(f" โ€ข ../../config/config.yml - Model and memory provider configuration") self.console.print() self.console.print("For detailed documentation, see:") self.console.print(" โ€ข Docs/quickstart.md") diff --git a/backends/advanced/run-test.sh b/backends/advanced/run-test.sh index 4f944256..e9544be6 100755 --- a/backends/advanced/run-test.sh +++ b/backends/advanced/run-test.sh @@ -47,6 +47,7 @@ _DEEPGRAM_API_KEY_OVERRIDE=${DEEPGRAM_API_KEY} _OPENAI_API_KEY_OVERRIDE=${OPENAI_API_KEY} _LLM_PROVIDER_OVERRIDE=${LLM_PROVIDER} _MEMORY_PROVIDER_OVERRIDE=${MEMORY_PROVIDER} +_CONFIG_FILE_OVERRIDE=${CONFIG_FILE} if [ -n "$DEEPGRAM_API_KEY" ] && [ -z "$_TRANSCRIPTION_PROVIDER_OVERRIDE" ]; then print_info "Using environment variables from CI/environment..." @@ -90,6 +91,15 @@ if [ -n "$_MEMORY_PROVIDER_OVERRIDE" ]; then export MEMORY_PROVIDER=$_MEMORY_PROVIDER_OVERRIDE print_info "Using command-line override: MEMORY_PROVIDER=$MEMORY_PROVIDER" fi +if [ -n "$_CONFIG_FILE_OVERRIDE" ]; then + export CONFIG_FILE=$_CONFIG_FILE_OVERRIDE + print_info "Using command-line override: CONFIG_FILE=$CONFIG_FILE" +fi + +# Set default CONFIG_FILE if not provided +# This allows testing with different provider combinations +# Usage: CONFIG_FILE=../../tests/configs/parakeet-ollama.yml ./run-test.sh +export CONFIG_FILE=${CONFIG_FILE:-../../config/config.yml} # Verify required environment variables based on configured providers TRANSCRIPTION_PROVIDER=${TRANSCRIPTION_PROVIDER:-deepgram} @@ -162,6 +172,9 @@ print_info "Using environment variables from .env file for test configuration" print_info "Cleaning test environment..." sudo rm -rf ./test_audio_chunks/ ./test_data/ ./test_debug_dir/ ./mongo_data_test/ ./qdrant_data_test/ ./test_neo4j/ || true +# Use unique project name to avoid conflicts with development environment +export COMPOSE_PROJECT_NAME="advanced-backend-test" + # Stop any existing test containers print_info "Stopping existing test containers..." docker compose -f docker-compose-test.yml down -v || true diff --git a/backends/advanced/start-workers.sh b/backends/advanced/start-workers.sh index a5ca2798..ad6bd6eb 100755 --- a/backends/advanced/start-workers.sh +++ b/backends/advanced/start-workers.sh @@ -51,23 +51,34 @@ start_workers() { uv run python -m advanced_omi_backend.workers.rq_worker_entry audio & AUDIO_PERSISTENCE_WORKER_PID=$! - # Only start Deepgram worker if DEEPGRAM_API_KEY is set - if [ -n "$DEEPGRAM_API_KEY" ]; then + # Determine which STT provider to use from config.yml + echo "๐Ÿ“‹ Checking config.yml for default STT provider..." + DEFAULT_STT=$(uv run python -c " +from advanced_omi_backend.model_registry import get_models_registry +registry = get_models_registry() +if registry and registry.defaults: + print(registry.defaults.get('stt', '')) +" 2>/dev/null || echo "") + + echo "๐Ÿ“‹ Configured STT provider: ${DEFAULT_STT:-none}" + + # Only start Deepgram worker if configured as default STT + if [[ "$DEFAULT_STT" == *"deepgram"* ]] && [ -n "$DEEPGRAM_API_KEY" ]; then echo "๐ŸŽต Starting audio stream Deepgram worker (1 worker for sequential processing)..." uv run python -m advanced_omi_backend.workers.audio_stream_deepgram_worker & AUDIO_STREAM_DEEPGRAM_WORKER_PID=$! else - echo "โญ๏ธ Skipping Deepgram stream worker (DEEPGRAM_API_KEY not set)" + echo "โญ๏ธ Skipping Deepgram stream worker (not configured as default STT or API key missing)" AUDIO_STREAM_DEEPGRAM_WORKER_PID="" fi - # Only start Parakeet worker if PARAKEET_ASR_URL is set - if [ -n "$PARAKEET_ASR_URL" ]; then + # Only start Parakeet worker if configured as default STT + if [[ "$DEFAULT_STT" == *"parakeet"* ]]; then echo "๐ŸŽต Starting audio stream Parakeet worker (1 worker for sequential processing)..." uv run python -m advanced_omi_backend.workers.audio_stream_parakeet_worker & AUDIO_STREAM_PARAKEET_WORKER_PID=$! else - echo "โญ๏ธ Skipping Parakeet stream worker (PARAKEET_ASR_URL not set)" + echo "โญ๏ธ Skipping Parakeet stream worker (not configured as default STT)" AUDIO_STREAM_PARAKEET_WORKER_PID="" fi diff --git a/config/README.md b/config/README.md new file mode 100644 index 00000000..e3a5cf3c --- /dev/null +++ b/config/README.md @@ -0,0 +1,106 @@ +# Chronicle Configuration + +This directory contains Chronicle's centralized configuration files. + +## Files + +- **`config.yml`** - Main configuration file (gitignored, user-specific) + - Contains model registry (LLM, STT, TTS, embeddings, vector store) + - Memory provider settings + - Service endpoints and API keys + +- **`config.yml.template`** - Template for new setups + - Use this to create your `config.yml` + - Contains placeholders with `${ENV_VAR:-default}` patterns + - No secrets included - safe to commit + +## Setup + +### First Time Setup + +```bash +# Option 1: Run the interactive wizard (recommended) +uv run --with-requirements setup-requirements.txt python wizard.py + +# Option 2: Manual setup +cp config/config.yml.template config/config.yml +# Edit config.yml to add your API keys and configure providers +``` + +### Environment Variable Substitution + +The config system supports environment variable substitution using `${VAR:-default}` syntax: + +```yaml +models: + - name: openai-llm + api_key: ${OPENAI_API_KEY:-} # Uses env var or empty string + model_url: ${OPENAI_BASE_URL:-https://api.openai.com/v1} # With fallback +``` + +## Configuration Sections + +### Defaults + +Specifies which models to use by default: + +```yaml +defaults: + llm: openai-llm # Default LLM model + embedding: openai-embed # Default embedding model + stt: stt-deepgram # Default speech-to-text + vector_store: vs-qdrant # Default vector database +``` + +### Models + +Array of model definitions - each model includes: +- `name`: Unique identifier +- `model_type`: llm, embedding, stt, tts, vector_store +- `model_provider`: openai, ollama, deepgram, parakeet, etc. +- `model_name`: Provider-specific model name +- `model_url`: API endpoint +- `api_key`: Authentication (use env vars!) +- `model_params`: Temperature, max_tokens, etc. + +### Memory + +Memory extraction and storage configuration: + +```yaml +memory: + provider: chronicle # chronicle, openmemory_mcp, or mycelia + timeout_seconds: 1200 + extraction: + enabled: true + prompt: "Custom extraction prompt..." +``` + +## Test Configurations + +For testing different provider combinations, see `tests/configs/`: +- These configs are version-controlled +- Use with `CONFIG_FILE` environment variable +- No secrets - only env var placeholders + +Example: +```bash +CONFIG_FILE=tests/configs/parakeet-ollama.yml ./backends/advanced/run-test.sh +``` + +## Hot Reload + +The memory configuration section supports hot reload - changes are picked up without service restart. Model registry changes require service restart. + +## Backups + +The setup wizard automatically backs up `config.yml` before making changes: +- Backups: `config.yml.backup.YYYYMMDD_HHMMSS` +- These are gitignored automatically + +## Documentation + +For detailed configuration guides, see: +- `/Docs/memory-configuration-guide.md` - Memory settings +- `/backends/advanced/Docs/quickstart.md` - Setup guide +- `/CLAUDE.md` - Project overview diff --git a/config.yml.template b/config/config.yml.template similarity index 100% rename from config.yml.template rename to config/config.yml.template diff --git a/extras/speaker-recognition/run-test.sh b/extras/speaker-recognition/run-test.sh index 6ac212fa..ac73de91 100755 --- a/extras/speaker-recognition/run-test.sh +++ b/extras/speaker-recognition/run-test.sh @@ -13,12 +13,12 @@ cleanup() { return fi cleanup_called=true - + print_info "Cleaning up on exit..." # Kill any background processes in this process group pkill -P $$ 2>/dev/null || true - # Clean up test containers - docker compose -f docker-compose-test.yml down -v 2>/dev/null || true + # Clean up test containers (use project name for consistency) + COMPOSE_PROJECT_NAME="speaker-recognition-test" docker compose -f docker-compose-test.yml down -v 2>/dev/null || true } # Set up signal traps for proper cleanup (but not EXIT to avoid double cleanup) @@ -124,6 +124,9 @@ uv sync --extra cpu --group test print_info "Environment variables configured for testing" +# Use unique project name to avoid conflicts with development environment +export COMPOSE_PROJECT_NAME="speaker-recognition-test" + # Clean test environment print_info "Cleaning test environment..." # Stop any existing test containers diff --git a/services.py b/services.py index 0deeff8a..0ffa014a 100755 --- a/services.py +++ b/services.py @@ -17,7 +17,7 @@ def load_config_yml(): """Load config.yml from repository root""" - config_path = Path(__file__).parent / 'config.yml' + config_path = Path(__file__).parent / 'config' / 'config.yml' if not config_path.exists(): return None @@ -25,7 +25,7 @@ def load_config_yml(): with open(config_path, 'r') as f: return yaml.safe_load(f) except Exception as e: - console.print(f"[yellow]โš ๏ธ Warning: Could not load config.yml: {e}[/yellow]") + console.print(f"[yellow]โš ๏ธ Warning: Could not load config/config.yml: {e}[/yellow]") return None SERVICES = { diff --git a/tests/configs/README.md b/tests/configs/README.md new file mode 100644 index 00000000..8b1e196f --- /dev/null +++ b/tests/configs/README.md @@ -0,0 +1,132 @@ +# Test Configuration Files + +This directory contains configuration variants for testing different provider combinations. + +## Available Test Configs + +### `deepgram-openai.yml` - Cloud Services +- **STT**: Deepgram Nova 3 +- **LLM**: OpenAI GPT-4o-mini +- **Embedding**: OpenAI text-embedding-3-small +- **Memory**: Chronicle native +- **Use Case**: Cloud-based testing when API credits available +- **Required**: `DEEPGRAM_API_KEY`, `OPENAI_API_KEY` + +### `parakeet-ollama.yml` - Full Local Stack +- **STT**: Parakeet ASR (local) +- **LLM**: Ollama llama3.1:latest +- **Embedding**: Ollama nomic-embed-text +- **Memory**: Chronicle native +- **Use Case**: Offline testing, no API keys needed +- **Required**: Parakeet ASR running on port 8767, Ollama running + +### `full-local.yml` - Alias +Symlink to `parakeet-ollama.yml` for convenience. + +## Usage + +### With run-test.sh + +```bash +# Test with Deepgram + OpenAI (cloud) +CONFIG_FILE=../../tests/configs/deepgram-openai.yml ./backends/advanced/run-test.sh + +# Test with Parakeet + Ollama (local) +CONFIG_FILE=../../tests/configs/parakeet-ollama.yml ./backends/advanced/run-test.sh + +# Using the full-local alias +CONFIG_FILE=../../tests/configs/full-local.yml ./backends/advanced/run-test.sh +``` + +### With Docker Compose + +```bash +# From backends/advanced/ +CONFIG_FILE=../../tests/configs/deepgram-openai.yml docker compose -f docker-compose-test.yml up +``` + +### Matrix Testing + +Test all configurations: + +```bash +for cfg in tests/configs/*.yml; do + echo "Testing with: $cfg" + CONFIG_FILE=$cfg ./backends/advanced/run-test.sh || exit 1 +done +``` + +## Creating New Test Configs + +When creating a new test configuration: + +1. **Name it descriptively**: `{stt}-{llm}.yml` (e.g., `mistral-openai.yml`) +2. **Use environment variables**: Always use `${VAR:-default}` pattern for secrets +3. **Set appropriate defaults**: Update the `defaults:` section to match your provider combo +4. **Include only required models**: Don't include models that aren't used +5. **Document requirements**: Update this README with required environment variables + +### Example Structure + +```yaml +# tests/configs/example-config.yml +defaults: + llm: provider-llm + embedding: provider-embed + stt: stt-provider + vector_store: vs-qdrant + +models: + - name: provider-llm + model_type: llm + model_provider: your_provider + api_key: ${YOUR_API_KEY:-} + # ... model config + + - name: stt-provider + model_type: stt + model_provider: your_stt_provider + api_key: ${YOUR_STT_API_KEY:-} + # ... stt config + +memory: + provider: chronicle + # ... memory config +``` + +## Environment Variables + +Test configs use environment variable substitution to avoid hardcoding secrets: + +- **Pattern**: `${VAR_NAME:-default_value}` +- **Example**: `api_key: ${OPENAI_API_KEY:-}` (empty string if not set) +- **Example**: `model_url: ${PARAKEET_ASR_URL:-http://localhost:8767}` (fallback to default) + +### Required by Config + +**deepgram-openai.yml**: +- `DEEPGRAM_API_KEY` - Deepgram transcription API key +- `OPENAI_API_KEY` - OpenAI LLM and embeddings API key + +**parakeet-ollama.yml**: +- `PARAKEET_ASR_URL` (optional) - Defaults to `http://localhost:8767` +- No API keys needed (all local services) + +## Best Practices + +1. **Never hardcode secrets**: Always use environment variables +2. **Test locally first**: Verify config works before adding to repo +3. **Document dependencies**: Update this README with service requirements +4. **Keep configs minimal**: Only include models actually used in tests +5. **Version control**: Test configs are tracked (no secrets), backups are ignored + +## Adding More Combinations + +As you add support for new providers, create corresponding test configs: + +- `mistral-openai.yml` - Mistral Voxtral STT + OpenAI LLM +- `deepgram-ollama.yml` - Deepgram STT + Local Ollama LLM +- `parakeet-openai.yml` - Local Parakeet STT + OpenAI LLM +- etc. + +Each new config should follow the naming convention and documentation pattern above. diff --git a/tests/configs/deepgram-openai.yml b/tests/configs/deepgram-openai.yml new file mode 100644 index 00000000..4cae5e7a --- /dev/null +++ b/tests/configs/deepgram-openai.yml @@ -0,0 +1,84 @@ +# Test Configuration: Deepgram (STT) + OpenAI (LLM) +# Cloud-based services - recommended for CI/testing when API credits available + +defaults: + llm: openai-llm + embedding: openai-embed + stt: stt-deepgram + vector_store: vs-qdrant + +models: + - name: openai-llm + description: OpenAI GPT-4o-mini + model_type: llm + model_provider: openai + api_family: openai + model_name: gpt-4o-mini + model_url: https://api.openai.com/v1 + api_key: ${OPENAI_API_KEY:-} + model_params: + temperature: 0.2 + max_tokens: 2000 + model_output: json + + - name: openai-embed + description: OpenAI text-embedding-3-small + model_type: embedding + model_provider: openai + api_family: openai + model_name: text-embedding-3-small + model_url: https://api.openai.com/v1 + api_key: ${OPENAI_API_KEY:-} + embedding_dimensions: 1536 + model_output: vector + + - name: vs-qdrant + description: Qdrant vector database + model_type: vector_store + model_provider: qdrant + api_family: qdrant + model_url: http://${QDRANT_BASE_URL:-qdrant}:${QDRANT_PORT:-6333} + model_params: + host: ${QDRANT_BASE_URL:-qdrant} + port: ${QDRANT_PORT:-6333} + collection_name: omi_memories + + - name: stt-deepgram + description: Deepgram Nova 3 (batch) + model_type: stt + model_provider: deepgram + api_family: http + model_url: https://api.deepgram.com/v1 + api_key: ${DEEPGRAM_API_KEY:-} + operations: + stt_transcribe: + method: POST + path: /listen + headers: + Authorization: Token ${DEEPGRAM_API_KEY:-} + Content-Type: audio/raw + query: + model: nova-3 + language: multi + smart_format: 'true' + punctuate: 'true' + diarize: 'true' + encoding: linear16 + sample_rate: 16000 + channels: '1' + response: + type: json + extract: + text: results.channels[0].alternatives[0].transcript + words: results.channels[0].alternatives[0].words + segments: results.channels[0].alternatives[0].paragraphs.paragraphs + +memory: + provider: chronicle + timeout_seconds: 1200 + extraction: + enabled: true + prompt: | + Extract important information from this conversation and return a JSON object with an array named "facts". + Include personal preferences, plans, names, dates, locations, numbers, and key details. + Keep items concise and useful. diff --git a/tests/configs/full-local.yml b/tests/configs/full-local.yml new file mode 120000 index 00000000..d2e90934 --- /dev/null +++ b/tests/configs/full-local.yml @@ -0,0 +1 @@ +parakeet-ollama.yml \ No newline at end of file diff --git a/tests/configs/parakeet-ollama.yml b/tests/configs/parakeet-ollama.yml new file mode 100644 index 00000000..a4ef958d --- /dev/null +++ b/tests/configs/parakeet-ollama.yml @@ -0,0 +1,73 @@ +# Test Configuration: Parakeet (STT) + Ollama (LLM) +# Full local stack - no API keys needed, runs entirely offline + +defaults: + llm: local-llm + embedding: local-embed + stt: stt-parakeet-batch + vector_store: vs-qdrant + +models: + - name: local-llm + description: Local Ollama LLM + model_type: llm + model_provider: ollama + api_family: openai + model_name: llama3.1:latest + model_url: http://localhost:11434/v1 + api_key: ${OPENAI_API_KEY:-ollama} + model_params: + temperature: 0.2 + max_tokens: 2000 + model_output: json + + - name: local-embed + description: Local embeddings via Ollama nomic-embed-text + model_type: embedding + model_provider: ollama + api_family: openai + model_name: nomic-embed-text:latest + model_url: http://localhost:11434/v1 + api_key: ${OPENAI_API_KEY:-ollama} + embedding_dimensions: 768 + model_output: vector + + - name: vs-qdrant + description: Qdrant vector database + model_type: vector_store + model_provider: qdrant + api_family: qdrant + model_url: http://${QDRANT_BASE_URL:-qdrant}:${QDRANT_PORT:-6333} + model_params: + host: ${QDRANT_BASE_URL:-qdrant} + port: ${QDRANT_PORT:-6333} + collection_name: omi_memories + + - name: stt-parakeet-batch + description: Parakeet NeMo ASR (batch) - local offline transcription + model_type: stt + model_provider: parakeet + api_family: http + model_url: ${PARAKEET_ASR_URL:-http://localhost:8767} + api_key: '' + operations: + stt_transcribe: + method: POST + path: /transcribe + content_type: multipart/form-data + response: + type: json + extract: + text: text + words: words + segments: segments + +memory: + provider: chronicle + timeout_seconds: 1200 + extraction: + enabled: true + prompt: | + Extract important information from this conversation and return a JSON object with an array named "facts". + Include personal preferences, plans, names, dates, locations, numbers, and key details. + Keep items concise and useful. diff --git a/tests/configs/parakeet-openai.yml b/tests/configs/parakeet-openai.yml new file mode 100644 index 00000000..f3147c33 --- /dev/null +++ b/tests/configs/parakeet-openai.yml @@ -0,0 +1,73 @@ +# Test Configuration: Parakeet (STT) + OpenAI (LLM) +# Hybrid stack - local transcription, cloud LLM + +defaults: + llm: openai-llm + embedding: openai-embed + stt: stt-parakeet-batch + vector_store: vs-qdrant + +models: + - name: openai-llm + description: OpenAI GPT-4o-mini + model_type: llm + model_provider: openai + api_family: openai + model_name: gpt-4o-mini + model_url: https://api.openai.com/v1 + api_key: ${OPENAI_API_KEY:-} + model_params: + temperature: 0.2 + max_tokens: 2000 + model_output: json + + - name: openai-embed + description: OpenAI text-embedding-3-small + model_type: embedding + model_provider: openai + api_family: openai + model_name: text-embedding-3-small + model_url: https://api.openai.com/v1 + api_key: ${OPENAI_API_KEY:-} + embedding_dimensions: 1536 + model_output: vector + + - name: vs-qdrant + description: Qdrant vector database + model_type: vector_store + model_provider: qdrant + api_family: qdrant + model_url: http://${QDRANT_BASE_URL:-qdrant}:${QDRANT_PORT:-6333} + model_params: + host: ${QDRANT_BASE_URL:-qdrant} + port: ${QDRANT_PORT:-6333} + collection_name: omi_memories + + - name: stt-parakeet-batch + description: Parakeet NeMo ASR (batch) - local offline transcription + model_type: stt + model_provider: parakeet + api_family: http + model_url: ${PARAKEET_ASR_URL:-http://localhost:8767} + api_key: '' + operations: + stt_transcribe: + method: POST + path: /transcribe + content_type: multipart/form-data + response: + type: json + extract: + text: text + words: words + segments: segments + +memory: + provider: chronicle + timeout_seconds: 1200 + extraction: + enabled: true + prompt: | + Extract important information from this conversation and return a JSON object with an array named "facts". + Include personal preferences, plans, names, dates, locations, numbers, and key details. + Keep items concise and useful. diff --git a/tests/integration/integration_test.robot b/tests/integration/integration_test.robot index d564a54e..d5af0388 100644 --- a/tests/integration/integration_test.robot +++ b/tests/integration/integration_test.robot @@ -11,6 +11,8 @@ Resource ../setup/teardown_keywords.robot Resource ../resources/session_keywords.robot Resource ../resources/audio_keywords.robot Resource ../resources/conversation_keywords.robot +Resource ../resources/memory_keywords.robot +Resource ../resources/queue_keywords.robot Variables ../setup/test_env.py Variables ../setup/test_data.py Suite Setup Suite Setup @@ -127,6 +129,44 @@ Audio Playback And Segment Timing Test Log All ${segment_count} segments have valid timestamps (0s - ${last_end}s) INFO Log Audio Playback And Segment Timing Test Completed Successfully INFO +End To End Pipeline With Memory Validation Test + [Documentation] Complete E2E test with memory extraction and OpenAI quality validation. + ... This test matches Python test_integration.py coverage exactly. + ... Separate from other tests to avoid breaking existing upload-only tests. + [Tags] e2e memory + [Timeout] 600s + + Log Starting End-to-End Pipeline Test with Memory Validation INFO + + # Phase 1: Upload audio and wait for complete processing + Log Uploading audio file and waiting for full processing INFO + ${conversation} ${memories}= Upload Audio File And Wait For Memory + ... ${TEST_AUDIO_FILE} + ... ${TEST_DEVICE_NAME} + + Set Global Variable ${TEST_CONVERSATION} ${conversation} + + # Phase 2: Verify transcription quality + Log Verifying transcription quality INFO + Verify Transcription Quality ${TEST_CONVERSATION} ${EXPECTED_TRANSCRIPT} + + # Phase 3: Verify memories were extracted + ${memory_count}= Get Length ${memories} + Should Be True ${memory_count} > 0 No memories extracted + Log Extracted ${memory_count} memories INFO + + # Phase 4: Verify memory quality with OpenAI (matches Python test!) + Log Validating memory quality with OpenAI INFO + Verify Memory Quality With OpenAI ${memories} ${EXPECTED_MEMORIES} + + # Phase 5: Verify chat integration + Log Verifying chat integration INFO + Verify Chat Integration api ${TEST_CONVERSATION} + + Log End-to-End Pipeline Test Completed Successfully INFO + Log โœ… Transcript verified INFO + Log โœ… ${memory_count} memories extracted and validated with OpenAI INFO + *** Keywords *** diff --git a/tests/resources/audio_keywords.robot b/tests/resources/audio_keywords.robot index 82c3d782..f3ae950d 100644 --- a/tests/resources/audio_keywords.robot +++ b/tests/resources/audio_keywords.robot @@ -69,6 +69,33 @@ Upload Audio File RETURN ${conversation} +Upload Audio File And Wait For Memory + [Documentation] Upload audio file and wait for complete processing including memory extraction. + ... This is for E2E testing - use Upload Audio File for upload-only tests. + [Arguments] ${audio_file_path} ${device_name}=robot-test ${folder}=. + + # Upload file (uses existing keyword) + ${conversation}= Upload Audio File ${audio_file_path} ${device_name} ${folder} + + # Get conversation ID to find memory job + ${conversation_id}= Set Variable ${conversation}[conversation_id] + Log Conversation ID: ${conversation_id} + + # Find memory job for this conversation + ${memory_jobs}= Get Jobs By Type And Conversation process_memory_job ${conversation_id} + Should Not Be Empty ${memory_jobs} No memory job found for conversation ${conversation_id} + + ${memory_job}= Set Variable ${memory_jobs}[0] + ${memory_job_id}= Set Variable ${memory_job}[job_id] + + Log Found memory job: ${memory_job_id} + + # Wait for memory extraction (uses keyword from memory_keywords.robot) + ${memories}= Wait For Memory Extraction ${memory_job_id} min_memories=1 + + RETURN ${conversation} ${memories} + + Get Cropped Audio Info [Documentation] Get cropped audio information for a conversation [Arguments] ${audio_uuid} diff --git a/tests/resources/memory_keywords.robot b/tests/resources/memory_keywords.robot index 4a02c40e..8c3f84c0 100644 --- a/tests/resources/memory_keywords.robot +++ b/tests/resources/memory_keywords.robot @@ -104,3 +104,140 @@ Verify Memory Extraction Should Be True ${api_memory_count} >= ${min_memories} Insufficient API memories: ${api_memory_count} Log Memory extraction verified: conversation=${conv_memory_count}, api=${api_memory_count} INFO + + +Wait For Memory Extraction + [Documentation] Wait for memory job to complete and verify memories extracted. + ... Fails fast if job doesn't exist, fails immediately, or service is unhealthy. + [Arguments] ${memory_job_id} ${min_memories}=1 ${timeout}=120 + + Log Waiting for memory job ${memory_job_id} to complete... + + # 1. Verify job exists before waiting (fail fast if job ID is invalid) + ${job_status}= Get Job Status ${memory_job_id} + Should Not Be Equal ${job_status} ${None} + ... Memory job ${memory_job_id} not found in queue - cannot wait for completion + + # 2. Check if job already failed (fail fast instead of waiting 120s) + ${current_status}= Set Variable ${job_status}[status] + IF '${current_status}' == 'failed' + ${error_info}= Evaluate $job_status.get('exc_info', 'Unknown error') + Fail Memory job ${memory_job_id} already failed: ${error_info} + END + + # 3. Wait for job completion with status monitoring + ${start_time}= Get Time epoch + ${end_time}= Evaluate ${start_time} + ${timeout} + + WHILE True + # Get current job status + ${job}= Get Job Status ${memory_job_id} + + # Handle job not found (e.g., expired from queue) + IF ${job} == ${None} + Fail Memory job ${memory_job_id} disappeared from queue during wait + END + + ${status}= Set Variable ${job}[status] + + # Success case - job completed + IF '${status}' == 'completed' or '${status}' == 'finished' + Log Memory job completed successfully + BREAK + END + + # Failure case - job failed (fail fast) + IF '${status}' == 'failed' + ${error_info}= Evaluate $job.get('exc_info', 'Unknown error') + Fail Memory job ${memory_job_id} failed during processing: ${error_info} + END + + # Timeout check + ${current_time}= Get Time epoch + IF ${current_time} >= ${end_time} + Fail Memory job ${memory_job_id} did not complete within ${timeout}s (last status: ${status}) + END + + # Log progress every iteration + Log Memory job status: ${status} (waiting...) DEBUG + + # Wait before next check + Sleep 5s + END + + # 4. Fetch memories from API with error handling + TRY + ${response}= GET On Session api /api/memories expected_status=200 + EXCEPT AS ${error} + Fail Failed to fetch memories from API: ${error} + END + + ${memories_data}= Set Variable ${response.json()} + ${memories}= Set Variable ${memories_data}[memories] + ${memory_count}= Get Length ${memories} + + # 5. Verify minimum memories were extracted + Should Be True ${memory_count} >= ${min_memories} + ... Expected at least ${min_memories} memories, found ${memory_count} + + Log Successfully extracted ${memory_count} memories + RETURN ${memories} + + +Check Memory Similarity With OpenAI + [Documentation] Use OpenAI to check if extracted memories match expected memories + [Arguments] ${actual_memories} ${expected_memories} ${openai_api_key} + + # Extract just the memory text from actual memories + ${actual_memory_texts}= Evaluate [mem.get('memory', '') for mem in $actual_memories] + + # Build OpenAI prompt (same as Python test) + ${prompt}= Catenate SEPARATOR=\n + ... Compare these two lists of memories to determine if they represent content from the same audio source. + ... + ... EXPECTED MEMORIES: + ... ${expected_memories} + ... + ... EXTRACTED MEMORIES: + ... ${actual_memory_texts} + ... + ... Respond in JSON format with: + ... {"similar": true/false, "reason": "brief explanation"} + + # Call OpenAI API + ${headers}= Create Dictionary Authorization=Bearer ${openai_api_key} Content-Type=application/json + ${payload}= Create Dictionary + ... model=gpt-4o-mini + ... messages=${{ [{"role": "user", "content": """${prompt}"""}] }} + ... response_format=${{ {"type": "json_object"} }} + + ${response}= POST https://api.openai.com/v1/chat/completions + ... headers=${headers} + ... json=${payload} + ... expected_status=200 + + ${result_json}= Set Variable ${response.json()} + ${content}= Set Variable ${result_json}[choices][0][message][content] + ${similarity_result}= Evaluate json.loads("""${content}""") json + + Log Memory similarity: ${similarity_result}[similar] INFO + Log Reason: ${similarity_result}[reason] INFO + + RETURN ${similarity_result} + + +Verify Memory Quality With OpenAI + [Documentation] Verify extracted memories match expected memories using OpenAI + [Arguments] ${actual_memories} ${expected_memories} + + # Get OpenAI API key from environment + ${openai_key}= Get Environment Variable OPENAI_API_KEY + + # Check similarity + ${result}= Check Memory Similarity With OpenAI ${actual_memories} ${expected_memories} ${openai_key} + + # Assert memories are similar + Should Be True ${result}[similar] == ${True} + ... Memory similarity check failed: ${result}[reason] + + Log โœ… Memory quality validated INFO diff --git a/tests/run-robot-tests.sh b/tests/run-robot-tests.sh index 0c264875..462377ed 100755 --- a/tests/run-robot-tests.sh +++ b/tests/run-robot-tests.sh @@ -42,6 +42,16 @@ print_info "============================" CLEANUP_CONTAINERS="${CLEANUP_CONTAINERS:-true}" OUTPUTDIR="${OUTPUTDIR:-results}" +# Set default CONFIG_FILE if not provided +# This allows testing with different provider combinations +# Usage: CONFIG_FILE=../tests/configs/parakeet-ollama.yml ./run-robot-tests.sh +export CONFIG_FILE="${CONFIG_FILE:-../config/config.yml}" + +# Convert CONFIG_FILE to absolute path (Docker Compose resolves relative paths from compose file location) +if [[ ! "$CONFIG_FILE" = /* ]]; then + CONFIG_FILE="$(cd "$(dirname "$CONFIG_FILE")" && pwd)/$(basename "$CONFIG_FILE")" +fi + # Load environment variables (CI or local) if [ -f "setup/.env.test" ] && [ -z "$DEEPGRAM_API_KEY" ]; then print_info "Loading environment variables from setup/.env.test..." @@ -69,6 +79,7 @@ fi print_info "DEEPGRAM_API_KEY length: ${#DEEPGRAM_API_KEY}" print_info "OPENAI_API_KEY length: ${#OPENAI_API_KEY}" +print_info "Using config file: $CONFIG_FILE" # Create test environment file if it doesn't exist if [ ! -f "setup/.env.test" ]; then @@ -100,6 +111,9 @@ cd ../backends/advanced print_info "Starting test infrastructure..." +# Use unique project name to avoid conflicts with development environment +export COMPOSE_PROJECT_NAME="advanced-backend-test" + # Ensure required config files exist # memory_config.yaml no longer used; memory settings live in config.yml @@ -109,7 +123,7 @@ docker compose -f docker-compose-test.yml down -v 2>/dev/null || true # Force remove any stuck containers with test names print_info "Removing any stuck test containers..." -docker rm -f advanced-mongo-test-1 advanced-redis-test-1 advanced-qdrant-test-1 advanced-chronicle-backend-test-1 advanced-workers-test-1 advanced-webui-test-1 2>/dev/null || true +docker rm -f advanced-backend-test-mongo-test-1 advanced-backend-test-redis-test-1 advanced-backend-test-qdrant-test-1 advanced-backend-test-chronicle-backend-test-1 advanced-backend-test-workers-test-1 advanced-backend-test-webui-test-1 2>/dev/null || true # Start infrastructure services (MongoDB, Redis, Qdrant) print_info "Starting MongoDB, Redis, and Qdrant (fresh containers)..." diff --git a/tests/setup/test_data.py b/tests/setup/test_data.py index 787f0399..6d73b265 100644 --- a/tests/setup/test_data.py +++ b/tests/setup/test_data.py @@ -36,6 +36,20 @@ # Expected content for transcript quality verification EXPECTED_TRANSCRIPT = "glass blowing" +# Expected memories for DIY Glass Blowing audio (from Python integration test) +# Source: backends/advanced/tests/assets/expected_memories.json +EXPECTED_MEMORIES = [ + "Nick assists significantly in the glass blowing process", + "Excitement and nervousness expressed during the process", + "Furnace contains about 400 pounds of liquid glass", + "Choice of color for the flower is light blue", + "Caitlin is mentioned as a participant", + "Class involves making a trumpet flower", + "Gravity is used as a tool in glass blowing", + "Nick did most of the turning during the demonstration", + "The video is sponsored by Squarespace." +] + # Expected segment timestamps for DIY Glass Blowing audio (4-minute version, 500 chunks) # These are the cropped audio timestamps after silence removal # Updated 2025-01-22 based on actual test output with streaming websocket processing diff --git a/wizard.py b/wizard.py index 05e97e59..53a0731a 100755 --- a/wizard.py +++ b/wizard.py @@ -314,19 +314,21 @@ def setup_git_hooks(): console.print(f"โš ๏ธ [yellow]Could not setup git hooks: {e} (optional)[/yellow]") def setup_config_file(): - """Setup config.yml from template if it doesn't exist""" - config_file = Path("config.yml") - config_template = Path("config.yml.template") + """Setup config/config.yml from template if it doesn't exist""" + config_file = Path("config/config.yml") + config_template = Path("config/config.yml.template") if not config_file.exists(): if config_template.exists(): import shutil + # Ensure config/ directory exists + config_file.parent.mkdir(parents=True, exist_ok=True) shutil.copy(config_template, config_file) - console.print("โœ… [green]Created config.yml from template[/green]") + console.print("โœ… [green]Created config/config.yml from template[/green]") else: - console.print("โš ๏ธ [yellow]config.yml.template not found, skipping config setup[/yellow]") + console.print("โš ๏ธ [yellow]config/config.yml.template not found, skipping config setup[/yellow]") else: - console.print("โ„น๏ธ [blue]config.yml already exists, keeping existing configuration[/blue]") + console.print("โ„น๏ธ [blue]config/config.yml already exists, keeping existing configuration[/blue]") def main(): """Main orchestration logic""" From 1bde611843e11e11c187c037fae1755228bca92b Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 1 Jan 2026 18:41:47 +0000 Subject: [PATCH 04/14] Add test requirements and clean up imports in wizard.py - Introduced a new `test-requirements.txt` file to manage testing dependencies. - Removed redundant import of `shutil` in `wizard.py` to improve code clarity. --- requirements.txt => test-requirements.txt | 0 wizard.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename requirements.txt => test-requirements.txt (100%) diff --git a/requirements.txt b/test-requirements.txt similarity index 100% rename from requirements.txt rename to test-requirements.txt diff --git a/wizard.py b/wizard.py index 53a0731a..b32e7790 100755 --- a/wizard.py +++ b/wizard.py @@ -4,6 +4,7 @@ Handles service selection and delegation only - no configuration duplication """ +import shutil import subprocess import sys from datetime import datetime @@ -320,7 +321,6 @@ def setup_config_file(): if not config_file.exists(): if config_template.exists(): - import shutil # Ensure config/ directory exists config_file.parent.mkdir(parents=True, exist_ok=True) shutil.copy(config_template, config_file) From e19d73bbd946f529dd7f8aa92e464f337f13e9ff Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Tue, 23 Dec 2025 18:30:53 +0000 Subject: [PATCH 05/14] Add ConfigManager for unified configuration management - Introduced a new `config_manager.py` module to handle reading and writing configurations from `config.yml` and `.env` files, ensuring backward compatibility. - Refactored `ChronicleSetup` in `backends/advanced/init.py` to utilize `ConfigManager` for loading and updating configurations, simplifying the setup process. - Removed redundant methods for loading and saving `config.yml` directly in `ChronicleSetup`, as these are now managed by `ConfigManager`. - Enhanced user feedback during configuration updates, including success messages for changes made to configuration files. --- backends/advanced/init.py | 100 ++++------- config_manager.py | 348 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 383 insertions(+), 65 deletions(-) create mode 100644 config_manager.py diff --git a/backends/advanced/init.py b/backends/advanced/init.py index 11390ff1..c0d390a9 100644 --- a/backends/advanced/init.py +++ b/backends/advanced/init.py @@ -36,8 +36,21 @@ def __init__(self, args=None): self.console.print("[red][ERROR][/red] Please run this script from the backends/advanced directory") sys.exit(1) - # Load config.yml if it exists - self.load_config_yml() + # Initialize ConfigManager + repo_root = Path.cwd().parent.parent # backends/advanced -> repo root + if str(repo_root) not in sys.path: + sys.path.insert(0, str(repo_root)) + + from config_manager import ConfigManager + + self.config_manager = ConfigManager(service_path="backends/advanced") + self.console.print(f"[blue][INFO][/blue] Using config.yml at: {self.config_manager.config_yml_path}") + + # Load existing config or create default structure + self.config_yml_data = self.config_manager.get_full_config() + if not self.config_yml_data: + self.console.print("[yellow][WARNING][/yellow] config.yml not found, will create default structure") + self.config_yml_data = self._get_default_config_structure() def print_header(self, title: str): """Print a colorful header""" @@ -126,21 +139,6 @@ def mask_api_key(self, key: str, show_chars: int = 5) -> str: return f"{key_clean[:show_chars]}{'*' * min(15, len(key_clean) - show_chars * 2)}{key_clean[-show_chars:]}" - def load_config_yml(self): - """Load config.yml from repository root""" - if not self.config_yml_path.exists(): - self.console.print(f"[yellow][WARNING][/yellow] config.yml not found at {self.config_yml_path}") - self.console.print("[yellow]Will create a new config.yml during setup[/yellow]") - self.config_yml_data = self._get_default_config_structure() - return - - try: - with open(self.config_yml_path, 'r') as f: - self.config_yml_data = yaml.safe_load(f) - self.console.print(f"[blue][INFO][/blue] Loaded existing config.yml") - except Exception as e: - self.console.print(f"[red][ERROR][/red] Failed to load config.yml: {e}") - self.config_yml_data = self._get_default_config_structure() def _get_default_config_structure(self) -> Dict[str, Any]: """Return default config.yml structure if file doesn't exist""" @@ -163,36 +161,6 @@ def _get_default_config_structure(self) -> Dict[str, Any]: } } - def save_config_yml(self): - """Save config.yml back to repository root""" - try: - # Backup existing config.yml if it exists - if self.config_yml_path.exists(): - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - backup_path = self.config_yml_path.parent / f"config.yml.backup.{timestamp}" - shutil.copy2(self.config_yml_path, backup_path) - self.console.print(f"[blue][INFO][/blue] Backed up config.yml to {backup_path.name}") - - # Write updated config - with open(self.config_yml_path, 'w') as f: - yaml.dump(self.config_yml_data, f, default_flow_style=False, sort_keys=False) - - self.console.print("[green][SUCCESS][/green] config.yml updated successfully") - except Exception as e: - self.console.print(f"[red][ERROR][/red] Failed to save config.yml: {e}") - raise - - def update_config_default(self, key: str, value: str): - """Update a default value in config.yml""" - if "defaults" not in self.config_yml_data: - self.config_yml_data["defaults"] = {} - self.config_yml_data["defaults"][key] = value - - def update_memory_config(self, updates: Dict[str, Any]): - """Update memory configuration in config.yml""" - if "memory" not in self.config_yml_data: - self.config_yml_data["memory"] = {} - self.config_yml_data["memory"].update(updates) def setup_authentication(self): """Configure authentication settings""" @@ -306,8 +274,8 @@ def setup_llm(self): if api_key: self.config["OPENAI_API_KEY"] = api_key # Update config.yml to use OpenAI models - self.update_config_default("llm", "openai-llm") - self.update_config_default("embedding", "openai-embed") + self.config_manager.update_config_defaults({"llm": "openai-llm", "embedding": "openai-embed"}) + self.config_yml_data = self.config_manager.get_full_config() # Reload to stay in sync self.console.print("[green][SUCCESS][/green] OpenAI configured in config.yml") self.console.print("[blue][INFO][/blue] Set defaults.llm: openai-llm") self.console.print("[blue][INFO][/blue] Set defaults.embedding: openai-embed") @@ -317,8 +285,8 @@ def setup_llm(self): elif choice == "2": self.console.print("[blue][INFO][/blue] Ollama selected") # Update config.yml to use Ollama models - self.update_config_default("llm", "local-llm") - self.update_config_default("embedding", "local-embed") + self.config_manager.update_config_defaults({"llm": "local-llm", "embedding": "local-embed"}) + self.config_yml_data = self.config_manager.get_full_config() # Reload to stay in sync self.console.print("[green][SUCCESS][/green] Ollama configured in config.yml") self.console.print("[blue][INFO][/blue] Set defaults.llm: local-llm") self.console.print("[blue][INFO][/blue] Set defaults.embedding: local-embed") @@ -327,7 +295,8 @@ def setup_llm(self): elif choice == "3": self.console.print("[blue][INFO][/blue] Skipping LLM setup - memory extraction disabled") # Disable memory extraction in config.yml - self.update_memory_config({"extraction": {"enabled": False}}) + self.config_manager.update_memory_config({"extraction": {"enabled": False}}) + self.config_yml_data = self.config_manager.get_full_config() # Reload to stay in sync def setup_memory(self): """Configure memory provider - updates config.yml""" @@ -347,9 +316,10 @@ def setup_memory(self): qdrant_url = self.prompt_value("Qdrant URL", "qdrant") self.config["QDRANT_BASE_URL"] = qdrant_url - # Update config.yml - self.update_memory_config({"provider": "chronicle"}) - self.console.print("[green][SUCCESS][/green] Chronicle memory provider configured in config.yml") + # Update config.yml (also updates .env automatically) + self.config_manager.update_memory_config({"provider": "chronicle"}) + self.config_yml_data = self.config_manager.get_full_config() # Reload to stay in sync + self.console.print("[green][SUCCESS][/green] Chronicle memory provider configured in config.yml and .env") elif choice == "2": self.console.print("[blue][INFO][/blue] OpenMemory MCP selected") @@ -359,8 +329,8 @@ def setup_memory(self): user_id = self.prompt_value("OpenMemory user ID", "openmemory") timeout = self.prompt_value("OpenMemory timeout (seconds)", "30") - # Update config.yml with OpenMemory MCP settings - self.update_memory_config({ + # Update config.yml with OpenMemory MCP settings (also updates .env automatically) + self.config_manager.update_memory_config({ "provider": "openmemory_mcp", "openmemory_mcp": { "server_url": mcp_url, @@ -369,7 +339,8 @@ def setup_memory(self): "timeout": int(timeout) } }) - self.console.print("[green][SUCCESS][/green] OpenMemory MCP configured in config.yml") + self.config_yml_data = self.config_manager.get_full_config() # Reload to stay in sync + self.console.print("[green][SUCCESS][/green] OpenMemory MCP configured in config.yml and .env") self.console.print("[yellow][WARNING][/yellow] Remember to start OpenMemory: cd ../../extras/openmemory-mcp && docker compose up -d") elif choice == "3": @@ -378,15 +349,16 @@ def setup_memory(self): mycelia_url = self.prompt_value("Mycelia API URL", "http://localhost:5173") timeout = self.prompt_value("Mycelia timeout (seconds)", "30") - # Update config.yml with Mycelia settings - self.update_memory_config({ + # Update config.yml with Mycelia settings (also updates .env automatically) + self.config_manager.update_memory_config({ "provider": "mycelia", "mycelia": { "api_url": mycelia_url, "timeout": int(timeout) } }) - self.console.print("[green][SUCCESS][/green] Mycelia memory provider configured in config.yml") + self.config_yml_data = self.config_manager.get_full_config() # Reload to stay in sync + self.console.print("[green][SUCCESS][/green] Mycelia memory provider configured in config.yml and .env") self.console.print("[yellow][WARNING][/yellow] Make sure Mycelia is running at the configured URL") def setup_optional_services(self): @@ -604,10 +576,8 @@ def generate_env_file(self): self.console.print("[green][SUCCESS][/green] .env file configured successfully with secure permissions") - # Save config.yml with all updates - self.console.print() - self.console.print("[blue][INFO][/blue] Saving configuration to config.yml...") - self.save_config_yml() + # Note: config.yml is automatically saved by ConfigManager when updates are made + self.console.print("[blue][INFO][/blue] Configuration saved to config.yml and .env (via ConfigManager)") def copy_config_templates(self): """Copy other configuration files""" diff --git a/config_manager.py b/config_manager.py new file mode 100644 index 00000000..2f64b082 --- /dev/null +++ b/config_manager.py @@ -0,0 +1,348 @@ +""" +Shared configuration manager for Chronicle. + +This module provides a unified interface for reading and writing configuration +across both config.yml (source of truth) and .env (backward compatibility). + +Key principles: +- config.yml is the source of truth for memory provider and model settings +- .env files are kept in sync for backward compatibility with legacy code +- All config updates should use this module to maintain consistency + +Usage: + # From any service in the project + from config_manager import ConfigManager + + # For backend service + config = ConfigManager(service_path="backends/advanced") + provider = config.get_memory_provider() + config.set_memory_provider("openmemory_mcp") + + # Auto-detects paths from cwd + config = ConfigManager() +""" + +import logging +import os +import shutil +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Optional + +import yaml + +logger = logging.getLogger(__name__) + + +class ConfigManager: + """Manages Chronicle configuration across config.yml and .env files.""" + + def __init__(self, service_path: Optional[str] = None, repo_root: Optional[Path] = None): + """ + Initialize ConfigManager. + + Args: + service_path: Path to service directory (e.g., "backends/advanced", "extras/speaker-recognition"). + If None, auto-detects from current working directory. + repo_root: Path to repository root. If None, auto-detects by finding config.yml. + """ + # Find repo root + if repo_root is None: + repo_root = self._find_repo_root() + self.repo_root = Path(repo_root) + + # Find service directory + if service_path is None: + service_path = self._detect_service_path() + self.service_path = self.repo_root / service_path if service_path else None + + # Paths + self.config_yml_path = self.repo_root / "config.yml" + self.env_path = self.service_path / ".env" if self.service_path else None + + logger.debug(f"ConfigManager initialized: repo_root={self.repo_root}, " + f"service_path={self.service_path}, config_yml={self.config_yml_path}") + + def _find_repo_root(self) -> Path: + """Find repository root by searching for config.yml.""" + current = Path.cwd() + + # Walk up until we find config.yml + while current != current.parent: + if (current / "config.yml").exists(): + return current + current = current.parent + + # Fallback to cwd if not found + logger.warning("Could not find config.yml, using current directory as repo root") + return Path.cwd() + + def _detect_service_path(self) -> Optional[str]: + """Auto-detect service path from current working directory.""" + cwd = Path.cwd() + + # Check if we're in a known service directory + known_services = [ + "backends/advanced", + "extras/speaker-recognition", + "extras/openmemory-mcp", + "extras/asr-services", + ] + + for service in known_services: + service_full_path = self.repo_root / service + if cwd == service_full_path or str(cwd).startswith(str(service_full_path)): + return service + + logger.debug("Could not auto-detect service path from cwd") + return None + + def _load_config_yml(self) -> Dict[str, Any]: + """Load config.yml file.""" + if not self.config_yml_path.exists(): + logger.warning(f"config.yml not found at {self.config_yml_path}") + return {} + + try: + with open(self.config_yml_path, 'r') as f: + return yaml.safe_load(f) or {} + except Exception as e: + logger.error(f"Failed to load config.yml: {e}") + return {} + + def _save_config_yml(self, config: Dict[str, Any]): + """Save config.yml file with backup.""" + try: + # Create backup + if self.config_yml_path.exists(): + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_path = self.config_yml_path.parent / f"config.yml.backup.{timestamp}" + shutil.copy2(self.config_yml_path, backup_path) + logger.info(f"Backed up config.yml to {backup_path.name}") + + # Write updated config + with open(self.config_yml_path, 'w') as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False) + + logger.info(f"Saved config.yml to {self.config_yml_path}") + + except Exception as e: + logger.error(f"Failed to save config.yml: {e}") + raise + + def _update_env_file(self, key: str, value: str): + """Update a single key in .env file.""" + if self.env_path is None: + logger.debug("No service path set, skipping .env update") + return + + if not self.env_path.exists(): + logger.warning(f".env file not found at {self.env_path}") + return + + try: + # Read current .env + with open(self.env_path, 'r') as f: + lines = f.readlines() + + # Update or add line + key_found = False + updated_lines = [] + + for line in lines: + if line.strip().startswith(f"{key}="): + updated_lines.append(f"{key}={value}\n") + key_found = True + else: + updated_lines.append(line) + + # If key wasn't found, add it + if not key_found: + updated_lines.append(f"\n# Auto-updated by ConfigManager\n{key}={value}\n") + + # Create backup + backup_path = f"{self.env_path}.bak" + shutil.copy2(self.env_path, backup_path) + logger.debug(f"Backed up .env to {backup_path}") + + # Write updated file + with open(self.env_path, 'w') as f: + f.writelines(updated_lines) + + # Update environment variable for current process + os.environ[key] = value + + logger.info(f"Updated {key}={value} in .env file") + + except Exception as e: + logger.error(f"Failed to update .env file: {e}") + raise + + def get_memory_provider(self) -> str: + """ + Get current memory provider from config.yml. + + Returns: + Memory provider name (chronicle, openmemory_mcp, or mycelia) + """ + config = self._load_config_yml() + provider = config.get("memory", {}).get("provider", "chronicle").lower() + + # Map legacy names + if provider in ("friend-lite", "friend_lite"): + provider = "chronicle" + + return provider + + def set_memory_provider(self, provider: str) -> Dict[str, Any]: + """ + Set memory provider in both config.yml and .env. + + This updates: + 1. config.yml: memory.provider field (source of truth) + 2. .env: MEMORY_PROVIDER variable (backward compatibility, if service_path set) + + Args: + provider: Memory provider name (chronicle, openmemory_mcp, or mycelia) + + Returns: + Dict with status and details of the update + + Raises: + ValueError: If provider is invalid + """ + # Validate provider + provider = provider.lower().strip() + valid_providers = ["chronicle", "openmemory_mcp", "mycelia"] + + if provider not in valid_providers: + raise ValueError( + f"Invalid provider '{provider}'. " + f"Valid providers: {', '.join(valid_providers)}" + ) + + # Update config.yml + config = self._load_config_yml() + + if "memory" not in config: + config["memory"] = {} + + config["memory"]["provider"] = provider + self._save_config_yml(config) + + # Update .env for backward compatibility (if we have a service path) + if self.env_path and self.env_path.exists(): + self._update_env_file("MEMORY_PROVIDER", provider) + + return { + "message": ( + f"Memory provider updated to '{provider}' in config.yml" + f"{' and .env' if self.env_path else ''}. " + "Please restart services for changes to take effect." + ), + "provider": provider, + "config_yml_path": str(self.config_yml_path), + "env_path": str(self.env_path) if self.env_path else None, + "requires_restart": True, + "status": "success" + } + + def get_memory_config(self) -> Dict[str, Any]: + """ + Get complete memory configuration from config.yml. + + Returns: + Full memory configuration dict + """ + config = self._load_config_yml() + return config.get("memory", {}) + + def update_memory_config(self, updates: Dict[str, Any]): + """ + Update memory configuration in config.yml. + + Args: + updates: Dict of updates to merge into memory config + """ + config = self._load_config_yml() + + if "memory" not in config: + config["memory"] = {} + + # Deep merge updates + config["memory"].update(updates) + + self._save_config_yml(config) + + # If provider was updated, also update .env + if "provider" in updates and self.env_path: + self._update_env_file("MEMORY_PROVIDER", updates["provider"]) + + def get_config_defaults(self) -> Dict[str, Any]: + """ + Get defaults configuration from config.yml. + + Returns: + Defaults configuration dict (llm, embedding, stt, tts, vector_store) + """ + config = self._load_config_yml() + return config.get("defaults", {}) + + def update_config_defaults(self, updates: Dict[str, str]): + """ + Update defaults configuration in config.yml. + + Args: + updates: Dict of updates to merge into defaults config + (e.g., {"llm": "openai-llm", "embedding": "openai-embed"}) + """ + config = self._load_config_yml() + + if "defaults" not in config: + config["defaults"] = {} + + # Update defaults + config["defaults"].update(updates) + + self._save_config_yml(config) + + def get_full_config(self) -> Dict[str, Any]: + """ + Get complete config.yml as dictionary. + + Returns: + Full configuration dict + """ + return self._load_config_yml() + + def save_full_config(self, config: Dict[str, Any]): + """ + Save complete config.yml from dictionary. + + Args: + config: Full configuration dict to save + """ + self._save_config_yml(config) + + +# Global singleton instance +_config_manager: Optional[ConfigManager] = None + + +def get_config_manager(service_path: Optional[str] = None) -> ConfigManager: + """ + Get global ConfigManager singleton instance. + + Args: + service_path: Optional service path for .env updates. + If None, uses cached instance or creates new one. + + Returns: + ConfigManager instance + """ + global _config_manager + + if _config_manager is None or service_path is not None: + _config_manager = ConfigManager(service_path=service_path) + + return _config_manager From 102836371032f22984c3b599b8141bf2214f83d3 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 1 Jan 2026 20:12:09 +0000 Subject: [PATCH 06/14] Refactor transcription provider configuration and enhance setup process - Updated `.env.template` to clarify speech-to-text configuration and removed deprecated options for Mistral. - Modified `docker-compose.yml` to streamline environment variable management by removing unused Mistral keys. - Enhanced `ChronicleSetup` in `init.py` to provide clearer user feedback and updated the transcription provider selection process to rely on `config.yml`. - Improved error handling in the websocket controller to determine the transcription provider from the model registry instead of environment variables. - Updated health check routes to reflect the new method of retrieving the transcription provider from `config.yml`. - Adjusted `config.yml.template` to include comments on transcription provider options for better user guidance. --- backends/advanced/.env.template | 12 ++-- backends/advanced/docker-compose.yml | 6 -- backends/advanced/init.py | 71 ++++++++++--------- .../controllers/websocket_controller.py | 32 ++++----- .../routers/modules/health_routes.py | 5 +- backends/advanced/start-workers.sh | 8 ++- config/config.yml.template | 5 +- 7 files changed, 69 insertions(+), 70 deletions(-) diff --git a/backends/advanced/.env.template b/backends/advanced/.env.template index 18a30d8a..a63ab6f5 100644 --- a/backends/advanced/.env.template +++ b/backends/advanced/.env.template @@ -45,18 +45,14 @@ OPENAI_MODEL=gpt-4o-mini # CHAT_TEMPERATURE=0.7 # ======================================== -# SPEECH-TO-TEXT CONFIGURATION (Choose one) +# SPEECH-TO-TEXT CONFIGURATION (API Keys Only) # ======================================== +# Provider selection is in config.yml (defaults.stt) -# Option 1: Deepgram (recommended for best transcription quality) +# Deepgram (cloud-based, recommended) DEEPGRAM_API_KEY= -# Option 2: Parakeet ASR service from extras/asr-services -# PARAKEET_ASR_URL=http://host.docker.internal:8767 - -# Optional: Specify which provider to use ('deepgram' or 'parakeet') -# If not set, will auto-select based on available configuration (Deepgram preferred) -# TRANSCRIPTION_PROVIDER= +# Note: Parakeet ASR URL configured in config.yml # ======================================== # SPEECH DETECTION CONFIGURATION diff --git a/backends/advanced/docker-compose.yml b/backends/advanced/docker-compose.yml index 80f27aae..b4094af3 100644 --- a/backends/advanced/docker-compose.yml +++ b/backends/advanced/docker-compose.yml @@ -15,9 +15,6 @@ services: - ../../config/config.yml:/app/config.yml # Removed :ro to allow UI config saving environment: - DEEPGRAM_API_KEY=${DEEPGRAM_API_KEY} - - MISTRAL_API_KEY=${MISTRAL_API_KEY} - - MISTRAL_MODEL=${MISTRAL_MODEL} - - TRANSCRIPTION_PROVIDER=${TRANSCRIPTION_PROVIDER} - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} - OLLAMA_BASE_URL=${OLLAMA_BASE_URL} - HF_TOKEN=${HF_TOKEN} @@ -68,9 +65,6 @@ services: - ../../config/config.yml:/app/config.yml # Removed :ro for consistency environment: - DEEPGRAM_API_KEY=${DEEPGRAM_API_KEY} - - MISTRAL_API_KEY=${MISTRAL_API_KEY} - - MISTRAL_MODEL=${MISTRAL_MODEL} - - TRANSCRIPTION_PROVIDER=${TRANSCRIPTION_PROVIDER} - PARAKEET_ASR_URL=${PARAKEET_ASR_URL} - OPENAI_API_KEY=${OPENAI_API_KEY} - GROQ_API_KEY=${GROQ_API_KEY} diff --git a/backends/advanced/init.py b/backends/advanced/init.py index c0d390a9..6a120499 100644 --- a/backends/advanced/init.py +++ b/backends/advanced/init.py @@ -175,16 +175,19 @@ def setup_authentication(self): self.console.print("[green][SUCCESS][/green] Admin account configured") def setup_transcription(self): - """Configure transcription provider""" + """Configure transcription provider - updates config.yml and .env""" self.print_section("Speech-to-Text Configuration") - + + self.console.print("[blue][INFO][/blue] Provider selection is configured in config.yml (defaults.stt)") + self.console.print("[blue][INFO][/blue] API keys are stored in .env") + self.console.print() + choices = { - "1": "Deepgram (recommended - high quality, requires API key)", - "2": "Mistral (Voxtral models - requires API key)", - "3": "Offline (Parakeet ASR - requires GPU, runs locally)", - "4": "None (skip transcription setup)" + "1": "Deepgram (recommended - high quality, cloud-based)", + "2": "Offline (Parakeet ASR - requires GPU, runs locally)", + "3": "None (skip transcription setup)" } - + choice = self.prompt_choice("Choose your transcription provider:", choices, "1") if choice == "1": @@ -202,44 +205,34 @@ def setup_transcription(self): api_key = self.prompt_value("Deepgram API key (leave empty to skip)", "") if api_key: - self.config["TRANSCRIPTION_PROVIDER"] = "deepgram" + # Write API key to .env self.config["DEEPGRAM_API_KEY"] = api_key - self.console.print("[green][SUCCESS][/green] Deepgram configured") - else: - self.console.print("[yellow][WARNING][/yellow] No API key provided - transcription will not work") - elif choice == "2": - self.config["TRANSCRIPTION_PROVIDER"] = "mistral" - self.console.print("[blue][INFO][/blue] Mistral selected") - self.console.print("Get your API key from: https://console.mistral.ai/") + # Update config.yml to use Deepgram + self.config_manager.update_config_defaults({"stt": "stt-deepgram"}) + self.config_yml_data = self.config_manager.get_full_config() # Reload - # Check for existing API key - existing_key = self.read_existing_env_value("MISTRAL_API_KEY") - if existing_key and existing_key not in ['your_mistral_api_key_here', 'your-mistral-key-here']: - masked_key = self.mask_api_key(existing_key) - prompt_text = f"Mistral API key ({masked_key}) [press Enter to reuse, or enter new]" - api_key_input = self.prompt_value(prompt_text, "") - api_key = api_key_input if api_key_input else existing_key - else: - api_key = self.prompt_value("Mistral API key (leave empty to skip)", "") - - model = self.prompt_value("Mistral model", "voxtral-mini-2507") - - if api_key: - self.config["MISTRAL_API_KEY"] = api_key - self.config["MISTRAL_MODEL"] = model - self.console.print("[green][SUCCESS][/green] Mistral configured") + self.console.print("[green][SUCCESS][/green] Deepgram configured in config.yml and .env") + self.console.print("[blue][INFO][/blue] Set defaults.stt: stt-deepgram") else: self.console.print("[yellow][WARNING][/yellow] No API key provided - transcription will not work") - elif choice == "3": - self.config["TRANSCRIPTION_PROVIDER"] = "parakeet" + elif choice == "2": self.console.print("[blue][INFO][/blue] Offline Parakeet ASR selected") parakeet_url = self.prompt_value("Parakeet ASR URL", "http://host.docker.internal:8767") + + # Write URL to .env for ${PARAKEET_ASR_URL} placeholder in config.yml self.config["PARAKEET_ASR_URL"] = parakeet_url + + # Update config.yml to use Parakeet + self.config_manager.update_config_defaults({"stt": "stt-parakeet-batch"}) + self.config_yml_data = self.config_manager.get_full_config() # Reload + + self.console.print("[green][SUCCESS][/green] Parakeet configured in config.yml and .env") + self.console.print("[blue][INFO][/blue] Set defaults.stt: stt-parakeet-batch") self.console.print("[yellow][WARNING][/yellow] Remember to start Parakeet service: cd ../../extras/asr-services && docker compose up parakeet") - elif choice == "4": + elif choice == "3": self.console.print("[blue][INFO][/blue] Skipping transcription setup") def setup_llm(self): @@ -592,7 +585,15 @@ def show_summary(self): self.console.print() self.console.print(f"โœ… Admin Account: {self.config.get('ADMIN_EMAIL', 'Not configured')}") - self.console.print(f"โœ… Transcription: {self.config.get('TRANSCRIPTION_PROVIDER', 'Not configured')}") + + # Show transcription from config.yml + stt_default = self.config_yml_data.get("defaults", {}).get("stt", "not set") + stt_model = next( + (m for m in self.config_yml_data.get("models", []) if m.get("name") == stt_default), + None + ) + stt_provider = stt_model.get("model_provider", "unknown") if stt_model else "not configured" + self.console.print(f"โœ… Transcription: {stt_provider} ({stt_default}) - config.yml") # Show LLM config from config.yml llm_default = self.config_yml_data.get("defaults", {}).get("llm", "not set") diff --git a/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py index b29ca88d..50ffc77f 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py @@ -302,22 +302,22 @@ async def _initialize_streaming_session( client_state.stream_audio_format = audio_format application_logger.info(f"๐Ÿ†” Created stream session: {client_state.stream_session_id}") - # Determine transcription provider from environment - transcription_provider = os.getenv("TRANSCRIPTION_PROVIDER", "").lower() - if transcription_provider == "parakeet": - provider = "parakeet" - elif transcription_provider == "deepgram": - provider = "deepgram" - else: - # Auto-detect: prefer Parakeet if URL is set, otherwise Deepgram - parakeet_url = os.getenv("PARAKEET_ASR_URL") - deepgram_key = os.getenv("DEEPGRAM_API_KEY") - if parakeet_url: - provider = "parakeet" - elif deepgram_key: - provider = "deepgram" - else: - raise ValueError("No transcription provider configured (DEEPGRAM_API_KEY or PARAKEET_ASR_URL required)") + # Determine transcription provider from config.yml + from advanced_omi_backend.model_registry import get_models_registry + + registry = get_models_registry() + if not registry: + raise ValueError("config.yml not found - cannot determine transcription provider") + + stt_model = registry.get_default("stt") + if not stt_model: + raise ValueError("No default STT model configured in config.yml (defaults.stt)") + + provider = stt_model.model_provider.lower() + if provider not in ["deepgram", "parakeet"]: + raise ValueError(f"Unsupported STT provider: {provider}. Expected: deepgram or parakeet") + + application_logger.info(f"๐Ÿ“‹ Using STT provider: {provider} (model: {stt_model.name})") # Initialize session tracking in Redis await audio_stream_producer.init_session( diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/health_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/health_routes.py index 5ffa5d6f..d6d9af5d 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/health_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/health_routes.py @@ -109,7 +109,10 @@ async def health_check(): if transcription_provider else "Not configured" ), - "transcription_provider": os.getenv("TRANSCRIPTION_PROVIDER", "auto-detect"), + "transcription_provider": ( + REGISTRY.get_default("stt").name if REGISTRY and REGISTRY.get_default("stt") + else "not configured" + ), "provider_type": ( transcription_provider.mode if transcription_provider else "none" ), diff --git a/backends/advanced/start-workers.sh b/backends/advanced/start-workers.sh index ad6bd6eb..2ed50727 100755 --- a/backends/advanced/start-workers.sh +++ b/backends/advanced/start-workers.sh @@ -57,13 +57,15 @@ start_workers() { from advanced_omi_backend.model_registry import get_models_registry registry = get_models_registry() if registry and registry.defaults: - print(registry.defaults.get('stt', '')) + stt_model = registry.get_default('stt') + if stt_model: + print(stt_model.model_provider or '') " 2>/dev/null || echo "") echo "๐Ÿ“‹ Configured STT provider: ${DEFAULT_STT:-none}" # Only start Deepgram worker if configured as default STT - if [[ "$DEFAULT_STT" == *"deepgram"* ]] && [ -n "$DEEPGRAM_API_KEY" ]; then + if [[ "$DEFAULT_STT" == "deepgram" ]] && [ -n "$DEEPGRAM_API_KEY" ]; then echo "๐ŸŽต Starting audio stream Deepgram worker (1 worker for sequential processing)..." uv run python -m advanced_omi_backend.workers.audio_stream_deepgram_worker & AUDIO_STREAM_DEEPGRAM_WORKER_PID=$! @@ -73,7 +75,7 @@ if registry and registry.defaults: fi # Only start Parakeet worker if configured as default STT - if [[ "$DEFAULT_STT" == *"parakeet"* ]]; then + if [[ "$DEFAULT_STT" == "parakeet" ]]; then echo "๐ŸŽต Starting audio stream Parakeet worker (1 worker for sequential processing)..." uv run python -m advanced_omi_backend.workers.audio_stream_parakeet_worker & AUDIO_STREAM_PARAKEET_WORKER_PID=$! diff --git a/config/config.yml.template b/config/config.yml.template index 37209d4b..7b43d042 100644 --- a/config/config.yml.template +++ b/config/config.yml.template @@ -2,6 +2,9 @@ defaults: llm: openai-llm embedding: openai-embed stt: stt-deepgram + # Transcription provider selection: + # - stt-deepgram: Cloud-based (requires DEEPGRAM_API_KEY in .env) + # - stt-parakeet-batch: Local ASR (requires Parakeet service running) tts: tts-http vector_store: vs-qdrant models: @@ -96,7 +99,7 @@ models: model_type: stt model_provider: parakeet api_family: http - model_url: http://172.17.0.1:8767 + model_url: http://${PARAKEET_ASR_URL:-172.17.0.1:8767} api_key: '' operations: stt_transcribe: From 129cd9518151e3b841b5c4486f3848ce03b36ada Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 1 Jan 2026 20:52:28 +0000 Subject: [PATCH 07/14] Enhance ConfigManager with deep merge functionality - Updated the `update_memory_config` method to perform a deep merge of updates into the memory configuration, ensuring nested dictionaries are merged correctly. - Added a new `_deep_merge` method to handle recursive merging of dictionaries, improving configuration management capabilities. --- config_manager.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/config_manager.py b/config_manager.py index 2f64b082..c9bf9a2a 100644 --- a/config_manager.py +++ b/config_manager.py @@ -262,15 +262,15 @@ def update_memory_config(self, updates: Dict[str, Any]): Update memory configuration in config.yml. Args: - updates: Dict of updates to merge into memory config + updates: Dict of updates to merge into memory config (deep merge) """ config = self._load_config_yml() if "memory" not in config: config["memory"] = {} - # Deep merge updates - config["memory"].update(updates) + # Deep merge updates recursively + self._deep_merge(config["memory"], updates) self._save_config_yml(config) @@ -278,6 +278,22 @@ def update_memory_config(self, updates: Dict[str, Any]): if "provider" in updates and self.env_path: self._update_env_file("MEMORY_PROVIDER", updates["provider"]) + def _deep_merge(self, base: dict, updates: dict) -> None: + """ + Recursively merge updates into base dictionary. + + Args: + base: Base dictionary to merge into (modified in-place) + updates: Updates to merge + """ + for key, value in updates.items(): + if key in base and isinstance(base[key], dict) and isinstance(value, dict): + # Recursively merge nested dictionaries + self._deep_merge(base[key], value) + else: + # Direct assignment for non-dict values + base[key] = value + def get_config_defaults(self) -> Dict[str, Any]: """ Get defaults configuration from config.yml. From afae13eb34ed8e29e1b72ad5b098fa41cf5628e9 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 1 Jan 2026 22:06:58 +0000 Subject: [PATCH 08/14] Refactor run-test.sh and enhance memory extraction tests - Removed deprecated environment variable handling for TRANSCRIPTION_PROVIDER in `run-test.sh`, streamlining the configuration process. - Introduced a new `run-custom.sh` script for executing Robot tests with custom configurations, improving test flexibility. - Enhanced memory extraction tests in `audio_keywords.robot` and `memory_keywords.robot` to include detailed assertions and result handling. - Updated `queue_keywords.robot` to fail fast if a job is in a 'failed' state when expecting 'completed', improving error handling. - Refactored `test_env.py` to load environment variables with correct precedence, ensuring better configuration management. --- backends/advanced/run-test.sh | 53 +++++++------- tests/resources/audio_keywords.robot | 23 ++++++- tests/resources/memory_keywords.robot | 99 ++++++++++++++++++++------- tests/resources/queue_keywords.robot | 7 ++ tests/run-custom.sh | 20 ++++++ tests/setup/test_env.py | 34 +++++---- 6 files changed, 167 insertions(+), 69 deletions(-) create mode 100755 tests/run-custom.sh diff --git a/backends/advanced/run-test.sh b/backends/advanced/run-test.sh index e9544be6..23717b0b 100755 --- a/backends/advanced/run-test.sh +++ b/backends/advanced/run-test.sh @@ -41,7 +41,6 @@ print_info "========================================" # Load environment variables (CI or local) # Priority: Command-line env vars > CI environment > .env.test > .env # Save any pre-existing environment variables to preserve command-line overrides -_TRANSCRIPTION_PROVIDER_OVERRIDE=${TRANSCRIPTION_PROVIDER} _PARAKEET_ASR_URL_OVERRIDE=${PARAKEET_ASR_URL} _DEEPGRAM_API_KEY_OVERRIDE=${DEEPGRAM_API_KEY} _OPENAI_API_KEY_OVERRIDE=${OPENAI_API_KEY} @@ -49,7 +48,7 @@ _LLM_PROVIDER_OVERRIDE=${LLM_PROVIDER} _MEMORY_PROVIDER_OVERRIDE=${MEMORY_PROVIDER} _CONFIG_FILE_OVERRIDE=${CONFIG_FILE} -if [ -n "$DEEPGRAM_API_KEY" ] && [ -z "$_TRANSCRIPTION_PROVIDER_OVERRIDE" ]; then +if [ -n "$DEEPGRAM_API_KEY" ]; then print_info "Using environment variables from CI/environment..." elif [ -f ".env.test" ]; then print_info "Loading environment variables from .env.test..." @@ -69,10 +68,6 @@ else fi # Restore command-line overrides (these take highest priority) -if [ -n "$_TRANSCRIPTION_PROVIDER_OVERRIDE" ]; then - export TRANSCRIPTION_PROVIDER=$_TRANSCRIPTION_PROVIDER_OVERRIDE - print_info "Using command-line override: TRANSCRIPTION_PROVIDER=$TRANSCRIPTION_PROVIDER" -fi if [ -n "$_PARAKEET_ASR_URL_OVERRIDE" ]; then export PARAKEET_ASR_URL=$_PARAKEET_ASR_URL_OVERRIDE print_info "Using command-line override: PARAKEET_ASR_URL=$PARAKEET_ASR_URL" @@ -101,35 +96,47 @@ fi # Usage: CONFIG_FILE=../../tests/configs/parakeet-ollama.yml ./run-test.sh export CONFIG_FILE=${CONFIG_FILE:-../../config/config.yml} -# Verify required environment variables based on configured providers -TRANSCRIPTION_PROVIDER=${TRANSCRIPTION_PROVIDER:-deepgram} +print_info "Using config file: $CONFIG_FILE" + +# Read STT provider from config.yml (source of truth) +STT_PROVIDER=$(uv run python -c " +from advanced_omi_backend.model_registry import get_models_registry +registry = get_models_registry() +if registry and registry.defaults: + stt_model = registry.get_default('stt') + if stt_model: + print(stt_model.model_provider or '') +" 2>/dev/null || echo "") + +# Fallback to environment variable for backward compatibility (will be removed) +if [ -z "$STT_PROVIDER" ]; then + STT_PROVIDER=${TRANSCRIPTION_PROVIDER:-deepgram} + print_warning "Could not read STT provider from config.yml, using TRANSCRIPTION_PROVIDER: $STT_PROVIDER" +fi + +# LLM provider can still use env var as it's not part of this refactor LLM_PROVIDER=${LLM_PROVIDER:-openai} print_info "Configured providers:" -print_info " TRANSCRIPTION_PROVIDER: $TRANSCRIPTION_PROVIDER" -print_info " LLM_PROVIDER: $LLM_PROVIDER" +print_info " STT Provider (from config.yml): $STT_PROVIDER" +print_info " LLM Provider: $LLM_PROVIDER" -# Check transcription provider API key -case "$TRANSCRIPTION_PROVIDER" in +# Check transcription provider API key based on config.yml +case "$STT_PROVIDER" in deepgram) if [ -z "$DEEPGRAM_API_KEY" ]; then - print_error "DEEPGRAM_API_KEY not set (required for TRANSCRIPTION_PROVIDER=deepgram)" + print_error "DEEPGRAM_API_KEY not set (required for STT provider: deepgram)" exit 1 fi print_info "DEEPGRAM_API_KEY length: ${#DEEPGRAM_API_KEY}" ;; - mistral) - if [ -z "$MISTRAL_API_KEY" ]; then - print_error "MISTRAL_API_KEY not set (required for TRANSCRIPTION_PROVIDER=mistral)" - exit 1 - fi - print_info "MISTRAL_API_KEY length: ${#MISTRAL_API_KEY}" - ;; - offline|parakeet) - print_info "Using offline/local transcription - no API key required" + parakeet) + print_info "Using Parakeet (local transcription) - no API key required" + PARAKEET_ASR_URL=${PARAKEET_ASR_URL:-http://localhost:8767} + print_info "PARAKEET_ASR_URL: $PARAKEET_ASR_URL" ;; *) - print_warning "Unknown TRANSCRIPTION_PROVIDER: $TRANSCRIPTION_PROVIDER" + print_warning "Unknown STT provider from config.yml: $STT_PROVIDER" ;; esac diff --git a/tests/resources/audio_keywords.robot b/tests/resources/audio_keywords.robot index f3ae950d..2d37fcbc 100644 --- a/tests/resources/audio_keywords.robot +++ b/tests/resources/audio_keywords.robot @@ -72,7 +72,8 @@ Upload Audio File Upload Audio File And Wait For Memory [Documentation] Upload audio file and wait for complete processing including memory extraction. ... This is for E2E testing - use Upload Audio File for upload-only tests. - [Arguments] ${audio_file_path} ${device_name}=robot-test ${folder}=. + ... Performs assertions inline to verify successful memory extraction. + [Arguments] ${audio_file_path} ${device_name}=robot-test ${folder}=. ${min_memories}=1 # Upload file (uses existing keyword) ${conversation}= Upload Audio File ${audio_file_path} ${device_name} ${folder} @@ -90,8 +91,24 @@ Upload Audio File And Wait For Memory Log Found memory job: ${memory_job_id} - # Wait for memory extraction (uses keyword from memory_keywords.robot) - ${memories}= Wait For Memory Extraction ${memory_job_id} min_memories=1 + # Wait for memory extraction (returns result dictionary) + ${result}= Wait For Memory Extraction ${memory_job_id} + + # Verify memory extraction succeeded + Should Be True ${result}[success] + ... Memory extraction failed: ${result.get('error_message', 'Unknown error')} + + # Verify job completed successfully + Should Be Equal As Strings ${result}[status] completed + ... Expected job status 'completed', got '${result}[status]' + + # Verify minimum memories were extracted + ${memory_count}= Set Variable ${result}[memory_count] + Should Be True ${memory_count} >= ${min_memories} + ... Expected at least ${min_memories} memories, found ${memory_count} + + ${memories}= Set Variable ${result}[memories] + Log Successfully extracted ${memory_count} memories RETURN ${conversation} ${memories} diff --git a/tests/resources/memory_keywords.robot b/tests/resources/memory_keywords.robot index 8c3f84c0..2ab79d9c 100644 --- a/tests/resources/memory_keywords.robot +++ b/tests/resources/memory_keywords.robot @@ -107,27 +107,50 @@ Verify Memory Extraction Wait For Memory Extraction - [Documentation] Wait for memory job to complete and verify memories extracted. - ... Fails fast if job doesn't exist, fails immediately, or service is unhealthy. - [Arguments] ${memory_job_id} ${min_memories}=1 ${timeout}=120 + [Documentation] Wait for memory job to complete and fetch extracted memories. + ... Returns a result dictionary with success status, job details, and memories. + ... Does not perform assertions - calling tests should verify the results. + ... + ... Return value structure: + ... { + ... 'success': True/False, + ... 'error_message': 'Error description' (only if success=False), + ... 'status': 'completed'/'failed'/'timeout'/'not_found', + ... 'job': {job object} (if available), + ... 'memories': [list of memories] (if successful), + ... 'memory_count': int (if successful) + ... } + [Arguments] ${memory_job_id} ${timeout}=120 Log Waiting for memory job ${memory_job_id} to complete... - # 1. Verify job exists before waiting (fail fast if job ID is invalid) + # 1. Check if job exists before waiting ${job_status}= Get Job Status ${memory_job_id} - Should Not Be Equal ${job_status} ${None} - ... Memory job ${memory_job_id} not found in queue - cannot wait for completion + IF ${job_status} == ${None} + ${result}= Create Dictionary + ... success=${False} + ... error_message=Memory job ${memory_job_id} not found in queue + ... status=not_found + RETURN ${result} + END - # 2. Check if job already failed (fail fast instead of waiting 120s) + # 2. Check if job already failed ${current_status}= Set Variable ${job_status}[status] IF '${current_status}' == 'failed' ${error_info}= Evaluate $job_status.get('exc_info', 'Unknown error') - Fail Memory job ${memory_job_id} already failed: ${error_info} + ${result}= Create Dictionary + ... success=${False} + ... error_message=Memory job already failed: ${error_info} + ... status=failed + ... job=${job_status} + RETURN ${result} END # 3. Wait for job completion with status monitoring ${start_time}= Get Time epoch ${end_time}= Evaluate ${start_time} + ${timeout} + ${final_job}= Set Variable ${job_status} + ${final_status}= Set Variable ${current_status} WHILE True # Get current job status @@ -135,10 +158,17 @@ Wait For Memory Extraction # Handle job not found (e.g., expired from queue) IF ${job} == ${None} - Fail Memory job ${memory_job_id} disappeared from queue during wait + ${result}= Create Dictionary + ... success=${False} + ... error_message=Memory job ${memory_job_id} disappeared from queue during wait + ... status=not_found + ... job=${final_job} + RETURN ${result} END ${status}= Set Variable ${job}[status] + ${final_job}= Set Variable ${job} + ${final_status}= Set Variable ${status} # Success case - job completed IF '${status}' == 'completed' or '${status}' == 'finished' @@ -146,16 +176,26 @@ Wait For Memory Extraction BREAK END - # Failure case - job failed (fail fast) + # Failure case - job failed IF '${status}' == 'failed' ${error_info}= Evaluate $job.get('exc_info', 'Unknown error') - Fail Memory job ${memory_job_id} failed during processing: ${error_info} + ${result}= Create Dictionary + ... success=${False} + ... error_message=Memory job failed during processing: ${error_info} + ... status=failed + ... job=${job} + RETURN ${result} END # Timeout check ${current_time}= Get Time epoch IF ${current_time} >= ${end_time} - Fail Memory job ${memory_job_id} did not complete within ${timeout}s (last status: ${status}) + ${result}= Create Dictionary + ... success=${False} + ... error_message=Memory job did not complete within ${timeout}s (last status: ${status}) + ... status=timeout + ... job=${job} + RETURN ${result} END # Log progress every iteration @@ -165,24 +205,33 @@ Wait For Memory Extraction Sleep 5s END - # 4. Fetch memories from API with error handling + # 4. Fetch memories from API TRY ${response}= GET On Session api /api/memories expected_status=200 + ${memories_data}= Set Variable ${response.json()} + ${memories}= Set Variable ${memories_data}[memories] + ${memory_count}= Get Length ${memories} + + # Return success result + ${result}= Create Dictionary + ... success=${True} + ... status=completed + ... job=${final_job} + ... memories=${memories} + ... memory_count=${memory_count} + + Log Successfully extracted ${memory_count} memories + RETURN ${result} EXCEPT AS ${error} - Fail Failed to fetch memories from API: ${error} + # Return error if API fetch fails + ${result}= Create Dictionary + ... success=${False} + ... error_message=Failed to fetch memories from API: ${error} + ... status=api_error + ... job=${final_job} + RETURN ${result} END - ${memories_data}= Set Variable ${response.json()} - ${memories}= Set Variable ${memories_data}[memories] - ${memory_count}= Get Length ${memories} - - # 5. Verify minimum memories were extracted - Should Be True ${memory_count} >= ${min_memories} - ... Expected at least ${min_memories} memories, found ${memory_count} - - Log Successfully extracted ${memory_count} memories - RETURN ${memories} - Check Memory Similarity With OpenAI [Documentation] Use OpenAI to check if extracted memories match expected memories diff --git a/tests/resources/queue_keywords.robot b/tests/resources/queue_keywords.robot index 32f8b7fa..3d709661 100644 --- a/tests/resources/queue_keywords.robot +++ b/tests/resources/queue_keywords.robot @@ -59,6 +59,7 @@ Get Job Status Check job status [Documentation] Check the status of a specific job by ID + ... Fails immediately if job is in 'failed' state when expecting 'completed' [Arguments] ${job_id} ${expected_status} ${job}= Get Job status ${job_id} @@ -69,6 +70,12 @@ Check job status ${actual_status}= Set Variable ${job}[status] Log Job ${job_id} status: ${actual_status} (expected: ${expected_status}) + # Fail fast if job is in failed state when we're expecting completed + IF '${actual_status}' == 'failed' and '${expected_status}' == 'completed' + ${error_msg}= Evaluate $job.get('exc_info') or $job.get('error', 'Unknown error') + Fail Job ${job_id} failed: ${error_msg} + END + Should Be Equal As Strings ${actual_status} ${expected_status} Job status is '${actual_status}', expected '${expected_status}' RETURN ${job} diff --git a/tests/run-custom.sh b/tests/run-custom.sh new file mode 100755 index 00000000..c1ce1317 --- /dev/null +++ b/tests/run-custom.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Quick wrapper for running Robot tests with custom configs +# Usage: ./run-custom.sh [parakeet-url] +# +# Examples: +# ./run-custom.sh parakeet-openai http://host.docker.internal:8767 +# ./run-custom.sh deepgram-openai +# ./run-custom.sh parakeet-ollama http://host.docker.internal:8767 + +set -e + +CONFIG_NAME="${1:-parakeet-openai}" +PARAKEET_URL="${2:-http://host.docker.internal:8767}" + +echo "Running Robot tests with config: ${CONFIG_NAME}" +echo "Parakeet ASR URL: ${PARAKEET_URL}" + +CONFIG_FILE="../tests/configs/${CONFIG_NAME}.yml" \ + PARAKEET_ASR_URL="${PARAKEET_URL}" \ + ./run-robot-tests.sh diff --git a/tests/setup/test_env.py b/tests/setup/test_env.py index fa3e0f9d..d11f2ff8 100644 --- a/tests/setup/test_env.py +++ b/tests/setup/test_env.py @@ -1,25 +1,23 @@ # Test Environment Configuration import os from pathlib import Path +from dotenv import load_dotenv -# Load .env file from backends/advanced directory if it exists -# This allows tests to work when run from VSCode or command line -def load_env_file(): - """Load environment variables from .env file if it exists.""" - # Look for .env in backends/advanced directory - env_file = Path(__file__).parent.parent.parent / "backends" / "advanced" / ".env" - if env_file.exists(): - with open(env_file) as f: - for line in f: - line = line.strip() - if line and not line.startswith('#') and '=' in line: - key, value = line.split('=', 1) - # Only set if not already in environment (CI takes precedence) - if key not in os.environ: - os.environ[key] = value - -# Load .env file (CI environment variables take precedence) -load_env_file() +# Load environment files with correct precedence: +# 1. Environment variables (highest priority - from shell, CI, etc.) +# 2. .env.test (test-specific configuration) +# 3. .env (default configuration) + +backend_dir = Path(__file__).parent.parent.parent / "backends" / "advanced" + +# Load in reverse order of precedence (since override=False won't overwrite existing vars) +# Load .env.test first (will set test-specific values) +load_dotenv(backend_dir / ".env.test", override=False) + +# Load .env second (will only fill in missing values, won't override .env.test or existing env vars) +load_dotenv(backend_dir / ".env", override=False) + +# Final precedence: environment variables > .env.test > .env # API Configuration API_URL = 'http://localhost:8001' # Use BACKEND_URL from test.env From 1fd98516c325d453aaa3e6909d3356a383f64aad Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 1 Jan 2026 22:33:54 +0000 Subject: [PATCH 09/14] unify tests to robot test, add some more clean up --- .github/workflows/README.md | 4 +- CLAUDE.md | 6 +- Docs/getting-started.md | 4 +- backends/advanced/Docs/quickstart.md | 4 +- backends/advanced/README.md | 21 +- backends/advanced/run-test.sh | 11 +- backends/advanced/tests/test_integration.py | 1591 ------------------- tests/integration/integration_test.robot | 2 +- 8 files changed, 32 insertions(+), 1611 deletions(-) delete mode 100644 backends/advanced/tests/test_integration.py diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3b645800..5e98cd18 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -86,6 +86,6 @@ uv sync --dev cp .env.template .env.test # Add your API keys to .env.test -# Run test (modify CACHED_MODE in test_integration.py if needed) -uv run pytest test_integration.py::test_full_pipeline_integration -v -s +# Run Robot Framework integration tests +uv run robot --outputdir test-results --loglevel INFO tests/integration/integration_test.robot ``` \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index e505b25a..abe20db6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -116,11 +116,11 @@ cp .env.template .env # Configure API keys # Manual test execution (for debugging) source .env && export DEEPGRAM_API_KEY && export OPENAI_API_KEY -uv run pytest tests/test_integration.py::test_full_pipeline_integration -v -s +uv run robot --outputdir test-results --loglevel INFO ../../tests/integration/integration_test.robot # Leave test containers running for debugging (don't auto-cleanup) CLEANUP_CONTAINERS=false source .env && export DEEPGRAM_API_KEY && export OPENAI_API_KEY -uv run pytest tests/test_integration.py::test_full_pipeline_integration -v -s +uv run robot --outputdir test-results --loglevel INFO ../../tests/integration/integration_test.robot # Manual cleanup when needed docker compose -f docker-compose-test.yml down -v @@ -390,7 +390,7 @@ docker compose up --build -d ### Testing Strategy - **Local Test Scripts**: Simplified scripts (`./run-test.sh`) mirror CI workflows for local development -- **End-to-End Integration**: `test_integration.py` validates complete audio processing pipeline +- **End-to-End Integration**: Robot Framework tests (`tests/integration/integration_test.robot`) validate complete audio processing pipeline - **Speaker Recognition Tests**: `test_speaker_service_integration.py` validates speaker identification - **Environment Flexibility**: Tests work with both local .env files and CI environment variables - **Automated Cleanup**: Test containers are automatically removed after execution diff --git a/Docs/getting-started.md b/Docs/getting-started.md index 506dd2f6..a923c99c 100644 --- a/Docs/getting-started.md +++ b/Docs/getting-started.md @@ -179,9 +179,9 @@ After configuration, verify everything works with the integration test suite: # Alternative: Manual test with detailed logging source .env && export DEEPGRAM_API_KEY OPENAI_API_KEY && \ - uv run pytest tests/test_integration.py -vv -s --log-cli-level=INFO + uv run robot --outputdir ../../test-results --loglevel INFO ../../tests/integration/integration_test.robot ``` -This end-to-end test validates the complete audio processing pipeline. +This end-to-end test validates the complete audio processing pipeline using Robot Framework. ## Using the System diff --git a/backends/advanced/Docs/quickstart.md b/backends/advanced/Docs/quickstart.md index 922fe9b7..0d681978 100644 --- a/backends/advanced/Docs/quickstart.md +++ b/backends/advanced/Docs/quickstart.md @@ -177,9 +177,9 @@ After configuration, verify everything works with the integration test suite: # Alternative: Manual test with detailed logging source .env && export DEEPGRAM_API_KEY OPENAI_API_KEY && \ - uv run pytest tests/test_integration.py -vv -s --log-cli-level=INFO + uv run robot --outputdir ../../test-results --loglevel INFO ../../tests/integration/integration_test.robot ``` -This end-to-end test validates the complete audio processing pipeline. +This end-to-end test validates the complete audio processing pipeline using Robot Framework. ## Using the System diff --git a/backends/advanced/README.md b/backends/advanced/README.md index ab86a22e..d493241c 100644 --- a/backends/advanced/README.md +++ b/backends/advanced/README.md @@ -100,14 +100,21 @@ See [Docs/HTTPS_SETUP.md](Docs/HTTPS_SETUP.md) for detailed configuration. To run integration tests with different transcription providers: ```bash -# Test with Parakeet ASR (offline transcription) -# Automatically starts test ASR service - no manual setup required -source .env && export DEEPGRAM_API_KEY && export OPENAI_API_KEY && TRANSCRIPTION_PROVIDER=parakeet uv run pytest tests/test_integration.py::test_full_pipeline_integration -v -s --tb=short +# Test with different configurations using config.yml files +# Test configs located in tests/configs/ -# Test with Deepgram (default) -source .env && export DEEPGRAM_API_KEY && export OPENAI_API_KEY && uv run pytest tests/test_integration.py::test_full_pipeline_integration -v -s --tb=short +# Test with Parakeet ASR + Ollama (offline, no API keys) +CONFIG_FILE=../../tests/configs/parakeet-ollama.yml ./run-test.sh + +# Test with Deepgram + OpenAI (cloud-based) +CONFIG_FILE=../../tests/configs/deepgram-openai.yml ./run-test.sh + +# Manual Robot Framework test execution +source .env && export DEEPGRAM_API_KEY OPENAI_API_KEY && \ + uv run robot --outputdir ../../test-results --loglevel INFO ../../tests/integration/integration_test.robot ``` **Prerequisites:** -- API keys configured in `.env` file -- For debugging: Set `CACHED_MODE = True` in test file to keep containers running +- API keys configured in `.env` file (for cloud providers) +- Test configurations in `tests/configs/` directory +- For debugging: Set `CLEANUP_CONTAINERS=false` environment variable to keep containers running diff --git a/backends/advanced/run-test.sh b/backends/advanced/run-test.sh index 23717b0b..17773dc1 100755 --- a/backends/advanced/run-test.sh +++ b/backends/advanced/run-test.sh @@ -205,9 +205,14 @@ fi # Set environment variables for the test export DOCKER_BUILDKIT=0 -# Run the integration test with extended timeout (mem0 needs time for comprehensive extraction) -print_info "Starting integration test (timeout: 15 minutes)..." -if timeout 900 uv run pytest tests/test_integration.py::test_full_pipeline_integration -v -s --tb=short --log-cli-level=INFO; then +# Configure Robot Framework test mode +# TEST_MODE=dev: Robot tests keep containers running (cleanup handled by run-test.sh) +# This allows CLEANUP_CONTAINERS flag to work as expected +export TEST_MODE=dev + +# Run the Robot Framework integration tests with extended timeout (mem0 needs time for comprehensive extraction) +print_info "Starting Robot Framework integration tests (timeout: 15 minutes)..." +if timeout 900 uv run robot --outputdir ../../test-results --loglevel INFO ../../tests/integration/integration_test.robot; then print_success "Integration tests completed successfully!" else TEST_EXIT_CODE=$? diff --git a/backends/advanced/tests/test_integration.py b/backends/advanced/tests/test_integration.py deleted file mode 100644 index 201eaafd..00000000 --- a/backends/advanced/tests/test_integration.py +++ /dev/null @@ -1,1591 +0,0 @@ -#!/usr/bin/env python3 -""" -End-to-end integration test for Chronicle backend with unified transcription support. - -This test validates the complete audio processing pipeline using isolated test environment: -1. Service startup with docker-compose-test.yml (isolated ports and databases) -2. ASR service startup (if Parakeet provider selected) -3. Authentication with test credentials -4. Audio file upload -5. Transcription (Deepgram API or Parakeet ASR service) -6. Memory extraction (OpenAI) -7. Data storage verification - -Run with: - # Deepgram API transcription (default) - source .env && export DEEPGRAM_API_KEY && export OPENAI_API_KEY && uv run pytest tests/test_integration.py::test_full_pipeline_integration -v -s - - # Parakeet ASR transcription (HTTP/WebSocket service) - source .env && export OPENAI_API_KEY && TRANSCRIPTION_PROVIDER=parakeet uv run pytest tests/test_integration.py::test_full_pipeline_integration -v -s - -Test Environment: -- Uses docker-compose-test.yml for service isolation -- Backend runs on port 8001 (vs dev 8000) -- MongoDB on port 27018 (vs dev 27017) -- Qdrant on ports 6335/6336 (vs dev 6333/6334) -- Parakeet ASR on port 8767 (parakeet provider) -- Test credentials configured via environment variables -- Provider selection via TRANSCRIPTION_PROVIDER environment variable -""" - -import asyncio -import json -import logging -import os -import shutil -import socket -import subprocess -import sys -import time -from pathlib import Path -from typing import Optional - -import openai -import pytest -import requests -from pymongo import MongoClient - -# Configure logging with immediate output (no buffering) -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - stream=sys.stdout, - force=True -) -logger = logging.getLogger(__name__) -# Ensure immediate output -logger.handlers[0].flush() if logger.handlers else None -from dotenv import load_dotenv - -# Test Configuration Flags -# REBUILD=True: Force rebuild of containers (useful when code changes) -# FRESH_RUN=True: Start with fresh data and containers (default) -# CLEANUP_CONTAINERS=True: Stop and remove containers after test (default) -REBUILD = os.environ.get("REBUILD", "true").lower() == "true" -FRESH_RUN = os.environ.get("FRESH_RUN", "true").lower() == "true" -CLEANUP_CONTAINERS = os.environ.get("CLEANUP_CONTAINERS", "true").lower() == "true" - -# Transcription Provider Configuration -# TRANSCRIPTION_PROVIDER: 'deepgram' (Deepgram API) or 'parakeet' (Parakeet ASR service) -TRANSCRIPTION_PROVIDER = os.environ.get("TRANSCRIPTION_PROVIDER", "deepgram") # Default to deepgram -# Get Parakeet URL from environment, fallback to port 8080 -PARAKEET_ASR_URL = os.environ.get("PARAKEET_ASR_URL", "http://host.docker.internal:8080") - -# Test Environment Configuration -# Base configuration for both providers -# NOTE: LLM configuration is now in config.yml (defaults.llm) -TEST_ENV_VARS_BASE = { - "AUTH_SECRET_KEY": "test-jwt-signing-key-for-integration-tests", - "ADMIN_PASSWORD": "test-admin-password-123", - "ADMIN_EMAIL": "test-admin@example.com", - "MONGODB_URI": "mongodb://localhost:27018", # Test port (database specified in backend) - "QDRANT_BASE_URL": "localhost", - "DISABLE_SPEAKER_RECOGNITION": "true", # Prevent segment duplication in tests -} - -# Deepgram provider configuration (API) -TEST_ENV_VARS_DEEPGRAM = { - **TEST_ENV_VARS_BASE, - "TRANSCRIPTION_PROVIDER": "deepgram", - # Deepgram API key loaded from environment -} - -# Parakeet provider configuration (HTTP/WebSocket ASR service) -TEST_ENV_VARS_PARAKEET = { - **TEST_ENV_VARS_BASE, - "TRANSCRIPTION_PROVIDER": "parakeet", - "PARAKEET_ASR_URL": PARAKEET_ASR_URL, -} - -# Select configuration based on provider -if TRANSCRIPTION_PROVIDER == "parakeet": - TEST_ENV_VARS = TEST_ENV_VARS_PARAKEET -else: # Default to deepgram - TEST_ENV_VARS = TEST_ENV_VARS_DEEPGRAM - -tests_dir = Path(__file__).parent - -# Test constants -BACKEND_URL = "http://localhost:8001" # Test backend port -TEST_AUDIO_PATH = tests_dir.parent.parent.parent / "extras/test-audios/DIY Experts Glass Blowing_16khz_mono_4min.wav" -TEST_AUDIO_PATH_PARAKEET = tests_dir / "assets" / "test_clip_10s.wav" # Shorter clip for parakeet testing -MAX_STARTUP_WAIT = 60 # seconds -PROCESSING_TIMEOUT = 300 # seconds for audio processing (5 minutes) - - -# Path to expected transcript file -EXPECTED_TRANSCRIPT_PATH = tests_dir / "assets/test_transcript.txt" - -# Path to expected memories file -EXPECTED_MEMORIES_PATH = tests_dir / "assets/expected_memories.json" - - -class IntegrationTestRunner: - """Manages the integration test lifecycle.""" - - def __init__(self): - print(f"๐Ÿ”ง Initializing IntegrationTestRunner", flush=True) - print(f" FRESH_RUN={FRESH_RUN}, CLEANUP_CONTAINERS={CLEANUP_CONTAINERS}, REBUILD={REBUILD}", flush=True) - print(f" TRANSCRIPTION_PROVIDER={TRANSCRIPTION_PROVIDER}", flush=True) - sys.stdout.flush() - - self.token: Optional[str] = None - self.services_started = False - self.services_started_by_test = False # Track if WE started the services - self.mongo_client: Optional[MongoClient] = None - self.fresh_run = FRESH_RUN # Use global configuration flag - self.cleanup_containers = CLEANUP_CONTAINERS # Use global cleanup flag - self.rebuild = REBUILD # Use global rebuild flag - self.asr_services_started = False # Track ASR services for parakeet provider - self.provider = TRANSCRIPTION_PROVIDER # Store provider type - - def load_expected_transcript(self) -> str: - """Load the expected transcript from the test assets file.""" - try: - # Use provider-specific expectations if available - if self.provider == "parakeet": - transcript_path = tests_dir / "assets/test_transcript_parakeet.txt" - if not transcript_path.exists(): - transcript_path = EXPECTED_TRANSCRIPT_PATH # Fallback to default - else: - transcript_path = EXPECTED_TRANSCRIPT_PATH - - with open(transcript_path, 'r', encoding='utf-8') as f: - return f.read().strip() - except FileNotFoundError: - logger.warning(f"โš ๏ธ Expected transcript file not found: {transcript_path}") - return "" - except Exception as e: - logger.warning(f"โš ๏ธ Error loading expected transcript: {e}") - return "" - - def load_expected_memories(self) -> list: - """Load the expected memories from the test assets file.""" - try: - # Use provider-specific expectations if available - if self.provider == "parakeet": - memories_path = tests_dir / "assets/expected_memories_parakeet.json" - if not memories_path.exists(): - memories_path = EXPECTED_MEMORIES_PATH # Fallback to default - else: - memories_path = EXPECTED_MEMORIES_PATH - - with open(memories_path, 'r', encoding='utf-8') as f: - import json - data = json.load(f) - # Handle both formats: list or dict with 'memories' key - if isinstance(data, list): - return data - elif isinstance(data, dict) and 'memories' in data: - return data['memories'] - else: - logger.warning(f"โš ๏ธ Unexpected memories file format: {type(data)}") - return [] - except FileNotFoundError: - logger.warning(f"โš ๏ธ Expected memories file not found: {memories_path}") - return [] - except Exception as e: - logger.warning(f"โš ๏ธ Error loading expected memories: {e}") - return [] - - def cleanup_test_data(self): - """Clean up test-specific data directories using lightweight Docker container.""" - if not self.fresh_run: - logger.info("๐Ÿ—‚๏ธ Skipping test data cleanup (reusing existing data)") - return - - logger.info("๐Ÿ—‚๏ธ Cleaning up test-specific data directories...") - - # Use lightweight Docker container to clean root-owned files - try: - result = subprocess.run([ - "docker", "run", "--rm", - "-v", f"{Path.cwd()}/data:/data", - "alpine:latest", - "sh", "-c", "rm -rf /data/test_*" - ], capture_output=True, text=True, timeout=30) - - if result.returncode == 0: - logger.info("โœ… Docker cleanup successful") - else: - logger.warning(f"Error during Docker cleanup: {result.stderr}") - - except Exception as e: - logger.warning(f"โš ๏ธ Docker cleanup failed: {e}") - logger.warning("๐Ÿ’ก Ensure Docker is running and accessible") - - logger.info("โœ“ Test data cleanup complete") - - def start_asr_services(self): - """Start ASR services for Parakeet transcription testing.""" - if self.provider != "parakeet": - logger.info(f"๐Ÿ”„ Skipping ASR services ({self.provider} provider uses API)") - return - - logger.info(f"๐Ÿš€ Starting Parakeet ASR service...") - - try: - asr_dir = Path(__file__).parent.parent.parent.parent / "extras/asr-services" - - # Stop any existing ASR services first - subprocess.run( - ["docker", "compose", "-f", "docker-compose-test.yml", "down"], - cwd=asr_dir, - capture_output=True - ) - - # Start Parakeet ASR service - result = subprocess.run( - ["docker", "compose", "-f", "docker-compose-test.yml", "up", "--build", "-d", "parakeet-asr-test"], - cwd=asr_dir, - capture_output=True, - text=True, - timeout=300 # 5 minute timeout for service startup - ) - - if result.returncode != 0: - logger.error(f"Failed to start Parakeet ASR service: {result.stderr}") - raise RuntimeError(f"Parakeet ASR service failed to start: {result.stderr}") - - self.asr_services_started = True - logger.info("โœ… Parakeet ASR service started successfully") - - except Exception as e: - logger.error(f"Error starting Parakeet ASR service: {e}") - raise - - def wait_for_asr_ready(self): - """Wait for ASR services to be ready.""" - if self.provider != "parakeet": - logger.info(f"๐Ÿ”„ Skipping ASR readiness check ({self.provider} provider uses API)") - return - - # Cascade failure check - don't wait for ASR if backend services failed - if not hasattr(self, 'services_started') or not self.services_started: - raise RuntimeError("Backend services are not running - cannot start ASR services") - - logger.info("๐Ÿ” Waiting for Parakeet ASR service to be ready...") - - start_time = time.time() - while time.time() - start_time < MAX_STARTUP_WAIT: - try: - # Check container status directly instead of HTTP health check - # This avoids the curl dependency issue in the container - result = subprocess.run( - ["docker", "ps", "--filter", "name=asr-services-parakeet-asr-test-1", "--format", "{{.Status}}"], - capture_output=True, - text=True, - timeout=10 - ) - - if result.returncode == 0 and result.stdout.strip(): - status = result.stdout.strip() - logger.debug(f"Container status: {status}") - - # Early exit on unhealthy containers - if "(unhealthy)" in status: - raise RuntimeError(f"Parakeet ASR container is unhealthy: {status}") - if "Exited" in status or "Dead" in status: - raise RuntimeError(f"Parakeet ASR container failed: {status}") - - # Look for 'Up' status and ideally '(healthy)' status - if "Up" in status: - # If container is healthy, we can skip the HTTP check - if "(healthy)" in status: - logger.info("โœ“ Parakeet ASR container is healthy") - return - # Additional check: try to connect to the service - try: - import requests - - # Use the same URL that the backend will use - response = requests.get(f"{PARAKEET_ASR_URL}/health", timeout=5) - if response.status_code == 200: - health_data = response.json() - if health_data.get("status") == "healthy": - logger.info("โœ“ Parakeet ASR service is healthy and accessible") - return - elif health_data.get("status") == "unhealthy": - raise RuntimeError(f"Parakeet ASR service reports unhealthy: {health_data}") - else: - logger.debug(f"Service responding but not ready: {health_data}") - elif response.status_code >= 500: - raise RuntimeError(f"Parakeet ASR service error: HTTP {response.status_code}") - elif response.status_code >= 400: - logger.warning(f"Parakeet ASR client error: HTTP {response.status_code}") - else: - logger.debug(f"Health check failed with status {response.status_code}") - except requests.exceptions.ConnectionError as e: - logger.debug(f"Connection failed, but container is up: {e}") - except Exception as e: - logger.debug(f"HTTP health check failed, but container is up: {e}") - else: - logger.debug(f"Container not ready yet: {status}") - else: - logger.debug("Container not found or not running") - - except Exception as e: - logger.debug(f"Container status check failed: {e}") - - time.sleep(2) - - raise RuntimeError("Parakeet ASR service failed to become ready within timeout") - - def cleanup_asr_services(self): - """Clean up ASR services.""" - if not self.asr_services_started: - return - - if not self.fresh_run: - logger.info("๐Ÿ”„ Skipping ASR services cleanup (reusing existing services)") - return - - logger.info("๐Ÿงน Cleaning up ASR services...") - - try: - asr_dir = Path(__file__).parent.parent.parent.parent / "extras/asr-services" - subprocess.run( - ["docker", "compose", "-f", "docker-compose-test.yml", "down"], - cwd=asr_dir, - capture_output=True - ) - logger.info("โœ… ASR services stopped") - except Exception as e: - logger.warning(f"Error stopping ASR services: {e}") - - def setup_environment(self): - """Set up environment variables for testing.""" - logger.info("Setting up test environment variables...") - - # Set test environment variables directly from TEST_ENV_VARS - logger.info("Setting test environment variables from TEST_ENV_VARS...") - for key, value in TEST_ENV_VARS.items(): - os.environ.setdefault(key, value) - logger.info(f"โœ“ {key} set") - - # Load API keys from .env file if not already in environment - if not os.environ.get('DEEPGRAM_API_KEY') or not os.environ.get('OPENAI_API_KEY'): - logger.info("Loading API keys from .env file...") - try: - # Try to load .env.test first (CI environment), then fall back to .env (local development) - env_test_path = '.env.test' - env_path = '.env' - - # Check if we're in the right directory (tests directory vs backend directory) - if not os.path.exists(env_test_path) and os.path.exists('../.env.test'): - env_test_path = '../.env.test' - if not os.path.exists(env_path) and os.path.exists('../.env'): - env_path = '../.env' - - if os.path.exists(env_test_path): - logger.info(f"Loading from {env_test_path}") - load_dotenv(env_test_path) - elif os.path.exists(env_path): - logger.info(f"Loading from {env_path}") - load_dotenv(env_path) - else: - logger.warning("No .env.test or .env file found") - except ImportError: - logger.warning("python-dotenv not available, relying on shell environment") - - # Debug: Log API key status (masked for security) - logger.info("API key status:") - for key in ["DEEPGRAM_API_KEY", "OPENAI_API_KEY"]: - value = os.environ.get(key) - if value: - masked_value = value[:4] + "*" * (len(value) - 8) + value[-4:] if len(value) > 8 else "***" - logger.info(f" โœ“ {key}: {masked_value}") - else: - logger.warning(f" โš ๏ธ {key}: NOT SET") - - # Log environment readiness based on provider type - deepgram_key = os.environ.get('DEEPGRAM_API_KEY') - openai_key = os.environ.get('OPENAI_API_KEY') - - # Validate based on transcription provider (streaming/batch architecture) - if self.provider == "deepgram": - # Deepgram provider validation (API-based) - if deepgram_key and openai_key: - logger.info("โœ“ All required keys for Deepgram transcription are available") - else: - logger.warning("โš ๏ธ Some keys missing for Deepgram transcription - test may fail") - if not deepgram_key: - logger.warning(" Missing DEEPGRAM_API_KEY (required for Deepgram transcription)") - if not openai_key: - logger.warning(" Missing OPENAI_API_KEY (required for memory processing)") - elif self.provider == "parakeet": - # Parakeet provider validation (local ASR service) - parakeet_url = os.environ.get('PARAKEET_ASR_URL') - if parakeet_url and openai_key: - logger.info("โœ“ All required configuration for Parakeet transcription is available") - logger.info(f" Using Parakeet ASR service at: {parakeet_url}") - else: - logger.warning("โš ๏ธ Missing configuration for Parakeet transcription - test may fail") - if not parakeet_url: - logger.warning(" Missing PARAKEET_ASR_URL (required for Parakeet ASR service)") - if not openai_key: - logger.warning(" Missing OPENAI_API_KEY (required for memory processing)") - else: - # Unknown or auto-select provider - check what's available - logger.info(f"Provider '{self.provider}' - checking available configuration...") - if deepgram_key and openai_key: - logger.info("โœ“ Deepgram configuration available") - elif os.environ.get('PARAKEET_ASR_URL') and openai_key: - logger.info("โœ“ Parakeet configuration available") - else: - logger.warning("โš ๏ธ No valid transcription provider configuration found") - if not openai_key: - logger.warning(" Missing OPENAI_API_KEY (required for memory processing)") - - def start_services(self): - """Start all services using docker compose.""" - logger.info("๐Ÿš€ Starting services with docker compose...") - - # Change to backend directory - os.chdir(tests_dir.parent) - - # Clean up test data directories first (unless cached) - self.cleanup_test_data() - - try: - # Check if test services are already running - check_result = subprocess.run(["docker", "compose", "-f", "docker-compose-test.yml", "ps", "-q"], capture_output=True, text=True) - running_services = check_result.stdout.strip().split('\n') if check_result.stdout.strip() else [] - - if len(running_services) > 0 and not self.rebuild: - logger.info(f"๐Ÿ”„ Found {len(running_services)} running test services") - # Check if test backend is healthy (only skip if not rebuilding) - try: - health_check = subprocess.run(["docker", "compose", "-f", "docker-compose-test.yml", "ps", "chronicle-backend-test"], capture_output=True, text=True) - if "healthy" in health_check.stdout or "Up" in health_check.stdout: - logger.info("โœ… Test services already running and healthy, skipping restart") - self.services_started = True - self.services_started_by_test = True # We'll manage test services - return - except: - pass - elif self.rebuild: - logger.info("๐Ÿ”จ Rebuild flag is True, will rebuild containers with latest code") - - logger.info("๐Ÿ”„ Need to start/restart test services...") - - # Handle container management based on rebuild and cached flags - if self.rebuild: - logger.info("๐Ÿ”จ Rebuild mode: stopping containers and rebuilding with latest code...") - # Stop existing test services and remove volumes for fresh rebuild - subprocess.run(["docker", "compose", "-f", "docker-compose-test.yml", "down", "-v"], capture_output=True) - elif not self.fresh_run: - logger.info("๐Ÿ”„ Reuse mode: restarting existing containers...") - subprocess.run(["docker", "compose", "-f", "docker-compose-test.yml", "restart"], capture_output=True) - else: - logger.info("๐Ÿ”„ Fresh mode: stopping containers and removing volumes...") - # Stop existing test services and remove volumes for fresh start - subprocess.run(["docker", "compose", "-f", "docker-compose-test.yml", "down", "-v"], capture_output=True) - - # memory_config.yaml deprecated; memory configuration provided via config.yml - - # Check if we're in CI environment - is_ci = os.environ.get("CI") == "true" or os.environ.get("GITHUB_ACTIONS") == "true" - - if is_ci: - # In CI, use simpler build process - logger.info("๐Ÿค– CI environment detected, using optimized build...") - if self.rebuild: - # Force rebuild in CI when rebuild flag is set with BuildKit disabled - env = os.environ.copy() - env['DOCKER_BUILDKIT'] = '0' - logger.info("๐Ÿ”จ Running Docker build command...") - build_result = subprocess.run(["docker", "compose", "-f", "docker-compose-test.yml", "build"], env=env) - if build_result.returncode != 0: - logger.error(f"โŒ Build failed with exit code {build_result.returncode}") - raise RuntimeError("Docker compose build failed") - cmd = ["docker", "compose", "-f", "docker-compose-test.yml", "up", "-d", "--no-build"] - else: - # Local development - use rebuild flag to determine build behavior - if self.rebuild: - cmd = ["docker", "compose", "-f", "docker-compose-test.yml", "up", "--build", "-d"] - logger.info("๐Ÿ”จ Local rebuild: will rebuild containers with latest code") - else: - cmd = ["docker", "compose", "-f", "docker-compose-test.yml", "up", "-d"] - logger.info("๐Ÿš€ Local start: using existing container images") - - # Start test services with BuildKit disabled to avoid bake issues - env = os.environ.copy() - env['DOCKER_BUILDKIT'] = '0' - logger.info(f"๐Ÿš€ Running Docker compose command: {' '.join(cmd)}") - result = subprocess.run(cmd, env=env, timeout=300) - - if result.returncode != 0: - logger.error(f"โŒ Failed to start services with exit code {result.returncode}") - - # Check individual container logs for better error details - logger.error("๐Ÿ” Checking individual container logs for details...") - try: - container_logs_result = subprocess.run( - ["docker", "compose", "-f", "docker-compose-test.yml", "logs", "--tail=50"], - capture_output=True, text=True, timeout=15 - ) - if container_logs_result.stdout: - logger.error("๐Ÿ“‹ Container logs:") - logger.error(container_logs_result.stdout) - if container_logs_result.stderr: - logger.error("๐Ÿ“‹ Container logs stderr:") - logger.error(container_logs_result.stderr) - except Exception as e: - logger.warning(f"Could not fetch container logs: {e}") - - # Check container status - logger.error("๐Ÿ” Checking container status...") - try: - status_result = subprocess.run( - ["docker", "compose", "-f", "docker-compose-test.yml", "ps"], - capture_output=True, text=True, timeout=10 - ) - if status_result.stdout: - logger.error("๐Ÿ“‹ Container status:") - logger.error(status_result.stdout) - except Exception as e: - logger.warning(f"Could not fetch container status: {e}") - - # Fail fast - no retry attempts - raise RuntimeError("Docker compose failed to start") - - self.services_started = True - self.services_started_by_test = True # Mark that we started the services - logger.info("โœ… Docker compose started successfully") - - except Exception as e: - logger.error(f"Error starting services: {e}") - raise - - def wait_for_services(self): - """Wait for all services to be ready with comprehensive health checks.""" - logger.info("๐Ÿ” Performing comprehensive service health validation...") - - start_time = time.time() - services_status = { - "backend": False, - "mongo": False, - "auth": False, - "readiness": False - } - - while time.time() - start_time < MAX_STARTUP_WAIT: - try: - # 1. Check backend basic health - if not services_status["backend"]: - try: - health_response = requests.get(f"{BACKEND_URL}/health", timeout=5) - if health_response.status_code == 200: - logger.info("โœ“ Backend health check passed") - services_status["backend"] = True - elif health_response.status_code >= 500: - raise RuntimeError(f"Backend service error: HTTP {health_response.status_code}") - elif health_response.status_code >= 400: - logger.warning(f"Backend client error: HTTP {health_response.status_code}") - except requests.exceptions.RequestException: - pass - - # 2. Check MongoDB connection via backend health check - if not services_status["mongo"] and services_status["backend"]: - try: - health_response = requests.get(f"{BACKEND_URL}/health", timeout=5) - if health_response.status_code == 200: - data = health_response.json() - mongo_health = data.get("services", {}).get("mongodb", {}) - if mongo_health.get("healthy", False): - logger.info("โœ“ MongoDB connection validated via backend health check") - services_status["mongo"] = True - except Exception: - pass - - # 3. Check comprehensive readiness (includes Qdrant validation) - if not services_status["readiness"] and services_status["backend"] and services_status["auth"]: - try: - readiness_response = requests.get(f"{BACKEND_URL}/readiness", timeout=5) - if readiness_response.status_code == 200: - data = readiness_response.json() - logger.info(f"๐Ÿ“‹ Readiness report: {json.dumps(data, indent=2)}") - - # Validate readiness data - backend validates Qdrant internally - if data.get("status") in ["healthy", "ready"]: - logger.info("โœ“ Backend reports all services ready (including Qdrant)") - services_status["readiness"] = True - elif data.get("status") == "unhealthy": - raise RuntimeError(f"Backend reports unhealthy status: {data}") - else: - logger.warning(f"โš ๏ธ Backend readiness check not fully healthy: {data}") - elif readiness_response.status_code >= 500: - raise RuntimeError(f"Backend readiness error: HTTP {readiness_response.status_code}") - elif readiness_response.status_code >= 400: - logger.warning(f"Backend readiness client error: HTTP {readiness_response.status_code}") - - except requests.exceptions.RequestException as e: - logger.debug(f"Readiness endpoint not ready yet: {e}") - - # 4. Check authentication endpoint - if not services_status["auth"] and services_status["backend"]: - try: - # Just check that the auth endpoint exists (will return error without credentials) - auth_response = requests.post(f"{BACKEND_URL}/auth/jwt/login", timeout=3) - # Expecting 422 (validation error) not connection error - if auth_response.status_code in [422, 400]: - logger.info("โœ“ Authentication endpoint accessible") - services_status["auth"] = True - except requests.exceptions.RequestException: - pass - - # 5. Final validation - all services ready - if all(services_status.values()): - logger.info("๐ŸŽ‰ All services validated and ready!") - return True - - # Log current status - ready_services = [name for name, status in services_status.items() if status] - pending_services = [name for name, status in services_status.items() if not status] - - elapsed = time.time() - start_time - logger.info(f"โณ Health check progress ({elapsed:.1f}s): โœ“ {ready_services} | โณ {pending_services}") - - except Exception as e: - logger.warning(f"โš ๏ธ Health check error: {e}") - - time.sleep(3) - - # Final status report - logger.error("โŒ Service readiness timeout!") - failed_services = [] - for service, status in services_status.items(): - status_emoji = "โœ“" if status else "โŒ" - logger.error(f" {status_emoji} {service}: {'Ready' if status else 'Not ready'}") - if not status: - failed_services.append(service) - - # Check for cascade failures - if backend failed, everything else will fail - if not services_status["backend"]: - logger.error("๐Ÿ’ฅ CRITICAL: Backend service failed - all dependent services will fail") - logger.error(" This indicates a fundamental infrastructure issue") - elif not services_status["mongo"]: - logger.error("๐Ÿ’ฅ CRITICAL: MongoDB connection failed - memory and auth will not work") - elif not services_status["readiness"]: - logger.error("๐Ÿ’ฅ WARNING: Readiness check failed - Qdrant or other dependencies may be down") - - raise TimeoutError(f"Services did not become ready in {MAX_STARTUP_WAIT}s. Failed services: {failed_services}") - - def authenticate(self): - """Authenticate and get admin token.""" - logger.info("๐Ÿ”‘ Authenticating as admin...") - - # Always use test credentials for test environment - logger.info("Using test environment credentials") - admin_email = TEST_ENV_VARS["ADMIN_EMAIL"] - admin_password = TEST_ENV_VARS["ADMIN_PASSWORD"] - - logger.info(f"Authenticating with email: {admin_email}") - - auth_url = f"{BACKEND_URL}/auth/jwt/login" - - response = requests.post( - auth_url, - data={ - 'username': admin_email, - 'password': admin_password - }, - headers={'Content-Type': 'application/x-www-form-urlencoded'} - ) - - if response.status_code != 200: - logger.error(f"Authentication failed with {admin_email}") - logger.error(f"Response: {response.text}") - raise RuntimeError(f"Authentication failed: {response.text}") - - data = response.json() - self.token = data.get('access_token') - - if not self.token: - raise RuntimeError("No access token received") - - logger.info("โœ“ Authentication successful") - - def upload_test_audio(self): - """Upload test audio file and monitor processing.""" - # Use different audio file for parakeet provider (shorter for faster testing) - audio_path = TEST_AUDIO_PATH_PARAKEET if self.provider == "parakeet" else TEST_AUDIO_PATH - - logger.info(f"๐Ÿ“ค Uploading test audio: {audio_path.name}") - - if not audio_path.exists(): - raise FileNotFoundError(f"Test audio file not found: {audio_path}") - - # Log audio file details - file_size = audio_path.stat().st_size - logger.info(f"๐Ÿ“Š Audio file size: {file_size:,} bytes ({file_size / (1024*1024):.2f} MB)") - - # Upload file - with open(audio_path, 'rb') as f: - files = {'files': (audio_path.name, f, 'audio/wav')} - data = {'device_name': 'integration_test'} - headers = {'Authorization': f'Bearer {self.token}'} - - logger.info("๐Ÿ“ค Sending upload request...") - response = requests.post( - f"{BACKEND_URL}/api/audio/upload", - files=files, - data=data, - headers=headers, - timeout=300 - ) - - logger.info(f"๐Ÿ“ค Upload response status: {response.status_code}") - - if response.status_code != 200: - raise RuntimeError(f"Upload failed: {response.text}") - - result = response.json() - logger.info(f"๐Ÿ“ค Upload response: {json.dumps(result, indent=2)}") - - # Extract client_id from response - client_id = result.get('client_id') - if not client_id: - raise RuntimeError("No client_id in upload response") - - logger.info(f"๐Ÿ“ค Generated client_id: {client_id}") - return result # Return full response with job IDs - - def verify_processing_results(self, upload_response: dict): - """Verify that audio was processed correctly using job tracking.""" - client_id = upload_response.get('client_id') - files = upload_response.get('files', []) - - if not files: - raise RuntimeError("No files in upload response") - - file_info = files[0] - transcript_job_id = file_info.get('transcript_job_id') - conversation_id = file_info.get('conversation_id') - - logger.info(f"๐Ÿ” Verifying processing results:") - logger.info(f" - Client ID: {client_id}") - logger.info(f" - Conversation ID: {conversation_id}") - logger.info(f" - Transcript Job ID: {transcript_job_id}") - - # Wait for transcription job to complete - logger.info("๐Ÿ” Waiting for transcription job to complete...") - start_time = time.time() - job_complete = False - - while time.time() - start_time < 60: # Wait up to 60 seconds for transcription - try: - # Check job status via queue API - response = requests.get( - f"{BACKEND_URL}/api/queue/jobs/{transcript_job_id}", - headers={"Authorization": f"Bearer {self.token}"}, - timeout=10 - ) - - if response.status_code == 200: - job_data = response.json() - status = job_data.get("status") - - if status == "completed": - logger.info(f"โœ… Transcription job completed successfully") - job_complete = True - break - elif status == "failed": - error = job_data.get("exc_info", "Unknown error") - logger.error(f"โŒ Transcription job failed: {error}") - break - else: - logger.info(f"โณ Job status: {status} ({time.time() - start_time:.1f}s)") - - else: - logger.warning(f"โš ๏ธ Job status check returned {response.status_code}") - - except Exception as e: - logger.warning(f"โš ๏ธ Error checking job status: {e}") - - time.sleep(5) - - if not job_complete: - raise AssertionError(f"Transcription job did not complete within 60 seconds. Last status: {status if 'status' in locals() else 'unknown'}") - - # Get the conversation via API - logger.info(f"๐Ÿ” Retrieving conversation...") - conversation = None - - try: - # Get conversations list - response = requests.get( - f"{BACKEND_URL}/api/conversations", - headers={"Authorization": f"Bearer {self.token}"}, - timeout=10 - ) - - if response.status_code == 200: - data = response.json() - conversations_list = data.get("conversations", []) - - # Find our conversation by conversation_id or client_id - for conv in conversations_list: - if conv.get('conversation_id') == conversation_id or conv.get('client_id') == client_id: - conversation = conv - logger.info(f"โœ… Found conversation in list: {conv.get('conversation_id')}") - break - - if not conversation: - logger.error(f"โŒ Conversation not found in list of {len(conversations_list)} conversations") - if conversations_list: - logger.error(f"๐Ÿ“Š Available conversations: {[c.get('conversation_id') for c in conversations_list[:5]]}") - else: - # Fetch full conversation details (list endpoint excludes transcript for performance) - logger.info(f"๐Ÿ” Fetching full conversation details...") - detail_response = requests.get( - f"{BACKEND_URL}/api/conversations/{conversation['conversation_id']}", - headers={"Authorization": f"Bearer {self.token}"}, - timeout=10 - ) - - if detail_response.status_code == 200: - conversation = detail_response.json()["conversation"] - logger.info(f"โœ… Retrieved full conversation details with transcript") - else: - logger.error(f"โŒ Failed to fetch conversation details: {detail_response.status_code}") - logger.error(f"Response: {detail_response.text}") - - else: - logger.error(f"โŒ Conversations API returned status: {response.status_code}") - logger.error(f"Response: {response.text}") - - except Exception as e: - logger.error(f"โŒ Error retrieving conversations: {e}", exc_info=True) - - if not conversation: - raise AssertionError(f"No conversation found for conversation_id: {conversation_id}") - - logger.info(f"โœ“ Conversation found: {conversation['audio_uuid']}") - - # Log conversation details - logger.info("๐Ÿ“‹ Conversation details:") - logger.info(f" - Audio UUID: {conversation['audio_uuid']}") - logger.info(f" - Client ID: {conversation.get('client_id')}") - logger.info(f" - Audio Path: {conversation.get('audio_path', 'N/A')}") - logger.info(f" - Timestamp: {conversation.get('timestamp', 'N/A')}") - - # Verify transcription (transcript is a string, segments is an array) - transcription = conversation.get('transcript', '') - segments = conversation.get('segments', []) - - logger.info(f"๐Ÿ“ Transcription details:") - logger.info(f" - Transcript length: {len(transcription)} characters") - logger.info(f" - Word count: {len(transcription.split()) if transcription else 0}") - logger.info(f" - Speaker segments: {len(segments)}") - - if transcription: - # Show first 200 characters of transcription - preview = transcription[:200] + "..." if len(transcription) > 200 else transcription - logger.info(f" - Preview: {preview}") - - # Load expected transcript for comparison - expected_transcript = self.load_expected_transcript() - logger.info(f" - Expected transcript length: {len(expected_transcript)} characters") - - # Log first 200 characters for comparison - logger.info(f" - Actual start: {transcription[:200]}...") - if expected_transcript: - logger.info(f" - Expected start: {expected_transcript[:200]}...") - - # Call OpenAI to verify transcript similarity - if os.environ.get("OPENAI_API_KEY") and expected_transcript: - similarity_result = self.check_transcript_similarity_simple(transcription, expected_transcript) - logger.info(f" - AI similarity assessment:") - logger.info(f" โ€ข Similar: {similarity_result.get('similar', 'unknown')}") - logger.info(f" โ€ข Reason: {similarity_result.get('reason', 'No reason provided')}") - - # Store result for validation - self.transcript_similarity_result = similarity_result - elif not expected_transcript: - logger.warning("โš ๏ธ No expected transcript available for comparison") - self.transcript_similarity_result = None - else: - logger.error("โŒ No transcription found") - - # Verify conversation has required fields - assert conversation.get('transcript'), "Conversation missing transcript" - assert len(conversation['transcript']) > 0, "Transcript is empty" - assert transcription.strip(), "Transcription text is empty" - - # Check for memory extraction (if LLM is configured) - if os.environ.get("OPENAI_API_KEY"): - logger.info("๐Ÿง  Checking for memory extraction...") - - # Check debug tracker for memory processing - response = requests.get( - f"{BACKEND_URL}/metrics", - headers={'Authorization': f'Bearer {self.token}'} - ) - - if response.status_code == 200: - metrics = response.json() - logger.info(f"๐Ÿ“Š System metrics: {json.dumps(metrics, indent=2)}") - - logger.info("โœ… Processing verification complete") - - return conversation, transcription - - def validate_memory_extraction(self, upload_response: dict): - """Validate that memory extraction worked correctly.""" - client_id = upload_response.get('client_id') - files = upload_response.get('files', []) - - logger.info(f"๐Ÿง  Validating memory extraction for client: {client_id}") - - # Get memory job ID from upload response - memory_job_id = files[0].get('memory_job_id') if files else None - if not memory_job_id: - raise RuntimeError("No memory_job_id in upload response") - - # Wait for memory processing to complete - client_memories = self.wait_for_memory_processing(memory_job_id, client_id) - - # Check if we're using OpenMemory MCP provider - memory_provider = os.environ.get("MEMORY_PROVIDER", "chronicle") - - if not client_memories: - if memory_provider == "openmemory_mcp": - # For OpenMemory MCP, check if there are any memories at all (deduplication is OK) - all_memories = self.get_memories_from_api() - if all_memories: - logger.info(f"โœ… OpenMemory MCP: Found {len(all_memories)} existing memories (deduplication successful)") - client_memories = all_memories # Use existing memories for validation - else: - raise AssertionError("No memories found in OpenMemory MCP - memory processing failed") - else: - raise AssertionError("No memories were extracted - memory processing failed") - - logger.info(f"โœ… Found {len(client_memories)} memories") - - # Load expected memories and compare - expected_memories = self.load_expected_memories() - if not expected_memories: - logger.warning("โš ๏ธ No expected memories available for comparison") - return client_memories - - # Use OpenAI to check if memories are similar - if os.environ.get("OPENAI_API_KEY"): - memory_similarity = self.check_memory_similarity_simple(client_memories, expected_memories) - logger.info(f"๐Ÿง  Memory similarity assessment:") - logger.info(f" โ€ข Similar: {memory_similarity.get('similar', 'unknown')}") - logger.info(f" โ€ข Reason: {memory_similarity.get('reason', 'No reason provided')}") - - # Store result for validation - self.memory_similarity_result = memory_similarity - else: - logger.warning("โš ๏ธ No OpenAI API key available for memory comparison") - self.memory_similarity_result = None - - return client_memories - - def check_transcript_similarity_simple(self, actual_transcript: str, expected_transcript: str) -> dict: - """Use OpenAI to check transcript similarity with simple boolean response.""" - try: - - client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY")) - - prompt = f""" - Compare these two transcripts to determine if they represent the same audio content. - - EXPECTED TRANSCRIPT: - "{expected_transcript}" - - ACTUAL TRANSCRIPT: - "{actual_transcript}" - - **MARK AS SIMILAR if:** - - Core content and topics match (e.g., glass blowing class, participants, activities) - - Key facts and events are present in both (names, numbers, objects, actions) - - Overall narrative flow is recognizable - - At least 70% semantic overlap exists - - **ACCEPTABLE DIFFERENCES (still mark as similar):** - - Minor word variations or ASR errors - - Different punctuation or capitalization - - Missing or extra filler words - - Small sections missing or repeated - - Slightly different word order - - Speaker diarization differences - - **ONLY MARK AS DISSIMILAR if:** - - Core content is fundamentally different - - Major sections (>30%) are missing or wrong - - It appears to be a different audio file entirely - - Respond in JSON format: - {{ - "reason": "brief explanation (1-3 sentences)" - "similar": true/false, - }} - """ - - response = client.chat.completions.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": prompt}], - response_format={"type": "json_object"} - ) - - response_text = (response.choices[0].message.content or "").strip() - - # Try to parse JSON response - try: - result = json.loads(response_text) - return result - except json.JSONDecodeError: - # If JSON parsing fails, return a basic result - return { - "similar": False, - "reason": f"Could not parse response: {response_text}" - } - - except Exception as e: - logger.warning(f"โš ๏ธ Could not check transcript similarity: {e}") - return { - "similar": False, - "reason": f"API call failed: {str(e)}" - } - - def check_memory_similarity_simple(self, actual_memories: list, expected_memories: list) -> dict: - """Use OpenAI to check if extracted memories are similar to expected memories.""" - try: - import openai - - client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY")) - - # Extract just the memory text from actual memories - actual_memory_texts = [mem.get('memory', '') for mem in actual_memories] - - prompt = f""" - Compare these two lists of memories to determine if they represent content from the same audio source and indicate successful memory extraction. - - **KEY CRITERIA FOR SIMILARITY (Return "similar": true if ANY of these are met):** - - 1. **Topic/Context Match**: Both lists should be about the same main activity/event (e.g., glass blowing class) - 2. **Core Facts Overlap**: At least 3-4 significant factual details should overlap (people, places, numbers, objects) - 3. **Semantic Coverage**: The same general knowledge should be captured, even if from different perspectives - - **ACCEPTABLE DIFFERENCES (Do NOT mark as dissimilar for these):** - - Different focus areas (one list more personal/emotional, other more technical/factual) - - Different level of detail (one more granular, other more high-level) - - Different speakers/participants emphasized - - Different organization or memory chunking - - Emotional vs factual framing of the same events - - Missing some details in either list (as long as core overlap exists) - - **MARK AS DISSIMILAR ONLY IF:** - - The memories seem to be from completely different audio/conversations - - No meaningful factual overlap (suggests wrong audio or major transcription failure) - - Core subject matter is entirely different - - **EVALUATION APPROACH:** - 1. Identify overlapping factual elements (people, places, objects, numbers, activities) - 2. Count significant semantic overlaps - 3. If 3+ substantial overlaps exist AND same general topic/context โ†’ mark as similar - 4. Focus on "are these from the same source" rather than "are these identical" - - EXPECTED MEMORIES: - {expected_memories} - - EXTRACTED MEMORIES: - {actual_memory_texts} - - Respond in JSON format with: - {{ - "reasoning": "detailed analysis of overlapping elements and why they indicate same/different source", - "reason": "brief explanation of the decision", - "similar": true/false - }} - """ - - logger.info(f"Making GPT-5-mini API call for memory similarity...") - response = client.chat.completions.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": prompt}], - response_format={"type": "json_object"} - ) - - response_text = (response.choices[0].message.content or "").strip() - logger.info(f"Memory similarity GPT-5-mini response: '{response_text}'") - - try: - result = json.loads(response_text) - return result - except json.JSONDecodeError as json_err: - # If JSON parsing fails, return a basic result - logger.error(f"JSON parsing failed: {json_err}") - logger.error(f"Response text that failed to parse: '{response_text}'") - return { - "reason": f"Could not parse response: {response_text}", - "similar": False, - } - - except Exception as e: - logger.error(f"โš ๏ธ Could not check memory similarity: {e}") - logger.error(f"Exception type: {type(e)}") - logger.error(f"Exception details: {str(e)}") - return { - "similar": False, - "reason": f"API call failed: {str(e)}" - } - - def get_memories_from_api(self) -> list: - """Fetch memories from the backend API.""" - try: - headers = {'Authorization': f'Bearer {self.token}'} - response = requests.get(f"{BACKEND_URL}/api/memories", headers=headers) - - if response.status_code == 200: - data = response.json() - return data.get('memories', []) - else: - logger.error(f"Failed to fetch memories: {response.status_code} - {response.text}") - return [] - except Exception as e: - logger.error(f"Error fetching memories: {e}") - return [] - - def wait_for_memory_processing(self, memory_job_id: str, client_id: str, timeout: int = 120): - """Wait for memory processing to complete using queue API.""" - logger.info(f"โณ Waiting for memory job {memory_job_id} to complete...") - - start_time = time.time() - job_complete = False - - while time.time() - start_time < timeout: - try: - # Check job status via queue API - response = requests.get( - f"{BACKEND_URL}/api/queue/jobs/{memory_job_id}", - headers={"Authorization": f"Bearer {self.token}"}, - timeout=10 - ) - - if response.status_code == 200: - job_data = response.json() - status = job_data.get("status") - - if status == "completed": - logger.info(f"โœ… Memory job completed successfully") - job_complete = True - break - elif status == "failed": - error = job_data.get("exc_info", "Unknown error") - logger.error(f"โŒ Memory job failed: {error}") - break - else: - logger.info(f"โณ Memory job status: {status} ({time.time() - start_time:.1f}s)") - - else: - logger.warning(f"โš ๏ธ Memory job status check returned {response.status_code}") - - except Exception as e: - logger.warning(f"โš ๏ธ Error checking memory job status: {e}") - - time.sleep(5) - - if not job_complete: - raise AssertionError(f"Memory job did not complete within {timeout} seconds. Last status: {status if 'status' in locals() else 'unknown'}") - - # Now fetch the memories from the API - memories = self.get_memories_from_api() - - # Filter by client_id for test isolation in fresh mode, or get all user memories in reuse mode - if not self.fresh_run: - # In reuse mode, get all user memories (API already filters by user_id) - user_memories = memories - if user_memories: - logger.info(f"โœ… Found {len(user_memories)} total user memories (reusing existing data)") - return user_memories - else: - # In fresh mode, filter by client_id for test isolation since we cleaned all data - client_memories = [mem for mem in memories if mem.get('metadata', {}).get('client_id') == client_id] - if client_memories: - logger.info(f"โœ… Found {len(client_memories)} memories for client {client_id}") - return client_memories - - logger.warning(f"โš ๏ธ No memories found after processing") - return [] - - async def create_chat_session(self, title: str = "Integration Test Session", description: str = "Testing memory integration") -> Optional[str]: - """Create a new chat session and return session ID.""" - logger.info(f"๐Ÿ“ Creating chat session: {title}") - - try: - response = requests.post( - f"{BACKEND_URL}/api/chat/sessions", - headers={"Authorization": f"Bearer {self.token}"}, - json={ - "title": title, - "description": description - }, - timeout=10 - ) - - if response.status_code == 200: - data = response.json() - session_id = data.get("session_id") - logger.info(f"โœ… Chat session created: {session_id}") - return session_id - else: - logger.error(f"โŒ Chat session creation failed: {response.status_code} - {response.text}") - return None - - except Exception as e: - logger.error(f"โŒ Error creating chat session: {e}") - return None - - async def send_chat_message(self, session_id: str, message: str) -> dict: - """Send a message to chat session and parse response.""" - logger.info(f"๐Ÿ’ฌ Sending message: {message}") - - try: - response = requests.post( - f"{BACKEND_URL}/api/chat/send", - headers={"Authorization": f"Bearer {self.token}"}, - json={ - "message": message, - "session_id": session_id - }, - timeout=30 - ) - - if response.status_code == 200: - # Parse SSE response - full_response = "" - memory_ids = [] - - for line in response.text.split('\n'): - if line.startswith('data: '): - try: - event_data = json.loads(line[6:]) - event_type = event_data.get("type") - - if event_type == "memory_context": - mem_ids = event_data.get("data", {}).get("memory_ids", []) - memory_ids.extend(mem_ids) - elif event_type == "content": - content = event_data.get("data", {}).get("content", "") - full_response += content - elif event_type == "done": - break - except json.JSONDecodeError: - pass - - logger.info(f"๐Ÿค– Response received ({len(full_response)} chars)") - if memory_ids: - logger.info(f"๐Ÿ“š Memories used: {len(memory_ids)} memory IDs") - - return { - "response": full_response, - "memories_used": memory_ids, - "success": True - } - else: - logger.error(f"โŒ Chat message failed: {response.status_code} - {response.text}") - return {"success": False, "error": response.text} - - except Exception as e: - logger.error(f"โŒ Error sending chat message: {e}") - return {"success": False, "error": str(e)} - - async def run_chat_conversation(self, session_id: str) -> bool: - """Run a test conversation with memory integration.""" - logger.info("๐ŸŽญ Starting chat conversation test...") - - # Test messages designed to trigger memory retrieval - test_messages = [ - "Hello! I'm testing the chat system with memory integration.", - "What do you know about glass blowing? Have I mentioned anything about it?", - ] - - memories_used_total = [] - - for i, message in enumerate(test_messages, 1): - logger.info(f"๐Ÿ“จ Message {i}/{len(test_messages)}") - result = await self.send_chat_message(session_id, message) - - if not result.get("success"): - logger.error(f"โŒ Chat message {i} failed: {result.get('error')}") - return False - - # Track memory usage - memories_used = result.get("memories_used", []) - memories_used_total.extend(memories_used) - - # Small delay between messages - time.sleep(1) - - logger.info(f"โœ… Chat conversation completed. Total memories used: {len(set(memories_used_total))}") - return True - - async def extract_memories_from_chat(self, session_id: str) -> dict: - """Extract memories from the chat session.""" - logger.info(f"๐Ÿง  Extracting memories from chat session: {session_id}") - - try: - response = requests.post( - f"{BACKEND_URL}/api/chat/sessions/{session_id}/extract-memories", - headers={"Authorization": f"Bearer {self.token}"}, - timeout=30 - ) - - if response.status_code == 200: - data = response.json() - if data.get("success"): - logger.info(f"โœ… Memory extraction successful: {data.get('count', 0)} memories created") - return data - else: - logger.warning(f"โš ๏ธ Memory extraction completed but no memories: {data.get('message', 'Unknown')}") - return data - else: - logger.error(f"โŒ Memory extraction failed: {response.status_code} - {response.text}") - return {"success": False, "error": response.text} - - except Exception as e: - logger.error(f"โŒ Error extracting memories from chat: {e}") - return {"success": False, "error": str(e)} - - def cleanup(self): - """Clean up test resources based on cached and rebuild flags.""" - logger.info("Cleaning up...") - - if self.mongo_client: - self.mongo_client.close() - - # Handle container cleanup based on cleanup_containers flag (rebuild flag doesn't affect cleanup) - if self.cleanup_containers and self.services_started_by_test: - logger.info("๐Ÿ”„ Cleanup mode: stopping test docker compose services...") - subprocess.run(["docker", "compose", "-f", "docker-compose-test.yml", "down", "-v"], capture_output=True) - logger.info("โœ“ Test containers stopped and volumes removed") - elif not self.cleanup_containers: - logger.info("๐Ÿ—‚๏ธ No cleanup: leaving containers running for debugging") - if self.rebuild: - logger.info(" (containers were rebuilt with latest code during this test)") - else: - logger.info("๐Ÿ”„ Test services were already running, leaving them as-is") - - logger.info("โœ“ Cleanup complete") - - -@pytest.fixture -def test_runner(): - """Pytest fixture for test runner.""" - runner = IntegrationTestRunner() - yield runner - runner.cleanup() - - -@pytest.mark.integration -def test_full_pipeline_integration(test_runner): - """Test the complete audio processing pipeline.""" - # Immediate output to confirm test is starting - print("๐Ÿš€ TEST STARTING - test_full_pipeline_integration", flush=True) - sys.stdout.flush() - sys.stderr.flush() - - try: - # Test timing tracking - test_start_time = time.time() - phase_times = {} - - # Immediate logging to debug environment - print("=" * 80, flush=True) - print("๐Ÿš€ STARTING INTEGRATION TEST", flush=True) - print("=" * 80, flush=True) - logger.info(f"Current working directory: {os.getcwd()}") - logger.info(f"Files in directory: {os.listdir('.')}") - logger.info(f"CI environment: {os.environ.get('CI', 'NOT SET')}") - logger.info(f"GITHUB_ACTIONS: {os.environ.get('GITHUB_ACTIONS', 'NOT SET')}") - sys.stdout.flush() - - # Phase 1: Environment setup - phase_start = time.time() - logger.info("๐Ÿ“‹ Phase 1: Setting up test environment...") - test_runner.setup_environment() - phase_times['env_setup'] = time.time() - phase_start - logger.info(f"โœ… Environment setup completed in {phase_times['env_setup']:.2f}s") - - # Phase 2: Service startup - phase_start = time.time() - logger.info("๐Ÿณ Phase 2: Starting services...") - test_runner.start_services() - phase_times['service_startup'] = time.time() - phase_start - logger.info(f"โœ… Service startup completed in {phase_times['service_startup']:.2f}s") - - # Phase 2b: ASR service startup (parakeet only) - phase_start = time.time() - logger.info(f"๐ŸŽค Phase 2b: Starting ASR services ({TRANSCRIPTION_PROVIDER} provider)...") - test_runner.start_asr_services() - phase_times['asr_startup'] = time.time() - phase_start - logger.info(f"โœ… ASR service startup completed in {phase_times['asr_startup']:.2f}s") - - # Phase 3: Wait for services - phase_start = time.time() - logger.info("โณ Phase 3: Waiting for services to be ready...") - test_runner.wait_for_services() - phase_times['service_readiness'] = time.time() - phase_start - logger.info(f"โœ… Service readiness check completed in {phase_times['service_readiness']:.2f}s") - - # Phase 3b: Wait for ASR services (parakeet only) - phase_start = time.time() - logger.info("โณ Phase 3b: Waiting for ASR services to be ready...") - test_runner.wait_for_asr_ready() - phase_times['asr_readiness'] = time.time() - phase_start - logger.info(f"โœ… ASR readiness check completed in {phase_times['asr_readiness']:.2f}s") - - # Phase 4: Authentication - phase_start = time.time() - logger.info("๐Ÿ”‘ Phase 4: Authentication...") - test_runner.authenticate() - phase_times['authentication'] = time.time() - phase_start - logger.info(f"โœ… Authentication completed in {phase_times['authentication']:.2f}s") - - # Phase 5: Audio upload and processing - phase_start = time.time() - logger.info("๐Ÿ“ค Phase 5: Audio upload...") - upload_response = test_runner.upload_test_audio() - client_id = upload_response.get('client_id') - phase_times['audio_upload'] = time.time() - phase_start - logger.info(f"โœ… Audio upload completed in {phase_times['audio_upload']:.2f}s") - - # Phase 6: Transcription processing - phase_start = time.time() - logger.info("๐ŸŽค Phase 6: Transcription processing...") - conversation, transcription = test_runner.verify_processing_results(upload_response) - phase_times['transcription_processing'] = time.time() - phase_start - logger.info(f"โœ… Transcription processing completed in {phase_times['transcription_processing']:.2f}s") - - # Phase 7: Memory extraction - phase_start = time.time() - logger.info("๐Ÿง  Phase 7: Memory extraction...") - memories = test_runner.validate_memory_extraction(upload_response) - phase_times['memory_extraction'] = time.time() - phase_start - logger.info(f"โœ… Memory extraction completed in {phase_times['memory_extraction']:.2f}s") - - # Phase 8: Chat with Memory Integration - # phase_start = time.time() - # logger.info("๐Ÿ’ฌ Phase 8: Chat with Memory Integration...") - - # # Create chat session - # session_id = asyncio.run(test_runner.create_chat_session( - # title="Integration Test Chat", - # description="Testing chat functionality with memory retrieval" - # )) - # assert session_id is not None, "Failed to create chat session" - - # # Run chat conversation - # chat_success = asyncio.run(test_runner.run_chat_conversation(session_id)) - # assert chat_success, "Chat conversation failed" - - # # Extract memories from chat session (optional - may create additional memories) - # chat_memory_result = asyncio.run(test_runner.extract_memories_from_chat(session_id)) - - # phase_times['chat_integration'] = time.time() - phase_start - # logger.info(f"โœ… Chat integration completed in {phase_times['chat_integration']:.2f}s") - - # Basic assertions - assert conversation is not None - assert len(conversation['transcript']) > 0 - assert transcription.strip() # Ensure we have actual text content - - # Transcript similarity assertion - if hasattr(test_runner, 'transcript_similarity_result') and test_runner.transcript_similarity_result: - assert test_runner.transcript_similarity_result.get('similar') == True, f"Transcript not similar enough: {test_runner.transcript_similarity_result.get('reason')}" - - # Memory validation assertions - assert memories is not None and len(memories) > 0, "No memories were extracted" - - # Memory similarity assertion - if hasattr(test_runner, 'memory_similarity_result') and test_runner.memory_similarity_result: - if test_runner.memory_similarity_result.get('similar') != True: - # Log transcript for debugging before failing - logger.error("=" * 80) - logger.error("โŒ MEMORY SIMILARITY CHECK FAILED - DEBUGGING INFO") - logger.error("=" * 80) - logger.error("๐Ÿ“ Generated Transcript:") - logger.error("-" * 60) - logger.error(transcription) - logger.error("-" * 60) - - # Format detailed error with both memory sets - expected_memories = test_runner.load_expected_memories() - extracted_memories = [mem.get('memory', '') for mem in memories] - - error_msg = f""" -Memory similarity check failed: -Reason: {test_runner.memory_similarity_result.get('reason', 'No reason provided')} -Reasoning: {test_runner.memory_similarity_result.get('reasoning', 'No detailed reasoning provided')} - -Expected memories ({len(expected_memories)}): -{chr(10).join(f" {i+1}. {mem}" for i, mem in enumerate(expected_memories))} - -Extracted memories ({len(extracted_memories)}): -{chr(10).join(f" {i+1}. {mem}" for i, mem in enumerate(extracted_memories))} - -Generated Transcript ({len(transcription)} chars): -{transcription[:500]}{'...' if len(transcription) > 500 else ''} -""" - assert False, error_msg - - # Calculate total test time - total_test_time = time.time() - test_start_time - phase_times['total_test'] = total_test_time - - # Log success with detailed timing - logger.info("=" * 80) - logger.info("๐ŸŽ‰ INTEGRATION TEST PASSED!") - logger.info("=" * 80) - logger.info(f"โฑ๏ธ TIMING BREAKDOWN:") - logger.info(f" ๐Ÿ“‹ Environment Setup: {phase_times['env_setup']:>6.2f}s") - logger.info(f" ๐Ÿณ Service Startup: {phase_times['service_startup']:>6.2f}s") - logger.info(f" โณ Service Readiness: {phase_times['service_readiness']:>6.2f}s") - logger.info(f" ๐Ÿ”‘ Authentication: {phase_times['authentication']:>6.2f}s") - logger.info(f" ๐Ÿ“ค Audio Upload: {phase_times['audio_upload']:>6.2f}s") - logger.info(f" ๐ŸŽค Transcription: {phase_times['transcription_processing']:>6.2f}s") - logger.info(f" ๐Ÿง  Memory Extraction: {phase_times['memory_extraction']:>6.2f}s") - # logger.info(f" ๐Ÿ’ฌ Chat Integration: {phase_times['chat_integration']:>6.2f}s") - logger.info(f" {'โ”€' * 35}") - logger.info(f" ๐Ÿ TOTAL TEST TIME: {total_test_time:>6.2f}s ({total_test_time/60:.1f}m)") - logger.info("") - logger.info(f"๐Ÿ“Š Test Results:") - logger.info(f" โœ… Audio file processed successfully") - logger.info(f" โœ… Transcription generated: {len(transcription)} characters") - logger.info(f" โœ… Word count: {len(transcription.split())}") - logger.info(f" โœ… Audio UUID: {conversation.get('audio_uuid')}") - logger.info(f" โœ… Client ID: {conversation.get('client_id')}") - logger.info(f" โœ… Memories extracted: {len(memories)}") - logger.info(f" โœ… Transcript similarity: {getattr(test_runner, 'transcript_similarity_result', {}).get('similar', 'N/A')}") - logger.info(f" โœ… Memory similarity: {getattr(test_runner, 'memory_similarity_result', {}).get('similar', 'N/A')}") - logger.info("") - logger.info("๐Ÿ“ Full Transcription:") - logger.info("-" * 60) - logger.info(transcription) - logger.info("-" * 60) - logger.info("") - logger.info("๐Ÿง  Extracted Memories:") - logger.info("-" * 60) - for i, memory in enumerate(memories[:10], 1): # Show first 10 memories - logger.info(f"{i}. {memory.get('memory', 'No content')}") - if len(memories) > 10: - logger.info(f"... and {len(memories) - 10} more memories") - logger.info("-" * 60) - logger.info("=" * 80) - - except Exception as e: - logger.error(f"Integration test failed: {e}") - raise - finally: - # Cleanup ASR services - test_runner.cleanup_asr_services() - - -if __name__ == "__main__": - # Run the test directly - pytest.main([__file__, "-v", "-s"]) diff --git a/tests/integration/integration_test.robot b/tests/integration/integration_test.robot index d5af0388..4b08381b 100644 --- a/tests/integration/integration_test.robot +++ b/tests/integration/integration_test.robot @@ -131,7 +131,7 @@ Audio Playback And Segment Timing Test End To End Pipeline With Memory Validation Test [Documentation] Complete E2E test with memory extraction and OpenAI quality validation. - ... This test matches Python test_integration.py coverage exactly. + ... Provides comprehensive integration testing of the entire audio processing pipeline. ... Separate from other tests to avoid breaking existing upload-only tests. [Tags] e2e memory [Timeout] 600s From 49235eac04c1d583bb10c774c6ccab275d82d6d0 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Fri, 2 Jan 2026 04:20:42 +0530 Subject: [PATCH 10/14] Update health check configuration in docker-compose-test.yml (#241) - Increased the number of retries from 5 to 10 for improved resilience during service readiness checks. - Extended the start period from 30s to 60s to allow more time for services to initialize before health checks commence. --- backends/advanced/docker-compose-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backends/advanced/docker-compose-test.yml b/backends/advanced/docker-compose-test.yml index 4d27c41e..3b0e1eaf 100644 --- a/backends/advanced/docker-compose-test.yml +++ b/backends/advanced/docker-compose-test.yml @@ -58,8 +58,8 @@ services: test: ["CMD", "curl", "-f", "http://localhost:8000/readiness"] interval: 10s timeout: 5s - retries: 5 - start_period: 30s + retries: 10 + start_period: 60s restart: unless-stopped webui-test: From 27e1d02a04bc6a9b846f40c690eb11cdd747ea0d Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 1 Jan 2026 23:01:23 +0000 Subject: [PATCH 11/14] Add step to create test configuration file in robot-tests.yml - Introduced a new step in the GitHub Actions workflow to copy the test configuration file from tests/configs/deepgram-openai.yml to a new config/config.yml. - Added logging to confirm the creation of the test config file, improving visibility during the test setup process. --- .github/workflows/robot-tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/robot-tests.yml b/.github/workflows/robot-tests.yml index 92073f7b..18ad71ac 100644 --- a/.github/workflows/robot-tests.yml +++ b/.github/workflows/robot-tests.yml @@ -94,6 +94,14 @@ jobs: TEST_DEVICE_NAME=robot-test EOF + - name: Create test config.yml + run: | + echo "Copying test configuration file..." + mkdir -p config + cp tests/configs/deepgram-openai.yml config/config.yml + echo "โœ“ Test config.yml created from tests/configs/deepgram-openai.yml" + ls -lh config/config.yml + - name: Start test environment working-directory: backends/advanced env: From 923c910a7504b7de9d00528f03339e8249ba5f17 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Thu, 1 Jan 2026 23:15:55 +0000 Subject: [PATCH 12/14] remove cache step since not required --- .github/workflows/robot-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/robot-tests.yml b/.github/workflows/robot-tests.yml index 18ad71ac..bac4c65a 100644 --- a/.github/workflows/robot-tests.yml +++ b/.github/workflows/robot-tests.yml @@ -61,7 +61,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@v4 From 89dafe6a09f70963b97a6f4519c86a5bf29bcb40 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Fri, 2 Jan 2026 00:06:41 +0000 Subject: [PATCH 13/14] coderabbit comments --- backends/advanced/init.py | 10 ++++------ config_manager.py | 10 +++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/backends/advanced/init.py b/backends/advanced/init.py index 6a120499..c68fa10f 100644 --- a/backends/advanced/init.py +++ b/backends/advanced/init.py @@ -22,6 +22,10 @@ from rich.prompt import Confirm, Prompt from rich.text import Text +# Add repo root to path for config_manager import +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from config_manager import ConfigManager + class ChronicleSetup: def __init__(self, args=None): @@ -37,12 +41,6 @@ def __init__(self, args=None): sys.exit(1) # Initialize ConfigManager - repo_root = Path.cwd().parent.parent # backends/advanced -> repo root - if str(repo_root) not in sys.path: - sys.path.insert(0, str(repo_root)) - - from config_manager import ConfigManager - self.config_manager = ConfigManager(service_path="backends/advanced") self.console.print(f"[blue][INFO][/blue] Using config.yml at: {self.config_manager.config_yml_path}") diff --git a/config_manager.py b/config_manager.py index c9bf9a2a..2999d4b4 100644 --- a/config_manager.py +++ b/config_manager.py @@ -57,24 +57,24 @@ def __init__(self, service_path: Optional[str] = None, repo_root: Optional[Path] self.service_path = self.repo_root / service_path if service_path else None # Paths - self.config_yml_path = self.repo_root / "config.yml" + self.config_yml_path = self.repo_root / "config" / "config.yml" self.env_path = self.service_path / ".env" if self.service_path else None logger.debug(f"ConfigManager initialized: repo_root={self.repo_root}, " f"service_path={self.service_path}, config_yml={self.config_yml_path}") def _find_repo_root(self) -> Path: - """Find repository root by searching for config.yml.""" + """Find repository root by searching for config/config.yml.""" current = Path.cwd() - # Walk up until we find config.yml + # Walk up until we find config/config.yml while current != current.parent: - if (current / "config.yml").exists(): + if (current / "config" / "config.yml").exists(): return current current = current.parent # Fallback to cwd if not found - logger.warning("Could not find config.yml, using current directory as repo root") + logger.warning("Could not find config/config.yml, using current directory as repo root") return Path.cwd() def _detect_service_path(self) -> Optional[str]: From 29f3124d4becdd53142fde53050fab084dbad509 Mon Sep 17 00:00:00 2001 From: Ankush Malaker <43288948+AnkushMalaker@users.noreply.github.com> Date: Fri, 2 Jan 2026 01:28:46 +0000 Subject: [PATCH 14/14] Refactor logging in exception handling across multiple modules - Updated logging statements to use `logger.exception` instead of `logger.error` for better error context in exception handling. - This change enhances the visibility of stack traces and error details during failures, improving debugging capabilities across various components including configuration management, audio services, and user authentication. --- .../src/advanced_omi_backend/app_factory.py | 32 ++++----- .../advanced/src/advanced_omi_backend/auth.py | 4 +- .../src/advanced_omi_backend/chat_service.py | 18 ++--- .../clients/audio_stream_client.py | 4 +- .../src/advanced_omi_backend/config.py | 8 +-- .../controllers/client_controller.py | 2 +- .../controllers/conversation_controller.py | 16 ++--- .../controllers/session_controller.py | 8 +-- .../controllers/system_controller.py | 6 +- .../controllers/user_controller.py | 20 ++---- .../controllers/websocket_controller.py | 8 +-- .../src/advanced_omi_backend/llm_client.py | 8 +-- .../src/advanced_omi_backend/models/job.py | 4 +- .../src/advanced_omi_backend/models/user.py | 4 +- .../routers/modules/chat_routes.py | 24 +++---- .../routers/modules/health_routes.py | 6 +- .../routers/modules/queue_routes.py | 70 +++++++++---------- .../services/audio_service.py | 2 +- .../services/audio_stream/aggregator.py | 4 +- .../services/audio_stream/consumer.py | 6 +- .../services/memory/providers/chronicle.py | 16 ++--- .../memory/providers/llm_providers.py | 14 ++-- .../services/memory/providers/mcp_client.py | 22 +++--- .../services/memory/providers/mycelia.py | 38 +++++----- .../memory/providers/openmemory_mcp.py | 26 +++---- .../memory/providers/vector_stores.py | 20 +++--- .../services/memory/service_factory.py | 6 +- .../services/transcription/deepgram.py | 2 +- .../transcription/parakeet_stream_consumer.py | 2 +- .../speaker_recognition_client.py | 32 ++++----- .../src/advanced_omi_backend/task_manager.py | 2 +- .../advanced_omi_backend/utils/audio_utils.py | 2 +- .../utils/conversation_utils.py | 12 ++-- .../workers/audio_jobs.py | 4 +- .../workers/audio_stream_deepgram_worker.py | 2 +- .../workers/audio_stream_parakeet_worker.py | 2 +- .../workers/conversation_jobs.py | 2 +- .../workers/memory_jobs.py | 2 +- config_manager.py | 6 +- extras/asr-services/client.py | 8 +-- extras/asr-services/enhanced_chunking.py | 8 +-- extras/asr-services/parakeet-offline.py | 14 ++-- .../tests/test_parakeet_service.py | 12 ++-- extras/havpe-relay/main.py | 24 +++---- extras/speaker-recognition/laptop_client.py | 14 ++-- .../scripts/download-pyannote.py | 2 +- .../scripts/enroll_speaker.py | 12 ++-- .../scripts/install-pytorch.py | 4 +- .../utils/audio_segment_manager.py | 2 +- .../utils/youtube_transcriber.py | 12 ++-- 50 files changed, 285 insertions(+), 293 deletions(-) diff --git a/backends/advanced/src/advanced_omi_backend/app_factory.py b/backends/advanced/src/advanced_omi_backend/app_factory.py index 7ccda184..5e47819f 100644 --- a/backends/advanced/src/advanced_omi_backend/app_factory.py +++ b/backends/advanced/src/advanced_omi_backend/app_factory.py @@ -62,23 +62,23 @@ async def lifespan(app: FastAPI): document_models=[User, Conversation, AudioFile], ) application_logger.info("Beanie initialized for all document models") - except Exception as e: - application_logger.error(f"Failed to initialize Beanie: {e}") + except Exception: + application_logger.exception(f"Failed to initialize Beanie") raise # Create admin user if needed try: await create_admin_user_if_needed() - except Exception as e: - application_logger.error(f"Failed to create admin user: {e}") + except Exception: + application_logger.exception(f"Failed to create admin user") # Don't raise here as this is not critical for startup # Sync admin user with Mycelia OAuth (if using Mycelia memory provider) try: from advanced_omi_backend.services.mycelia_sync import sync_admin_on_startup await sync_admin_on_startup() - except Exception as e: - application_logger.error(f"Failed to sync admin with Mycelia OAuth: {e}") + except Exception: + application_logger.exception(f"Failed to sync admin with Mycelia OAuth") # Don't raise here as this is not critical for startup # Initialize Redis connection for RQ @@ -87,8 +87,8 @@ async def lifespan(app: FastAPI): redis_conn.ping() application_logger.info("Redis connection established for RQ") application_logger.info("RQ workers can be started with: rq worker transcription memory default") - except Exception as e: - application_logger.error(f"Failed to connect to Redis for RQ: {e}") + except Exception: + application_logger.exception(f"Failed to connect to Redis for RQ") application_logger.warning("RQ queue system will not be available - check Redis connection") # Initialize audio stream service for Redis Streams @@ -97,8 +97,8 @@ async def lifespan(app: FastAPI): await audio_service.connect() application_logger.info("Audio stream service connected to Redis Streams") application_logger.info("Audio stream workers can be started with: python -m advanced_omi_backend.workers.audio_stream_worker") - except Exception as e: - application_logger.error(f"Failed to connect audio stream service: {e}") + except Exception: + application_logger.exception(f"Failed to connect audio stream service") application_logger.warning("Redis Streams audio processing will not be available") # Initialize Redis client for audio streaming producer (used by WebSocket handlers) @@ -137,8 +137,8 @@ async def lifespan(app: FastAPI): try: from advanced_omi_backend.controllers.websocket_controller import cleanup_client_state await cleanup_client_state(client_id) - except Exception as e: - application_logger.error(f"Error cleaning up client {client_id}: {e}") + except Exception: + application_logger.exception(f"Error cleaning up client {client_id}") # RQ workers shut down automatically when process ends # No special cleanup needed for Redis connections @@ -148,16 +148,16 @@ async def lifespan(app: FastAPI): audio_service = get_audio_stream_service() await audio_service.disconnect() application_logger.info("Audio stream service disconnected") - except Exception as e: - application_logger.error(f"Error disconnecting audio stream service: {e}") + except Exception: + application_logger.exception(f"Error disconnecting audio stream service") # Close Redis client for audio streaming producer try: if hasattr(app.state, 'redis_audio_stream') and app.state.redis_audio_stream: await app.state.redis_audio_stream.close() application_logger.info("Redis client for audio streaming producer closed") - except Exception as e: - application_logger.error(f"Error closing Redis audio streaming client: {e}") + except Exception: + application_logger.exception(f"Error closing Redis audio streaming client") # Stop metrics collection and save final report application_logger.info("Metrics collection stopped") diff --git a/backends/advanced/src/advanced_omi_backend/auth.py b/backends/advanced/src/advanced_omi_backend/auth.py index 7c68d0b4..0e95e771 100644 --- a/backends/advanced/src/advanced_omi_backend/auth.py +++ b/backends/advanced/src/advanced_omi_backend/auth.py @@ -238,7 +238,7 @@ async def create_admin_user_if_needed(): ) except Exception as e: - logger.error(f"Failed to create admin user: {e}", exc_info=True) + logger.exception(f"Failed to create admin user: {e}") async def websocket_auth(websocket, token: Optional[str] = None) -> Optional[User]: @@ -262,7 +262,7 @@ async def websocket_auth(websocket, token: Optional[str] = None) -> Optional[Use else: logger.warning(f"Token validated but user inactive or not found: user={user}") except Exception as e: - logger.error(f"WebSocket auth with query token failed: {type(e).__name__}: {e}", exc_info=True) + logger.exception(f"WebSocket auth with query token failed: {type(e).__name__}: {e}") # Try cookie authentication logger.debug("Attempting WebSocket auth with cookie.") diff --git a/backends/advanced/src/advanced_omi_backend/chat_service.py b/backends/advanced/src/advanced_omi_backend/chat_service.py index b77f864a..993f9a29 100644 --- a/backends/advanced/src/advanced_omi_backend/chat_service.py +++ b/backends/advanced/src/advanced_omi_backend/chat_service.py @@ -161,7 +161,7 @@ async def initialize(self): logger.info("Chat service initialized successfully") except Exception as e: - logger.error(f"Failed to initialize chat service: {e}") + logger.exception(f"Failed to initialize chat service: {e}") raise async def create_session(self, user_id: str, title: str = None) -> ChatSession: @@ -273,10 +273,10 @@ async def add_message(self, message: ChatMessage) -> bool: {"session_id": message.session_id, "user_id": message.user_id}, {"$set": update_data} ) - + return True except Exception as e: - logger.error(f"Failed to add message to session {message.session_id}: {e}") + logger.exception(f"Failed to add message to session {message.session_id}: {e}") return False async def get_relevant_memories(self, query: str, user_id: str) -> List[Dict]: @@ -290,7 +290,7 @@ async def get_relevant_memories(self, query: str, user_id: str) -> List[Dict]: logger.info(f"Retrieved {len(memories)} relevant memories for query: {query[:50]}...") return memories except Exception as e: - logger.error(f"Failed to retrieve memories for user {user_id}: {e}") + logger.exception(f"Failed to retrieve memories for user {user_id}: {e}") return [] async def format_conversation_context( @@ -421,7 +421,7 @@ async def generate_response_stream( } except Exception as e: - logger.error(f"Error generating response for session {session_id}: {e}") + logger.exception(f"Error generating response for session {session_id}: {e}") yield { "type": "error", "data": {"error": str(e)}, @@ -440,7 +440,7 @@ async def update_session_title(self, session_id: str, user_id: str, title: str) ) return result.modified_count > 0 except Exception as e: - logger.error(f"Failed to update session title: {e}") + logger.exception(f"Failed to update session title: {e}") return False async def get_chat_statistics(self, user_id: str) -> Dict: @@ -467,7 +467,7 @@ async def get_chat_statistics(self, user_id: str) -> Dict: "last_chat": latest_session["updated_at"] if latest_session else None } except Exception as e: - logger.error(f"Failed to get chat statistics for user {user_id}: {e}") + logger.exception(f"Failed to get chat statistics for user {user_id}: {e}") return {"total_sessions": 0, "total_messages": 0, "last_chat": None} async def extract_memories_from_session(self, session_id: str, user_id: str) -> Tuple[bool, List[str], int]: @@ -528,9 +528,9 @@ async def extract_memories_from_session(self, session_id: str, user_id: str) -> else: logger.error(f"โŒ Failed to extract memories from chat session {session_id}") return False, [], 0 - + except Exception as e: - logger.error(f"Failed to extract memories from session {session_id}: {e}") + logger.exception(f"Failed to extract memories from session {session_id}: {e}") return False, [], 0 diff --git a/backends/advanced/src/advanced_omi_backend/clients/audio_stream_client.py b/backends/advanced/src/advanced_omi_backend/clients/audio_stream_client.py index af89fd51..a8ac0004 100644 --- a/backends/advanced/src/advanced_omi_backend/clients/audio_stream_client.py +++ b/backends/advanced/src/advanced_omi_backend/clients/audio_stream_client.py @@ -435,7 +435,7 @@ async def _connect_and_start(): logger.info(f"Stream {stream_id} started for {device_name}") except Exception as e: session.error = str(e) - logger.error(f"Stream {stream_id} failed to start: {e}") + logger.exception(f"Stream {stream_id} failed to start: {e}") future = asyncio.run_coroutine_threadsafe(_connect_and_start(), loop) future.result(timeout=10) # Wait for connection @@ -553,4 +553,4 @@ def cleanup_all(self): try: self.stop_stream(stream_id) except Exception as e: - logger.warning(f"Error stopping stream {stream_id}: {e}") + logger.exception(f"Error stopping stream {stream_id}: {e}") diff --git a/backends/advanced/src/advanced_omi_backend/config.py b/backends/advanced/src/advanced_omi_backend/config.py index 2b07a8d4..c61e9c6d 100644 --- a/backends/advanced/src/advanced_omi_backend/config.py +++ b/backends/advanced/src/advanced_omi_backend/config.py @@ -88,8 +88,8 @@ def load_diarization_settings_from_file(): config_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy(template_path, config_path) logger.info(f"Created diarization config from template at {config_path}") - except Exception as e: - logger.warning(f"Could not copy template to {config_path}: {e}") + except Exception: + logger.exception(f"Could not copy template to {config_path}") # Load from file if it exists if config_path.exists(): @@ -99,7 +99,7 @@ def load_diarization_settings_from_file(): logger.info(f"Loaded diarization settings from {config_path}") return _diarization_settings except Exception as e: - logger.error(f"Error loading diarization settings from {config_path}: {e}") + logger.exception(f"Error loading diarization settings from {config_path}") # Fall back to defaults _diarization_settings = DEFAULT_DIARIZATION_SETTINGS.copy() @@ -127,7 +127,7 @@ def save_diarization_settings_to_file(settings): logger.info(f"Saved diarization settings to {config_path}") return True except Exception as e: - logger.error(f"Error saving diarization settings to {config_path}: {e}") + logger.exception(f"Error saving diarization settings to {config_path}") return False diff --git a/backends/advanced/src/advanced_omi_backend/controllers/client_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/client_controller.py index b400d3ed..7fde807e 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/client_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/client_controller.py @@ -46,7 +46,7 @@ async def get_active_clients(user: User, client_manager: ClientManager): } except Exception as e: - logger.error(f"Error getting active clients: {e}") + logger.exception(f"Error getting active clients") return JSONResponse( status_code=500, content={"error": "Failed to get active clients"}, diff --git a/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py index b9533391..eab182a6 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/conversation_controller.py @@ -77,7 +77,7 @@ async def close_current_conversation(client_id: str, user: User, client_manager: ) except Exception as e: - logger.error(f"Error closing conversation for client {client_id}: {e}") + logger.exception(f"Error closing conversation for client {client_id}") return JSONResponse( content={"error": f"Failed to close conversation: {str(e)}"}, status_code=500, @@ -128,7 +128,7 @@ async def get_conversation(conversation_id: str, user: User): return {"conversation": response} except Exception as e: - logger.error(f"Error fetching conversation {conversation_id}: {e}") + logger.exception(f"Error fetching conversation {conversation_id}") return JSONResponse(status_code=500, content={"error": "Error fetching conversation"}) @@ -267,7 +267,7 @@ async def delete_conversation(conversation_id: str, user: User): ) except Exception as e: - logger.error(f"Error deleting conversation {conversation_id}: {e}") + logger.exception(f"Error deleting conversation {conversation_id}") return JSONResponse( status_code=500, content={"error": f"Failed to delete conversation: {str(e)}"} @@ -400,7 +400,7 @@ async def reprocess_transcript(conversation_id: str, user: User): }) except Exception as e: - logger.error(f"Error starting transcript reprocessing: {e}") + logger.exception(f"Error starting transcript reprocessing") return JSONResponse(status_code=500, content={"error": "Error starting transcript reprocessing"}) @@ -465,7 +465,7 @@ async def reprocess_memory(conversation_id: str, transcript_version_id: str, use }) except Exception as e: - logger.error(f"Error starting memory reprocessing: {e}") + logger.exception(f"Error starting memory reprocessing") return JSONResponse(status_code=500, content={"error": "Error starting memory reprocessing"}) @@ -501,7 +501,7 @@ async def activate_transcript_version(conversation_id: str, version_id: str, use }) except Exception as e: - logger.error(f"Error activating transcript version: {e}") + logger.exception(f"Error activating transcript version") return JSONResponse(status_code=500, content={"error": "Error activating transcript version"}) @@ -534,7 +534,7 @@ async def activate_memory_version(conversation_id: str, version_id: str, user: U }) except Exception as e: - logger.error(f"Error activating memory version: {e}") + logger.exception(f"Error activating memory version") return JSONResponse(status_code=500, content={"error": "Error activating memory version"}) @@ -577,5 +577,5 @@ async def get_conversation_version_history(conversation_id: str, user: User): return JSONResponse(content=history) except Exception as e: - logger.error(f"Error fetching version history: {e}") + logger.exception(f"Error fetching version history") return JSONResponse(status_code=500, content={"error": "Error fetching version history"}) diff --git a/backends/advanced/src/advanced_omi_backend/controllers/session_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/session_controller.py index a3836898..20a7db7b 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/session_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/session_controller.py @@ -63,7 +63,7 @@ async def get_session_info(redis_client, session_id: str) -> Optional[Dict]: } except Exception as e: - logger.error(f"Error getting session info for {session_id}: {e}") + logger.exception(f"Error getting session info for {session_id}") return None @@ -99,7 +99,7 @@ async def get_all_sessions(redis_client, limit: int = 100) -> List[Dict]: return sessions except Exception as e: - logger.error(f"Error getting all sessions: {e}") + logger.exception(f"Error getting all sessions") return [] @@ -119,7 +119,7 @@ async def get_session_conversation_count(redis_client, session_id: str) -> int: conversation_count_bytes = await redis_client.get(conversation_count_key) return int(conversation_count_bytes.decode()) if conversation_count_bytes else 0 except Exception as e: - logger.error(f"Error getting conversation count for session {session_id}: {e}") + logger.exception(f"Error getting conversation count for session {session_id}") return 0 @@ -141,7 +141,7 @@ async def increment_session_conversation_count(redis_client, session_id: str) -> logger.info(f"๐Ÿ“Š Conversation count for session {session_id}: {count}") return count except Exception as e: - logger.error(f"Error incrementing conversation count for session {session_id}: {e}") + logger.exception(f"Error incrementing conversation count for session {session_id}") return 0 diff --git a/backends/advanced/src/advanced_omi_backend/controllers/system_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/system_controller.py index cbf45783..19a16c00 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/system_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/system_controller.py @@ -285,7 +285,7 @@ async def update_memory_config_raw(config_yaml: str): try: new_mem = yaml.safe_load(config_yaml) or {} except yaml.YAMLError as e: - raise ValueError(f"Invalid YAML syntax: {str(e)}") + raise ValueError(f"Invalid YAML syntax: {str(e)}") from e cfg_path = _find_config_path() if not os.path.exists(cfg_path): @@ -322,7 +322,7 @@ async def validate_memory_config(config_yaml: str): try: parsed = yaml.safe_load(config_yaml) except yaml.YAMLError as e: - raise HTTPException(status_code=400, detail=f"Invalid YAML syntax: {str(e)}") + raise HTTPException(status_code=400, detail=f"Invalid YAML syntax: {str(e)}") from e if not isinstance(parsed, dict): raise HTTPException(status_code=400, detail="Configuration must be a YAML object") # Minimal checks @@ -333,7 +333,7 @@ async def validate_memory_config(config_yaml: str): raise except Exception as e: logger.exception("Error validating memory config") - raise HTTPException(status_code=500, detail=f"Error validating memory config: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error validating memory config: {str(e)}") from e async def reload_memory_config(): diff --git a/backends/advanced/src/advanced_omi_backend/controllers/user_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/user_controller.py index a1b9c140..4f4d52d0 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/user_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/user_controller.py @@ -32,8 +32,8 @@ async def get_users(): users.append(user) return users except Exception as e: - logger.error(f"Error fetching users: {e}") - raise HTTPException(status_code=500, detail="Error fetching users") + logger.exception(f"Error fetching users: {e}") + raise HTTPException(status_code=500, detail="Error fetching users") from e async def create_user(user_data: UserCreate): @@ -69,10 +69,7 @@ async def create_user(user_data: UserCreate): ) except Exception as e: - import traceback - error_details = traceback.format_exc() - logger.error(f"Error creating user: {e}") - logger.error(f"Full traceback: {error_details}") + logger.exception(f"Error creating user: {e}") return JSONResponse( status_code=500, content={"message": f"Error creating user: {str(e)}"}, @@ -86,7 +83,7 @@ async def update_user(user_id: str, user_data: UserUpdate): try: object_id = ObjectId(user_id) except Exception as e: - logger.error(f"Invalid ObjectId format for user_id {user_id}: {e}") + logger.exception(f"Invalid ObjectId format for user_id {user_id}: {e}") return JSONResponse( status_code=400, content={"message": f"Invalid user_id format: {user_id}. Must be a valid ObjectId."}, @@ -122,10 +119,7 @@ async def update_user(user_id: str, user_data: UserUpdate): ) except Exception as e: - import traceback - error_details = traceback.format_exc() - logger.error(f"Error updating user: {e}") - logger.error(f"Full traceback: {error_details}") + logger.exception(f"Error updating user: {e}") return JSONResponse( status_code=500, content={"message": f"Error updating user: {str(e)}"}, @@ -188,7 +182,7 @@ async def delete_user( ) deleted_data["memories_deleted"] = memory_count except Exception as mem_error: - logger.error(f"Error deleting memories for user {user_id}: {mem_error}") + logger.exception(f"Error deleting memories for user {user_id}: {mem_error}") deleted_data["memories_deleted"] = 0 deleted_data["memory_deletion_error"] = str(mem_error) @@ -211,7 +205,7 @@ async def delete_user( ) except Exception as e: - logger.error(f"Error deleting user {user_id}: {e}") + logger.exception(f"Error deleting user {user_id}: {e}") return JSONResponse( status_code=500, content={"message": f"Error deleting user: {str(e)}"}, diff --git a/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py b/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py index 50ffc77f..2b562995 100644 --- a/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py +++ b/backends/advanced/src/advanced_omi_backend/controllers/websocket_controller.py @@ -144,8 +144,8 @@ async def cleanup_client_state(client_id: str): logger.info(f"๐Ÿงน Cleaned up job tracking key for client {client_id}") else: logger.debug(f"No speech detection job found for client {client_id}") - except Exception as e: - logger.warning(f"โš ๏ธ Error during job cancellation for client {client_id}: {e}") + except Exception: + logger.exception(f"โš ๏ธ Error during job cancellation for client {client_id}") # Mark all active sessions for this client as complete AND delete Redis streams try: @@ -264,8 +264,8 @@ async def _setup_websocket_connection( ready_msg = json.dumps({"type": "ready", "message": "WebSocket connection established"}) + "\n" await ws.send_text(ready_msg) application_logger.debug(f"โœ… Sent ready message to {client_id}") - except Exception as e: - application_logger.error(f"Failed to send ready message to {client_id}: {e}") + except Exception: + application_logger.exception(f"Failed to send ready message to {client_id}") # Create client state client_state = await create_client_state(client_id, user, device_name) diff --git a/backends/advanced/src/advanced_omi_backend/llm_client.py b/backends/advanced/src/advanced_omi_backend/llm_client.py index 5b6e907d..1a6238d1 100644 --- a/backends/advanced/src/advanced_omi_backend/llm_client.py +++ b/backends/advanced/src/advanced_omi_backend/llm_client.py @@ -80,10 +80,10 @@ def __init__( self.client = OpenAI(api_key=self.api_key, base_url=self.base_url) self.logger.info(f"OpenAI client initialized (no tracing), base_url: {self.base_url}") except ImportError: - self.logger.error("OpenAI library not installed. Install with: pip install openai") + self.logger.exception("OpenAI library not installed. Install with: pip install openai") raise except Exception as e: - self.logger.error(f"Failed to initialize OpenAI client: {e}") + self.logger.exception(f"Failed to initialize OpenAI client: {e}") raise def generate( @@ -107,7 +107,7 @@ def generate( response = self.client.chat.completions.create(**params) return response.choices[0].message.content.strip() except Exception as e: - self.logger.error(f"Error generating completion: {e}") + self.logger.exception(f"Error generating completion: {e}") raise def health_check(self) -> Dict: @@ -130,7 +130,7 @@ def health_check(self) -> Dict: "api_key_configured": bool(self.api_key and self.api_key != "dummy"), } except Exception as e: - self.logger.error(f"Health check failed: {e}") + self.logger.exception(f"Health check failed: {e}") return { "status": "โŒ Failed", "error": str(e), diff --git a/backends/advanced/src/advanced_omi_backend/models/job.py b/backends/advanced/src/advanced_omi_backend/models/job.py index b295782c..a462ad12 100644 --- a/backends/advanced/src/advanced_omi_backend/models/job.py +++ b/backends/advanced/src/advanced_omi_backend/models/job.py @@ -60,8 +60,8 @@ async def _ensure_beanie_initialized(): _beanie_initialized = True logger.info("โœ… Beanie initialized in RQ worker process") - except Exception as e: - logger.error(f"โŒ Failed to initialize Beanie in RQ worker: {e}") + except Exception: + logger.exception(f"โŒ Failed to initialize Beanie in RQ worker") raise diff --git a/backends/advanced/src/advanced_omi_backend/models/user.py b/backends/advanced/src/advanced_omi_backend/models/user.py index b0ced195..603a47d5 100644 --- a/backends/advanced/src/advanced_omi_backend/models/user.py +++ b/backends/advanced/src/advanced_omi_backend/models/user.py @@ -111,8 +111,8 @@ async def get_user_by_id(user_id: str) -> Optional[User]: """Get user by MongoDB ObjectId string.""" try: return await User.get(PydanticObjectId(user_id)) - except Exception as e: - logger.error(f"Failed to get user by ID {user_id}: {e}") + except Exception: + logger.exception(f"Failed to get user by ID {user_id}") # Re-raise for proper error handling upstream raise diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/chat_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/chat_routes.py index d0c64904..cfece48d 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/chat_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/chat_routes.py @@ -83,7 +83,7 @@ async def create_chat_session( updated_at=session.updated_at.isoformat() ) except Exception as e: - logger.error(f"Failed to create chat session for user {current_user.id}: {e}") + logger.exception(f"Failed to create chat session for user {current_user.id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to create chat session" @@ -122,7 +122,7 @@ async def get_chat_sessions( return session_responses except Exception as e: - logger.error(f"Failed to get chat sessions for user {current_user.id}: {e}") + logger.exception(f"Failed to get chat sessions for user {current_user.id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to retrieve chat sessions" @@ -154,7 +154,7 @@ async def get_chat_session( except HTTPException: raise except Exception as e: - logger.error(f"Failed to get chat session {session_id} for user {current_user.id}: {e}") + logger.exception(f"Failed to get chat session {session_id} for user {current_user.id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to retrieve chat session" @@ -201,7 +201,7 @@ async def update_chat_session( except HTTPException: raise except Exception as e: - logger.error(f"Failed to update chat session {session_id} for user {current_user.id}: {e}") + logger.exception(f"Failed to update chat session {session_id} for user {current_user.id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to update chat session" @@ -228,7 +228,7 @@ async def delete_chat_session( except HTTPException: raise except Exception as e: - logger.error(f"Failed to delete chat session {session_id} for user {current_user.id}: {e}") + logger.exception(f"Failed to delete chat session {session_id} for user {current_user.id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to delete chat session" @@ -273,7 +273,7 @@ async def get_session_messages( except HTTPException: raise except Exception as e: - logger.error(f"Failed to get messages for session {session_id}, user {current_user.id}: {e}") + logger.exception(f"Failed to get messages for session {session_id}, user {current_user.id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to retrieve messages" @@ -319,7 +319,7 @@ async def event_stream(): yield "data: [DONE]\n\n" except Exception as e: - logger.error(f"Error in streaming response: {e}") + logger.exception(f"Error in streaming response: {e}") error_event = { "type": "error", "data": {"error": str(e)}, @@ -340,7 +340,7 @@ async def event_stream(): except HTTPException: raise except Exception as e: - logger.error(f"Failed to process message for user {current_user.id}: {e}") + logger.exception(f"Failed to process message for user {current_user.id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to process message" @@ -362,7 +362,7 @@ async def get_chat_statistics( last_chat=stats["last_chat"].isoformat() if stats["last_chat"] else None ) except Exception as e: - logger.error(f"Failed to get chat statistics for user {current_user.id}: {e}") + logger.exception(f"Failed to get chat statistics for user {current_user.id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to retrieve chat statistics" @@ -398,9 +398,9 @@ async def extract_memories_from_session( "count": 0, "message": "Failed to extract memories from chat session" } - + except Exception as e: - logger.error(f"Failed to extract memories from session {session_id} for user {current_user.id}: {e}") + logger.exception(f"Failed to extract memories from session {session_id} for user {current_user.id}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to extract memories from chat session" @@ -422,7 +422,7 @@ async def chat_health_check(): "timestamp": time.time() } except Exception as e: - logger.error(f"Chat service health check failed: {e}") + logger.exception(f"Chat service health check failed: {e}") raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Chat service is not available" diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/health_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/health_routes.py index d6d9af5d..d482dd9b 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/health_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/health_routes.py @@ -65,7 +65,7 @@ async def auth_health_check(): await asyncio.wait_for(memory_service.test_connection(), timeout=2.0) memory_status = "ok" except Exception as e: - logger.warning(f"Memory service health check failed: {e}") + logger.exception(f"Memory service health check failed: {e}") memory_status = "degraded" else: memory_status = "unavailable" @@ -77,7 +77,7 @@ async def auth_health_check(): "timestamp": int(time.time()) } except Exception as e: - logger.error(f"Auth health check failed: {e}") + logger.exception(f"Auth health check failed: {e}") return JSONResponse( status_code=500, content={ @@ -500,7 +500,7 @@ async def readiness_check(): await asyncio.wait_for(mongo_client.admin.command("ping"), timeout=2.0) return JSONResponse(content={"status": "ready", "timestamp": int(time.time())}, status_code=200) except Exception as e: - logger.error(f"Readiness check failed: {e}") + logger.exception(f"Readiness check failed: {e}") return JSONResponse( content={"status": "not_ready", "error": str(e), "timestamp": int(time.time())}, status_code=503 diff --git a/backends/advanced/src/advanced_omi_backend/routers/modules/queue_routes.py b/backends/advanced/src/advanced_omi_backend/routers/modules/queue_routes.py index 2da3767b..7677e656 100644 --- a/backends/advanced/src/advanced_omi_backend/routers/modules/queue_routes.py +++ b/backends/advanced/src/advanced_omi_backend/routers/modules/queue_routes.py @@ -46,7 +46,7 @@ async def list_jobs( return result except Exception as e: - logger.error(f"Failed to list jobs: {e}") + logger.exception(f"Failed to list jobs: {e}") return {"error": "Failed to list jobs", "jobs": [], "pagination": {"total": 0, "limit": limit, "offset": offset, "has_more": False}} @@ -87,8 +87,8 @@ async def get_job_status( # Re-raise HTTPException unchanged (e.g., 403 Forbidden) raise except Exception as e: - logger.error(f"Failed to get job status {job_id}: {e}") - raise HTTPException(status_code=404, detail="Job not found") + logger.exception(f"Failed to get job status {job_id}: {e}") + raise HTTPException(status_code=404, detail="Job not found") from e @router.get("/jobs/{job_id}") @@ -138,8 +138,8 @@ async def get_job( # Re-raise HTTPException unchanged (e.g., 403 Forbidden) raise except Exception as e: - logger.error(f"Failed to get job {job_id}: {e}") - raise HTTPException(status_code=404, detail="Job not found") + logger.exception(f"Failed to get job {job_id}: {e}") + raise HTTPException(status_code=404, detail="Job not found") from e @router.delete("/jobs/{job_id}") @@ -181,8 +181,8 @@ async def cancel_job( # Re-raise HTTPException unchanged (e.g., 403 Forbidden) raise except Exception as e: - logger.error(f"Failed to cancel/delete job {job_id}: {e}") - raise HTTPException(status_code=404, detail=f"Job not found or could not be cancelled: {str(e)}") + logger.exception(f"Failed to cancel/delete job {job_id}: {e}") + raise HTTPException(status_code=404, detail=f"Job not found or could not be cancelled: {str(e)}") from e @router.get("/jobs/by-session/{session_id}") @@ -330,8 +330,8 @@ def process_job_and_dependents(job, queue_name, base_status): } except Exception as e: - logger.error(f"Failed to get jobs for session {session_id}: {e}") - raise HTTPException(status_code=500, detail=f"Failed to get jobs for session: {str(e)}") + logger.exception(f"Failed to get jobs for session {session_id}: {e}") + raise HTTPException(status_code=500, detail=f"Failed to get jobs for session: {str(e)}") from e @router.get("/stats") @@ -344,7 +344,7 @@ async def get_queue_stats_endpoint( return stats except Exception as e: - logger.error(f"Failed to get queue stats: {e}") + logger.exception(f"Failed to get queue stats: {e}") return {"total_jobs": 0, "queued_jobs": 0, "processing_jobs": 0, "completed_jobs": 0, "failed_jobs": 0, "cancelled_jobs": 0, "deferred_jobs": 0} @@ -377,8 +377,8 @@ async def get_queue_worker_details( return status except Exception as e: - logger.error(f"Failed to get queue worker details: {e}") - raise HTTPException(status_code=500, detail=f"Failed to get worker details: {str(e)}") + logger.exception(f"Failed to get queue worker details: {e}") + raise HTTPException(status_code=500, detail=f"Failed to get worker details: {str(e)}") from e @router.get("/streams") @@ -472,7 +472,7 @@ async def get_stream_info(stream_key): "groups": groups_info } except Exception as e: - logger.error(f"Error getting info for stream {stream_key}: {e}") + logger.exception(f"Error getting info for stream {stream_key}: {e}") return None # Fetch all stream info in parallel @@ -486,7 +486,7 @@ async def get_stream_info(stream_key): } except Exception as e: - logger.error(f"Failed to get stream stats: {e}", exc_info=True) + logger.exception(f"Failed to get stream stats: {e}") return { "error": str(e), "total_streams": 0, @@ -538,7 +538,7 @@ async def flush_jobs( job.delete() total_removed += 1 except Exception as e: - logger.error(f"Error deleting job {job_id}: {e}") + logger.exception(f"Error deleting job {job_id}: {e}") if "failed" in request.statuses: registry = FailedJobRegistry(queue=queue) @@ -549,7 +549,7 @@ async def flush_jobs( job.delete() total_removed += 1 except Exception as e: - logger.error(f"Error deleting job {job_id}: {e}") + logger.exception(f"Error deleting job {job_id}: {e}") if "cancelled" in request.statuses: registry = CanceledJobRegistry(queue=queue) @@ -560,7 +560,7 @@ async def flush_jobs( job.delete() total_removed += 1 except Exception as e: - logger.error(f"Error deleting job {job_id}: {e}") + logger.exception(f"Error deleting job {job_id}: {e}") return { "total_removed": total_removed, @@ -569,8 +569,8 @@ async def flush_jobs( } except Exception as e: - logger.error(f"Failed to flush jobs: {e}") - raise HTTPException(status_code=500, detail=f"Failed to flush jobs: {str(e)}") + logger.exception(f"Failed to flush jobs: {e}") + raise HTTPException(status_code=500, detail=f"Failed to flush jobs: {str(e)}") from e @router.post("/flush-all") @@ -661,12 +661,12 @@ async def flush_all_jobs( except Exception as e: # Job might already be deleted or not exist - try to remove from registry anyway - logger.warning(f"Error deleting job {job_id}: {e}") + logger.exception(f"Error deleting job {job_id}: {e}") try: registry.remove(job_id) logger.info(f"Removed stale job reference {job_id} from {registry_name} registry") except Exception as reg_error: - logger.error(f"Could not remove {job_id} from registry: {reg_error}") + logger.exception(f"Could not remove {job_id} from registry: {reg_error}") # Also clean up audio streams and consumer locks deleted_keys = 0 @@ -715,8 +715,8 @@ async def flush_all_jobs( } except Exception as e: - logger.error(f"Failed to flush all jobs: {e}") - raise HTTPException(status_code=500, detail=f"Failed to flush all jobs: {str(e)}") + logger.exception(f"Failed to flush all jobs: {e}") + raise HTTPException(status_code=500, detail=f"Failed to flush all jobs: {str(e)}") from e @router.get("/sessions") @@ -767,7 +767,7 @@ async def get_redis_sessions( "conversation_count": conversation_count }) except Exception as e: - logger.error(f"Error getting session info for {key}: {e}") + logger.exception(f"Error getting session info for {key}: {e}") await redis_client.close() @@ -777,8 +777,8 @@ async def get_redis_sessions( } except Exception as e: - logger.error(f"Failed to get sessions: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to get sessions: {str(e)}") + logger.exception(f"Failed to get sessions: {e}") + raise HTTPException(status_code=500, detail=f"Failed to get sessions: {str(e)}") from e @router.post("/sessions/clear") @@ -820,7 +820,7 @@ async def clear_old_sessions( deleted_count += 1 logger.info(f"Deleted old session: {key.decode()}") except Exception as e: - logger.error(f"Error processing session {key}: {e}") + logger.exception(f"Error processing session {key}: {e}") await redis_client.close() @@ -830,8 +830,8 @@ async def clear_old_sessions( } except Exception as e: - logger.error(f"Failed to clear sessions: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to clear sessions: {str(e)}") + logger.exception(f"Failed to clear sessions: {e}") + raise HTTPException(status_code=500, detail=f"Failed to clear sessions: {str(e)}") from e @router.get("/dashboard") @@ -918,7 +918,7 @@ async def fetch_jobs_by_status(status_name: str, limit: int = 100): return all_jobs except Exception as e: - logger.error(f"Error fetching {status_name} jobs: {e}") + logger.exception(f"Error fetching {status_name} jobs: {e}") return [] async def fetch_stats(): @@ -926,7 +926,7 @@ async def fetch_stats(): try: return get_job_stats() except Exception as e: - logger.error(f"Error fetching stats: {e}") + logger.exception(f"Error fetching stats: {e}") return {"total_jobs": 0, "queued_jobs": 0, "processing_jobs": 0, "completed_jobs": 0, "failed_jobs": 0} async def fetch_streaming_status(): @@ -937,7 +937,7 @@ async def fetch_streaming_status(): # Use the actual request object from the parent function return await session_controller.get_streaming_status(request) except Exception as e: - logger.error(f"Error fetching streaming status: {e}") + logger.exception(f"Error fetching streaming status: {e}") return {"active_sessions": [], "stream_health": {}, "rq_queues": {}} async def fetch_session_jobs(session_id: str): @@ -1029,7 +1029,7 @@ def get_job_status(job): return {"session_id": session_id, "jobs": all_jobs} except Exception as e: - logger.error(f"Error fetching jobs for session {session_id}: {e}") + logger.exception(f"Error fetching jobs for session {session_id}: {e}") return {"session_id": session_id, "jobs": []} # Execute all fetches in parallel @@ -1095,5 +1095,5 @@ def get_job_status(job): } except Exception as e: - logger.error(f"Failed to get dashboard data: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to get dashboard data: {str(e)}") + logger.exception(f"Failed to get dashboard data: {e}") + raise HTTPException(status_code=500, detail=f"Failed to get dashboard data: {str(e)}") from e diff --git a/backends/advanced/src/advanced_omi_backend/services/audio_service.py b/backends/advanced/src/advanced_omi_backend/services/audio_service.py index 992ede75..1d1bcd86 100644 --- a/backends/advanced/src/advanced_omi_backend/services/audio_service.py +++ b/backends/advanced/src/advanced_omi_backend/services/audio_service.py @@ -75,7 +75,7 @@ async def _ensure_consumer_groups(self): # We'll create them dynamically in publish_audio_chunk pass except Exception as e: - logger.error(f"Error ensuring consumer groups: {e}") + logger.exception(f"Error ensuring consumer groups: {e}") async def publish_audio_chunk( self, diff --git a/backends/advanced/src/advanced_omi_backend/services/audio_stream/aggregator.py b/backends/advanced/src/advanced_omi_backend/services/audio_stream/aggregator.py index 26b985ab..67a6d2f1 100644 --- a/backends/advanced/src/advanced_omi_backend/services/audio_stream/aggregator.py +++ b/backends/advanced/src/advanced_omi_backend/services/audio_stream/aggregator.py @@ -75,7 +75,7 @@ async def get_session_results(self, session_id: str) -> list[dict]: return results except Exception as e: - logger.error(f"๐Ÿ”„ Error getting results for session {session_id}: {e}") + logger.exception(f"๐Ÿ”„ Error getting results for session {session_id}: {e}") return [] async def get_combined_results(self, session_id: str) -> dict: @@ -203,5 +203,5 @@ async def get_realtime_results( return results, new_last_id except Exception as e: - logger.error(f"๐Ÿ”„ Error getting realtime results for session {session_id}: {e}") + logger.exception(f"๐Ÿ”„ Error getting realtime results for session {session_id}: {e}") return [], last_id diff --git a/backends/advanced/src/advanced_omi_backend/services/audio_stream/consumer.py b/backends/advanced/src/advanced_omi_backend/services/audio_stream/consumer.py index 8ae0646b..406d858e 100644 --- a/backends/advanced/src/advanced_omi_backend/services/audio_stream/consumer.py +++ b/backends/advanced/src/advanced_omi_backend/services/audio_stream/consumer.py @@ -110,7 +110,7 @@ async def release_stream(self, stream_name: str): await self.stream_locks[stream_name].release() logger.info(f"๐Ÿ”“ Released stream: {stream_name}") except Exception as e: - logger.warning(f"Failed to release lock for {stream_name}: {e}") + logger.exception(f"Failed to release lock for {stream_name}: {e}") finally: del self.stream_locks[stream_name] @@ -120,7 +120,7 @@ async def renew_stream_locks(self): try: await lock.reacquire() except Exception as e: - logger.warning(f"Failed to renew lock for {stream_name}: {e}") + logger.exception(f"Failed to renew lock for {stream_name}: {e}") # Lock expired, remove from our list del self.stream_locks[stream_name] if stream_name in self.active_streams: @@ -340,7 +340,7 @@ async def start_consuming(self): break else: # Other ResponseError - log and continue - logger.error(f"โžก๏ธ [{self.consumer_name}] Redis ResponseError: {e}") + logger.exception(f"โžก๏ธ [{self.consumer_name}] Redis ResponseError: {e}") await asyncio.sleep(1) diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/providers/chronicle.py b/backends/advanced/src/advanced_omi_backend/services/memory/providers/chronicle.py index a0974e21..b78a1641 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/providers/chronicle.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/providers/chronicle.py @@ -95,7 +95,7 @@ async def initialize(self) -> None: ) except Exception as e: - memory_logger.error(f"Memory service initialization failed: {e}") + memory_logger.exception(f"Memory service initialization failed: {e}") raise async def add_memory( @@ -204,7 +204,7 @@ async def add_memory( memory_logger.error(f"โฐ Memory processing timed out for {source_id}") raise e except Exception as e: - memory_logger.error(f"โŒ Add memory failed for {source_id}: {e}") + memory_logger.exception(f"โŒ Add memory failed for {source_id}: {e}") raise e async def search_memories(self, query: str, user_id: str, limit: int = 10, score_threshold: float = 0.0) -> List[MemoryEntry]: @@ -241,7 +241,7 @@ async def search_memories(self, query: str, user_id: str, limit: int = 10, score return results except Exception as e: - memory_logger.error(f"Search memories failed: {e}") + memory_logger.exception(f"Search memories failed: {e}") return [] async def get_all_memories(self, user_id: str, limit: int = 100) -> List[MemoryEntry]: @@ -265,7 +265,7 @@ async def get_all_memories(self, user_id: str, limit: int = 100) -> List[MemoryE memory_logger.info(f"๐Ÿ“š Retrieved {len(memories)} memories for user {user_id}") return memories except Exception as e: - memory_logger.error(f"Get all memories failed: {e}") + memory_logger.exception(f"Get all memories failed: {e}") return [] async def count_memories(self, user_id: str) -> Optional[int]: @@ -287,7 +287,7 @@ async def count_memories(self, user_id: str) -> Optional[int]: memory_logger.info(f"๐Ÿ”ข Total {count} memories for user {user_id}") return count except Exception as e: - memory_logger.error(f"Count memories failed: {e}") + memory_logger.exception(f"Count memories failed: {e}") return None async def delete_memory(self, memory_id: str, user_id: Optional[str] = None, user_email: Optional[str] = None) -> bool: @@ -308,7 +308,7 @@ async def delete_memory(self, memory_id: str, user_id: Optional[str] = None, use memory_logger.info(f"๐Ÿ—‘๏ธ Deleted memory {memory_id}") return success except Exception as e: - memory_logger.error(f"Delete memory failed: {e}") + memory_logger.exception(f"Delete memory failed: {e}") return False async def delete_all_user_memories(self, user_id: str) -> int: @@ -328,7 +328,7 @@ async def delete_all_user_memories(self, user_id: str) -> int: memory_logger.info(f"๐Ÿ—‘๏ธ Deleted {count} memories for user {user_id}") return count except Exception as e: - memory_logger.error(f"Delete user memories failed: {e}") + memory_logger.exception(f"Delete user memories failed: {e}") return 0 async def test_connection(self) -> bool: @@ -342,7 +342,7 @@ async def test_connection(self) -> bool: await self.initialize() return True except Exception as e: - memory_logger.error(f"Connection test failed: {e}") + memory_logger.exception(f"Connection test failed: {e}") return False def shutdown(self) -> None: diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/providers/llm_providers.py b/backends/advanced/src/advanced_omi_backend/services/memory/providers/llm_providers.py index e8ab92bb..19dc1969 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/providers/llm_providers.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/providers/llm_providers.py @@ -199,7 +199,7 @@ async def extract_memories(self, text: str, prompt: str) -> List[str]: return cleaned_facts except Exception as e: - memory_logger.error(f"OpenAI memory extraction failed: {e}") + memory_logger.exception(f"OpenAI memory extraction failed: {e}") return [] async def _process_chunk(self, system_prompt: str, chunk: str, index: int) -> List[str]: @@ -242,7 +242,7 @@ async def _process_chunk(self, system_prompt: str, chunk: str, index: int) -> Li return _parse_memories_content(facts) except Exception as e: - memory_logger.error(f"Error processing chunk {index}: {e}") + memory_logger.exception(f"Error processing chunk {index}: {e}") return [] async def generate_embeddings(self, texts: List[str]) -> List[List[float]]: @@ -261,7 +261,7 @@ async def generate_embeddings(self, texts: List[str]) -> List[List[float]]: return [data.embedding for data in response.data] except Exception as e: - memory_logger.error(f"OpenAI embedding generation failed: {e}") + memory_logger.exception(f"OpenAI embedding generation failed: {e}") raise e async def test_connection(self) -> bool: @@ -276,11 +276,11 @@ async def test_connection(self) -> bool: await client.models.list() return True except Exception as e: - memory_logger.error(f"OpenAI connection test failed: {e}") + memory_logger.exception(f"OpenAI connection test failed: {e}") return False - + except Exception as e: - memory_logger.error(f"OpenAI connection test failed: {e}") + memory_logger.exception(f"OpenAI connection test failed: {e}") return False async def propose_memory_actions( @@ -330,7 +330,7 @@ async def propose_memory_actions( return result except Exception as e: - memory_logger.error(f"OpenAI propose_memory_actions failed: {e}") + memory_logger.exception(f"OpenAI propose_memory_actions failed: {e}") return {} diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/providers/mcp_client.py b/backends/advanced/src/advanced_omi_backend/services/memory/providers/mcp_client.py index 971c41f3..f21e541a 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/providers/mcp_client.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/providers/mcp_client.py @@ -169,10 +169,10 @@ async def add_memories(self, text: str) -> List[str]: return [str(uuid.uuid4())] except httpx.HTTPError as e: - memory_logger.error(f"HTTP error adding memories: {e}") + memory_logger.exception(f"HTTP error adding memories: {e}") raise MCPError(f"HTTP error: {e}") except Exception as e: - memory_logger.error(f"Error adding memories: {e}") + memory_logger.exception(f"Error adding memories: {e}") raise MCPError(f"Failed to add memories: {e}") async def search_memory(self, query: str, limit: int = 10) -> List[Dict[str, Any]]: @@ -244,7 +244,7 @@ async def search_memory(self, query: str, limit: int = 10) -> List[Dict[str, Any return formatted_memories[:limit] except Exception as e: - memory_logger.error(f"Error searching memories: {e}") + memory_logger.exception(f"Error searching memories: {e}") return [] async def list_memories(self, limit: int = 100) -> List[Dict[str, Any]]: @@ -313,7 +313,7 @@ async def list_memories(self, limit: int = 100) -> List[Dict[str, Any]]: return formatted_memories except Exception as e: - memory_logger.error(f"Error listing memories: {e}") + memory_logger.exception(f"Error listing memories: {e}") return [] async def delete_all_memories(self) -> int: @@ -358,7 +358,7 @@ async def delete_all_memories(self) -> int: return len(memory_ids) except Exception as e: - memory_logger.error(f"Error deleting all memories: {e}") + memory_logger.exception(f"Error deleting all memories: {e}") return 0 async def get_memory(self, memory_id: str) -> Optional[Dict[str, Any]]: @@ -398,10 +398,10 @@ async def get_memory(self, memory_id: str) -> Optional[Dict[str, Any]]: except httpx.HTTPStatusError as e: if e.response.status_code == 404: return None - memory_logger.error(f"HTTP error getting memory: {e}") + memory_logger.exception(f"HTTP error getting memory: {e}") return None except Exception as e: - memory_logger.error(f"Error getting memory: {e}") + memory_logger.exception(f"Error getting memory: {e}") return None async def update_memory( @@ -445,10 +445,10 @@ async def update_memory( return True except httpx.HTTPStatusError as e: - memory_logger.error(f"HTTP error updating memory: {e.response.status_code}") + memory_logger.exception(f"HTTP error updating memory: {e.response.status_code}") return False except Exception as e: - memory_logger.error(f"Error updating memory: {e}") + memory_logger.exception(f"Error updating memory: {e}") return False async def delete_memory(self, memory_id: str, user_id: Optional[str] = None, user_email: Optional[str] = None) -> bool: @@ -473,7 +473,7 @@ async def delete_memory(self, memory_id: str, user_id: Optional[str] = None, use return True except Exception as e: - memory_logger.warning(f"Error deleting memory {memory_id}: {e}") + memory_logger.exception(f"Error deleting memory {memory_id}: {e}") return False async def test_connection(self) -> bool: @@ -500,7 +500,7 @@ async def test_connection(self) -> bool: return False except Exception as e: - memory_logger.error(f"OpenMemory server connection test failed: {e}") + memory_logger.exception(f"OpenMemory server connection test failed: {e}") return False diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/providers/mycelia.py b/backends/advanced/src/advanced_omi_backend/services/memory/providers/mycelia.py index 6ace9ad6..a117f30a 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/providers/mycelia.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/providers/mycelia.py @@ -101,7 +101,7 @@ async def initialize(self) -> None: memory_logger.info("โœ… Mycelia memory service initialized successfully") except Exception as e: - memory_logger.error(f"โŒ Failed to initialize Mycelia service: {e}") + memory_logger.exception(f"โŒ Failed to initialize Mycelia service: {e}") raise RuntimeError(f"Mycelia initialization failed: {e}") async def _get_user_jwt(self, user_id: str, user_email: Optional[str] = None) -> str: @@ -277,12 +277,12 @@ async def _extract_memories_via_llm( memory_logger.info(f"๐Ÿง  Extracted {len(facts)} facts from transcript via OpenAI") return facts except json.JSONDecodeError as e: - memory_logger.error(f"Failed to parse LLM response as JSON: {e}") + memory_logger.exception(f"Failed to parse LLM response as JSON: {e}") memory_logger.error(f"LLM response was: {content[:300]}") return [] except Exception as e: - memory_logger.error(f"Failed to extract memories via OpenAI: {e}") + memory_logger.exception(f"Failed to extract memories via OpenAI: {e}") raise RuntimeError(f"OpenAI memory extraction failed: {e}") from e async def _extract_temporal_entity_via_llm( @@ -352,16 +352,16 @@ async def _extract_temporal_entity_via_llm( return temporal_entity except json.JSONDecodeError as e: - memory_logger.error(f"โŒ Failed to parse temporal extraction JSON: {e}") + memory_logger.exception(f"โŒ Failed to parse temporal extraction JSON: {e}") memory_logger.error(f"Content (first 300 chars): {content[:300]}") return None except Exception as e: - memory_logger.error(f"Failed to validate temporal entity: {e}") + memory_logger.exception(f"Failed to validate temporal entity: {e}") memory_logger.error(f"Data: {content[:300] if content else 'None'}") return None except Exception as e: - memory_logger.error(f"Failed to extract temporal data via OpenAI: {e}") + memory_logger.exception(f"Failed to extract temporal data via OpenAI: {e}") # Don't fail the entire memory creation if temporal extraction fails return None @@ -489,7 +489,7 @@ async def add_memory( return (False, []) except Exception as e: - memory_logger.error(f"Failed to add memory via Mycelia: {e}") + memory_logger.exception(f"Failed to add memory via Mycelia: {e}") return (False, []) async def search_memories( @@ -541,7 +541,7 @@ async def search_memories( return memories except Exception as e: - memory_logger.error(f"Failed to search memories via Mycelia: {e}") + memory_logger.exception(f"Failed to search memories via Mycelia: {e}") return [] async def get_all_memories(self, user_id: str, limit: int = 100) -> List[MemoryEntry]: @@ -574,7 +574,7 @@ async def get_all_memories(self, user_id: str, limit: int = 100) -> List[MemoryE return memories except Exception as e: - memory_logger.error(f"Failed to get memories via Mycelia: {e}") + memory_logger.exception(f"Failed to get memories via Mycelia: {e}") return [] async def count_memories(self, user_id: str) -> Optional[int]: @@ -606,7 +606,7 @@ async def count_memories(self, user_id: str) -> Optional[int]: return response.json() except Exception as e: - memory_logger.error(f"Failed to count memories via Mycelia: {e}") + memory_logger.exception(f"Failed to count memories via Mycelia: {e}") return None async def get_memory( @@ -643,7 +643,7 @@ async def get_memory( return None except Exception as e: - memory_logger.error(f"Failed to get memory via Mycelia: {e}") + memory_logger.exception(f"Failed to get memory via Mycelia: {e}") return None async def update_memory( @@ -721,7 +721,7 @@ async def update_memory( return False except Exception as e: - memory_logger.error(f"Failed to update memory via Mycelia: {e}") + memory_logger.exception(f"Failed to update memory via Mycelia: {e}") return False async def delete_memory( @@ -758,7 +758,7 @@ async def delete_memory( return False except Exception as e: - memory_logger.error(f"Failed to delete memory via Mycelia: {e}") + memory_logger.exception(f"Failed to delete memory via Mycelia: {e}") return False async def delete_all_user_memories(self, user_id: str) -> int: @@ -793,7 +793,7 @@ async def delete_all_user_memories(self, user_id: str) -> int: return deleted_count except Exception as e: - memory_logger.error(f"Failed to delete user memories via Mycelia: {e}") + memory_logger.exception(f"Failed to delete user memories via Mycelia: {e}") return 0 async def test_connection(self) -> bool: @@ -814,7 +814,7 @@ async def test_connection(self) -> bool: return response.status_code == 200 except Exception as e: - memory_logger.error(f"Mycelia connection test failed: {e}") + memory_logger.exception(f"Mycelia connection test failed: {e}") return False async def aclose(self) -> None: @@ -825,7 +825,7 @@ async def aclose(self) -> None: await self._client.aclose() memory_logger.info("โœ… Mycelia HTTP client closed successfully") except Exception as e: - memory_logger.error(f"Error closing Mycelia HTTP client: {e}") + memory_logger.exception(f"Error closing Mycelia HTTP client: {e}") self._initialized = False def shutdown(self) -> None: @@ -851,14 +851,14 @@ def shutdown(self) -> None: asyncio.ensure_future(self.aclose(), loop=loop) memory_logger.info("โœ… Close operation scheduled on running event loop") except Exception as e: - memory_logger.error(f"Error scheduling close on running loop: {e}") + memory_logger.exception(f"Error scheduling close on running loop: {e}") else: # No running loop, safe to use run_until_complete try: asyncio.get_event_loop().run_until_complete(self.aclose()) except Exception as e: - memory_logger.error(f"Error during shutdown: {e}") + memory_logger.exception(f"Error during shutdown: {e}") except Exception as e: - memory_logger.error(f"Unexpected error during shutdown: {e}") + memory_logger.exception(f"Unexpected error during shutdown: {e}") self._initialized = False diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/providers/openmemory_mcp.py b/backends/advanced/src/advanced_omi_backend/services/memory/providers/openmemory_mcp.py index 2fe34164..01190fca 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/providers/openmemory_mcp.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/providers/openmemory_mcp.py @@ -100,7 +100,7 @@ async def initialize(self) -> None: ) except Exception as e: - memory_logger.error(f"OpenMemory MCP service initialization failed: {e}") + memory_logger.exception(f"OpenMemory MCP service initialization failed: {e}") raise RuntimeError(f"Initialization failed: {e}") async def add_memory( @@ -175,10 +175,10 @@ async def add_memory( return True, [] except MCPError as e: - memory_logger.error(f"โŒ OpenMemory MCP error for {source_id}: {e}") + memory_logger.exception(f"โŒ OpenMemory MCP error for {source_id}: {e}") raise e except Exception as e: - memory_logger.error(f"โŒ OpenMemory MCP service failed for {source_id}: {e}") + memory_logger.exception(f"โŒ OpenMemory MCP service failed for {source_id}: {e}") raise e async def search_memories( @@ -226,10 +226,10 @@ async def search_memories( return memory_entries except MCPError as e: - memory_logger.error(f"Search memories failed: {e}") + memory_logger.exception(f"Search memories failed: {e}") return [] except Exception as e: - memory_logger.error(f"Search memories failed: {e}") + memory_logger.exception(f"Search memories failed: {e}") return [] finally: # Restore original user context @@ -273,10 +273,10 @@ async def get_all_memories( return memory_entries except MCPError as e: - memory_logger.error(f"Get all memories failed: {e}") + memory_logger.exception(f"Get all memories failed: {e}") return [] except Exception as e: - memory_logger.error(f"Get all memories failed: {e}") + memory_logger.exception(f"Get all memories failed: {e}") return [] finally: # Restore original user_id @@ -313,7 +313,7 @@ async def get_memory(self, memory_id: str, user_id: Optional[str] = None) -> Opt return memory_entry except Exception as e: - memory_logger.error(f"Failed to get memory: {e}") + memory_logger.exception(f"Failed to get memory: {e}") return None finally: # Restore original user_id @@ -358,7 +358,7 @@ async def update_memory( return success except Exception as e: - memory_logger.error(f"Failed to update memory: {e}") + memory_logger.exception(f"Failed to update memory: {e}") return False finally: # Restore original user_id @@ -382,7 +382,7 @@ async def delete_memory(self, memory_id: str, user_id: Optional[str] = None, use memory_logger.info(f"๐Ÿ—‘๏ธ Deleted memory {memory_id} via MCP") return success except Exception as e: - memory_logger.error(f"Delete memory failed: {e}") + memory_logger.exception(f"Delete memory failed: {e}") return False async def delete_all_user_memories(self, user_id: str) -> int: @@ -407,7 +407,7 @@ async def delete_all_user_memories(self, user_id: str) -> int: return count except Exception as e: - memory_logger.error(f"Delete user memories failed: {e}") + memory_logger.exception(f"Delete user memories failed: {e}") return 0 finally: # Restore original user_id @@ -424,7 +424,7 @@ async def test_connection(self) -> bool: await self.initialize() return await self.mcp_client.test_connection() except Exception as e: - memory_logger.error(f"Connection test failed: {e}") + memory_logger.exception(f"Connection test failed: {e}") return False def shutdown(self) -> None: @@ -494,7 +494,7 @@ def _mcp_result_to_memory_entry(self, mcp_result: Dict[str, Any], user_id: str) ) except Exception as e: - memory_logger.error(f"Failed to convert MCP result to MemoryEntry: {e}") + memory_logger.exception(f"Failed to convert MCP result to MemoryEntry: {e}") return None async def _update_database_relationships( diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/providers/vector_stores.py b/backends/advanced/src/advanced_omi_backend/services/memory/providers/vector_stores.py index cf153472..b36ecf10 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/providers/vector_stores.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/providers/vector_stores.py @@ -92,7 +92,7 @@ async def initialize(self) -> None: f"Collection {self.collection_name} exists with correct dimensions ({self.embedding_dims})" ) except Exception as e: - memory_logger.warning(f"Error checking collection info: {e}. Recreating...") + memory_logger.exception(f"Error checking collection info: {e}. Recreating...") try: await self.client.delete_collection(self.collection_name) except: @@ -114,7 +114,7 @@ async def initialize(self) -> None: ) except Exception as e: - memory_logger.error(f"Qdrant initialization failed: {e}") + memory_logger.exception(f"Qdrant initialization failed: {e}") raise async def add_memories(self, memories: List[MemoryEntry]) -> List[str]: @@ -144,7 +144,7 @@ async def add_memories(self, memories: List[MemoryEntry]) -> List[str]: return [] except Exception as e: - memory_logger.error(f"Qdrant add memories failed: {e}") + memory_logger.exception(f"Qdrant add memories failed: {e}") return [] async def search_memories(self, query_embedding: List[float], user_id: str, limit: int, score_threshold: float = 0.0) -> List[MemoryEntry]: @@ -202,7 +202,7 @@ async def search_memories(self, query_embedding: List[float], user_id: str, limi return memories except Exception as e: - memory_logger.error(f"Qdrant search failed: {e}") + memory_logger.exception(f"Qdrant search failed: {e}") return [] async def get_memories(self, user_id: str, limit: int) -> List[MemoryEntry]: @@ -237,7 +237,7 @@ async def get_memories(self, user_id: str, limit: int) -> List[MemoryEntry]: return memories except Exception as e: - memory_logger.error(f"Qdrant get memories failed: {e}") + memory_logger.exception(f"Qdrant get memories failed: {e}") return [] async def delete_memory(self, memory_id: str, user_id: Optional[str] = None, user_email: Optional[str] = None) -> bool: @@ -264,7 +264,7 @@ async def delete_memory(self, memory_id: str, user_id: Optional[str] = None, use return True except Exception as e: - memory_logger.error(f"Qdrant delete memory failed: {e}") + memory_logger.exception(f"Qdrant delete memory failed: {e}") return False async def delete_user_memories(self, user_id: str) -> int: @@ -293,7 +293,7 @@ async def delete_user_memories(self, user_id: str) -> int: return count except Exception as e: - memory_logger.error(f"Qdrant delete user memories failed: {e}") + memory_logger.exception(f"Qdrant delete user memories failed: {e}") return 0 async def test_connection(self) -> bool: @@ -305,7 +305,7 @@ async def test_connection(self) -> bool: return False except Exception as e: - memory_logger.error(f"Qdrant connection test failed: {e}") + memory_logger.exception(f"Qdrant connection test failed: {e}") return False async def update_memory( @@ -350,7 +350,7 @@ async def update_memory( ) return True except Exception as e: - memory_logger.error(f"Qdrant update memory failed: {e}") + memory_logger.exception(f"Qdrant update memory failed: {e}") return False async def count_memories(self, user_id: str) -> int: @@ -376,7 +376,7 @@ async def count_memories(self, user_id: str) -> int: return result.count except Exception as e: - memory_logger.error(f"Qdrant count memories failed: {e}") + memory_logger.exception(f"Qdrant count memories failed: {e}") return 0 diff --git a/backends/advanced/src/advanced_omi_backend/services/memory/service_factory.py b/backends/advanced/src/advanced_omi_backend/services/memory/service_factory.py index 5607d8ff..ce057ea8 100644 --- a/backends/advanced/src/advanced_omi_backend/services/memory/service_factory.py +++ b/backends/advanced/src/advanced_omi_backend/services/memory/service_factory.py @@ -104,8 +104,8 @@ def get_memory_service() -> MemoryServiceBase: memory_logger.info(f"โœ… Global memory service created: {type(_memory_service).__name__}") except Exception as e: - memory_logger.error(f"โŒ Failed to create memory service: {e}") - raise RuntimeError(f"Memory service creation failed: {e}") + memory_logger.exception(f"โŒ Failed to create memory service: {e}") + raise RuntimeError(f"Memory service creation failed: {e}") from e return _memory_service @@ -119,7 +119,7 @@ def shutdown_memory_service() -> None: _memory_service.shutdown() memory_logger.info("๐Ÿ”„ Memory service shut down") except Exception as e: - memory_logger.error(f"Error shutting down memory service: {e}") + memory_logger.exception(f"Error shutting down memory service: {e}") finally: _memory_service = None diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/deepgram.py b/backends/advanced/src/advanced_omi_backend/services/transcription/deepgram.py index 03b2936d..7b4e0a8b 100644 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/deepgram.py +++ b/backends/advanced/src/advanced_omi_backend/services/transcription/deepgram.py @@ -76,7 +76,7 @@ async def transcribe_audio(inner_self, audio_data: bytes, sample_rate: int) -> d } except Exception as e: - logger.error(f"Deepgram transcription failed: {e}", exc_info=True) + logger.exception(f"Deepgram transcription failed: {e}") raise # Instantiate the concrete consumer diff --git a/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet_stream_consumer.py b/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet_stream_consumer.py index f629cefd..c572d5ef 100644 --- a/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet_stream_consumer.py +++ b/backends/advanced/src/advanced_omi_backend/services/transcription/parakeet_stream_consumer.py @@ -74,7 +74,7 @@ async def transcribe_audio(inner_self, audio_data: bytes, sample_rate: int) -> d } except Exception as e: - logger.error(f"Parakeet transcription failed: {e}", exc_info=True) + logger.exception(f"Parakeet transcription failed: {e}") raise # Instantiate the concrete consumer diff --git a/backends/advanced/src/advanced_omi_backend/speaker_recognition_client.py b/backends/advanced/src/advanced_omi_backend/speaker_recognition_client.py index 50b12645..8a8c93c2 100644 --- a/backends/advanced/src/advanced_omi_backend/speaker_recognition_client.py +++ b/backends/advanced/src/advanced_omi_backend/speaker_recognition_client.py @@ -143,16 +143,16 @@ async def diarize_identify_match( return result except ClientConnectorError as e: - logger.error(f"๐ŸŽค Failed to connect to speaker recognition service: {e}") + logger.exception(f"๐ŸŽค Failed to connect to speaker recognition service: {e}") return {} except asyncio.TimeoutError as e: - logger.error(f"๐ŸŽค Timeout connecting to speaker recognition service: {e}") + logger.exception(f"๐ŸŽค Timeout connecting to speaker recognition service: {e}") return {} except aiohttp.ClientError as e: - logger.warning(f"๐ŸŽค Client error during speaker recognition: {e}") + logger.exception(f"๐ŸŽค Client error during speaker recognition: {e}") return {} except Exception as e: - logger.error(f"๐ŸŽค Error during speaker recognition: {e}") + logger.exception(f"๐ŸŽค Error during speaker recognition: {e}") return {} async def diarize_and_identify( @@ -264,18 +264,16 @@ async def diarize_and_identify( return result except ClientConnectorError as e: - logger.error(f"๐ŸŽค [DIARIZE] โŒ Failed to connect to speaker recognition service at {self.service_url}: {e}") + logger.exception(f"๐ŸŽค [DIARIZE] โŒ Failed to connect to speaker recognition service at {self.service_url}: {e}") return {} except asyncio.TimeoutError as e: - logger.error(f"๐ŸŽค [DIARIZE] โŒ Timeout connecting to speaker recognition service: {e}") + logger.exception(f"๐ŸŽค [DIARIZE] โŒ Timeout connecting to speaker recognition service: {e}") return {} except aiohttp.ClientError as e: - logger.warning(f"๐ŸŽค [DIARIZE] โŒ Client error during speaker recognition: {e}") + logger.exception(f"๐ŸŽค [DIARIZE] โŒ Client error during speaker recognition: {e}") return {} except Exception as e: - logger.error(f"๐ŸŽค [DIARIZE] โŒ Error during speaker diarization and identification: {e}") - import traceback - logger.debug(traceback.format_exc()) + logger.exception(f"๐ŸŽค [DIARIZE] โŒ Error during speaker diarization and identification: {e}") return {} async def identify_speakers(self, audio_path: str, segments: List[Dict]) -> Dict[str, str]: @@ -351,10 +349,10 @@ async def identify_speakers(self, audio_path: str, segments: List[Dict]) -> Dict return speaker_mapping except aiohttp.ClientError as e: - logger.warning(f"Failed to connect to speaker recognition service: {e}") + logger.exception(f"Failed to connect to speaker recognition service: {e}") return {} except Exception as e: - logger.error(f"Error during speaker identification: {e}") + logger.exception(f"Error during speaker identification: {e}") return {} def _process_diarization_result( @@ -408,7 +406,7 @@ def _process_diarization_result( return speaker_mapping except Exception as e: - logger.error(f"๐ŸŽค Error processing diarization result: {e}") + logger.exception(f"๐ŸŽค Error processing diarization result: {e}") return {} async def get_enrolled_speakers(self, user_id: Optional[str] = None) -> Dict: @@ -440,10 +438,10 @@ async def get_enrolled_speakers(self, user_id: Optional[str] = None) -> Dict: return result except aiohttp.ClientError as e: - logger.warning(f"๐ŸŽค Failed to connect to speaker recognition service: {e}") + logger.exception(f"๐ŸŽค Failed to connect to speaker recognition service: {e}") return {"speakers": []} except Exception as e: - logger.error(f"๐ŸŽค Error getting enrolled speakers: {e}") + logger.exception(f"๐ŸŽค Error getting enrolled speakers: {e}") return {"speakers": []} async def check_if_enrolled_speaker_present( @@ -574,7 +572,7 @@ async def check_if_enrolled_speaker_present( return (False, result) # Return both boolean and speaker recognition results except Exception as e: - logger.error(f"๐ŸŽค [SPEAKER CHECK] โŒ Speaker recognition check failed: {e}", exc_info=True) + logger.exception(f"๐ŸŽค [SPEAKER CHECK] โŒ Speaker recognition check failed: {e}") return (False, {}) # Fail closed - don't create conversation on error finally: @@ -622,5 +620,5 @@ async def health_check(self) -> bool: return False except Exception as e: - logger.error(f"Error during speaker service health check: {e}") + logger.exception(f"Error during speaker service health check: {e}") return False diff --git a/backends/advanced/src/advanced_omi_backend/task_manager.py b/backends/advanced/src/advanced_omi_backend/task_manager.py index b93a397d..587b3832 100644 --- a/backends/advanced/src/advanced_omi_backend/task_manager.py +++ b/backends/advanced/src/advanced_omi_backend/task_manager.py @@ -175,7 +175,7 @@ async def _periodic_cleanup(self): logger.info(f"Long-running tasks: {', '.join(long_running[:5])}") except Exception as e: - logger.error(f"Error in periodic cleanup: {e}", exc_info=True) + logger.exception(f"Error in periodic cleanup: {e}") except asyncio.CancelledError: logger.info("Periodic cleanup cancelled") diff --git a/backends/advanced/src/advanced_omi_backend/utils/audio_utils.py b/backends/advanced/src/advanced_omi_backend/utils/audio_utils.py index 3a3b554d..f849d7db 100644 --- a/backends/advanced/src/advanced_omi_backend/utils/audio_utils.py +++ b/backends/advanced/src/advanced_omi_backend/utils/audio_utils.py @@ -381,7 +381,7 @@ def write_pcm_to_wav( ) except Exception as e: - logger.error(f"โŒ Failed to write PCM to WAV: {e}") + logger.exception(f"โŒ Failed to write PCM to WAV") raise diff --git a/backends/advanced/src/advanced_omi_backend/utils/conversation_utils.py b/backends/advanced/src/advanced_omi_backend/utils/conversation_utils.py index b2cddf4c..83a7bf3c 100644 --- a/backends/advanced/src/advanced_omi_backend/utils/conversation_utils.py +++ b/backends/advanced/src/advanced_omi_backend/utils/conversation_utils.py @@ -188,8 +188,8 @@ async def generate_title(text: str, segments: Optional[list] = None) -> str: title = await async_generate(prompt, temperature=0.3) return title.strip().strip('"').strip("'") or "Conversation" - except Exception as e: - logger.warning(f"Failed to generate LLM title: {e}") + except Exception: + logger.exception(f"Failed to generate LLM title") # Fallback to simple title generation words = text.split()[:6] title = " ".join(words) @@ -256,8 +256,8 @@ async def generate_short_summary(text: str, segments: Optional[list] = None) -> summary = await async_generate(prompt, temperature=0.3) return summary.strip().strip('"').strip("'") or "No content" - except Exception as e: - logger.warning(f"Failed to generate LLM short summary: {e}") + except Exception: + logger.exception(f"Failed to generate LLM short summary") # Fallback to simple summary generation return ( conversation_text[:120] + "..." @@ -355,8 +355,8 @@ async def generate_detailed_summary(text: str, segments: Optional[list] = None) summary = await async_generate(prompt, temperature=0.3) return summary.strip().strip('"').strip("'") or "No meaningful content to summarize" - except Exception as e: - logger.warning(f"Failed to generate detailed summary: {e}") + except Exception: + logger.exception(f"Failed to generate detailed summary") # Fallback to returning cleaned transcript lines = conversation_text.split("\n") cleaned = "\n".join(line.strip() for line in lines if line.strip()) diff --git a/backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py index 56df7149..98c08ab4 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py @@ -181,7 +181,7 @@ async def process_cropping_job( } except Exception as e: - logger.error(f"โŒ RQ: Audio cropping failed for conversation {conversation_id}: {e}") + logger.exception(f"โŒ RQ: Audio cropping failed for conversation {conversation_id}: {e}") raise @@ -229,7 +229,7 @@ async def audio_streaming_persistence_job( logger.info(f"๐Ÿ“ฆ Created audio persistence consumer group for {audio_stream_name}") except Exception as e: if "BUSYGROUP" not in str(e): - logger.warning(f"Failed to create audio consumer group: {e}") + logger.exception(f"Failed to create audio consumer group: {e}") logger.debug(f"Audio consumer group already exists for {audio_stream_name}") # Job control diff --git a/backends/advanced/src/advanced_omi_backend/workers/audio_stream_deepgram_worker.py b/backends/advanced/src/advanced_omi_backend/workers/audio_stream_deepgram_worker.py index a58682c1..95364ebf 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/audio_stream_deepgram_worker.py +++ b/backends/advanced/src/advanced_omi_backend/workers/audio_stream_deepgram_worker.py @@ -69,7 +69,7 @@ def signal_handler(signum, frame): await consumer.start_consuming() except Exception as e: - logger.error(f"Worker error: {e}", exc_info=True) + logger.exception(f"Worker error: {e}") sys.exit(1) finally: await redis_client.aclose() diff --git a/backends/advanced/src/advanced_omi_backend/workers/audio_stream_parakeet_worker.py b/backends/advanced/src/advanced_omi_backend/workers/audio_stream_parakeet_worker.py index 56f2f26b..656dcd76 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/audio_stream_parakeet_worker.py +++ b/backends/advanced/src/advanced_omi_backend/workers/audio_stream_parakeet_worker.py @@ -83,7 +83,7 @@ def signal_handler(signum, _frame): await consumer.stop() except Exception as e: - logger.error(f"Worker error: {e}", exc_info=True) + logger.exception(f"Worker error: {e}") sys.exit(1) finally: await redis_client.aclose() diff --git a/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py index d2b8c4fd..e74e25f3 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py @@ -156,7 +156,7 @@ async def handle_end_of_conversation( redis_conn.set(f"speech_detection_job:{client_id}", speech_job.id, ex=3600) logger.info(f"๐Ÿ“Œ Stored speech detection job ID for client {client_id}") except Exception as e: - logger.warning(f"โš ๏ธ Failed to store job ID for {client_id}: {e}") + logger.exception(f"โš ๏ธ Failed to store job ID for {client_id}: {e}") logger.info(f"โœ… Enqueued speech detection job {speech_job.id}") else: diff --git a/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py b/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py index 8b64d690..c9c4824e 100644 --- a/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py +++ b/backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py @@ -223,7 +223,7 @@ async def process_memory_job(conversation_id: str, *, redis_client=None) -> Dict {"memory_id": memory_id, "text": memory_text[:200]} ) except Exception as e: - logger.warning(f"Failed to fetch memory details for UI: {e}") + logger.exception(f"Failed to fetch memory details for UI: {e}") current_job.meta.update( { diff --git a/config_manager.py b/config_manager.py index 2999d4b4..9ccb5b6d 100644 --- a/config_manager.py +++ b/config_manager.py @@ -107,7 +107,7 @@ def _load_config_yml(self) -> Dict[str, Any]: with open(self.config_yml_path, 'r') as f: return yaml.safe_load(f) or {} except Exception as e: - logger.error(f"Failed to load config.yml: {e}") + logger.exception(f"Failed to load config.yml: {e}") return {} def _save_config_yml(self, config: Dict[str, Any]): @@ -127,7 +127,7 @@ def _save_config_yml(self, config: Dict[str, Any]): logger.info(f"Saved config.yml to {self.config_yml_path}") except Exception as e: - logger.error(f"Failed to save config.yml: {e}") + logger.exception(f"Failed to save config.yml: {e}") raise def _update_env_file(self, key: str, value: str): @@ -175,7 +175,7 @@ def _update_env_file(self, key: str, value: str): logger.info(f"Updated {key}={value} in .env file") except Exception as e: - logger.error(f"Failed to update .env file: {e}") + logger.exception(f"Failed to update .env file: {e}") raise def get_memory_provider(self) -> str: diff --git a/extras/asr-services/client.py b/extras/asr-services/client.py index 51f33f84..acfa6a0d 100644 --- a/extras/asr-services/client.py +++ b/extras/asr-services/client.py @@ -44,7 +44,7 @@ async def write_transcript(text: str, output_file: Path | None = None): f.write(f"{text}\n") logger.info(f"Transcript written to: {output_file}") except Exception as e: - logger.error(f"Failed to write to output file {output_file}: {e}") + logger.exception(f"Failed to write to output file {output_file}: {e}") async def run_mic_transcription(asr_url: str, device_index: int | None = None, output_file: Path | None = None): @@ -253,7 +253,7 @@ async def file_reader(): logger.info("Stopping file reader...") return except Exception as e: - logger.error(f"Error reading file: {e}") + logger.exception(f"Error reading file: {e}") raise async def transcriptions(): @@ -370,7 +370,7 @@ async def main(): with open(output_file, 'w', encoding='utf-8') as f: f.write("") # Clear file except Exception as e: - logger.error(f"Failed to initialize output file {output_file}: {e}") + logger.exception(f"Failed to initialize output file {output_file}: {e}") return 1 try: @@ -384,7 +384,7 @@ async def main(): except KeyboardInterrupt: print("\nExiting...") except Exception as e: - logger.error(f"Error: {e}") + logger.exception(f"Error: {e}") return 1 return 0 diff --git a/extras/asr-services/enhanced_chunking.py b/extras/asr-services/enhanced_chunking.py index fab9d927..636494b3 100644 --- a/extras/asr-services/enhanced_chunking.py +++ b/extras/asr-services/enhanced_chunking.py @@ -144,7 +144,7 @@ def get_timestamped_results(self): return [self.merged_hypothesis] if self.merged_hypothesis else self.all_hypotheses except Exception as e: - logger.error(f"Hypothesis joining FAILED: {e}") + logger.exception(f"Hypothesis joining FAILED: {e}") raise e # Don't silently fall back def _join_hypotheses(self, hypotheses): @@ -354,7 +354,7 @@ def extract_timestamps_from_hypotheses_native(hypotheses: List[Hypothesis], chun return words except Exception as e: - logger.error(f"Native timestamp extraction FAILED: {e}") + logger.exception(f"Native timestamp extraction FAILED: {e}") raise e # Don't silently fall back @@ -466,7 +466,7 @@ def extract_timestamps_from_hypotheses(hypotheses: List[Hypothesis], chunk_start return words except Exception as e: - logger.error(f"Critical error in extract_timestamps_from_hypotheses: {e}") + logger.exception(f"Critical error in extract_timestamps_from_hypotheses: {e}") return [] @@ -595,5 +595,5 @@ async def transcribe_with_enhanced_chunking(model, audio_file_path: str, return response except Exception as e: - logger.error(f"Enhanced chunking failed: {e}") + logger.exception(f"Enhanced chunking failed: {e}") raise \ No newline at end of file diff --git a/extras/asr-services/parakeet-offline.py b/extras/asr-services/parakeet-offline.py index 111411d5..dba768aa 100644 --- a/extras/asr-services/parakeet-offline.py +++ b/extras/asr-services/parakeet-offline.py @@ -127,7 +127,7 @@ async def warmup(self) -> None: ) # 0.1s silence logger.info("Shared NeMo ASR model warmed up successfully.") except Exception as e: - logger.error(f"Error during ASR model warm-up: {e}") + logger.exception(f"Error during ASR model warm-up: {e}") async def _transcribe_chunked(self, speech: Sequence[AudioChunk]) -> dict: """Chunked transcription method for long audio.""" @@ -194,7 +194,7 @@ async def _transcribe_chunked(self, speech: Sequence[AudioChunk]) -> dict: try: os.unlink(tmpfile_name) except Exception as e: - logger.warning(f"Failed to delete temporary file {tmpfile_name}: {e}") + logger.exception(f"Failed to delete temporary file {tmpfile_name}: {e}") async def _transcribe(self, speech: Sequence[AudioChunk]) -> dict: """Internal transcription method that returns structured result.""" @@ -282,9 +282,9 @@ async def _transcribe(self, speech: Sequence[AudioChunk]) -> dict: else: logger.warning("NeMo returned empty results") return {"text": "", "words": [], "segments": []} - + except Exception as e: - logger.error(f"Error during transcription: {e}") + logger.exception(f"Error during transcription: {e}") # Re-raise the exception so HTTP endpoint can return proper error code raise e finally: @@ -384,9 +384,9 @@ async def add_audio_chunk(self, audio_data: bytes, rate: int, width: int, channe if "end" in speech_dict: vad_trigger = True break - + except Exception as e: - logger.warning(f"VAD processing error: {e}") + logger.exception(f"VAD processing error: {e}") return time_trigger or buffer_trigger or vad_trigger @@ -648,7 +648,7 @@ async def websocket_endpoint(websocket: WebSocket): except WebSocketDisconnect: logger.info("WebSocket disconnected") except Exception as e: - logger.error(f"WebSocket error: {e}") + logger.exception(f"WebSocket error: {e}") finally: # Cleanup if session_id and session_id in active_sessions: diff --git a/extras/asr-services/tests/test_parakeet_service.py b/extras/asr-services/tests/test_parakeet_service.py index 4c94af12..a03d47ac 100644 --- a/extras/asr-services/tests/test_parakeet_service.py +++ b/extras/asr-services/tests/test_parakeet_service.py @@ -77,9 +77,9 @@ def start_service(self): self.service_started = True logger.info("โœ… Parakeet ASR service started successfully") - + except Exception as e: - logger.error(f"Error starting Parakeet service: {e}") + logger.exception(f"Error starting Parakeet service: {e}") raise def cleanup_service(self): @@ -101,9 +101,9 @@ def cleanup_service(self): logger.warning(f"Service cleanup may have failed: {result.stderr}") else: logger.info("โœ… Parakeet ASR service cleaned up") - + except Exception as e: - logger.warning(f"Error during service cleanup: {e}") + logger.exception(f"Error during service cleanup: {e}") finally: self.service_started = False @@ -152,9 +152,9 @@ def transcribe_audio_file(self, audio_path: Path) -> Dict[str, Any]: result = response.json() logger.info(f"โœ… Transcription completed: {len(result.get('text', ''))} chars") return result - + except Exception as e: - logger.error(f"โŒ Transcription failed: {e}") + logger.exception(f"โŒ Transcription failed: {e}") raise @pytest.fixture diff --git a/extras/havpe-relay/main.py b/extras/havpe-relay/main.py index eac6d58b..8f26dffa 100644 --- a/extras/havpe-relay/main.py +++ b/extras/havpe-relay/main.py @@ -103,10 +103,10 @@ async def get_jwt_token(username: str, password: str, backend_url: str) -> Optio logger.error("โŒ Authentication request timed out") return None except httpx.RequestError as e: - logger.error(f"โŒ Authentication request failed: {e}") + logger.exception(f"โŒ Authentication request failed: {e}") return None except Exception as e: - logger.error(f"โŒ Unexpected authentication error: {e}") + logger.exception(f"โŒ Unexpected authentication error: {e}") return None @@ -232,7 +232,7 @@ async def read(self) -> Optional[AudioChunk]: ) except Exception as e: - logger.error(f"Error processing ESP32 audio data: {e}") + logger.exception(f"Error processing ESP32 audio data: {e}") return None @@ -246,7 +246,7 @@ async def ensure_socket_connection(socket_client: SocketClient) -> bool: logger.info("โœ… Authenticated WebSocket connection established") return True except Exception as e: - logger.error(f"โŒ Failed to connect to WebSocket: {e}") + logger.exception(f"โŒ Failed to connect to WebSocket: {e}") if attempt < max_retries - 1: await exponential_backoff_sleep(attempt) else: @@ -299,10 +299,10 @@ async def send_with_retry(socket_client: SocketClient, chunk: AudioChunk) -> tup # Check for authentication-related errors if any(auth_err in error_str for auth_err in ['401', 'unauthorized', 'forbidden', 'authentication']): - logger.warning(f"โŒ Authentication error detected: {e}") + logger.exception(f"โŒ Authentication error detected: {e}") return False, True # Failed, needs new auth token - - logger.warning(f"โš ๏ธ Failed to send chunk (attempt {attempt + 1}): {e}") + + logger.exception(f"โš ๏ธ Failed to send chunk (attempt {attempt + 1}): {e}") if attempt < max_retries - 1: if await ensure_socket_connection(socket_client): continue # Try again with reconnected client @@ -359,7 +359,7 @@ async def process_esp32_audio( try: await file_sink.write(chunk) except Exception as e: - logger.warning(f"โš ๏ธ Failed to write to file sink: {e}") + logger.exception(f"โš ๏ธ Failed to write to file sink: {e}") # Send to authenticated backend if socket_client: @@ -405,7 +405,7 @@ async def process_esp32_audio( logger.info("๐Ÿ›‘ ESP32 audio processor cancelled") raise except Exception as e: - logger.error(f"โŒ Error in ESP32 audio processor: {e}") + logger.exception(f"โŒ Error in ESP32 audio processor: {e}") raise @@ -459,7 +459,7 @@ async def run_audio_processor(args, esp32_file_sink): logger.info("๐Ÿ›‘ Interrupted โ€“ stopping") break except Exception as e: - logger.error(f"โŒ Audio processor error: {e}") + logger.exception(f"โŒ Audio processor error: {e}") logger.info(f"๐Ÿ”„ Restarting with exponential backoff...") await exponential_backoff_sleep(retry_attempt) retry_attempt += 1 @@ -551,7 +551,7 @@ async def main(): logger.error("๐Ÿ’ก Update AUTH_USERNAME and AUTH_PASSWORD constants or use command line arguments") return except Exception as e: - logger.error(f"โŒ Authentication test error: {e}") + logger.exception(f"โŒ Authentication test error: {e}") logger.error("๐Ÿ’ก Make sure the backend is running and accessible") return @@ -585,7 +585,7 @@ async def main(): except KeyboardInterrupt: logger.info("Interrupted โ€“ shutting down") except Exception as e: - logger.error(f"Fatal error: {e}") + logger.exception(f"Fatal error: {e}") finally: logger.info("Recording session ended") diff --git a/extras/speaker-recognition/laptop_client.py b/extras/speaker-recognition/laptop_client.py index 53ff7c09..a0e7c486 100644 --- a/extras/speaker-recognition/laptop_client.py +++ b/extras/speaker-recognition/laptop_client.py @@ -286,7 +286,7 @@ async def cmd_enroll(args): logger.error(f"โŒ Failed to enroll speaker - unexpected response: {result}") except Exception as e: - logger.error(f"Error during enrollment: {e}") + logger.exception(f"Error during enrollment: {e}") finally: # Clean up temporary file only if we recorded it if cleanup_audio: @@ -321,7 +321,7 @@ async def cmd_identify(args): save_audio(audio_path, args.save_file) except Exception as e: - logger.error(f"Error during identification: {e}") + logger.exception(f"Error during identification: {e}") finally: # Clean up temporary file Path(audio_path).unlink(missing_ok=True) @@ -355,7 +355,7 @@ async def cmd_verify(args): save_audio(audio_path, args.save_file) except Exception as e: - logger.error(f"Error during verification: {e}") + logger.exception(f"Error during verification: {e}") finally: # Clean up temporary file Path(audio_path).unlink(missing_ok=True) @@ -381,7 +381,7 @@ async def cmd_list(args): logger.info("๐Ÿ“‹ No speakers enrolled") except Exception as e: - logger.error(f"Error listing speakers: {e}") + logger.exception(f"Error listing speakers: {e}") async def cmd_remove(args): @@ -402,7 +402,7 @@ async def cmd_remove(args): logger.error(f"โŒ Failed to remove speaker: {args.speaker_id}") except Exception as e: - logger.error(f"Error removing speaker: {e}") + logger.exception(f"Error removing speaker: {e}") async def cmd_diarize(args): @@ -466,7 +466,7 @@ async def cmd_diarize(args): save_audio(audio_path, args.save_file) except Exception as e: - logger.error(f"Error during diarization: {e}") + logger.exception(f"Error during diarization: {e}") finally: # Clean up temporary file only if we recorded it if cleanup_audio: @@ -555,7 +555,7 @@ def main(): except KeyboardInterrupt: logger.info("Operation cancelled by user") except Exception as e: - logger.error(f"Unexpected error: {e}") + logger.exception(f"Unexpected error: {e}") if __name__ == "__main__": diff --git a/extras/speaker-recognition/scripts/download-pyannote.py b/extras/speaker-recognition/scripts/download-pyannote.py index b2c51394..02dd589a 100755 --- a/extras/speaker-recognition/scripts/download-pyannote.py +++ b/extras/speaker-recognition/scripts/download-pyannote.py @@ -42,7 +42,7 @@ def download_models(): return True except Exception as e: - logger.warning(f"Failed to download models during build (will download at runtime): {e}") + logger.exception(f"Failed to download models during build (will download at runtime): {e}") return True # Don't fail the build if __name__ == "__main__": diff --git a/extras/speaker-recognition/scripts/enroll_speaker.py b/extras/speaker-recognition/scripts/enroll_speaker.py index f73e4634..d3fbd54d 100755 --- a/extras/speaker-recognition/scripts/enroll_speaker.py +++ b/extras/speaker-recognition/scripts/enroll_speaker.py @@ -50,7 +50,7 @@ def check_service_health_with_url(service_url): logger.info(f"โœ… Speaker service is running (Device: {data.get('device', 'unknown')}, Speakers: {data.get('speakers', 0)})") return True except Exception as e: - logger.error(f"โŒ Cannot connect to speaker service at {service_url}: {e}") + logger.exception(f"โŒ Cannot connect to speaker service at {service_url}: {e}") logger.info("Make sure the speaker service is running: docker compose up speaker-recognition") return False @@ -94,7 +94,7 @@ def enroll_single_file(file_path: str, speaker_id: str, speaker_name: str, start return False except Exception as e: - logger.error(f"โŒ Error during enrollment: {e}") + logger.exception(f"โŒ Error during enrollment: {e}") return False @@ -131,7 +131,7 @@ def enroll_multiple_files(file_paths: List[str], speaker_id: str, speaker_name: return False except Exception as e: - logger.error(f"โŒ Error during batch enrollment: {e}") + logger.exception(f"โŒ Error during batch enrollment: {e}") return False @@ -200,7 +200,7 @@ def download_youtube_audio(url: str, start: Optional[float] = None, end: Optiona return output_path except Exception as e: - logger.error(f"โŒ Error downloading YouTube audio: {e}") + logger.exception(f"โŒ Error downloading YouTube audio: {e}") return None @@ -232,7 +232,7 @@ def list_speakers() -> bool: return False except Exception as e: - logger.error(f"โŒ Error listing speakers: {e}") + logger.exception(f"โŒ Error listing speakers: {e}") return False @@ -251,7 +251,7 @@ def delete_speaker(speaker_id: str) -> bool: return False except Exception as e: - logger.error(f"โŒ Error deleting speaker: {e}") + logger.exception(f"โŒ Error deleting speaker: {e}") return False diff --git a/extras/speaker-recognition/scripts/install-pytorch.py b/extras/speaker-recognition/scripts/install-pytorch.py index 7953db58..e71919de 100755 --- a/extras/speaker-recognition/scripts/install-pytorch.py +++ b/extras/speaker-recognition/scripts/install-pytorch.py @@ -43,10 +43,10 @@ def install_pytorch(): return True except subprocess.CalledProcessError as e: - logger.error(f"Failed to install PyTorch: {e}") + logger.exception(f"Failed to install PyTorch: {e}") return False except Exception as e: - logger.error(f"Unexpected error installing PyTorch: {e}") + logger.exception(f"Unexpected error installing PyTorch: {e}") return False if __name__ == "__main__": diff --git a/extras/speaker-recognition/src/simple_speaker_recognition/utils/audio_segment_manager.py b/extras/speaker-recognition/src/simple_speaker_recognition/utils/audio_segment_manager.py index 437a5c3a..99c6f905 100644 --- a/extras/speaker-recognition/src/simple_speaker_recognition/utils/audio_segment_manager.py +++ b/extras/speaker-recognition/src/simple_speaker_recognition/utils/audio_segment_manager.py @@ -45,7 +45,7 @@ def _load_manifest(self, user_id: int, speaker_id: str) -> Dict[str, Any]: with open(manifest_path, 'r') as f: return json.load(f) except Exception as e: - logger.error(f"Error loading manifest: {e}") + logger.exception(f"Error loading manifest: {e}") return self._create_empty_manifest(speaker_id) else: return self._create_empty_manifest(speaker_id) diff --git a/extras/speaker-recognition/src/simple_speaker_recognition/utils/youtube_transcriber.py b/extras/speaker-recognition/src/simple_speaker_recognition/utils/youtube_transcriber.py index 5571a00f..53e58a30 100644 --- a/extras/speaker-recognition/src/simple_speaker_recognition/utils/youtube_transcriber.py +++ b/extras/speaker-recognition/src/simple_speaker_recognition/utils/youtube_transcriber.py @@ -88,7 +88,7 @@ def download_audio(self, url: str) -> Tuple[str, str]: return audio_path, title except Exception as e: - self.logger.error(f"Audio download failed for {url}: {str(e)}", exc_info=True) + self.logger.exception(f"Audio download failed for {url}: {str(e)}") raise def segment_audio(self, audio_path: str, title: str) -> List[str]: @@ -134,7 +134,7 @@ def segment_audio(self, audio_path: str, title: str) -> List[str]: return segments except Exception as e: - self.logger.error(f"Audio segmentation failed for {audio_path}: {str(e)}", exc_info=True) + self.logger.exception(f"Audio segmentation failed for {audio_path}: {str(e)}") raise def transcribe_audio(self, audio_path: str, use_cache: bool = True, diarize: bool = True) -> dict: @@ -219,7 +219,7 @@ def to_dict(self): return None except Exception as e: - self.logger.error(f"Transcription error for {audio_path}: {str(e)}", exc_info=True) + self.logger.exception(f"Transcription error for {audio_path}: {str(e)}") return None def save_raw_json(self, response, title: str, segment_num: int): @@ -245,7 +245,7 @@ def save_raw_json(self, response, title: str, segment_num: int): self.logger.info(f"Saved raw JSON to: {filepath}") except Exception as e: - self.logger.error(f"Failed to save raw JSON for {title}-{segment_num}: {str(e)}", exc_info=True) + self.logger.exception(f"Failed to save raw JSON for {title}-{segment_num}: {str(e)}") def format_transcript(self, response, title: str, segment_num: int): """Format transcript with speaker info and timestamps""" @@ -340,7 +340,7 @@ def format_transcript(self, response, title: str, segment_num: int): self.logger.info(f"Saved transcript to: {filepath}") except Exception as e: - self.logger.error(f"Failed to format transcript for {title}-{segment_num}: {str(e)}", exc_info=True) + self.logger.exception(f"Failed to format transcript for {title}-{segment_num}: {str(e)}") async def process_youtube_url(self, url: str): """Process a single YouTube URL through the entire pipeline""" @@ -354,7 +354,7 @@ async def process_youtube_url(self, url: str): title = self.sanitize_filename(info['title']) duration = info.get('duration', 0) except Exception as e: - self.logger.error(f"Failed to extract video info for {url}: {str(e)}") + self.logger.exception(f"Failed to extract video info for {url}: {str(e)}") raise # Check if audio segments already exist