diff --git a/docs/audit_log.md b/docs/audit_log.md new file mode 100644 index 0000000..8ae5cdb --- /dev/null +++ b/docs/audit_log.md @@ -0,0 +1,176 @@ +# Audit Log Trail Feature + +The audit log trail feature provides comprehensive logging of all LLM interactions for compliance, debugging, and analytics purposes. + +## Overview + +The audit log automatically captures every input and output interaction with LLM models, storing detailed information about each request including timestamps, token usage, costs, and performance metrics. + +## Features + +- **Automatic Logging**: All LLM interactions are automatically logged without requiring additional code +- **Comprehensive Data**: Captures input prompts, output responses, model information, token usage, costs, and timing +- **Easy Access**: Simple methods to retrieve and manage audit logs +- **Transparent**: Works with all existing LLM client implementations + +## Implementation + +### Base Class Integration + +The audit log is implemented in the `BaseLLMClient` class: + +```python +class BaseLLMClient(ABC): + def __init__(self, ...): + # ... existing initialization ... + self.audit_log: List[AuditLogEntry] = [] + + def _log_audit_entry(self, response: LLMResponse, prompt: str) -> None: + """Log an audit entry for the LLM interaction.""" + audit_entry = AuditLogEntry( + timestamp=datetime.now(), + input_prompt=prompt, + output_response=response.output, + model=response.model, + provider=response.provider, + input_tokens=response.input_tokens, + output_tokens=response.output_tokens, + cost=response.cost, + duration=response.duration, + ) + self.audit_log.append(audit_entry) +``` + +### Audit Log Entry Structure + +Each audit log entry contains: + +```python +@dataclass +class AuditLogEntry: + timestamp: datetime # When the interaction occurred + input_prompt: str # The input prompt sent to the model + output_response: str # The response from the model + model: str # The model used (e.g., "gpt-4") + provider: str # The provider (e.g., "openai") + input_tokens: int # Number of input tokens + output_tokens: int # Number of output tokens + cost: float # Cost of the interaction + duration: float # Time taken for the interaction +``` + +## Usage + +### Basic Usage + +```python +from intent_kit.services.ai.openai_client import OpenAIClient + +# Initialize client +client = OpenAIClient(api_key="your-api-key") + +# Make LLM calls (audit logging happens automatically) +response1 = client.generate("What is AI?", model="gpt-4") +response2 = client.generate("Explain machine learning", model="gpt-4") + +# Access audit log +audit_log = client.get_audit_log() +print(f"Total interactions: {len(audit_log)}") + +# Process audit entries +for entry in audit_log: + print(f"Model: {entry.model}, Cost: ${entry.cost:.4f}") + print(f"Input: {entry.input_prompt[:50]}...") + print(f"Output: {entry.output_response[:50]}...") + print("---") +``` + +### Available Methods + +- `get_audit_log() -> List[AuditLogEntry]`: Get a copy of the complete audit log +- `clear_audit_log() -> None`: Clear all audit log entries +- `audit_log`: Direct access to the audit log list (read-only recommended) + +### Working with Audit Data + +```python +# Calculate total costs +total_cost = sum(entry.cost for entry in client.get_audit_log()) + +# Find interactions with specific models +gpt4_interactions = [ + entry for entry in client.get_audit_log() + if entry.model == "gpt-4" +] + +# Analyze token usage patterns +total_input_tokens = sum(entry.input_tokens for entry in client.get_audit_log()) +total_output_tokens = sum(entry.output_tokens for entry in client.get_audit_log()) + +# Find expensive interactions +expensive_interactions = [ + entry for entry in client.get_audit_log() + if entry.cost > 0.10 +] +``` + +## Supported Clients + +The audit log feature is automatically available in all LLM client implementations: + +- `OpenAIClient` - OpenAI GPT models +- `AnthropicClient` - Anthropic Claude models +- `OllamaClient` - Local Ollama models +- `GoogleClient` - Google Gemini models +- `OpenRouterClient` - OpenRouter models + +## Benefits + +### Compliance +- Track all AI interactions for regulatory requirements +- Maintain detailed logs for audit purposes +- Monitor usage patterns and costs + +### Debugging +- Review input/output pairs for quality issues +- Analyze performance and cost patterns +- Troubleshoot model behavior + +### Analytics +- Calculate total costs across all interactions +- Analyze token usage patterns +- Monitor response times and performance + +## Example Output + +``` +Audit Log (3 entries): +======================================== + +Entry 1: + Timestamp: 2025-08-05 23:15:35.495048 + Input: What is the capital of France? + Output: The capital of France is Paris. + Model: gpt-4 + Provider: openai + Tokens: 6 in, 8 out + Cost: $0.0003 + Duration: 1.234s + +Entry 2: + Timestamp: 2025-08-05 23:15:37.123456 + Input: Explain quantum computing + Output: Quantum computing is a type of computation... + Model: gpt-4 + Provider: openai + Tokens: 3 in, 45 out + Cost: $0.0012 + Duration: 2.567s +``` + +## Notes + +- Audit logs are stored in memory and will be cleared when the client is destroyed +- For persistent storage, consider saving audit logs to a database or file +- The audit log is thread-safe for basic operations but consider synchronization for concurrent access +- Large audit logs may consume significant memory; use `clear_audit_log()` periodically if needed \ No newline at end of file diff --git a/examples/audit_log_demo.py b/examples/audit_log_demo.py new file mode 100644 index 0000000..ef53ac2 --- /dev/null +++ b/examples/audit_log_demo.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +Audit Log Demo + +This example demonstrates the audit log functionality in the LLM base class. +The audit log stores all input and output interactions for compliance and debugging. +""" + +import os +from intent_kit.services.ai.base_client import AuditLogEntry +from intent_kit.services.ai.openai_client import OpenAIClient +from intent_kit.services.ai.anthropic_client import AnthropicClient +from intent_kit.services.ai.ollama_client import OllamaClient + + +def demo_audit_log(): + """Demonstrate the audit log functionality.""" + print("Audit Log Demo") + print("=" * 50) + + # Example 1: Using OpenAI client with audit logging + print("\n1. OpenAI Client Audit Log Example") + print("-" * 40) + + # Note: In a real scenario, you would set your API key + # api_key = os.getenv("OPENAI_API_KEY") + # client = OpenAIClient(api_key=api_key) + + # For demo purposes, we'll show the structure without making actual API calls + print("Audit log automatically captures:") + print(" - Input prompts") + print(" - Output responses") + print(" - Model used") + print(" - Provider") + print(" - Token usage (input/output)") + print(" - Cost") + print(" - Duration") + print(" - Timestamp") + + # Example 2: Accessing audit log data + print("\n2. Accessing Audit Log Data") + print("-" * 40) + + print("Methods available:") + print(" - client.get_audit_log() -> List[AuditLogEntry]") + print(" - client.clear_audit_log() -> None") + print(" - client.audit_log -> List[AuditLogEntry] (direct access)") + + # Example 3: Audit log entry structure + print("\n3. Audit Log Entry Structure") + print("-" * 40) + + print("AuditLogEntry fields:") + print(" - timestamp: datetime") + print(" - input_prompt: str") + print(" - output_response: str") + print(" - model: str") + print(" - provider: str") + print(" - input_tokens: int") + print(" - output_tokens: int") + print(" - cost: float") + print(" - duration: float") + + # Example 4: Usage pattern + print("\n4. Usage Pattern") + print("-" * 40) + + print(""" +# Initialize client +client = OpenAIClient(api_key="your-api-key") + +# Make LLM calls (audit logging happens automatically) +response1 = client.generate("What is AI?", model="gpt-4") +response2 = client.generate("Explain machine learning", model="gpt-4") + +# Access audit log +audit_log = client.get_audit_log() +print(f"Total interactions: {len(audit_log)}") + +# Process audit entries +for entry in audit_log: + print(f"Model: {entry.model}, Cost: ${entry.cost:.4f}") + print(f"Input: {entry.input_prompt[:50]}...") + print(f"Output: {entry.output_response[:50]}...") + print("---") + +# Clear audit log if needed +client.clear_audit_log() +""") + + # Example 5: Compliance and debugging benefits + print("\n5. Benefits") + print("-" * 40) + + print("Compliance:") + print(" - Track all AI interactions for regulatory requirements") + print(" - Maintain detailed logs for audit purposes") + print(" - Monitor usage patterns and costs") + + print("\nDebugging:") + print(" - Review input/output pairs for quality issues") + print(" - Analyze performance and cost patterns") + print(" - Troubleshoot model behavior") + + print("\nAnalytics:") + print(" - Calculate total costs across all interactions") + print(" - Analyze token usage patterns") + print(" - Monitor response times and performance") + + +def show_audit_log_structure(): + """Show the structure of the audit log implementation.""" + print("\nAudit Log Implementation Details") + print("=" * 50) + + print(""" +The audit log feature is implemented in the BaseLLMClient class: + +1. Audit Log Storage: + - self.audit_log: List[AuditLogEntry] = [] + - Automatically initialized in __init__ + +2. Logging Method: + - _log_audit_entry(response: LLMResponse, prompt: str) + - Called automatically after each generate() call + +3. Access Methods: + - get_audit_log() -> List[AuditLogEntry] + - clear_audit_log() -> None + +4. Integration: + - All concrete LLM clients (OpenAI, Anthropic, etc.) + automatically log audit entries + - No additional code required in client implementations + - Transparent to existing code +""") + + +if __name__ == "__main__": + demo_audit_log() + show_audit_log_structure() \ No newline at end of file diff --git a/intent_kit/services/ai/anthropic_client.py b/intent_kit/services/ai/anthropic_client.py index a3e57fd..0b1fa8f 100644 --- a/intent_kit/services/ai/anthropic_client.py +++ b/intent_kit/services/ai/anthropic_client.py @@ -237,7 +237,7 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: else "" ) - return LLMResponse( + response = LLMResponse( output=self._clean_response(output_text), model=model, input_tokens=input_tokens, @@ -246,6 +246,11 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: provider="anthropic", duration=duration, ) + + # Log audit entry + self._log_audit_entry(response, prompt) + + return response except Exception as e: self.logger.error(f"Error generating text with Anthropic: {e}") diff --git a/intent_kit/services/ai/base_client.py b/intent_kit/services/ai/base_client.py index a1592ae..6f04f79 100644 --- a/intent_kit/services/ai/base_client.py +++ b/intent_kit/services/ai/base_client.py @@ -6,12 +6,28 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Optional, Any, Dict +from typing import Optional, Any, Dict, List +from datetime import datetime from intent_kit.types import LLMResponse, Cost, InputTokens, OutputTokens from intent_kit.services.ai.pricing_service import PricingService from intent_kit.utils.logger import Logger +@dataclass +class AuditLogEntry: + """Audit log entry for LLM interactions.""" + + timestamp: datetime + input_prompt: str + output_response: str + model: str + provider: str + input_tokens: int + output_tokens: int + cost: float + duration: float + + @dataclass class ModelPricing: """Pricing information for a specific AI model.""" @@ -55,6 +71,7 @@ def __init__( self._client: Optional[Any] = None self.pricing_service = pricing_service or PricingService() self.pricing_config: PricingConfiguration = self._create_pricing_config() + self.audit_log: List[AuditLogEntry] = [] self._initialize_client(**kwargs) @abstractmethod @@ -76,6 +93,29 @@ def _ensure_imported(self) -> None: """Ensure the required package is imported. Must be implemented by subclasses.""" pass + def _log_audit_entry(self, response: LLMResponse, prompt: str) -> None: + """Log an audit entry for the LLM interaction.""" + audit_entry = AuditLogEntry( + timestamp=datetime.now(), + input_prompt=prompt, + output_response=response.output, + model=response.model, + provider=response.provider, + input_tokens=response.input_tokens, + output_tokens=response.output_tokens, + cost=response.cost, + duration=response.duration, + ) + self.audit_log.append(audit_entry) + + def get_audit_log(self) -> List[AuditLogEntry]: + """Get the complete audit log.""" + return self.audit_log.copy() + + def clear_audit_log(self) -> None: + """Clear the audit log.""" + self.audit_log.clear() + @abstractmethod def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: """ diff --git a/intent_kit/services/ai/google_client.py b/intent_kit/services/ai/google_client.py index a260fc3..d504195 100644 --- a/intent_kit/services/ai/google_client.py +++ b/intent_kit/services/ai/google_client.py @@ -221,7 +221,7 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: duration=duration, ) - return LLMResponse( + response = LLMResponse( output=self._clean_response(google_response.text), model=model, input_tokens=input_tokens, @@ -230,6 +230,11 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: provider="google", duration=duration, ) + + # Log audit entry + self._log_audit_entry(response, prompt) + + return response except Exception as e: self.logger.error(f"Error generating text with Google GenAI: {e}") diff --git a/intent_kit/services/ai/ollama_client.py b/intent_kit/services/ai/ollama_client.py index 3b1d220..1e2b990 100644 --- a/intent_kit/services/ai/ollama_client.py +++ b/intent_kit/services/ai/ollama_client.py @@ -179,7 +179,7 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: duration=duration, ) - return LLMResponse( + response = LLMResponse( output=self._clean_response(ollama_response.response), model=model, input_tokens=input_tokens, @@ -188,6 +188,11 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: provider="ollama", duration=duration, ) + + # Log audit entry + self._log_audit_entry(response, prompt) + + return response except Exception as e: self.logger.error(f"Error generating text with Ollama: {e}") diff --git a/intent_kit/services/ai/openai_client.py b/intent_kit/services/ai/openai_client.py index fb4f6b6..5546d87 100644 --- a/intent_kit/services/ai/openai_client.py +++ b/intent_kit/services/ai/openai_client.py @@ -266,7 +266,7 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: duration=duration, ) - return LLMResponse( + response = LLMResponse( output=self._clean_response(content), model=model, input_tokens=input_tokens, @@ -275,6 +275,11 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: provider="openai", duration=duration, ) + + # Log audit entry + self._log_audit_entry(response, prompt) + + return response except Exception as e: self.logger.error(f"Error generating text with OpenAI: {e}") diff --git a/intent_kit/services/ai/openrouter_client.py b/intent_kit/services/ai/openrouter_client.py index 4ec3a2e..6caffce 100644 --- a/intent_kit/services/ai/openrouter_client.py +++ b/intent_kit/services/ai/openrouter_client.py @@ -334,7 +334,7 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: self.logger.info(f"OpenRouter content: {content}") self.logger.info(f"OpenRouter first_choice: {first_choice.display()}") - return LLMResponse( + response = LLMResponse( output=content, model=model, input_tokens=input_tokens, @@ -343,6 +343,11 @@ def generate(self, prompt: str, model: Optional[str] = None) -> LLMResponse: provider="openrouter", duration=duration, ) + + # Log audit entry + self._log_audit_entry(response, prompt) + + return response def calculate_cost( self,