diff --git a/SkinSenseAI/src/screens/ChatScreen.js b/SkinSenseAI/src/screens/ChatScreen.js index 756cffc..405d61c 100644 --- a/SkinSenseAI/src/screens/ChatScreen.js +++ b/SkinSenseAI/src/screens/ChatScreen.js @@ -7,8 +7,10 @@ import { KeyboardAvoidingView, Platform, Alert, + Image, } from "react-native"; import { KeyboardAwareScrollView } from "react-native-keyboard-aware-scroll-view"; +import * as ImagePicker from "expo-image-picker"; import { LinearGradient } from "expo-linear-gradient"; import { StatusBar } from "expo-status-bar"; @@ -21,6 +23,7 @@ export default function ChatScreen({ navigation, route }) { const [inputMessage, setInputMessage] = useState(""); const [isLoading, setIsLoading] = useState(false); const [sessionId, setSessionId] = useState(null); + const [selectedImage, setSelectedImage] = useState(null); const scrollViewRef = useRef(); const textInputRef = useRef(); @@ -69,24 +72,28 @@ export default function ChatScreen({ navigation, route }) { }; const sendMessage = async () => { - if (!inputMessage.trim() || !sessionId || isLoading) return; + if ((!inputMessage.trim() && !selectedImage) || !sessionId || isLoading) return; const userMessage = { id: `temp-${Date.now()}`, - message: inputMessage.trim(), + message: inputMessage.trim() || "📷 Shared an image", is_user: true, + image_url: selectedImage?.uri, created_at: new Date().toISOString(), }; setMessages((prev) => [...prev, userMessage]); - const messageToSend = inputMessage.trim(); + const messageToSend = inputMessage.trim() || "What can you tell me about this image?"; + const imageToSend = selectedImage; setInputMessage(""); + setSelectedImage(null); setIsLoading(true); try { const aiResponse = await ApiService.sendChatMessage( sessionId, - messageToSend + messageToSend, + imageToSend ); setMessages((prev) => [ @@ -107,6 +114,37 @@ export default function ChatScreen({ navigation, route }) { } }; + const pickImage = async () => { + try { + // Request permission + const permissionResult = await ImagePicker.requestMediaLibraryPermissionsAsync(); + + if (permissionResult.granted === false) { + Alert.alert("Permission Required", "Please allow access to your photo library to upload images."); + return; + } + + // Pick image + const result = await ImagePicker.launchImageLibraryAsync({ + mediaTypes: ImagePicker.MediaTypeOptions.Images, + allowsEditing: true, + aspect: [4, 3], + quality: 0.8, + }); + + if (!result.canceled && result.assets && result.assets.length > 0) { + setSelectedImage({ + uri: result.assets[0].uri, + mimeType: result.assets[0].mimeType || 'image/jpeg', + fileName: result.assets[0].fileName || `image_${Date.now()}.jpg`, + }); + } + } catch (error) { + console.error("Image picker error:", error); + Alert.alert("Error", "Failed to pick image. Please try again."); + } + }; + const formatTime = (dateString) => { const date = new Date(dateString); return date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }); @@ -147,6 +185,13 @@ export default function ChatScreen({ navigation, route }) { : "rgba(255, 255, 255, 0.08)", }} > + {message.image_url && ( + + )} + {/* Image Preview */} + {selectedImage && ( + + + + + + Image selected + + setSelectedImage(null)} + className="w-8 h-8 rounded-full items-center justify-center" + style={{ + backgroundColor: "rgba(255, 255, 255, 0.05)", + borderWidth: 1, + borderColor: "rgba(255, 255, 255, 0.1)", + }} + > + + + + )} + + + + + None: + # Add image_url column to chat_messages table + op.add_column('chat_messages', sa.Column('image_url', sa.String(length=500), nullable=True)) + + +def downgrade() -> None: + # Remove image_url column from chat_messages table + op.drop_column('chat_messages', 'image_url') diff --git a/fastapi-backend/app/crud/chat.py b/fastapi-backend/app/crud/chat.py index d4fe1a3..ad46d26 100644 --- a/fastapi-backend/app/crud/chat.py +++ b/fastapi-backend/app/crud/chat.py @@ -45,7 +45,8 @@ def add_message_to_session( session_id: UUID, message: str, is_user: bool, - user_id: int + user_id: int, + image_url: Optional[str] = None ) -> ChatMessage: """Add a message to a chat session.""" # Verify session belongs to user @@ -56,7 +57,8 @@ def add_message_to_session( chat_message = ChatMessage( session_id=session_id, message=message, - is_user=is_user + is_user=is_user, + image_url=image_url ) db.add(chat_message) diff --git a/fastapi-backend/app/models/chat.py b/fastapi-backend/app/models/chat.py index 9d0ad16..dec4d72 100644 --- a/fastapi-backend/app/models/chat.py +++ b/fastapi-backend/app/models/chat.py @@ -27,6 +27,7 @@ class ChatMessage(Base): session_id = Column(UUID(as_uuid=True), ForeignKey("chat_sessions.id"), nullable=False) message = Column(Text, nullable=False) is_user = Column(Boolean, nullable=False) # True for user, False for AI + image_url = Column(String(500), nullable=True) # Optional image attachment created_at = Column(DateTime(timezone=True), server_default=func.now()) # Relationships diff --git a/fastapi-backend/app/routers/chat.py b/fastapi-backend/app/routers/chat.py index 598f4c1..14c10d2 100644 --- a/fastapi-backend/app/routers/chat.py +++ b/fastapi-backend/app/routers/chat.py @@ -1,7 +1,8 @@ -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form from sqlalchemy.orm import Session from typing import List, Optional from uuid import UUID +import base64 from app.core.database import get_db from app.schemas.chat import ( @@ -95,6 +96,7 @@ async def get_chat_session_detail( id=msg.id, message=msg.message, is_user=msg.is_user, + image_url=msg.image_url, created_at=msg.created_at ) for msg in session.messages @@ -112,11 +114,12 @@ async def get_chat_session_detail( @router.post("/sessions/{session_id}/messages", response_model=ChatMessageResponse) async def send_message( session_id: UUID, - message_data: ChatMessageCreate, + message: str = Form(...), + image: Optional[UploadFile] = File(None), current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db) ): - """Send a message and get AI response.""" + """Send a message and get AI response. Supports optional image upload.""" try: # Verify session exists @@ -124,13 +127,23 @@ async def send_message( if not session: raise HTTPException(status_code=404, detail="Chat session not found") - # Add user message + # Process image if provided + image_data = None + image_url = None + if image: + image_data = await image.read() + # Store image as base64 data URL for simplicity + image_base64 = base64.b64encode(image_data).decode('utf-8') + image_url = f"data:{image.content_type};base64,{image_base64}" + + # Add user message with image user_message = add_message_to_session( db=db, session_id=session_id, - message=message_data.message, + message=message, is_user=True, - user_id=current_user.id + user_id=current_user.id, + image_url=image_url ) # Get conversation context @@ -159,13 +172,14 @@ async def send_message( enhanced_skin_concerns = f"{current_user.skin_concerns or ''}\n{allergen_context}\n{issue_context}".strip() - # Generate AI response using Gemini with enhanced context + # Generate AI response using Gemini with enhanced context and optional image gemini_service = GeminiChatService() ai_response = gemini_service.generate_chat_response( - user_message=message_data.message, + user_message=message, skin_type=current_user.skin_type, skin_concerns=enhanced_skin_concerns, - conversation_history=recent_messages + conversation_history=recent_messages, + image_data=image_data ) # Add AI response @@ -180,13 +194,14 @@ async def send_message( # Extract insights and update skin memory gemini_service = GeminiChatService() await gemini_service._extract_and_update_memory( - db, current_user.id, message_data.message, ai_response + db, current_user.id, message, ai_response ) return ChatMessageResponse( id=ai_message.id, message=ai_message.message, is_user=ai_message.is_user, + image_url=ai_message.image_url, created_at=ai_message.created_at ) diff --git a/fastapi-backend/app/schemas/chat.py b/fastapi-backend/app/schemas/chat.py index 6fc3b3c..f39d684 100644 --- a/fastapi-backend/app/schemas/chat.py +++ b/fastapi-backend/app/schemas/chat.py @@ -5,11 +5,13 @@ class ChatMessageCreate(BaseModel): message: str + image_url: Optional[str] = None class ChatMessageResponse(BaseModel): id: UUID message: str is_user: bool + image_url: Optional[str] = None created_at: datetime class Config: diff --git a/fastapi-backend/app/services/gemini_chat.py b/fastapi-backend/app/services/gemini_chat.py index 8ec9a42..7690ac3 100644 --- a/fastapi-backend/app/services/gemini_chat.py +++ b/fastapi-backend/app/services/gemini_chat.py @@ -75,7 +75,8 @@ def generate_chat_response( user_message: str, skin_type: str = None, skin_concerns: str = None, - conversation_history: List = None + conversation_history: List = None, + image_data: bytes = None ) -> str: """Generate AI chat response - this method is called by the router""" try: @@ -110,14 +111,24 @@ def generate_chat_response( 4. Always prioritize safety and suggest consulting dermatologists for serious issues 5. Keep responses concise but informative 6. If you detect new skin issues or concerns, acknowledge them appropriately +7. If an image is provided, analyze it in the context of skincare (product, skin condition, etc.) Current message from user: {user_message} +{"The user has also shared an image for analysis." if image_data else ""} Please provide a helpful response: """ - # Generate AI response - response = self.model.generate_content(system_prompt) + # Generate AI response with or without image + if image_data: + # Convert bytes to PIL Image + from PIL import Image + import io + image = Image.open(io.BytesIO(image_data)) + response = self.model.generate_content([system_prompt, image]) + else: + response = self.model.generate_content(system_prompt) + return response.text except Exception as e: diff --git a/readme.md b/readme.md index 7e7e049..21f8ede 100644 --- a/readme.md +++ b/readme.md @@ -47,6 +47,8 @@ - **Contextual Advice**: Personalized recommendations based on your skin profile - **Product Suggestions**: Smart recommendations for your specific needs - **Memory Integration**: AI remembers your history for better advice +- **Image Upload Support**: Share product photos or skin selfies for visual analysis +- **Multi-Modal Analysis**: Combine text and images for comprehensive skincare advice ### 📊 **Skin Memory System** - **Issue Tracking**: Monitor skin problems and their progress @@ -260,6 +262,10 @@ PUT /api/v1/skin/profile POST /api/v1/chat/sessions GET /api/v1/chat/sessions POST /api/v1/chat/{session_id}/messages + # Supports multipart/form-data with optional image upload + # Parameters: + # - message: string (required) + # - image: file (optional) ``` ### Response Format