Add Image Upload Support to AI Chat for Enhanced Skincare Analysis

SkinSenseAI/src/screens/ChatScreen.js

@@ -7,8 +7,10 @@ import {
   KeyboardAvoidingView,
   Platform,
   Alert,
+  Image,
 } from "react-native";
 import { KeyboardAwareScrollView } from "react-native-keyboard-aware-scroll-view";
+import * as ImagePicker from "expo-image-picker";
 
 import { LinearGradient } from "expo-linear-gradient";
 import { StatusBar } from "expo-status-bar";
@@ -21,6 +23,7 @@ export default function ChatScreen({ navigation, route }) {
   const [inputMessage, setInputMessage] = useState("");
   const [isLoading, setIsLoading] = useState(false);
   const [sessionId, setSessionId] = useState(null);
+  const [selectedImage, setSelectedImage] = useState(null);
   const scrollViewRef = useRef();
   const textInputRef = useRef();
 
@@ -69,24 +72,28 @@ export default function ChatScreen({ navigation, route }) {
   };
 
   const sendMessage = async () => {
-    if (!inputMessage.trim() || !sessionId || isLoading) return;
+    if ((!inputMessage.trim() && !selectedImage) || !sessionId || isLoading) return;
 
     const userMessage = {
       id: `temp-${Date.now()}`,
-      message: inputMessage.trim(),
+      message: inputMessage.trim() || "📷 Shared an image",
       is_user: true,
+      image_url: selectedImage?.uri,
       created_at: new Date().toISOString(),
     };
 
     setMessages((prev) => [...prev, userMessage]);
-    const messageToSend = inputMessage.trim();
+    const messageToSend = inputMessage.trim() || "What can you tell me about this image?";
+    const imageToSend = selectedImage;
     setInputMessage("");
+    setSelectedImage(null);
     setIsLoading(true);
 
     try {
       const aiResponse = await ApiService.sendChatMessage(
         sessionId,
-        messageToSend
+        messageToSend,
+        imageToSend
       );
 
       setMessages((prev) => [
@@ -107,6 +114,37 @@ export default function ChatScreen({ navigation, route }) {
     }
   };
 
+  const pickImage = async () => {
+    try {
+      // Request permission
+      const permissionResult = await ImagePicker.requestMediaLibraryPermissionsAsync();
+
+      if (permissionResult.granted === false) {
+        Alert.alert("Permission Required", "Please allow access to your photo library to upload images.");
+        return;
+      }
+
+      // Pick image
+      const result = await ImagePicker.launchImageLibraryAsync({
+        mediaTypes: ImagePicker.MediaTypeOptions.Images,
+        allowsEditing: true,
+        aspect: [4, 3],
+        quality: 0.8,
+      });
+
+      if (!result.canceled && result.assets && result.assets.length > 0) {
+        setSelectedImage({
+          uri: result.assets[0].uri,
+          mimeType: result.assets[0].mimeType || 'image/jpeg',
+          fileName: result.assets[0].fileName || `image_${Date.now()}.jpg`,
+        });
+      }
+    } catch (error) {
+      console.error("Image picker error:", error);
+      Alert.alert("Error", "Failed to pick image. Please try again.");
+    }
+  };
+
   const formatTime = (dateString) => {
     const date = new Date(dateString);
     return date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" });
@@ -147,6 +185,13 @@ export default function ChatScreen({ navigation, route }) {
               : "rgba(255, 255, 255, 0.08)",
           }}
         >
+          {message.image_url && (
+            <Image
+              source={{ uri: message.image_url }}
+              className="w-full h-48 rounded-lg mb-2"
+              resizeMode="cover"
+            />
+          )}
           <Text
             className={`text-base leading-relaxed ${
               isUser ? "text-cyan-100" : "text-gray-100"
@@ -288,7 +333,54 @@ export default function ChatScreen({ navigation, route }) {
               borderTopColor: "rgba(255, 255, 255, 0.1)",
             }}
           >
+            {/* Image Preview */}
+            {selectedImage && (
+              <View className="mb-3 flex-row items-center">
+                <View
+                  className="rounded-xl overflow-hidden mr-2"
+                  style={{
+                    borderWidth: 1,
+                    borderColor: "rgba(0, 245, 255, 0.3)",
+                  }}
+                >
+                  <Image
+                    source={{ uri: selectedImage.uri }}
+                    className="w-16 h-16"
+                    resizeMode="cover"
+                  />
+                </View>
+                <Text className="text-gray-400 text-sm flex-1">
+                  Image selected
+                </Text>
+                <TouchableOpacity
+                  onPress={() => setSelectedImage(null)}
+                  className="w-8 h-8 rounded-full items-center justify-center"
+                  style={{
+                    backgroundColor: "rgba(255, 255, 255, 0.05)",
+                    borderWidth: 1,
+                    borderColor: "rgba(255, 255, 255, 0.1)",
+                  }}
+                >
+                  <Ionicons name="close" size={16} color="white" />
+                </TouchableOpacity>
+              </View>
+            )}
+
             <View className="flex-row items-end space-x-3">
+              <TouchableOpacity
+                onPress={pickImage}
+                disabled={isLoading}
+                className="w-12 h-12 rounded-full items-center justify-center"
+                style={{
+                  backgroundColor: "rgba(0, 245, 255, 0.1)",
+                  borderWidth: 1,
+                  borderColor: "rgba(0, 245, 255, 0.2)",
+                  opacity: isLoading ? 0.5 : 1,
+                }}
+              >
+                <Ionicons name="image-outline" size={20} color="#00f5ff" />
+              </TouchableOpacity>
+
               <View
                 className="flex-1 rounded-2xl px-4 py-3"
                 style={{
@@ -311,10 +403,10 @@ export default function ChatScreen({ navigation, route }) {
 
               <TouchableOpacity
                 onPress={sendMessage}
-                disabled={!inputMessage.trim() || isLoading}
+                disabled={(!inputMessage.trim() && !selectedImage) || isLoading}
                 className="w-12 h-12 rounded-full items-center justify-center overflow-hidden"
                 style={{
-                  opacity: !inputMessage.trim() || isLoading ? 0.5 : 1,
+                  opacity: (!inputMessage.trim() && !selectedImage) || isLoading ? 0.5 : 1,
                 }}
               >
                 <LinearGradient

SkinSenseAI/src/services/api.js

@@ -481,14 +481,34 @@ class ApiService {
     }
   }
 
-  async sendChatMessage(sessionId, message) {
+  async sendChatMessage(sessionId, message, imageFile = null) {
     try {
-      return await this.makeRequest(`/chat/sessions/${sessionId}/messages`, {
-        method: "POST",
-        body: JSON.stringify({
-          message: message,
-        }),
+      const formData = new FormData();
+      formData.append('message', message);
+
+      if (imageFile) {
+        formData.append('image', {
+          uri: imageFile.uri,
+          type: imageFile.mimeType || 'image/jpeg',
+          name: imageFile.fileName || 'chat_image.jpg',
+        });
+      }
+
+      const response = await fetch(`${this.baseURL}/chat/sessions/${sessionId}/messages`, {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${await this.getAuthToken()}`,
+          // Don't set Content-Type for FormData, let the browser set it with boundary
+        },
+        body: formData,
       });
+
+      if (!response.ok) {
+        const errorData = await response.json();
+        throw new Error(errorData.detail || `HTTP error! status: ${response.status}`);
+      }
+
+      return await response.json();
     } catch (error) {
       console.error("Send chat message error:", error);
       throw error;

fastapi-backend/alembic/versions/b1c2d3e4f5a6_add_image_url_to_chat_messages.py

@@ -0,0 +1,27 @@
+"""Add image_url to chat_messages
+
+Revision ID: b1c2d3e4f5a6
+Revises: a16f8f6c4860
+Create Date: 2025-10-09 23:44:00.000000
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision: str = 'b1c2d3e4f5a6'
+down_revision: Union[str, None] = 'a16f8f6c4860'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add image_url column to chat_messages table
+    op.add_column('chat_messages', sa.Column('image_url', sa.String(length=500), nullable=True))
+
+
+def downgrade() -> None:
+    # Remove image_url column from chat_messages table
+    op.drop_column('chat_messages', 'image_url')

fastapi-backend/app/crud/chat.py

@@ -45,7 +45,8 @@ def add_message_to_session(
     session_id: UUID, 
     message: str, 
     is_user: bool,
-    user_id: int
+    user_id: int,
+    image_url: Optional[str] = None
 ) -> ChatMessage:
     """Add a message to a chat session."""
     # Verify session belongs to user
@@ -56,7 +57,8 @@ def add_message_to_session(
     chat_message = ChatMessage(
         session_id=session_id,
         message=message,
-        is_user=is_user
+        is_user=is_user,
+        image_url=image_url
     )
     db.add(chat_message)
 

fastapi-backend/app/models/chat.py

@@ -27,6 +27,7 @@ class ChatMessage(Base):
     session_id = Column(UUID(as_uuid=True), ForeignKey("chat_sessions.id"), nullable=False)
     message = Column(Text, nullable=False)
     is_user = Column(Boolean, nullable=False)  # True for user, False for AI
+    image_url = Column(String(500), nullable=True)  # Optional image attachment
     created_at = Column(DateTime(timezone=True), server_default=func.now())
 
     # Relationships

fastapi-backend/app/routers/chat.py

@@ -1,7 +1,8 @@
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form
 from sqlalchemy.orm import Session
 from typing import List, Optional
 from uuid import UUID
+import base64
 
 from app.core.database import get_db
 from app.schemas.chat import (
@@ -95,6 +96,7 @@ async def get_chat_session_detail(
             id=msg.id,
             message=msg.message,
             is_user=msg.is_user,
+            image_url=msg.image_url,
             created_at=msg.created_at
         )
         for msg in session.messages
@@ -112,25 +114,36 @@ async def get_chat_session_detail(
 @router.post("/sessions/{session_id}/messages", response_model=ChatMessageResponse)
 async def send_message(
     session_id: UUID,
-    message_data: ChatMessageCreate,
+    message: str = Form(...),
+    image: Optional[UploadFile] = File(None),
     current_user: User = Depends(get_current_active_user),
     db: Session = Depends(get_db)
 ):
-    """Send a message and get AI response."""
+    """Send a message and get AI response. Supports optional image upload."""
 
     try:
         # Verify session exists
         session = get_chat_session(db, session_id, current_user.id)
         if not session:
             raise HTTPException(status_code=404, detail="Chat session not found")
 
-        # Add user message
+        # Process image if provided
+        image_data = None
+        image_url = None
+        if image:
+            image_data = await image.read()
+            # Store image as base64 data URL for simplicity
+            image_base64 = base64.b64encode(image_data).decode('utf-8')
+            image_url = f"data:{image.content_type};base64,{image_base64}"
+
+        # Add user message with image
         user_message = add_message_to_session(
             db=db,
             session_id=session_id,
-            message=message_data.message,
+            message=message,
             is_user=True,
-            user_id=current_user.id
+            user_id=current_user.id,
+            image_url=image_url
         )
 
         # Get conversation context
@@ -159,13 +172,14 @@ async def send_message(
 
         enhanced_skin_concerns = f"{current_user.skin_concerns or ''}\n{allergen_context}\n{issue_context}".strip()
 
-        # Generate AI response using Gemini with enhanced context
+        # Generate AI response using Gemini with enhanced context and optional image
         gemini_service = GeminiChatService()
         ai_response = gemini_service.generate_chat_response(
-            user_message=message_data.message,
+            user_message=message,
             skin_type=current_user.skin_type,
             skin_concerns=enhanced_skin_concerns,
-            conversation_history=recent_messages
+            conversation_history=recent_messages,
+            image_data=image_data
         )
 
         # Add AI response
@@ -180,13 +194,14 @@ async def send_message(
         # Extract insights and update skin memory
         gemini_service = GeminiChatService()
         await gemini_service._extract_and_update_memory(
-            db, current_user.id, message_data.message, ai_response
+            db, current_user.id, message, ai_response
         )
 
         return ChatMessageResponse(
             id=ai_message.id,
             message=ai_message.message,
             is_user=ai_message.is_user,
+            image_url=ai_message.image_url,
             created_at=ai_message.created_at
         )
 

fastapi-backend/app/schemas/chat.py

@@ -5,11 +5,13 @@
 
 class ChatMessageCreate(BaseModel):
     message: str
+    image_url: Optional[str] = None
 
 class ChatMessageResponse(BaseModel):
     id: UUID
     message: str
     is_user: bool
+    image_url: Optional[str] = None
     created_at: datetime
 
     class Config: