diff --git a/SkinSenseAI/src/screens/ChatScreen.js b/SkinSenseAI/src/screens/ChatScreen.js
index 756cffc..405d61c 100644
--- a/SkinSenseAI/src/screens/ChatScreen.js
+++ b/SkinSenseAI/src/screens/ChatScreen.js
@@ -7,8 +7,10 @@ import {
KeyboardAvoidingView,
Platform,
Alert,
+ Image,
} from "react-native";
import { KeyboardAwareScrollView } from "react-native-keyboard-aware-scroll-view";
+import * as ImagePicker from "expo-image-picker";
import { LinearGradient } from "expo-linear-gradient";
import { StatusBar } from "expo-status-bar";
@@ -21,6 +23,7 @@ export default function ChatScreen({ navigation, route }) {
const [inputMessage, setInputMessage] = useState("");
const [isLoading, setIsLoading] = useState(false);
const [sessionId, setSessionId] = useState(null);
+ const [selectedImage, setSelectedImage] = useState(null);
const scrollViewRef = useRef();
const textInputRef = useRef();
@@ -69,24 +72,28 @@ export default function ChatScreen({ navigation, route }) {
};
const sendMessage = async () => {
- if (!inputMessage.trim() || !sessionId || isLoading) return;
+ if ((!inputMessage.trim() && !selectedImage) || !sessionId || isLoading) return;
const userMessage = {
id: `temp-${Date.now()}`,
- message: inputMessage.trim(),
+ message: inputMessage.trim() || "📷 Shared an image",
is_user: true,
+ image_url: selectedImage?.uri,
created_at: new Date().toISOString(),
};
setMessages((prev) => [...prev, userMessage]);
- const messageToSend = inputMessage.trim();
+ const messageToSend = inputMessage.trim() || "What can you tell me about this image?";
+ const imageToSend = selectedImage;
setInputMessage("");
+ setSelectedImage(null);
setIsLoading(true);
try {
const aiResponse = await ApiService.sendChatMessage(
sessionId,
- messageToSend
+ messageToSend,
+ imageToSend
);
setMessages((prev) => [
@@ -107,6 +114,37 @@ export default function ChatScreen({ navigation, route }) {
}
};
+ const pickImage = async () => {
+ try {
+ // Request permission
+ const permissionResult = await ImagePicker.requestMediaLibraryPermissionsAsync();
+
+ if (permissionResult.granted === false) {
+ Alert.alert("Permission Required", "Please allow access to your photo library to upload images.");
+ return;
+ }
+
+ // Pick image
+ const result = await ImagePicker.launchImageLibraryAsync({
+ mediaTypes: ImagePicker.MediaTypeOptions.Images,
+ allowsEditing: true,
+ aspect: [4, 3],
+ quality: 0.8,
+ });
+
+ if (!result.canceled && result.assets && result.assets.length > 0) {
+ setSelectedImage({
+ uri: result.assets[0].uri,
+ mimeType: result.assets[0].mimeType || 'image/jpeg',
+ fileName: result.assets[0].fileName || `image_${Date.now()}.jpg`,
+ });
+ }
+ } catch (error) {
+ console.error("Image picker error:", error);
+ Alert.alert("Error", "Failed to pick image. Please try again.");
+ }
+ };
+
const formatTime = (dateString) => {
const date = new Date(dateString);
return date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" });
@@ -147,6 +185,13 @@ export default function ChatScreen({ navigation, route }) {
: "rgba(255, 255, 255, 0.08)",
}}
>
+ {message.image_url && (
+
+ )}
+ {/* Image Preview */}
+ {selectedImage && (
+
+
+
+
+
+ Image selected
+
+ setSelectedImage(null)}
+ className="w-8 h-8 rounded-full items-center justify-center"
+ style={{
+ backgroundColor: "rgba(255, 255, 255, 0.05)",
+ borderWidth: 1,
+ borderColor: "rgba(255, 255, 255, 0.1)",
+ }}
+ >
+
+
+
+ )}
+
+
+
+
+
None:
+ # Add image_url column to chat_messages table
+ op.add_column('chat_messages', sa.Column('image_url', sa.String(length=500), nullable=True))
+
+
+def downgrade() -> None:
+ # Remove image_url column from chat_messages table
+ op.drop_column('chat_messages', 'image_url')
diff --git a/fastapi-backend/app/crud/chat.py b/fastapi-backend/app/crud/chat.py
index d4fe1a3..ad46d26 100644
--- a/fastapi-backend/app/crud/chat.py
+++ b/fastapi-backend/app/crud/chat.py
@@ -45,7 +45,8 @@ def add_message_to_session(
session_id: UUID,
message: str,
is_user: bool,
- user_id: int
+ user_id: int,
+ image_url: Optional[str] = None
) -> ChatMessage:
"""Add a message to a chat session."""
# Verify session belongs to user
@@ -56,7 +57,8 @@ def add_message_to_session(
chat_message = ChatMessage(
session_id=session_id,
message=message,
- is_user=is_user
+ is_user=is_user,
+ image_url=image_url
)
db.add(chat_message)
diff --git a/fastapi-backend/app/models/chat.py b/fastapi-backend/app/models/chat.py
index 9d0ad16..dec4d72 100644
--- a/fastapi-backend/app/models/chat.py
+++ b/fastapi-backend/app/models/chat.py
@@ -27,6 +27,7 @@ class ChatMessage(Base):
session_id = Column(UUID(as_uuid=True), ForeignKey("chat_sessions.id"), nullable=False)
message = Column(Text, nullable=False)
is_user = Column(Boolean, nullable=False) # True for user, False for AI
+ image_url = Column(String(500), nullable=True) # Optional image attachment
created_at = Column(DateTime(timezone=True), server_default=func.now())
# Relationships
diff --git a/fastapi-backend/app/routers/chat.py b/fastapi-backend/app/routers/chat.py
index 598f4c1..14c10d2 100644
--- a/fastapi-backend/app/routers/chat.py
+++ b/fastapi-backend/app/routers/chat.py
@@ -1,7 +1,8 @@
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form
from sqlalchemy.orm import Session
from typing import List, Optional
from uuid import UUID
+import base64
from app.core.database import get_db
from app.schemas.chat import (
@@ -95,6 +96,7 @@ async def get_chat_session_detail(
id=msg.id,
message=msg.message,
is_user=msg.is_user,
+ image_url=msg.image_url,
created_at=msg.created_at
)
for msg in session.messages
@@ -112,11 +114,12 @@ async def get_chat_session_detail(
@router.post("/sessions/{session_id}/messages", response_model=ChatMessageResponse)
async def send_message(
session_id: UUID,
- message_data: ChatMessageCreate,
+ message: str = Form(...),
+ image: Optional[UploadFile] = File(None),
current_user: User = Depends(get_current_active_user),
db: Session = Depends(get_db)
):
- """Send a message and get AI response."""
+ """Send a message and get AI response. Supports optional image upload."""
try:
# Verify session exists
@@ -124,13 +127,23 @@ async def send_message(
if not session:
raise HTTPException(status_code=404, detail="Chat session not found")
- # Add user message
+ # Process image if provided
+ image_data = None
+ image_url = None
+ if image:
+ image_data = await image.read()
+ # Store image as base64 data URL for simplicity
+ image_base64 = base64.b64encode(image_data).decode('utf-8')
+ image_url = f"data:{image.content_type};base64,{image_base64}"
+
+ # Add user message with image
user_message = add_message_to_session(
db=db,
session_id=session_id,
- message=message_data.message,
+ message=message,
is_user=True,
- user_id=current_user.id
+ user_id=current_user.id,
+ image_url=image_url
)
# Get conversation context
@@ -159,13 +172,14 @@ async def send_message(
enhanced_skin_concerns = f"{current_user.skin_concerns or ''}\n{allergen_context}\n{issue_context}".strip()
- # Generate AI response using Gemini with enhanced context
+ # Generate AI response using Gemini with enhanced context and optional image
gemini_service = GeminiChatService()
ai_response = gemini_service.generate_chat_response(
- user_message=message_data.message,
+ user_message=message,
skin_type=current_user.skin_type,
skin_concerns=enhanced_skin_concerns,
- conversation_history=recent_messages
+ conversation_history=recent_messages,
+ image_data=image_data
)
# Add AI response
@@ -180,13 +194,14 @@ async def send_message(
# Extract insights and update skin memory
gemini_service = GeminiChatService()
await gemini_service._extract_and_update_memory(
- db, current_user.id, message_data.message, ai_response
+ db, current_user.id, message, ai_response
)
return ChatMessageResponse(
id=ai_message.id,
message=ai_message.message,
is_user=ai_message.is_user,
+ image_url=ai_message.image_url,
created_at=ai_message.created_at
)
diff --git a/fastapi-backend/app/schemas/chat.py b/fastapi-backend/app/schemas/chat.py
index 6fc3b3c..f39d684 100644
--- a/fastapi-backend/app/schemas/chat.py
+++ b/fastapi-backend/app/schemas/chat.py
@@ -5,11 +5,13 @@
class ChatMessageCreate(BaseModel):
message: str
+ image_url: Optional[str] = None
class ChatMessageResponse(BaseModel):
id: UUID
message: str
is_user: bool
+ image_url: Optional[str] = None
created_at: datetime
class Config:
diff --git a/fastapi-backend/app/services/gemini_chat.py b/fastapi-backend/app/services/gemini_chat.py
index 8ec9a42..7690ac3 100644
--- a/fastapi-backend/app/services/gemini_chat.py
+++ b/fastapi-backend/app/services/gemini_chat.py
@@ -75,7 +75,8 @@ def generate_chat_response(
user_message: str,
skin_type: str = None,
skin_concerns: str = None,
- conversation_history: List = None
+ conversation_history: List = None,
+ image_data: bytes = None
) -> str:
"""Generate AI chat response - this method is called by the router"""
try:
@@ -110,14 +111,24 @@ def generate_chat_response(
4. Always prioritize safety and suggest consulting dermatologists for serious issues
5. Keep responses concise but informative
6. If you detect new skin issues or concerns, acknowledge them appropriately
+7. If an image is provided, analyze it in the context of skincare (product, skin condition, etc.)
Current message from user: {user_message}
+{"The user has also shared an image for analysis." if image_data else ""}
Please provide a helpful response:
"""
- # Generate AI response
- response = self.model.generate_content(system_prompt)
+ # Generate AI response with or without image
+ if image_data:
+ # Convert bytes to PIL Image
+ from PIL import Image
+ import io
+ image = Image.open(io.BytesIO(image_data))
+ response = self.model.generate_content([system_prompt, image])
+ else:
+ response = self.model.generate_content(system_prompt)
+
return response.text
except Exception as e:
diff --git a/readme.md b/readme.md
index 7e7e049..21f8ede 100644
--- a/readme.md
+++ b/readme.md
@@ -47,6 +47,8 @@
- **Contextual Advice**: Personalized recommendations based on your skin profile
- **Product Suggestions**: Smart recommendations for your specific needs
- **Memory Integration**: AI remembers your history for better advice
+- **Image Upload Support**: Share product photos or skin selfies for visual analysis
+- **Multi-Modal Analysis**: Combine text and images for comprehensive skincare advice
### 📊 **Skin Memory System**
- **Issue Tracking**: Monitor skin problems and their progress
@@ -260,6 +262,10 @@ PUT /api/v1/skin/profile
POST /api/v1/chat/sessions
GET /api/v1/chat/sessions
POST /api/v1/chat/{session_id}/messages
+ # Supports multipart/form-data with optional image upload
+ # Parameters:
+ # - message: string (required)
+ # - image: file (optional)
```
### Response Format