diff --git a/Backend/ar_model/pose_service.py b/Backend/ar_model/pose_service.py new file mode 100644 index 00000000..deb38824 --- /dev/null +++ b/Backend/ar_model/pose_service.py @@ -0,0 +1,69 @@ +from flask import Flask, request, jsonify +import cv2 +import mediapipe as mp +import base64 +import numpy as np +import logging +import traceback + +# Set up logging for debugging and error tracking +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + +app = Flask(__name__) +mp_pose = mp.solutions.pose + +# Initialize MediaPipe Pose model with higher confidence thresholds +pose = mp_pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7) + +@app.route('/process_frame', methods=['POST']) +def process_frame(): + try: + # Log incoming request + logger.info('[DEBUG] Received request to process frame') + + # Get JSON data from request + data = request.json + if not data or 'frame' not in data: + logger.error('[ERROR] Invalid request: Missing frame data') + return jsonify({'error': 'Missing frame data'}), 400 + + # Extract and log frame data + frame_base64 = data['frame'] + logger.debug('[DEBUG] Frame base64 length: %d', len(frame_base64)) + + # Decode base64 string to image + frame = base64.b64decode(frame_base64) + image = cv2.imdecode(np.frombuffer(frame, np.uint8), cv2.IMREAD_COLOR) + if image is None: + logger.error('[ERROR] Failed to decode image') + return jsonify({'error': 'Failed to decode image'}), 400 + + # Log image details + logger.debug('[DEBUG] Image decoded, shape: %s', image.shape) + + # Convert image to RGB for MediaPipe processing + image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + results = pose.process(image_rgb) + + # Process pose detection results + if results.pose_landmarks: + landmarks = [ + {'name': f'landmark_{i}', 'x': lm.x, 'y': lm.y, 'z': lm.z} + for i, lm in enumerate(results.pose_landmarks.landmark) + ] + logger.info('[DEBUG] Pose landmarks detected: %d', len(landmarks)) + return jsonify({'landmarks': landmarks}) + else: + logger.warning('[WARN] No pose landmarks detected') + return jsonify({'landmarks': []}) + + except Exception as e: + # Log detailed error information + logger.error('[ERROR] Error processing frame: %s', str(e)) + logger.error('[ERROR] Traceback: %s', traceback.format_exc()) + return jsonify({'error': str(e)}), 500 + +if __name__ == '__main__': + logger.info('[INFO] Starting pose detection service on http://0.0.0.0:5002') + app.run(host='0.0.0.0', port=5002, debug=True) \ No newline at end of file diff --git a/Backend/controllers/arController.js b/Backend/controllers/arController.js index a730ca29..69b17e66 100644 --- a/Backend/controllers/arController.js +++ b/Backend/controllers/arController.js @@ -89,4 +89,83 @@ const getTherapyDetails = async (req, res) => { } }; -module.exports = { getARRecommendations, getTherapyDetails }; \ No newline at end of file +// Process Frame for Pose Detection API +const processFrame = async (req, res) => { + const { frame } = req.body; + + if (!frame) { + console.error("[ERROR] No frame data provided in request."); + return res.status(400).json({ error: "Frame data is required" }); + } + + console.log("[DEBUG] Received request to process frame"); + + try { + // Forward the frame to the pose detection service + const response = await axios.post('http://localhost:5002/process_frame', { frame }, { + timeout: 5000, + }); + + console.log("[DEBUG] Pose detection service response:", JSON.stringify(response.data, null, 2)); + res.status(200).json(response.data); + } catch (error) { + console.error(`[ERROR] Failed to process frame: ${error.message}`); + console.error("[DEBUG] Stack trace:", error.stack); + res.status(500).json({ error: "Failed to process frame", details: error.message }); + } +}; + +// Fetch target pose landmarks +const getTherapyPoseLandmarks = async (req, res) => { + const { therapyName } = req.params; + + if (!therapyName) { + console.error("[ERROR] No therapy name provided in request."); + return res.status(400).json({ error: "Therapy name is required" }); + } + + console.log(`[DEBUG] Fetching pose landmarks for therapy: ${therapyName}`); + + try { + console.log("[DEBUG] Firestore db object:", db ? "Initialized" : "Not initialized"); + + const normalizedTherapyName = therapyName + .trim() + .replace(/%20/g, " ") + .replace(/-/g, " "); + + console.log(`[DEBUG] Normalized therapy name: ${normalizedTherapyName}`); + + if (!db) { + throw new Error("Firestore db is not initialized"); + } + + const therapyRef = doc(db, "therapies", normalizedTherapyName); + console.log("[DEBUG] Therapy reference created:", therapyRef.path); + + const therapySnapshot = await getDoc(therapyRef); + + if (!therapySnapshot.exists()) { + console.error(`[ERROR] No therapy details found for: ${normalizedTherapyName}`); + return res.status(404).json({ error: `Therapy "${therapyName}" not found in database` }); + } + + const therapyData = therapySnapshot.data(); + console.log("[DEBUG] Fetched therapy data:", JSON.stringify(therapyData, null, 2)); + + // Assuming landmarks are stored as an array of {name, x, y, z} objects + const landmarks = therapyData.landmarks || []; + if (!landmarks.length) { + console.warn(`[WARN] No landmarks found for ${normalizedTherapyName}`); + } + + console.log("[DEBUG] Fetched pose landmarks:", JSON.stringify(landmarks, null, 2)); + res.status(200).json({ landmarks }); + } catch (error) { + console.error(`[ERROR] Failed to fetch therapy pose landmarks: ${error.message}`); + console.error("[DEBUG] Stack trace:", error.stack); + res.status(500).json({ error: "Failed to fetch therapy pose landmarks", details: error.message }); + } +}; + +module.exports = { getARRecommendations, getTherapyDetails, processFrame, getTherapyPoseLandmarks }; \ No newline at end of file diff --git a/Backend/index.js b/Backend/index.js index 2c0a31f8..2d591c55 100644 --- a/Backend/index.js +++ b/Backend/index.js @@ -16,12 +16,15 @@ const { deleteHealthData, } = require("./controllers/healthController"); const { getChatRecommendation } = require("./controllers/chatController"); -const { getARRecommendations, getTherapyDetails } = require("./controllers/arController"); -const fileUpload = require("express-fileupload"); +const { getARRecommendations, getTherapyDetails, processFrame, getTherapyPoseLandmarks } = require("./controllers/arController"); const app = express(); + +// Custom CORS configuration app.use(cors({ origin: "*" })); -app.use(express.json()); + +// Increase body size limit for all routes (or apply to specific routes) +app.use(express.json({ limit: "2mb" })); // Set to 2MB for all requests app.use("/uploads", express.static(path.join(__dirname, "uploads"))); app.use(fileUpload()); @@ -41,6 +44,8 @@ app.delete("/healthData/:userId", deleteHealthData); // AR Therapy Routes app.get("/ar_therapy/:userId", getARRecommendations); app.get("/therapy_details/:therapyName", getTherapyDetails); +app.post("/process_frame", express.json({ limit: "2mb" }), processFrame); +app.get("/therapy_landmarks/:therapyName", getTherapyPoseLandmarks); // Chatbot Routes app.post('/healthChat/:userId', getChatRecommendation); @@ -56,5 +61,9 @@ app.listen(PORT, () => { // Error handling middleware app.use((err, req, res, next) => { console.error("[ERROR] Server Error:", err.stack); - res.status(500).json({ error: "Internal server error", message: err.message }); + if (err.type === 'entity.too.large') { + res.status(413).json({ error: "Payload too large", message: "Request body exceeds the allowed limit (2MB)" }); + } else { + res.status(500).json({ error: "Internal server error", message: err.message }); + } }); \ No newline at end of file diff --git a/CeylonCare/app.json b/CeylonCare/app.json index 67009df4..ac8a71eb 100644 --- a/CeylonCare/app.json +++ b/CeylonCare/app.json @@ -4,29 +4,32 @@ "slug": "CeylonCare", "version": "1.0.0", "assetBundlePatterns": [ - "assets/videos/*" + "**/*"], + "assets": [ + "./assets/ar_poses/Downward_Dog.glb" ], "orientation": "portrait", "icon": "", + "jsEngine": "jsc", "scheme": "myapp", "userInterfaceStyle": "automatic", "newArchEnabled": true, "ios": { "supportsTablet": true, "infoPlist": { - "NSMicrophoneUsageDescription": "We need microphone access to record your voice for the chatbot.", - "ITSAppUsesNonExemptEncryption": false - }, - "bundleIdentifier": "com.amfdo.CeylonCare" + "NSCameraUsageDescription": "This app uses the camera for real-time pose detection to guide your yoga practice.", + "NSMicrophoneUsageDescription": "We need microphone access to record your voice for the chatbot." + } }, "android": { "adaptiveIcon": { - "foregroundImage": "", + "foregroundImage": "./assets/images/logo.png", "backgroundColor": "#ffffff" }, "permissions": [ + "android.permission.CAMERA", "android.permission.RECORD_AUDIO", - "android.permission.CAMERA" + "android.permission.WEBVIEW_MEDIA" ], "package": "com.amfdo.CeylonCare" }, @@ -39,20 +42,22 @@ [ "expo-splash-screen", { - "image": "", + "image": "./assets/images/logo.png", "imageWidth": 200, "resizeMode": "contain", "backgroundColor": "#ffffff" } ], - "expo-asset", - "expo-font", [ "expo-camera", { - "cameraPermission": "Allow this app to use the camera for AR features." + "cameraPermission": "Allow $(PRODUCT_NAME) to access your camera", + "microphonePermission": "Allow $(PRODUCT_NAME) to access your microphone", + "recordAudioAndroid": true } ], + "expo-asset", + "expo-font", "expo-router" ], "extra": { diff --git a/CeylonCare/app/_layout.tsx b/CeylonCare/app/_layout.tsx index eec80e2c..fc0014c2 100644 --- a/CeylonCare/app/_layout.tsx +++ b/CeylonCare/app/_layout.tsx @@ -1,23 +1,43 @@ -import React from 'react'; -import { createStackNavigator } from '@react-navigation/stack'; +import React from "react"; +import { createStackNavigator } from "@react-navigation/stack"; +import { RouteProp } from "@react-navigation/native"; -// Pages - Validate imports -import Register from './pages/Register'; -import SplashScreen from './pages/SplashScreen'; -import Login from './pages/Login'; -import ForgotPassword from './pages/ForgotPassword'; -import Onboarding from './pages/Onboarding'; -import Home from './pages/Home'; -import Profile from './pages/profile/Profile'; -import ProfileDetails from './pages/profile/ProfileDetails'; -import HealthDetails from './pages/profile/HealthDetails'; -import PrivacyPolicy from './pages/profile/PrivacyPolicy'; -import ChatScreen from './pages/Chatbot_component/ChatScreen'; -import TherapyRecommendations from './pages/AR_component/TherapyRecommendations'; -import TherapyDetails from './pages/AR_component/TherapyDetails'; -import ARAvatarScreen from './pages/AR_component/ARAvatarScreen'; +// Pages +import Register from "./pages/Register"; +import SplashScreen from "./pages/SplashScreen"; +import Login from "./pages/Login"; +import ForgotPassword from "./pages/ForgotPassword"; +import Onboarding from "./pages/Onboarding"; +import Home from "./pages/Home"; +import Profile from "./pages/profile/Profile"; +import ProfileDetails from "./pages/profile/ProfileDetails"; +import HealthDetails from "./pages/profile/HealthDetails"; +import PrivacyPolicy from "./pages/profile/PrivacyPolicy"; +import ChatScreen from "./pages/Chatbot_component/ChatScreen"; +import TherapyRecommendations from "./pages/AR_component/TherapyRecommendations"; +import TherapyDetails from "./pages/AR_component/TherapyDetails"; +import ARAvatarScreen from "./pages/AR_component/ARAvatarScreen"; +import CameraTest from "./pages/AR_component/CameraTest"; -const Stack = createStackNavigator(); +type RootStackParamList = { + Splash: undefined; + Register: undefined; + Login: undefined; + ForgotPassword: undefined; + Onboarding: undefined; + Home: undefined; + Profile: undefined; + ProfileDetails: undefined; + HealthDetails: undefined; + PrivacyPolicy: undefined; + TherapyRecommendations: undefined; + TherapyDetails: { therapyName: string }; + ARAvatarScreen: undefined; + CameraTest: undefined; + ChatScreen: undefined; +}; + +const Stack = createStackNavigator(); // Validate all screen components const validateScreenComponent = (name: string, component: any) => { @@ -123,6 +143,13 @@ const StackNavigator = () => { component={ARAvatarScreen} options={{ headerShown: false }} /> + + + {/* Chatbot_Component */} = ({ isAnimating, url }) => { + // Define a dummy texture loader to bypass texture loading + const dummyTextureLoader = { + load: (url: string, onLoad: (texture: THREE.Texture) => void) => { + onLoad(new THREE.Texture()); // Return a blank texture immediately + }, + setCrossOrigin: () => {}, + }; -// Define Recommendation Type -type Recommendation = { - name: string; - ar_pose?: string; - correct_pose_landmarks?: PoseLandmark[]; -}; + const gltf = useGLTF( + url || '', + true, + true, + (loader: any) => { + loader.textureLoader = dummyTextureLoader; + } + ); -// Define Pose Landmark Type -type PoseLandmark = { - name: string; - x: number; - y: number; - z?: number; -}; + const meshRef = useRef(null!); + const { actions, mixer } = useAnimations(gltf.animations, gltf.scene); -const ARAvatarScreen = ({ route, navigation }) => { - const { recommendations = [{ name: route.params?.therapyName, ar_pose: route.params?.arPoseUrl }] } = route.params || {}; - console.log('[DEBUG] Route params received:', JSON.stringify(route.params, null, 2)); + useEffect(() => { + if (gltf.scene) { + gltf.scene.traverse((child) => { + if (child instanceof THREE.Mesh) { + child.material = new THREE.MeshBasicMaterial({ color: 0x00ff00 }); // Green + } + }); + } + }, [gltf]); - // State variables - const [permission, requestPermission] = useCameraPermissions(); - const [isCameraReady, setIsCameraReady] = useState(false); - const [isWebViewValid, setIsWebViewValid] = useState(false); - const [cameraError, setCameraError] = useState(null); - const [webViewError, setWebViewError] = useState(null); - const [isModelLoaded, setIsModelLoaded] = useState(false); - const [isAnimating, setIsAnimating] = useState(false); - const [isWebViewBridgeReady, setIsWebViewBridgeReady] = useState(false); - const [currentPoseIndex, setCurrentPoseIndex] = useState(0); - const [poseRecommendations, setPoseRecommendations] = useState(recommendations); - const [landmarks, setLandmarks] = useState([]); - const [feedback, setFeedback] = useState('Please ensure your full body is visible in the camera frame.'); - const [poseDetectionError, setPoseDetectionError] = useState(null); - const [isPoseWebViewInitialized, setIsPoseWebViewInitialized] = useState(false); - const [useCameraView, setUseCameraView] = useState(true); - const [zoom, setZoom] = useState(0); // Zoom level state - const webViewRef = useRef(null); - const poseWebViewRef = useRef(null); + useEffect(() => { + console.log('[DEBUG] GLTF model loaded with URL:', url, 'GLTF object:', gltf); + if (!gltf.scene) { + console.error('[ERROR] GLTF scene is null or undefined for URL:', url); + // Avoid triggering Alert here to prevent UI display + } + }, [gltf, url]); - // Lock to landscape on mount, unlock on unmount useEffect(() => { - console.log('[DEBUG] ARAvatarScreen component mounted'); - const lockOrientation = async () => { - try { - console.log('[DEBUG] Attempting to lock screen to landscape'); - const currentOrientation = await ScreenOrientation.getOrientationAsync(); - console.log('[DEBUG] Current orientation:', currentOrientation); - await ScreenOrientation.lockAsync(ScreenOrientation.OrientationLock.LANDSCAPE); - console.log('[INFO] Screen successfully locked to landscape'); - } catch (error) { - console.error('[ERROR] Failed to lock orientation:', error.message); + if (isAnimating && mixer && actions) { + const action = actions['WarriorII'] || actions[Object.keys(actions)[0]]; + if (action) { + console.log('[DEBUG] Starting animation'); + action.reset().play(); + } else { + console.warn('[WARN] No valid animation action found'); } - }; + } else if (!isAnimating && mixer) { + console.log('[DEBUG] Stopping animation'); + mixer.stopAllAction(); + } + }, [isAnimating, actions, mixer]); - const unlockOrientation = async () => { - try { - console.log('[DEBUG] Attempting to unlock screen orientation'); - await ScreenOrientation.unlockAsync(); - console.log('[INFO] Screen orientation successfully unlocked'); - } catch (error) { - console.error('[ERROR] Failed to unlock orientation:', error.message); + useFrame((state, delta) => { + if (meshRef.current && gltf.scene && isAnimating && mixer) { + mixer.update(delta); + } + }); + + return gltf.scene ? ( + + ) : ( + + + + + ); +}; + +const ARAvatarScreen: React.FC<{ + route: { params?: { arPoseUrl?: string; therapyName?: string } }; + navigation: { goBack: () => void }; +}> = ({ route, navigation }) => { + const { arPoseUrl = 'https://res.cloudinary.com/dmwaockgw/image/upload/v1741590782/warrior_II_lp2hfq.glb', therapyName = 'Warrior II' } = route.params || {}; + + const [feedback, setFeedback] = useState('Please stand 2-3 meters back.'); + const [isCameraReady, setIsCameraReady] = useState(false); + const [permission, requestPermission] = useCameraPermissions(); + const [isAnimating, setIsAnimating] = useState(false); + const [modelUri, setModelUri] = useState(null); + const intervalRef = useRef | null>(null); + const cameraRef = useRef(null); + + // Custom error logging to capture but not display specific errors + useEffect(() => { + const originalError = console.error; + console.error = (...args) => { + if (typeof args[0] === 'string' && args[0].includes("THREE.GLTFLoader: Couldn't load texture")) { + // Log the error to console for debugging but do not trigger UI feedback + originalError('[SUPPRESSED FROM UI] THREE.GLTFLoader: Couldn’t load texture', ...args.slice(1)); + } else { + originalError(...args); // Pass through other errors normally } }; - - lockOrientation(); return () => { - console.log('[DEBUG] ARAvatarScreen component unmounting'); - unlockOrientation(); + console.error = originalError; // Restore original function on cleanup }; }, []); - // Validate components - useEffect(() => { - console.log('[DEBUG] Validating essential components'); - if (!Camera.CameraView) { - console.error('[ERROR] CameraView component unavailable'); - setCameraError('Camera component unavailable'); - } else { - console.log('[INFO] CameraView component validated'); - } - if (!WebView) { - console.error('[ERROR] WebView component unavailable'); - setIsWebViewValid(false); - } else { - console.log('[INFO] WebView component validated'); - setIsWebViewValid(true); - } - if (!navigation?.navigate) { - console.error('[ERROR] Navigation prop is invalid'); - } else { - console.log('[INFO] Navigation prop validated'); - } - }, [navigation]); - - // Fetch therapy details from backend API useEffect(() => { - const fetchTherapyDetails = async () => { - console.log('[DEBUG] Fetching therapy details for recommendations'); - try { - const updatedRecommendations = await Promise.all( - recommendations.map(async (therapy: Recommendation) => { - console.log('[DEBUG] Fetching details for therapy:', therapy.name); - try { - const response = await axios.get(`http://192.168.60.107:5000/therapy_details/${encodeURIComponent(therapy.name)}`, { - timeout: 10000, - }); - console.log('[DEBUG] API response for therapy details:', JSON.stringify(response.data, null, 2)); - const data = response.data; - if (!data.ar_pose) { - console.warn('[WARN] No ar_pose provided for:', therapy.name); - } - if (!data.correct_pose_landmarks || !data.correct_pose_landmarks.length) { - console.warn('[WARN] No correct_pose_landmarks provided for:', therapy.name); - } - return { - ...therapy, - ar_pose: data.ar_pose || therapy.ar_pose, - correct_pose_landmarks: data.correct_pose_landmarks || [], - }; - } catch (error) { - console.error('[ERROR] Failed to fetch details for therapy:', therapy.name, error.message); - return therapy; - } - }) - ); - setPoseRecommendations(updatedRecommendations); - console.log('[INFO] Updated recommendations with therapy details:', JSON.stringify(updatedRecommendations, null, 2)); - } catch (error) { - console.error('[ERROR] Failed to fetch therapy details:', error.message); + const loadAsset = async () => { + if (arPoseUrl && arPoseUrl.startsWith('http')) { + setModelUri(arPoseUrl); + } else { + setModelUri(null); } }; + loadAsset(); + }, [arPoseUrl]); - if (recommendations.length > 0) { - console.log('[DEBUG] Recommendations length > 0, initiating fetch'); - fetchTherapyDetails(); - } else { - console.warn('[WARN] No recommendations provided'); - } - }, [recommendations]); - - // Handle camera permission useEffect(() => { - if (!permission) { - console.log('[DEBUG] Camera permission not yet initialized'); - return; - } - if (permission.status !== 'granted') { - console.log('[DEBUG] Camera permission not granted, requesting...'); - requestPermission(); - } else { - console.log('[INFO] Camera permission already granted'); - } - }, [permission, requestPermission]); + ScreenOrientation.lockAsync(ScreenOrientation.OrientationLock.LANDSCAPE); + return () => { + if (intervalRef.current) clearInterval(intervalRef.current); + ScreenOrientation.unlockAsync(); + }; + }, []); - // Monitor pose detection useEffect(() => { - console.log('[DEBUG] Monitoring pose detection, landmarks length:', landmarks.length); - const timeout = setTimeout(() => { - if (!landmarks.length && !poseDetectionError && isPoseWebViewInitialized) { - console.warn('[WARN] No pose landmarks detected after 10 seconds'); - setPoseDetectionError('Pose detection failed: No landmarks detected. Ensure your full body is visible or try disabling camera feed.'); - setUseCameraView(false); // Disable CameraView to test Pose WebView - } - }, 10000); - - return () => clearTimeout(timeout); - }, [landmarks, poseDetectionError, isPoseWebViewInitialized]); - - // Toggle animation with WebView communication - const toggleAnimation = () => { - if (!isModelLoaded) { - console.log('[DEBUG] Animation toggle ignored: model not loaded'); - return; + if (isCameraReady && cameraRef.current) { + intervalRef.current = setInterval(captureFrame, 2000); } - setIsAnimating((prev) => { - const newState = !prev; - console.log('[DEBUG] Toggling animation state to:', newState); - if (isWebViewBridgeReady && webViewRef.current) { - const message = newState ? 'start' : 'stop'; - console.log('[DEBUG] Sending message to AR WebView:', message); - try { - webViewRef.current.postMessage(message); - console.log('[INFO] Message sent to AR WebView successfully'); - } catch (error) { - console.error('[ERROR] Failed to send message to AR WebView:', error.message); - } - } else { - console.warn('[WARN] AR WebView bridge not ready or ref missing'); - } - return newState; - }); - }; + }, [isCameraReady]); - // Switch to a different pose - const switchPose = (index: number) => { - console.log('[DEBUG] Attempting to switch pose to index:', index); - if (index < 0 || index >= poseRecommendations.length) { - console.warn('[WARN] Invalid pose index:', index); + const captureFrame = async () => { + if (!cameraRef.current || !permission?.granted) { + setFeedback('Camera not ready or permission denied.'); return; } - setCurrentPoseIndex(index); - setIsModelLoaded(false); - setLandmarks([]); - setFeedback('Please ensure your full body is visible in the camera frame.'); - setPoseDetectionError(null); - if (webViewRef.current && isWebViewBridgeReady) { - console.log('[DEBUG] Stopping current animation before switching'); - webViewRef.current.postMessage('stop'); - } - console.log('[INFO] Switched to pose:', poseRecommendations[index].name); - }; - - // Compare user pose with correct pose and provide feedback - useEffect(() => { - const comparePoses = () => { - const currentPose = poseRecommendations[currentPoseIndex]; - console.log('[DEBUG] Comparing poses for:', currentPose?.name); - if (!currentPose) { - console.log('[DEBUG] Skipping comparison: currentPose is undefined'); - setFeedback('Error: No pose selected'); - return; - } - if (!landmarks.length) { - console.log('[DEBUG] Skipping comparison: No detected landmarks'); - setFeedback('No pose detected. Please step back to ensure your full body is visible in the camera.'); - return; - } - if (!currentPose.correct_pose_landmarks || !currentPose.correct_pose_landmarks.length) { - console.log('[DEBUG] Skipping comparison: No correct pose landmarks'); - setFeedback('Error: No reference pose data available. Please contact support.'); - return; - } - console.log('[DEBUG] Detected landmarks:', JSON.stringify(landmarks, null, 2)); - console.log('[DEBUG] Correct pose landmarks:', JSON.stringify(currentPose.correct_pose_landmarks, null, 2)); - - let feedbackMessage = ''; - currentPose.correct_pose_landmarks.forEach((correct: PoseLandmark) => { - const detected = landmarks.find((l: PoseLandmark) => l.name === correct.name); - if (detected) { - const xDiff = Math.abs(detected.x - correct.x); - const yDiff = Math.abs(detected.y - correct.y); - console.log(`[DEBUG] Comparing ${correct.name}: xDiff=${xDiff}, yDiff=${yDiff}`); - if (xDiff > 0.1 || yDiff > 0.1) { - feedbackMessage += `Adjust ${correct.name} position\n`; - } - } else { - console.warn(`[WARN] Landmark ${correct.name} not detected in landmarks`); - feedbackMessage += `Cannot detect ${correct.name}\n`; - } - }); - - const finalFeedback = feedbackMessage || 'Pose looks good!'; - setFeedback(finalFeedback); - console.log('[INFO] Feedback set to:', finalFeedback); - }; - - comparePoses(); - }, [landmarks, currentPoseIndex, poseRecommendations]); - - // End session and navigate back - const handleEndSession = async () => { - console.log('[DEBUG] Initiating end session'); try { - if (webViewRef.current && isWebViewBridgeReady) { - console.log('[DEBUG] Sending stop message to AR WebView'); - webViewRef.current.postMessage('stop'); - } - console.log('[DEBUG] Locking orientation to portrait'); - await ScreenOrientation.lockAsync(ScreenOrientation.OrientationLock.PORTRAIT_UP); - console.log('[INFO] Orientation locked to portrait, navigating back'); - navigation.goBack(); + await retryAsync(async () => { + if (!cameraRef.current) { + throw new Error('Camera is not available.'); + } + const photo = await cameraRef.current.takePictureAsync({ base64: true, quality: 0.3 }); + const response = await axios.post('http://192.168.8.134:5000/process_frame', { frame: photo.base64 }, { timeout: 5000 }); + const landmarks = response.data.landmarks; + setFeedback(landmarks?.length > 0 ? 'Pose detected successfully!' : 'No pose detected.'); + }, 3, 1000); } catch (error) { - console.error('[ERROR] Failed to end session:', error.message); + console.error('[ERROR] Frame capture error:', error); + setFeedback('Error capturing frame.'); } }; - // Zoom In - const zoomIn = () => { - setZoom(prevZoom => Math.min(prevZoom + 0.1, 1)); - console.log('[DEBUG] Zooming in, new zoom level:', Math.min(zoom + 0.1, 1)); - }; - - // Zoom Out - const zoomOut = () => { - setZoom(prevZoom => Math.max(prevZoom - 0.1, 0)); - console.log('[DEBUG] Zooming out, new zoom level:', Math.max(zoom - 0.1, 0)); + const retryAsync = async (fn: () => Promise, retries: number, delay: number) => { + try { + await fn(); + } catch (error) { + if (retries > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + await retryAsync(fn, retries - 1, delay * 2); + } else { + throw error; + } + } }; - // Permission loading state - if (!permission) { - console.log('[DEBUG] Waiting for camera permission initialization'); - return ( - - - Checking camera permission... - - ); - } - - // Permission denied state - if (permission.status !== 'granted') { - console.log('[DEBUG] Camera permission denied'); + if (!permission?.granted) { return ( - - Camera permission required - - Retry + + Camera permission required. + + Grant Permission ); } - const currentPose = poseRecommendations[currentPoseIndex]; - const arPoseUrl = currentPose?.ar_pose; - console.log('[DEBUG] Current AR pose URL:', arPoseUrl); - - // AR WebView HTML - const getARWebViewHtml = () => { - if (!arPoseUrl) { - console.warn('[WARN] No AR pose URL available'); - return '

No AR pose URL available

'; - } - console.log('[DEBUG] Generating AR WebView HTML with URL:', arPoseUrl); - return ` - - - - - - - - - - - - - - - - - - `; - }; - - // Pose Estimation WebView HTML with Canvas Overlay - const getPoseEstimationHtml = () => { - console.log('[DEBUG] Generating Pose Estimation WebView HTML'); - return ` - - - - - - - - - - - - `; - }; - return ( - {/* Camera Feed */} - {useCameraView && Camera.CameraView && !cameraError ? ( - { - console.log('[INFO] Camera is ready'); - setIsCameraReady(true); - }} - onMountError={(error) => { - console.error('[ERROR] Camera mount error:', error.message); - setCameraError(error.message); - }} - /> - ) : useCameraView ? ( - - {cameraError || 'Camera unavailable'} - - ) : ( - - Camera feed disabled to attempt pose detection - - )} - - {/* AR WebView */} - {isWebViewValid && arPoseUrl && !webViewError ? ( - console.log('[DEBUG] AR WebView loading started')} - onLoadEnd={() => console.log('[INFO] AR WebView HTML fully loaded')} - onError={(e) => { - console.error('[ERROR] AR WebView load error:', e.nativeEvent.description); - setWebViewError(e.nativeEvent.description); - }} - onMessage={(event) => { - const data = event.nativeEvent.data; - console.log('[DEBUG] AR WebView message:', data); - if (data === 'bridge-ready') { - setIsWebViewBridgeReady(true); - console.log('[INFO] AR WebView bridge is ready'); - } else if (data === 'model-loaded') { - setIsModelLoaded(true); - console.log('[INFO] AR model fully loaded'); - } else if (data.startsWith('model-error:')) { - console.error('[ERROR] AR model load failed:', data); - setWebViewError('Failed to load AR model'); - } - }} - /> - ) : ( - - {webViewError || 'AR WebView error'} - - )} - - {/* Pose Estimation WebView with Canvas Overlay */} - {isCameraReady && ( - { - const data = event.nativeEvent.data; - console.log('[DEBUG] Pose WebView message received:', data); - try { - if (data === 'Pose library initialized') { - setIsPoseWebViewInitialized(true); - console.log('[INFO] Pose WebView initialized'); - return; - } - if (data.startsWith('Error:')) { - console.error('[ERROR] Pose detection error:', data); - setPoseDetectionError(data); - return; - } - if (data === 'No landmarks detected') { - console.log('[DEBUG] No landmarks detected by MediaPipe'); - setLandmarks([]); - return; - } - const detectedLandmarks = JSON.parse(data); - console.log('[INFO] Detected landmarks:', JSON.stringify(detectedLandmarks, null, 2)); - setLandmarks(detectedLandmarks); - setPoseDetectionError(null); - } catch (error) { - console.error('[ERROR] Failed to parse pose landmarks:', error.message); - setPoseDetectionError('Failed to parse pose data'); - } - }} - onError={(e) => { - console.error('[ERROR] Pose WebView error:', e.nativeEvent.description); - setPoseDetectionError('Pose WebView failed to load'); - }} - onLoadStart={() => console.log('[DEBUG] Pose WebView loading started')} - onLoadEnd={() => console.log('[INFO] Pose WebView HTML loaded')} - /> - )} - - {/* Feedback and Error Display */} - {(feedback || poseDetectionError) && ( - - - {poseDetectionError || feedback} - - - )} - - {/* Header */} - - { - console.log('[DEBUG] Back button pressed'); - navigation.goBack(); - }} - > - - - AR Avatar Viewer - - - {/* Pose Switching UI */} - - {poseRecommendations.map((therapy, index) => ( - switchPose(index)} - > - {therapy.name} - - ))} - - - {/* Loading Overlay */} - {!isModelLoaded && ( - - - Loading AR Avatar... - - )} - - {/* Zoom Controls */} - - - - - - - - - - {/* Start/Pause Button */} - - - {isAnimating ? 'Pause' : 'Start'} + setIsCameraReady(true)} /> + + + + + + {feedback} + navigation.goBack()} style={styles.backButton}> + Back - - {/* End Button */} - - - End + setIsAnimating((prev) => !prev)} style={[styles.startButton, isAnimating ? styles.startButtonActive : null]}> + {isAnimating ? 'Stop Animation' : 'Start Animation'} ); }; const styles = StyleSheet.create({ - container: { - flex: 1, - backgroundColor: '#000', - }, - header: { - height: 60, - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'center', - paddingTop: 10, - }, - backButton: { - position: 'absolute', - left: 15, - top: 20, - }, - headerText: { - fontSize: 20, - color: 'white', - fontWeight: '600', - }, - errorContainer: { - flex: 1, - justifyContent: 'center', - alignItems: 'center', - }, - errorText: { - color: 'red', - textAlign: 'center', - fontSize: 16, - }, - loadingContainer: { - flex: 1, - justifyContent: 'center', - alignItems: 'center', - backgroundColor: '#fff', - }, - loadingOverlay: { - position: 'absolute', - top: 0, - left: 0, - right: 0, - bottom: 0, - justifyContent: 'center', - alignItems: 'center', - backgroundColor: 'rgba(0, 0, 0, 0.5)', - }, - loadingText: { - color: 'white', - marginTop: 10, - fontSize: 16, - }, - fabPlay: { - position: 'absolute', - bottom: 20, - right: 20, - flexDirection: 'row', - alignItems: 'center', - backgroundColor: '#00BBD3', - padding: 15, - borderRadius: 30, - elevation: 5, - shadowColor: '#000', - shadowOffset: { width: 0, height: 2 }, - shadowOpacity: 0.3, - shadowRadius: 3, - }, - fabEnd: { - position: 'absolute', - bottom: 20, - left: 20, - flexDirection: 'row', - alignItems: 'center', - backgroundColor: '#FF4D4F', - padding: 15, - borderRadius: 30, - elevation: 5, - shadowColor: '#000', - shadowOffset: { width: 0, height: 2 }, - shadowOpacity: 0.3, - shadowRadius: 3, - }, - fabText: { - color: 'white', - marginLeft: 10, - fontSize: 16, - }, - fabDisabled: { - opacity: 0.5, - }, - retryButton: { - marginTop: 20, - backgroundColor: '#00BBD3', - paddingVertical: 10, - paddingHorizontal: 20, - borderRadius: 25, - }, - retryButtonText: { - color: 'white', - fontSize: 16, - }, - poseSwitchContainer: { - position: 'absolute', - bottom: 100, - flexDirection: 'row', - backgroundColor: 'rgba(0, 0, 0, 0.5)', - padding: 5, - }, - poseButton: { - padding: 10, - margin: 5, - backgroundColor: '#00BBD3', - borderRadius: 5, - }, - activePoseButton: { - backgroundColor: '#33E4DB', - }, - poseButtonText: { - color: 'white', - fontSize: 14, - }, - feedbackContainer: { - position: 'absolute', - top: 100, - backgroundColor: 'rgba(0, 0, 0, 0.7)', - padding: 10, - borderRadius: 5, - alignSelf: 'center', - }, - feedbackText: { - color: 'white', - fontSize: 16, - textAlign: 'center', - }, - zoomControls: { - position: 'absolute', - bottom: 150, - right: 20, - flexDirection: 'column', - }, - zoomButton: { - backgroundColor: '#00BBD3', - padding: 10, - borderRadius: 5, - marginVertical: 5, - }, + container: { flex: 1, backgroundColor: '#000', justifyContent: 'flex-end' }, + camera: { position: 'absolute', top: 0, left: 0, width: '100%', height: '70%', zIndex: 0 }, + canvas: { position: 'absolute', top: 0, left: 0, width: '100%', height: '100%', zIndex: 1 }, + feedback: { position: 'absolute', bottom: 150, width: '100%', textAlign: 'center', color: 'white', fontSize: 18, backgroundColor: 'rgba(0,0,0,0.5)', padding: 5, zIndex: 2 }, + message: { textAlign: 'center', paddingBottom: 10, color: 'white', fontSize: 16 }, + button: { padding: 10, backgroundColor: '#00BBD3', borderRadius: 5, alignSelf: 'center', zIndex: 2 }, + buttonText: { color: 'white', fontSize: 16 }, + backButton: { position: 'absolute', bottom: 10, left: 10, padding: 15, backgroundColor: '#00BBD3', borderRadius: 25, zIndex: 2 }, + startButton: { position: 'absolute', bottom: 70, alignSelf: 'center', padding: 15, backgroundColor: '#00BBD3', borderRadius: 25, zIndex: 2 }, + startButtonActive: { backgroundColor: '#FF4444' }, }); export default ARAvatarScreen; \ No newline at end of file diff --git a/CeylonCare/app/pages/AR_component/CameraTest.tsx b/CeylonCare/app/pages/AR_component/CameraTest.tsx new file mode 100644 index 00000000..9a4528e9 --- /dev/null +++ b/CeylonCare/app/pages/AR_component/CameraTest.tsx @@ -0,0 +1,69 @@ +import { CameraView, CameraType, useCameraPermissions } from 'expo-camera'; +import { useState } from 'react'; +import { Button, StyleSheet, Text, TouchableOpacity, View } from 'react-native'; + +export default function App() { + const [facing, setFacing] = useState('back'); + const [permission, requestPermission] = useCameraPermissions(); + + if (!permission) { + // Camera permissions are still loading. + return ; + } + + if (!permission.granted) { + // Camera permissions are not granted yet. + return ( + + We need your permission to show the camera +