Component 03 it21223976 yeshmantha w n

Backend/ar_model/pose_service.py

@@ -0,0 +1,69 @@
+from flask import Flask, request, jsonify
+import cv2
+import mediapipe as mp
+import base64
+import numpy as np
+import logging
+import traceback
+
+# Set up logging for debugging and error tracking
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+app = Flask(__name__)
+mp_pose = mp.solutions.pose
+
+# Initialize MediaPipe Pose model with higher confidence thresholds
+pose = mp_pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7)
+
+@app.route('/process_frame', methods=['POST'])
+def process_frame():
+    try:
+        # Log incoming request
+        logger.info('[DEBUG] Received request to process frame')
+
+        # Get JSON data from request
+        data = request.json
+        if not data or 'frame' not in data:
+            logger.error('[ERROR] Invalid request: Missing frame data')
+            return jsonify({'error': 'Missing frame data'}), 400
+
+        # Extract and log frame data
+        frame_base64 = data['frame']
+        logger.debug('[DEBUG] Frame base64 length: %d', len(frame_base64))
+
+        # Decode base64 string to image
+        frame = base64.b64decode(frame_base64)
+        image = cv2.imdecode(np.frombuffer(frame, np.uint8), cv2.IMREAD_COLOR)
+        if image is None:
+            logger.error('[ERROR] Failed to decode image')
+            return jsonify({'error': 'Failed to decode image'}), 400
+
+        # Log image details
+        logger.debug('[DEBUG] Image decoded, shape: %s', image.shape)
+
+        # Convert image to RGB for MediaPipe processing
+        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        results = pose.process(image_rgb)
+
+        # Process pose detection results
+        if results.pose_landmarks:
+            landmarks = [
+                {'name': f'landmark_{i}', 'x': lm.x, 'y': lm.y, 'z': lm.z}
+                for i, lm in enumerate(results.pose_landmarks.landmark)
+            ]
+            logger.info('[DEBUG] Pose landmarks detected: %d', len(landmarks))
+            return jsonify({'landmarks': landmarks})
+        else:
+            logger.warning('[WARN] No pose landmarks detected')
+            return jsonify({'landmarks': []})
+
+    except Exception as e:
+        # Log detailed error information
+        logger.error('[ERROR] Error processing frame: %s', str(e))
+        logger.error('[ERROR] Traceback: %s', traceback.format_exc())
+        return jsonify({'error': str(e)}), 500
+
+if __name__ == '__main__':
+    logger.info('[INFO] Starting pose detection service on http://0.0.0.0:5002')
+    app.run(host='0.0.0.0', port=5002, debug=True)

Backend/controllers/arController.js

@@ -89,4 +89,83 @@ const getTherapyDetails = async (req, res) => {
   }
 };
 
-module.exports = { getARRecommendations, getTherapyDetails };
+// Process Frame for Pose Detection API
+const processFrame = async (req, res) => {
+  const { frame } = req.body;
+
+  if (!frame) {
+    console.error("[ERROR] No frame data provided in request.");
+    return res.status(400).json({ error: "Frame data is required" });
+  }
+
+  console.log("[DEBUG] Received request to process frame");
+
+  try {
+    // Forward the frame to the pose detection service
+    const response = await axios.post('http://localhost:5002/process_frame', { frame }, {
+      timeout: 5000,
+    });
+
+    console.log("[DEBUG] Pose detection service response:", JSON.stringify(response.data, null, 2));
+    res.status(200).json(response.data);
+  } catch (error) {
+    console.error(`[ERROR] Failed to process frame: ${error.message}`);
+    console.error("[DEBUG] Stack trace:", error.stack);
+    res.status(500).json({ error: "Failed to process frame", details: error.message });
+  }
+};
+
+// Fetch target pose landmarks
+const getTherapyPoseLandmarks = async (req, res) => {
+  const { therapyName } = req.params;
+
+  if (!therapyName) {
+    console.error("[ERROR] No therapy name provided in request.");
+    return res.status(400).json({ error: "Therapy name is required" });
+  }
+
+  console.log(`[DEBUG] Fetching pose landmarks for therapy: ${therapyName}`);
+
+  try {
+    console.log("[DEBUG] Firestore db object:", db ? "Initialized" : "Not initialized");
+
+    const normalizedTherapyName = therapyName
+      .trim()
+      .replace(/%20/g, " ")
+      .replace(/-/g, " ");
+
+    console.log(`[DEBUG] Normalized therapy name: ${normalizedTherapyName}`);
+
+    if (!db) {
+      throw new Error("Firestore db is not initialized");
+    }
+
+    const therapyRef = doc(db, "therapies", normalizedTherapyName);
+    console.log("[DEBUG] Therapy reference created:", therapyRef.path);
+
+    const therapySnapshot = await getDoc(therapyRef);
+
+    if (!therapySnapshot.exists()) {
+      console.error(`[ERROR] No therapy details found for: ${normalizedTherapyName}`);
+      return res.status(404).json({ error: `Therapy "${therapyName}" not found in database` });
+    }
+
+    const therapyData = therapySnapshot.data();
+    console.log("[DEBUG] Fetched therapy data:", JSON.stringify(therapyData, null, 2));
+
+    // Assuming landmarks are stored as an array of {name, x, y, z} objects
+    const landmarks = therapyData.landmarks || [];
+    if (!landmarks.length) {
+      console.warn(`[WARN] No landmarks found for ${normalizedTherapyName}`);
+    }
+
+    console.log("[DEBUG] Fetched pose landmarks:", JSON.stringify(landmarks, null, 2));
+    res.status(200).json({ landmarks });
+  } catch (error) {
+    console.error(`[ERROR] Failed to fetch therapy pose landmarks: ${error.message}`);
+    console.error("[DEBUG] Stack trace:", error.stack);
+    res.status(500).json({ error: "Failed to fetch therapy pose landmarks", details: error.message });
+  }
+};
+
+module.exports = { getARRecommendations, getTherapyDetails, processFrame, getTherapyPoseLandmarks };

Backend/index.js

@@ -16,12 +16,15 @@ const {
   deleteHealthData,
 } = require("./controllers/healthController");
 const { getChatRecommendation } = require("./controllers/chatController");
-const { getARRecommendations, getTherapyDetails } = require("./controllers/arController");
-const fileUpload = require("express-fileupload");
+const { getARRecommendations, getTherapyDetails, processFrame, getTherapyPoseLandmarks } = require("./controllers/arController");
 
 const app = express();
+
+// Custom CORS configuration
 app.use(cors({ origin: "*" }));
-app.use(express.json());
+
+// Increase body size limit for all routes (or apply to specific routes)
+app.use(express.json({ limit: "2mb" })); // Set to 2MB for all requests
 app.use("/uploads", express.static(path.join(__dirname, "uploads")));
 app.use(fileUpload());
 
@@ -41,6 +44,8 @@ app.delete("/healthData/:userId", deleteHealthData);
 // AR Therapy Routes
 app.get("/ar_therapy/:userId", getARRecommendations);
 app.get("/therapy_details/:therapyName", getTherapyDetails);
+app.post("/process_frame", express.json({ limit: "2mb" }), processFrame);
+app.get("/therapy_landmarks/:therapyName", getTherapyPoseLandmarks);
 
 // Chatbot Routes
 app.post('/healthChat/:userId', getChatRecommendation);
@@ -56,5 +61,9 @@ app.listen(PORT, () => {
 // Error handling middleware
 app.use((err, req, res, next) => {
   console.error("[ERROR] Server Error:", err.stack);
-  res.status(500).json({ error: "Internal server error", message: err.message });
+  if (err.type === 'entity.too.large') {
+    res.status(413).json({ error: "Payload too large", message: "Request body exceeds the allowed limit (2MB)" });
+  } else {
+    res.status(500).json({ error: "Internal server error", message: err.message });
+  }
 });

CeylonCare/app.json

@@ -4,29 +4,32 @@
     "slug": "CeylonCare",
     "version": "1.0.0",
     "assetBundlePatterns": [
-      "assets/videos/*"
+      "**/*"],
+    "assets": [
+      "./assets/ar_poses/Downward_Dog.glb"
     ],
     "orientation": "portrait",
     "icon": "",
+    "jsEngine": "jsc",
     "scheme": "myapp",
     "userInterfaceStyle": "automatic",
     "newArchEnabled": true,
     "ios": {
       "supportsTablet": true,
       "infoPlist": {
-        "NSMicrophoneUsageDescription": "We need microphone access to record your voice for the chatbot.",
-        "ITSAppUsesNonExemptEncryption": false
-      },
-      "bundleIdentifier": "com.amfdo.CeylonCare"
+        "NSCameraUsageDescription": "This app uses the camera for real-time pose detection to guide your yoga practice.",
+        "NSMicrophoneUsageDescription": "We need microphone access to record your voice for the chatbot."
+      }
     },
     "android": {
       "adaptiveIcon": {
-        "foregroundImage": "",
+        "foregroundImage": "./assets/images/logo.png",
         "backgroundColor": "#ffffff"
       },
       "permissions": [
+        "android.permission.CAMERA",
         "android.permission.RECORD_AUDIO",
-        "android.permission.CAMERA"
+        "android.permission.WEBVIEW_MEDIA"
       ],
       "package": "com.amfdo.CeylonCare"
     },
@@ -39,20 +42,22 @@
       [
         "expo-splash-screen",
         {
-          "image": "",
+          "image": "./assets/images/logo.png",
           "imageWidth": 200,
           "resizeMode": "contain",
           "backgroundColor": "#ffffff"
         }
       ],
-      "expo-asset",
-      "expo-font",
       [
         "expo-camera",
         {
-          "cameraPermission": "Allow this app to use the camera for AR features."
+          "cameraPermission": "Allow $(PRODUCT_NAME) to access your camera",
+          "microphonePermission": "Allow $(PRODUCT_NAME) to access your microphone",
+          "recordAudioAndroid": true
         }
       ],
+      "expo-asset",
+      "expo-font",
       "expo-router"
     ],
     "extra": {

CeylonCare/app/_layout.tsx

@@ -1,23 +1,43 @@
-import React from 'react';
-import { createStackNavigator } from '@react-navigation/stack';
+import React from "react";
+import { createStackNavigator } from "@react-navigation/stack";
+import { RouteProp } from "@react-navigation/native";
 
-// Pages - Validate imports
-import Register from './pages/Register';
-import SplashScreen from './pages/SplashScreen';
-import Login from './pages/Login';
-import ForgotPassword from './pages/ForgotPassword';
-import Onboarding from './pages/Onboarding';
-import Home from './pages/Home';
-import Profile from './pages/profile/Profile';
-import ProfileDetails from './pages/profile/ProfileDetails';
-import HealthDetails from './pages/profile/HealthDetails';
-import PrivacyPolicy from './pages/profile/PrivacyPolicy';
-import ChatScreen from './pages/Chatbot_component/ChatScreen';
-import TherapyRecommendations from './pages/AR_component/TherapyRecommendations';
-import TherapyDetails from './pages/AR_component/TherapyDetails';
-import ARAvatarScreen from './pages/AR_component/ARAvatarScreen';
+// Pages
+import Register from "./pages/Register";
+import SplashScreen from "./pages/SplashScreen";
+import Login from "./pages/Login";
+import ForgotPassword from "./pages/ForgotPassword";
+import Onboarding from "./pages/Onboarding";
+import Home from "./pages/Home";
+import Profile from "./pages/profile/Profile";
+import ProfileDetails from "./pages/profile/ProfileDetails";
+import HealthDetails from "./pages/profile/HealthDetails";
+import PrivacyPolicy from "./pages/profile/PrivacyPolicy";
+import ChatScreen from "./pages/Chatbot_component/ChatScreen";
+import TherapyRecommendations from "./pages/AR_component/TherapyRecommendations";
+import TherapyDetails from "./pages/AR_component/TherapyDetails";
+import ARAvatarScreen from "./pages/AR_component/ARAvatarScreen";
+import CameraTest from "./pages/AR_component/CameraTest";
 
-const Stack = createStackNavigator();
+type RootStackParamList = {
+  Splash: undefined;
+  Register: undefined;
+  Login: undefined;
+  ForgotPassword: undefined;
+  Onboarding: undefined;
+  Home: undefined;
+  Profile: undefined;
+  ProfileDetails: undefined;
+  HealthDetails: undefined;
+  PrivacyPolicy: undefined;
+  TherapyRecommendations: undefined;
+  TherapyDetails: { therapyName: string };
+  ARAvatarScreen: undefined;
+  CameraTest: undefined;
+  ChatScreen: undefined;
+};
+
+const Stack = createStackNavigator<RootStackParamList>();
 
 // Validate all screen components
 const validateScreenComponent = (name: string, component: any) => {
@@ -123,6 +143,13 @@ const StackNavigator = () => {
         component={ARAvatarScreen}
         options={{ headerShown: false }}
       />
+
+      <Stack.Screen
+        name="CameraTest"
+        component={CameraTest}
+        options={{ headerShown: false }}
+      />
+
       {/* Chatbot_Component */}
       <Stack.Screen
         name="ChatScreen"