linking everything together

Author: Aayush-Agnihotri

.env.example

@@ -4,6 +4,7 @@ WORKERS=
 GOOGLE_SHEETS_API_KEY=
 FREEDGE_SHEET_ID=
 FREEDGE_APPROVED_EMAILS=
+SERVER_URL=
 
 # Server Configuration
 PORT=8000

prisma/migrations/20251120022949/migration.sql

@@ -362,6 +362,27 @@ async function processAllEateries(
   });
 }
 
+async function getAllEateriesData() {
+  return await prisma.eatery.findMany({
+    include: {
+      events: {
+        include: {
+          menu: {
+            include: {
+              items: {
+                include: {
+                  dietaryPreferences: true,
+                  allergens: true,
+                },
+              },
+            },
+          },
+        },
+      },
+    },
+  });
+}
+
 export async function main() {
   const startTime = Date.now();
   console.log('Starting scraper at', new Date(startTime).toString(), '\n');
@@ -467,6 +488,48 @@ export async function main() {
     console.error('✗ Error during database transaction:', error);
     throw error;
   }
+
+  // Send newly populated data to server
+  console.log('[Scheduler] Scraper run finished\n');
+  try {
+    console.log('[Scheduler] Fetching all eatery data for server update...');
+    const startFetchTime = Date.now();
+    const allEateryData = await getAllEateriesData();
+    const fetchDuration = ((Date.now() - startFetchTime) / 1000).toFixed(2);
+    console.log(
+      `[Scheduler] Fetched ${allEateryData.length} eateries in ${fetchDuration}s`,
+    );
+
+    console.log('[Scheduler] Updating server cache with new eatery data...');
+    const startUpdateTime = Date.now();
+    const response = await fetch(
+      `${process.env.SERVER_URL}/internal/cache/`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          [process.env.CACHE_REFRESH_HEADER!]:
+            process.env.CACHE_REFRESH_SECRET!,
+        },
+        body: JSON.stringify({ eateries: allEateryData }),
+      },
+    );
+    if (!response.ok) {
+      throw new Error(
+        `Server responded with status ${response.status} during cache update`,
+      );
+    }
+    const updateDuration = ((Date.now() - startUpdateTime) / 1000).toFixed(2);
+    console.log(
+      `[Scheduler] Server cache updated successfully in ${updateDuration}s`,
+    );
+  }
+  catch (error) {
+    console.error(
+      '[Scheduler] Failed to update server cache with new eatery data:',
+      error,
+    );
+  }
 
   const totalDuration = ((Date.now() - startTime) / 1000).toFixed(2);
   console.log(`\n✅ Dining data scraped successfully in ${totalDuration}s`);

prisma/scraper.ts

@@ -22,7 +22,7 @@ app.set('trust proxy', 1);
 
 app.use(requestLogger);
 app.use(helmet());
-app.use(express.json({ limit: '10kb' }));
+app.use(express.json({ limit: '2mb' }));
 app.use(ipRateLimiter);
 
 const router = express.Router();

src/server.ts

@@ -68,11 +68,11 @@ export function getEateriesEtag(): string {
 
 export async function refreshCacheFromDB() {
   clearAppCache();
-  
+
   // Temporarily suppress console logs for this query
   const originalLog = console.log;
   console.log = () => {};
-  
+
   const eateries = await prisma.eatery.findMany({
     include: {
       events: {
@@ -87,18 +87,17 @@ export async function refreshCacheFromDB() {
               },
             },
           },
-          userEventVotes: true,
         },
         orderBy: {
           startTimestamp: 'asc',
         },
       },
     },
   });
-  
+
   // Restore console.log
   console.log = originalLog;
-  
+
   populateCache(eateries);
 }