refactor: use $facet to merge services count query into single pipeline

src/app/api/services/route.ts

@@ -37,9 +37,8 @@ interface AggregationStage {
   $addFields?: Record<string, unknown>;
   $unwind?: string | { path: string; preserveNullAndEmptyArrays?: boolean };
   $project?: Record<string, number | string>;
-  $skip?: number;
-  $limit?: number;
   $count?: string;
+  $facet?: Record<string, Record<string, unknown>[]>;
 }
 
 // This function is now replaced by loadAccommodationDataFromDatabase() from utils/accommodationData.ts
@@ -314,11 +313,10 @@ export async function GET(req: Request) {
       }
     });
 
-    // Add pagination
-    pipeline.push(
-      { $skip: (page - 1) * limit },
-      { $limit: limit }
-    );
+    // Pagination is applied in JavaScript after combining services with
+    // accommodation results (which come from a separate collection).
+    // Both sources use $geoNear for distance, and must be interleaved
+    // by distance before slicing.
 
     // Load filtered accommodation data from database (only what we need)
     const accommodationData = await loadFilteredAccommodationData({
@@ -342,18 +340,21 @@ export async function GET(req: Request) {
       }
     }
 
-    // Remove pagination from pipeline for accommodation integration
-    const pipelineWithoutPagination = [...pipeline];
-    pipelineWithoutPagination.splice(-2, 2); // Remove skip and limit
+    // Single query: fetch results and count in one round trip using $facet
+    const facetPipeline: AggregationStage[] = [
+      ...pipeline,
+      {
+        $facet: {
+          results: [{ $match: {} }],
+          totalCount: [{ $count: 'count' }]
+        }
+      }
+    ];
 
-    // Use aggregation pipeline for regular services
-    const services = await servicesCol.aggregate(pipelineWithoutPagination).toArray();
-
-    // Get total count of services using optimized pipeline
-    const countPipeline = [...pipelineWithoutPagination];
-    countPipeline.push({ $count: "total" });
-    const countResult = await servicesCol.aggregate(countPipeline).toArray();
-    const servicesTotal = countResult.length > 0 ? countResult[0].total : 0;
+    const [facetResult] = await servicesCol.aggregate(facetPipeline).toArray();
+    const typedResult = facetResult as { results: Record<string, unknown>[]; totalCount: Array<{ count: number }> };
+    const services = typedResult.results;
+    const servicesTotal = typedResult.totalCount[0]?.count || 0;
 
     // Combine services and accommodation results
     const combinedResults: Array<Record<string, unknown> & { distance?: number }> = [...services, ...accommodationResults];

src/config/constants.ts

@@ -6,6 +6,8 @@ export const DEFAULT_SERVICE_LIMIT = 20;
 export const DEFAULT_SERVICE_PROVIDER_LIMIT = 20;
 export const DEFAULT_ACCOMMODATION_LIMIT = 50;
 export const DEFAULT_ORG_SEARCH_LIMIT = 10;
+// Must load all results because services and accommodation are interleaved by distance client-side.
+// Raised from 100 to 500 for dense urban areas.
 export const MAX_SERVICES_FETCH_LIMIT = 500;
 export const FALLBACK_SERVICES_LIMIT = 50;
 export const DEFAULT_SEARCH_RADIUS_KM = 5;

tests/__tests__/api/services-geospatial.test.ts

@@ -154,56 +154,59 @@ describe('GET /api/services - Error Handling and Geospatial Tests', () => {
     });
 
     it('handles edge case coordinates when database is available', async () => {
-      // Mock successful database connection
+      // Mock successful database connection with $facet result structure
+      const emptyFacetResult = [{ results: [], totalCount: [] }];
       const mockClient = {
         db: jest.fn().mockReturnValue({
           collection: jest.fn().mockReturnValue({
             aggregate: jest.fn().mockReturnValue({
-              toArray: jest.fn().mockResolvedValue([])
+              toArray: jest.fn().mockResolvedValue(emptyFacetResult)
             })
           })
         })
       };
       mockGetClientPromise.mockResolvedValue(mockClient);
-      
+
       const req = new Request('http://localhost/api/services?lat=-89.9&lng=179.9&radius=10');
       const res = await GET(req);
       const json = await res.json();
-      
+
       expect(res.status).toBe(200);
       expect(json.status).toBe('success');
       expect(Array.isArray(json.results)).toBe(true);
     });
 
     it('handles very large radius values when database is available', async () => {
-      // Mock successful database connection
+      // Mock successful database connection with $facet result structure
+      const emptyFacetResult = [{ results: [], totalCount: [] }];
       const mockClient = {
         db: jest.fn().mockReturnValue({
           collection: jest.fn().mockReturnValue({
             aggregate: jest.fn().mockReturnValue({
-              toArray: jest.fn().mockResolvedValue([])
+              toArray: jest.fn().mockResolvedValue(emptyFacetResult)
             })
           })
         })
       };
       mockGetClientPromise.mockResolvedValue(mockClient);
-      
+
       const req = new Request('http://localhost/api/services?lat=53.8008&lng=-1.5491&radius=20000');
       const res = await GET(req);
       const json = await res.json();
-      
+
       expect(res.status).toBe(200);
       expect(json.status).toBe('success');
       expect(Array.isArray(json.results)).toBe(true);
     });
 
     it('handles very small radius values when database is available', async () => {
-      // Mock successful database connection
+      // Mock successful database connection with $facet result structure
+      const emptyFacetResult = [{ results: [], totalCount: [] }];
       const mockClient = {
         db: jest.fn().mockReturnValue({
           collection: jest.fn().mockReturnValue({
             aggregate: jest.fn().mockReturnValue({
-              toArray: jest.fn().mockResolvedValue([])
+              toArray: jest.fn().mockResolvedValue(emptyFacetResult)
             })
           })
         })

tests/__tests__/api/services.test.ts

@@ -80,66 +80,31 @@ const createMockServicesCollection = (shouldThrow = false, customData?: any[], a
   const mockAggregate = jest.fn().mockImplementation((pipeline: any[]) => ({
     toArray: jest.fn().mockImplementation(async () => {
       if (shouldThrow) throw new Error('Database error');
-
-      // Check if this is a count pipeline (contains $count stage)
-      const hasCountStage = pipeline.some(stage => stage.$count);
-      if (hasCountStage) {
-        // Apply same filtering logic for count
-        let dataToCount = aggregateData || customData || mockServices;
-
-        // Apply location filtering if specified in pipeline
-        const matchStage = pipeline.find(stage => stage.$match);
-        if (matchStage && matchStage.$match['Address.City']) {
-          const cityRegex = matchStage.$match['Address.City'].$regex;
-          dataToCount = dataToCount.filter(service => 
-            cityRegex.test(service.Address?.City || '')
-          );
-        }
-
-        // Apply category filtering if specified in pipeline
-        if (matchStage && matchStage.$match['ParentCategoryKey']) {
-          const categoryRegex = matchStage.$match['ParentCategoryKey'].$regex;
-          dataToCount = dataToCount.filter(service => 
-            categoryRegex.test(service.ParentCategoryKey || '')
-          );
-        }
-
-        return [{ total: dataToCount.length }];
-      }
-
+
       // Simulate filtering based on the pipeline stages
       let filteredServices = aggregateData || customData || mockServices;
-      
+
       // Apply location filtering if specified in pipeline
       const matchStage = pipeline.find(stage => stage.$match);
       if (matchStage && matchStage.$match['Address.City']) {
         const cityRegex = matchStage.$match['Address.City'].$regex;
-        filteredServices = filteredServices.filter(service => 
+        filteredServices = filteredServices.filter(service =>
           cityRegex.test(service.Address?.City || '')
         );
       }
-      
+
       // Apply category filtering if specified in pipeline
       if (matchStage && matchStage.$match['ParentCategoryKey']) {
         const categoryRegex = matchStage.$match['ParentCategoryKey'].$regex;
-        filteredServices = filteredServices.filter(service => 
+        filteredServices = filteredServices.filter(service =>
           categoryRegex.test(service.ParentCategoryKey || '')
         );
       }
-
-      // Apply pagination
-      const skipStage = pipeline.find(stage => stage.$skip);
-      const limitStage = pipeline.find(stage => stage.$limit);
-      const skip = skipStage?.$skip || 0;
-      const limit = limitStage?.$limit || filteredServices.length;
-
-      filteredServices = filteredServices.slice(skip, skip + limit);
-
-      // Return with provider data from lookup
-      return filteredServices.map(service => ({
+
+      // Enrich with provider data
+      const enriched = filteredServices.map(service => ({
         ...service,
         distance: service.distance || 5000,
-        // Add provider data from lookup
         provider: mockProviders.find(p => p.Key === service.ServiceProviderKey),
         name: service.Title || service.ServiceProviderName || service.name,
         description: service.Description || service.Info || service.description,
@@ -149,6 +114,17 @@ const createMockServicesCollection = (shouldThrow = false, customData?: any[], a
           isVerified: mockProviders.find(p => p.Key === service.ServiceProviderKey)?.IsVerified || false
         } : null
       }));
+
+      // If pipeline contains $facet, return wrapped structure
+      const hasFacet = pipeline.some(stage => stage.$facet);
+      if (hasFacet) {
+        return [{
+          results: enriched,
+          totalCount: enriched.length > 0 ? [{ count: enriched.length }] : []
+        }];
+      }
+
+      return enriched;
     }),
   }));