fix: catalog and extractable records paginated fetch

confd/production.tmpl

@@ -10,5 +10,6 @@ window._env_ = {
   DEFAULT_TERRAIN_PROVIDER_URL: '{{ getv "/configuration/default/terrain/provider/url" "" }}',
   CSW_3D_URL: '{{ getv "/configuration/csw/3d/url" ""}}',
   EXTRACTABLE_MANAGER_URL: '{{ getv "/configuration/extractable/manager/url" ""}}',
-  SHOW_POI_TOOL: {{ getv "/configuration/show/poi/tool" "true"}}
+  SHOW_POI_TOOL: {{ getv "/configuration/show/poi/tool" "true"}},
+  NUMBER_OF_RECORDS_PER_PAGE: {{ getv "/configuration/number/of/records/per/page" "200"}}
 };

confd/production.toml

@@ -13,5 +13,6 @@ keys = [
   "/configuration/default/terrain/provider/url",
   "/configuration/csw/3d/url",
   "/configuration/extractable/manager/url",
-  "/configuration/show/poi/tool"
+  "/configuration/show/poi/tool",
+  "/configuration/number/of/records/per/page"
 ]

helm/templates/configmap.yaml

@@ -22,6 +22,7 @@
   CONFIGURATION_CSW_3D_URL: {{ quote .Values.env.csw3DUrl }}
   CONFIGURATION_EXTRACTABLE_MANAGER_URL: {{ quote .Values.env.extractableManagerUrl }}
   CONFIGURATION_SHOW_POI_TOOL: {{ quote .Values.env.showPOITool }}
+  NUMBER_OF_RECORDS_PER_PAGE: {{ quote .Values.env.numberOfRecordsPerPage }}
 {{- end }}
 
 {{ include "mc-chart.configmap" (dict "MAIN_OBJECT_BLOCK" $MAIN_OBJECT_BLOCK "COMPONENT_NAME" $COMPONENT_NAME "DATA" $DATA "WITH_TELEMETRY_TRACING" false "WITH_TELEMETRY_METRICS" false "context" .)}}

helm/values.yaml

@@ -113,6 +113,7 @@ env:
   csw3DUrl: ''
   extractableManagerUrl: ''
   showPOITool: true
+  numberOfRecordsPerPage: 200
 route:
   enabled: true
   annotations:

src/common/services/CatalogService.ts

@@ -1,41 +1,81 @@
 import { createCatalogTree } from '../../components/common/Tree/TreeGroup';
 import { IDENTIFIER_FIELD } from '../../components/Wizard/Wizard.types';
 import appConfig from '../../utils/Config';
-import { get3DRecordsXML, parse3DQueryResults } from '../../utils/cswQueryBuilder';
+import {
+  get3DRecordsXML,
+  getNumberOfMatchedRecords,
+  parse3DQueryResults,
+} from '../../utils/cswQueryBuilder';
 import { loadingUpdater } from '../../utils/loadingUpdater';
 import { execute } from '../../utils/requestHandler';
-import { ExtractableRecord } from './ExtractableService';
+import { ExtractableRecord, ExtractableResponse } from './ExtractableService';
+
+const PAGE_SIZE = appConfig.numberOfRecordsPerPage;
+const EXTRACTABLE_PAGE_SIZE = appConfig.numberOfExtractablesPerPage;
+
+const fetchAll3DRecordsParallel = async () => {
+  const numberOfRecordsXml = get3DRecordsXML('hits', 0);
+
+  const resNumberOfRecords = await execute(`${appConfig.csw3dUrl}`, 'POST', {
+    data: numberOfRecordsXml,
+  });
+
+  const totalRecords = getNumberOfMatchedRecords(resNumberOfRecords as string);
+  const totalPages = Math.ceil(totalRecords / PAGE_SIZE);
+
+  const promises = Array.from({ length: totalPages }, (_, i) => {
+    const startPosition = i * PAGE_SIZE + 1;
+    const xml = get3DRecordsXML('results', PAGE_SIZE, startPosition);
+    return execute(appConfig.csw3dUrl, 'POST', { data: xml });
+  });
+
+  const responses = await Promise.all(promises);
+
+  const allRecords = responses.flatMap(
+    (res) => parse3DQueryResults(res as string) as Record<string, unknown>[]
+  );
+
+  return allRecords;
+};
+
+const fetchExtractable = async () => {
+  let extract: ExtractableRecord[] = [];
+  let startIndex = 1;
+
+  while (startIndex > 0) {
+    const extractableResponse: ExtractableResponse = (await execute(
+      `${appConfig.extractableManagerUrl}/records?startPosition=${startIndex}&maxRecords=${EXTRACTABLE_PAGE_SIZE}`,
+      'GET'
+    )) as unknown as ExtractableResponse;
+    if (Array.isArray(extractableResponse.records)) {
+      extract.push(...extractableResponse.records);
+      startIndex = extractableResponse.nextRecord as number;
+    }
+  }
+  return extract;
+};
 
 export const fetchCatalog = async (setLoading: loadingUpdater) => {
-  let parsed, extractables;
+  let records;
+  let extractables: ExtractableRecord[] = [];
+
   try {
     setLoading(true);
-    const data = get3DRecordsXML();
-    const records = await execute(`${appConfig.csw3dUrl}`, 'POST', { data });
-    parsed = parse3DQueryResults(records as string) as Record<string, unknown>[];
-    extractables = await execute(
-      `${appConfig.extractableManagerUrl}/records?startPosition=1&maxRecords=1000`,
-      'GET'
-    );
+    records = await fetchAll3DRecordsParallel();
+    extractables = await fetchExtractable();
   } catch (error) {
     console.error('Failed to fetch catalog/extractable data:', error);
   } finally {
-    const catalogRecords = Array.isArray(parsed) ? parsed : [];
-    const extractablesPayload = extractables as { records?: ExtractableRecord[] } | undefined;
-    const extractablesList = extractablesPayload?.records;
-    const extractablesRecords: ExtractableRecord[] = Array.isArray(extractablesList)
-      ? extractablesList
-      : [];
-    const enriched = enrichRecords(catalogRecords, extractablesRecords);
+    const catalogRecords = Array.isArray(records) ? records : [];
+    const enriched = enrichRecords(catalogRecords, extractables);
     setLoading(false);
     return {
       data: createCatalogTree(enriched),
       sumAll: catalogRecords.length,
-      sumExtractable:
-        catalogRecords.length > 0 ? extractablesRecords.length : catalogRecords.length,
+      sumExtractable: catalogRecords.length > 0 ? extractables.length : catalogRecords.length,
       sumNotExtractable:
         catalogRecords.length > 0
-          ? catalogRecords.length - extractablesRecords.length
+          ? catalogRecords.length - extractables.length
           : catalogRecords.length,
     };
   }

src/common/services/ExtractableService.ts

@@ -2,6 +2,13 @@ import appConfig from '../../utils/Config';
 import { loadingUpdater } from '../../utils/loadingUpdater';
 import { execute } from '../../utils/requestHandler';
 
+export interface ExtractableResponse {
+  nextRecord: number;
+  numberOfRecords: number;
+  numberOfRecordsReturned: number;
+  records: ExtractableRecord[];
+}
+
 export interface ExtractableRecord {
   id: string;
   recordName: string;

src/utils/Config.ts

@@ -18,6 +18,7 @@ const DEFAULT_TERRAIN_PROVIDER_URL = (window as any)._env_.DEFAULT_TERRAIN_PROVI
 const CSW_3D_URL = (window as any)._env_.CSW_3D_URL;
 const EXTRACTABLE_MANAGER_URL = (window as any)._env_.EXTRACTABLE_MANAGER_URL;
 const SHOW_POI_TOOL = (window as any)._env_.SHOW_POI_TOOL;
+const NUMBER_OF_RECORDS_PER_PAGE = (window as any)._env_.NUMBER_OF_RECORDS_PER_PAGE;
 
 const enrichBaseMaps = (baseMaps: IBaseMaps): IBaseMaps => {
   return {
@@ -56,6 +57,8 @@ class Config {
   public csw3dUrl = CSW_3D_URL;
   public extractableManagerUrl = EXTRACTABLE_MANAGER_URL;
   public showPOITool = SHOW_POI_TOOL;
+  public numberOfRecordsPerPage = NUMBER_OF_RECORDS_PER_PAGE;
+  public numberOfExtractablesPerPage = 500;
 }
 
 const appConfig = new Config(); // Singleton

src/utils/cswQueryBuilder.ts

@@ -1,6 +1,18 @@
 import { XMLParser } from 'fast-xml-parser';
+import appConfig from './Config';
 
-export const get3DRecordsXML = () => {
+const extractSearchResults = (xml: string) => {
+  const parser = new XMLParser({ ignoreAttributes: false });
+  const parsed = parser.parse(xml);
+
+  return parsed?.['csw:GetRecordsResponse']?.['csw:SearchResults'];
+};
+
+export const get3DRecordsXML = (
+  resultType: string = 'results',
+  maxRecords: number = appConfig.numberOfRecordsPerPage,
+  startPosition: number = 1
+) => {
   return `
     <csw:GetRecords
       xmlns="http://www.opengis.net/cat/csw/2.0.2"
@@ -12,7 +24,9 @@ export const get3DRecordsXML = () => {
       xmlns:dct="http://purl.org/dc/terms/"
       service="CSW"
       version="2.0.2"
-      resultType="results"
+      resultType="${resultType}"
+      maxRecords="${maxRecords}"
+      startPosition="${startPosition}"
       outputSchema="http://schema.mapcolonies.com/3d">
         <csw:Query typeNames="csw:Record">
           <csw:ElementSetName>full</csw:ElementSetName>
@@ -22,14 +36,12 @@ export const get3DRecordsXML = () => {
 
 export const parse3DQueryResults = (xml: string): Record<string, unknown>[] | null => {
   let retValue = null;
-  const parser = new XMLParser({ ignoreAttributes: false });
-  const parsedQuery = parser.parse(xml);
-  const recordsResult = parsedQuery['csw:GetRecordsResponse']['csw:SearchResults'];
-  if (recordsResult['@_numberOfRecordsMatched'] === '0') {
+  const searchResults = extractSearchResults(xml);
+  if (searchResults?.['@_numberOfRecordsMatched'] === '0') {
     console.error(`Didn't find matched IDs!`);
     return retValue;
   }
-  const records = parsedQuery['csw:GetRecordsResponse']['csw:SearchResults']['mc:MC3DRecord'];
+  const records = searchResults['mc:MC3DRecord'];
   if (Array.isArray(records)) {
     retValue = records;
   } else {
@@ -40,3 +52,8 @@ export const parse3DQueryResults = (xml: string): Record<string, unknown>[] | nu
     ['3DPhotoRealistic', 'PointCloud'].includes(record['mc:productType'])
   );
 };
+
+export const getNumberOfMatchedRecords = (xml: string): number => {
+  const searchResults = extractSearchResults(xml);
+  return Number(searchResults?.['@_numberOfRecordsMatched'] ?? 0);
+};