From 691040657f938b405be58213b1632c360569bcf8 Mon Sep 17 00:00:00 2001 From: Eliav Maimon Date: Thu, 26 Feb 2026 12:43:36 +0200 Subject: [PATCH 1/4] fix: parallel fetch records --- confd/production.tmpl | 3 +- confd/production.toml | 3 +- helm/templates/configmap.yaml | 1 + helm/values.yaml | 1 + src/common/services/CatalogService.ts | 54 ++++++++++++++++++++++----- src/utils/Config.ts | 2 + src/utils/cswQueryBuilder.ts | 37 +++++++++++++----- 7 files changed, 79 insertions(+), 22 deletions(-) diff --git a/confd/production.tmpl b/confd/production.tmpl index 11e0010..99b37eb 100644 --- a/confd/production.tmpl +++ b/confd/production.tmpl @@ -10,5 +10,6 @@ window._env_ = { DEFAULT_TERRAIN_PROVIDER_URL: '{{ getv "/configuration/default/terrain/provider/url" "" }}', CSW_3D_URL: '{{ getv "/configuration/csw/3d/url" ""}}', EXTRACTABLE_MANAGER_URL: '{{ getv "/configuration/extractable/manager/url" ""}}', - SHOW_POI_TOOL: {{ getv "/configuration/show/poi/tool" "true"}} + SHOW_POI_TOOL: {{ getv "/configuration/show/poi/tool" "true"}}, + NUMBER_OF_RECORDS_PER_PAGE: {{ getv "/configuration/number/of/records/per/page" "200"}} }; diff --git a/confd/production.toml b/confd/production.toml index ac2bbf4..00456a4 100644 --- a/confd/production.toml +++ b/confd/production.toml @@ -13,5 +13,6 @@ keys = [ "/configuration/default/terrain/provider/url", "/configuration/csw/3d/url", "/configuration/extractable/manager/url", - "/configuration/show/poi/tool" + "/configuration/show/poi/tool", + "/configuration/number/of/records/per/page" ] diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 4d8d3e1..cec3747 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -22,6 +22,7 @@ CONFIGURATION_CSW_3D_URL: {{ quote .Values.env.csw3DUrl }} CONFIGURATION_EXTRACTABLE_MANAGER_URL: {{ quote .Values.env.extractableManagerUrl }} CONFIGURATION_SHOW_POI_TOOL: {{ quote .Values.env.showPOITool }} + NUMBER_OF_RECORDS_PER_PAGE: {{ quote .Values.env.numberOfRecordsPerPage }} {{- end }} {{ include "mc-chart.configmap" (dict "MAIN_OBJECT_BLOCK" $MAIN_OBJECT_BLOCK "COMPONENT_NAME" $COMPONENT_NAME "DATA" $DATA "WITH_TELEMETRY_TRACING" false "WITH_TELEMETRY_METRICS" false "context" .)}} diff --git a/helm/values.yaml b/helm/values.yaml index 6d187f6..262a970 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -113,6 +113,7 @@ env: csw3DUrl: '' extractableManagerUrl: '' showPOITool: true + numberOfRecordsPerPage: 200 route: enabled: true annotations: diff --git a/src/common/services/CatalogService.ts b/src/common/services/CatalogService.ts index e8aac77..cd21dc9 100644 --- a/src/common/services/CatalogService.ts +++ b/src/common/services/CatalogService.ts @@ -1,22 +1,56 @@ import { createCatalogTree } from '../../components/common/Tree/TreeGroup'; import { IDENTIFIER_FIELD } from '../../components/Wizard/Wizard.types'; import appConfig from '../../utils/Config'; -import { get3DRecordsXML, parse3DQueryResults } from '../../utils/cswQueryBuilder'; +import { get3DRecordsXML, getNumberOfMatchedRecords, parse3DQueryResults } from '../../utils/cswQueryBuilder'; import { loadingUpdater } from '../../utils/loadingUpdater'; import { execute } from '../../utils/requestHandler'; import { ExtractableRecord } from './ExtractableService'; +const PAGE_SIZE = appConfig.numberOfRecordsPerPage; + +export const fetchAll3DRecordsParallel = async () => { + const numberOfRecordsXml = get3DRecordsXML('hits', 0); + + const resNumberOfRecords = await execute( + `${appConfig.csw3dUrl}`, + 'POST', + { data: numberOfRecordsXml } + ); + + const totalRecords = getNumberOfMatchedRecords(resNumberOfRecords as string); + const totalPages = Math.ceil(totalRecords / PAGE_SIZE); + + const promises = Array.from({ length: totalPages }, (_, i) => { + const startPosition = i * PAGE_SIZE + 1; + + const xml = get3DRecordsXML( + 'results', + PAGE_SIZE, + startPosition, + ); + + return execute( + appConfig.csw3dUrl, + 'POST', + { data: xml } + ); + }); + + const responses = await Promise.all(promises); + + const allRecords = responses.flatMap((res) => + parse3DQueryResults(res as string) as Record[] + ); + + return allRecords; +}; + export const fetchCatalog = async (setLoading: loadingUpdater) => { - let parsed, extractables; + let records, extractables; try { setLoading(true); - const data = get3DRecordsXML(); - const records = await execute( - `${appConfig.csw3dUrl}`, - 'POST', - { data } - ); - parsed = parse3DQueryResults(records as string) as Record[]; + records = await fetchAll3DRecordsParallel(); + extractables = await execute( `${appConfig.extractableManagerUrl}/records?startPosition=1&maxRecords=1000`, 'GET' @@ -24,7 +58,7 @@ export const fetchCatalog = async (setLoading: loadingUpdater) => { } catch (error) { console.error('Failed to fetch catalog/extractable data:', error); } finally { - const catalogRecords = Array.isArray(parsed) ? parsed : []; + const catalogRecords = Array.isArray(records) ? records : []; const extractablesPayload = extractables as { records?: ExtractableRecord[] } | undefined; const extractablesList = extractablesPayload?.records; const extractablesRecords: ExtractableRecord[] = Array.isArray(extractablesList) ? extractablesList : []; diff --git a/src/utils/Config.ts b/src/utils/Config.ts index 0c23501..ed30fd4 100644 --- a/src/utils/Config.ts +++ b/src/utils/Config.ts @@ -13,6 +13,7 @@ const DEFAULT_TERRAIN_PROVIDER_URL = (window as any)._env_.DEFAULT_TERRAIN_PROVI const CSW_3D_URL = (window as any)._env_.CSW_3D_URL; const EXTRACTABLE_MANAGER_URL = (window as any)._env_.EXTRACTABLE_MANAGER_URL; const SHOW_POI_TOOL = (window as any)._env_.SHOW_POI_TOOL; +const NUMBER_OF_RECORDS_PER_PAGE = (window as any)._env_.NUMBER_OF_RECORDS_PER_PAGE; const enrichBaseMaps = (baseMaps: IBaseMaps): IBaseMaps => { return { @@ -48,6 +49,7 @@ class Config { public csw3dUrl = CSW_3D_URL; public extractableManagerUrl = EXTRACTABLE_MANAGER_URL; public showPOITool = SHOW_POI_TOOL; + public numberOfRecordsPerPage = NUMBER_OF_RECORDS_PER_PAGE; } const appConfig = new Config(); // Singleton diff --git a/src/utils/cswQueryBuilder.ts b/src/utils/cswQueryBuilder.ts index 6740aba..2951323 100644 --- a/src/utils/cswQueryBuilder.ts +++ b/src/utils/cswQueryBuilder.ts @@ -1,6 +1,18 @@ import { XMLParser } from 'fast-xml-parser'; +import appConfig from './Config'; -export const get3DRecordsXML = () => { +const extractSearchResults = (xml: string) => { + const parser = new XMLParser({ ignoreAttributes: false }); + const parsed = parser.parse(xml); + + return parsed?.['csw:GetRecordsResponse']?.['csw:SearchResults']; +}; + +export const get3DRecordsXML = ( + resultType: string = 'results', + maxRecords: number = appConfig.numberOfRecordsPerPage, + startPosition: number = 1 +) => { return ` { xmlns:dct="http://purl.org/dc/terms/" service="CSW" version="2.0.2" - resultType="results" + resultType="${resultType}" + maxRecords="${maxRecords}" + startPosition="${startPosition}" outputSchema="http://schema.mapcolonies.com/3d"> full @@ -21,21 +35,24 @@ export const get3DRecordsXML = () => { }; export const parse3DQueryResults = (xml: string): Record[] | null => { - let retValue = null; - const parser = new XMLParser({ ignoreAttributes: false }); - const parsedQuery = parser.parse(xml); - const recordsResult = parsedQuery['csw:GetRecordsResponse']['csw:SearchResults']; - if (recordsResult['@_numberOfRecordsMatched'] === '0') { + let retValue = null; + const searchResults = extractSearchResults(xml); + if (searchResults?.['@_numberOfRecordsMatched'] === '0') { console.error(`Didn't find matched IDs!`); return retValue; } - const records = parsedQuery['csw:GetRecordsResponse']['csw:SearchResults']['mc:MC3DRecord']; + const records = searchResults['mc:MC3DRecord']; if (Array.isArray(records)) { retValue = records; } else { - retValue = [records]; + retValue = [records]; } - return retValue.filter((record) => ['3DPhotoRealistic','PointCloud'].includes(record['mc:productType'])); + return retValue.filter((record) => ['3DPhotoRealistic', 'PointCloud'].includes(record['mc:productType'])); }; + +export const getNumberOfMatchedRecords = (xml: string): number => { + const searchResults = extractSearchResults(xml); + return Number(searchResults?.['@_numberOfRecordsMatched'] ?? 0); +} From 207a2fd04720dacfdd1e464b9c3ae9f98b31a15a Mon Sep 17 00:00:00 2001 From: Eliav Maimon Date: Thu, 26 Feb 2026 14:46:58 +0200 Subject: [PATCH 2/4] fix: pagination fetch extractable --- src/common/services/CatalogService.ts | 43 ++++++++++++++--------- src/common/services/ExtractableService.ts | 7 ++++ src/utils/Config.ts | 1 + 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/common/services/CatalogService.ts b/src/common/services/CatalogService.ts index cf76dc5..84adefb 100644 --- a/src/common/services/CatalogService.ts +++ b/src/common/services/CatalogService.ts @@ -4,11 +4,12 @@ import appConfig from '../../utils/Config'; import { get3DRecordsXML, getNumberOfMatchedRecords, parse3DQueryResults } from '../../utils/cswQueryBuilder'; import { loadingUpdater } from '../../utils/loadingUpdater'; import { execute } from '../../utils/requestHandler'; -import { ExtractableRecord } from './ExtractableService'; +import { ExtractableRecord, ExtractableResponse } from './ExtractableService'; const PAGE_SIZE = appConfig.numberOfRecordsPerPage; +const EXTRACTABLE_PAGE_SIZE = appConfig.numberOfExtractablesPerPage; -export const fetchAll3DRecordsParallel = async () => { +const fetchAll3DRecordsParallel = async () => { const numberOfRecordsXml = get3DRecordsXML('hits', 0); const resNumberOfRecords = await execute( @@ -45,35 +46,45 @@ export const fetchAll3DRecordsParallel = async () => { return allRecords; }; +const fetchExtractable = async () => { + let extract: ExtractableRecord[] = []; + let startIndex = 1; + + while (startIndex > 0) { + const extractableResponse: ExtractableResponse = await execute( + `${appConfig.extractableManagerUrl}/records?startPosition=${startIndex}&maxRecords=${EXTRACTABLE_PAGE_SIZE}`, + 'GET' + ) as unknown as ExtractableResponse; + if (Array.isArray(extractableResponse.records)) { + extract.push(...extractableResponse.records); + startIndex = extractableResponse.nextRecord as number; + } + } + return extract; +} + export const fetchCatalog = async (setLoading: loadingUpdater) => { - let records, extractables; + let records; + let extractables: ExtractableRecord[] = []; + try { setLoading(true); records = await fetchAll3DRecordsParallel(); - - extractables = await execute( - `${appConfig.extractableManagerUrl}/records?startPosition=1&maxRecords=1000`, - 'GET' - ); + extractables = await fetchExtractable(); } catch (error) { console.error('Failed to fetch catalog/extractable data:', error); } finally { const catalogRecords = Array.isArray(records) ? records : []; - const extractablesPayload = extractables as { records?: ExtractableRecord[] } | undefined; - const extractablesList = extractablesPayload?.records; - const extractablesRecords: ExtractableRecord[] = Array.isArray(extractablesList) - ? extractablesList - : []; - const enriched = enrichRecords(catalogRecords, extractablesRecords); + const enriched = enrichRecords(catalogRecords, extractables); setLoading(false); return { data: createCatalogTree(enriched), sumAll: catalogRecords.length, sumExtractable: - catalogRecords.length > 0 ? extractablesRecords.length : catalogRecords.length, + catalogRecords.length > 0 ? extractables.length : catalogRecords.length, sumNotExtractable: catalogRecords.length > 0 - ? catalogRecords.length - extractablesRecords.length + ? catalogRecords.length - extractables.length : catalogRecords.length, }; } diff --git a/src/common/services/ExtractableService.ts b/src/common/services/ExtractableService.ts index 5995e38..32eb69b 100644 --- a/src/common/services/ExtractableService.ts +++ b/src/common/services/ExtractableService.ts @@ -2,6 +2,13 @@ import appConfig from '../../utils/Config'; import { loadingUpdater } from '../../utils/loadingUpdater'; import { execute } from '../../utils/requestHandler'; +export interface ExtractableResponse { + nextRecord: number; + numberOfRecords: number; + numberOfRecordsReturned: number; + records: ExtractableRecord[] +} + export interface ExtractableRecord { id: string; recordName: string; diff --git a/src/utils/Config.ts b/src/utils/Config.ts index 253da9d..4a87da1 100644 --- a/src/utils/Config.ts +++ b/src/utils/Config.ts @@ -58,6 +58,7 @@ class Config { public extractableManagerUrl = EXTRACTABLE_MANAGER_URL; public showPOITool = SHOW_POI_TOOL; public numberOfRecordsPerPage = NUMBER_OF_RECORDS_PER_PAGE; + public numberOfExtractablesPerPage = 500; } const appConfig = new Config(); // Singleton From 790e9a6ad6f299a3a6989e079cc6233a24fec0c3 Mon Sep 17 00:00:00 2001 From: Eliav Maimon Date: Thu, 26 Feb 2026 14:50:40 +0200 Subject: [PATCH 3/4] fix: prettier --- src/common/services/CatalogService.ts | 39 ++++++++++------------- src/common/services/ExtractableService.ts | 2 +- src/utils/cswQueryBuilder.ts | 2 +- 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/common/services/CatalogService.ts b/src/common/services/CatalogService.ts index 84adefb..bb3a36d 100644 --- a/src/common/services/CatalogService.ts +++ b/src/common/services/CatalogService.ts @@ -1,7 +1,11 @@ import { createCatalogTree } from '../../components/common/Tree/TreeGroup'; import { IDENTIFIER_FIELD } from '../../components/Wizard/Wizard.types'; import appConfig from '../../utils/Config'; -import { get3DRecordsXML, getNumberOfMatchedRecords, parse3DQueryResults } from '../../utils/cswQueryBuilder'; +import { + get3DRecordsXML, + getNumberOfMatchedRecords, + parse3DQueryResults, +} from '../../utils/cswQueryBuilder'; import { loadingUpdater } from '../../utils/loadingUpdater'; import { execute } from '../../utils/requestHandler'; import { ExtractableRecord, ExtractableResponse } from './ExtractableService'; @@ -12,11 +16,9 @@ const EXTRACTABLE_PAGE_SIZE = appConfig.numberOfExtractablesPerPage; const fetchAll3DRecordsParallel = async () => { const numberOfRecordsXml = get3DRecordsXML('hits', 0); - const resNumberOfRecords = await execute( - `${appConfig.csw3dUrl}`, - 'POST', - { data: numberOfRecordsXml } - ); + const resNumberOfRecords = await execute(`${appConfig.csw3dUrl}`, 'POST', { + data: numberOfRecordsXml, + }); const totalRecords = getNumberOfMatchedRecords(resNumberOfRecords as string); const totalPages = Math.ceil(totalRecords / PAGE_SIZE); @@ -24,23 +26,15 @@ const fetchAll3DRecordsParallel = async () => { const promises = Array.from({ length: totalPages }, (_, i) => { const startPosition = i * PAGE_SIZE + 1; - const xml = get3DRecordsXML( - 'results', - PAGE_SIZE, - startPosition, - ); + const xml = get3DRecordsXML('results', PAGE_SIZE, startPosition); - return execute( - appConfig.csw3dUrl, - 'POST', - { data: xml } - ); + return execute(appConfig.csw3dUrl, 'POST', { data: xml }); }); const responses = await Promise.all(promises); - const allRecords = responses.flatMap((res) => - parse3DQueryResults(res as string) as Record[] + const allRecords = responses.flatMap( + (res) => parse3DQueryResults(res as string) as Record[] ); return allRecords; @@ -51,17 +45,17 @@ const fetchExtractable = async () => { let startIndex = 1; while (startIndex > 0) { - const extractableResponse: ExtractableResponse = await execute( + const extractableResponse: ExtractableResponse = (await execute( `${appConfig.extractableManagerUrl}/records?startPosition=${startIndex}&maxRecords=${EXTRACTABLE_PAGE_SIZE}`, 'GET' - ) as unknown as ExtractableResponse; + )) as unknown as ExtractableResponse; if (Array.isArray(extractableResponse.records)) { extract.push(...extractableResponse.records); startIndex = extractableResponse.nextRecord as number; } } return extract; -} +}; export const fetchCatalog = async (setLoading: loadingUpdater) => { let records; @@ -80,8 +74,7 @@ export const fetchCatalog = async (setLoading: loadingUpdater) => { return { data: createCatalogTree(enriched), sumAll: catalogRecords.length, - sumExtractable: - catalogRecords.length > 0 ? extractables.length : catalogRecords.length, + sumExtractable: catalogRecords.length > 0 ? extractables.length : catalogRecords.length, sumNotExtractable: catalogRecords.length > 0 ? catalogRecords.length - extractables.length diff --git a/src/common/services/ExtractableService.ts b/src/common/services/ExtractableService.ts index 32eb69b..6d65ccb 100644 --- a/src/common/services/ExtractableService.ts +++ b/src/common/services/ExtractableService.ts @@ -6,7 +6,7 @@ export interface ExtractableResponse { nextRecord: number; numberOfRecords: number; numberOfRecordsReturned: number; - records: ExtractableRecord[] + records: ExtractableRecord[]; } export interface ExtractableRecord { diff --git a/src/utils/cswQueryBuilder.ts b/src/utils/cswQueryBuilder.ts index 7569d54..07f552d 100644 --- a/src/utils/cswQueryBuilder.ts +++ b/src/utils/cswQueryBuilder.ts @@ -56,4 +56,4 @@ export const parse3DQueryResults = (xml: string): Record[] | nu export const getNumberOfMatchedRecords = (xml: string): number => { const searchResults = extractSearchResults(xml); return Number(searchResults?.['@_numberOfRecordsMatched'] ?? 0); -} +}; From 8cdf07274ea2b380c781bca9e8db2196408311e9 Mon Sep 17 00:00:00 2001 From: Eliav Maimon Date: Thu, 26 Feb 2026 14:58:54 +0200 Subject: [PATCH 4/4] chore: remove spaces --- src/common/services/CatalogService.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/common/services/CatalogService.ts b/src/common/services/CatalogService.ts index bb3a36d..75871a5 100644 --- a/src/common/services/CatalogService.ts +++ b/src/common/services/CatalogService.ts @@ -25,9 +25,7 @@ const fetchAll3DRecordsParallel = async () => { const promises = Array.from({ length: totalPages }, (_, i) => { const startPosition = i * PAGE_SIZE + 1; - const xml = get3DRecordsXML('results', PAGE_SIZE, startPosition); - return execute(appConfig.csw3dUrl, 'POST', { data: xml }); });