diff --git a/confd/production.tmpl b/confd/production.tmpl index 11e0010..99b37eb 100644 --- a/confd/production.tmpl +++ b/confd/production.tmpl @@ -10,5 +10,6 @@ window._env_ = { DEFAULT_TERRAIN_PROVIDER_URL: '{{ getv "/configuration/default/terrain/provider/url" "" }}', CSW_3D_URL: '{{ getv "/configuration/csw/3d/url" ""}}', EXTRACTABLE_MANAGER_URL: '{{ getv "/configuration/extractable/manager/url" ""}}', - SHOW_POI_TOOL: {{ getv "/configuration/show/poi/tool" "true"}} + SHOW_POI_TOOL: {{ getv "/configuration/show/poi/tool" "true"}}, + NUMBER_OF_RECORDS_PER_PAGE: {{ getv "/configuration/number/of/records/per/page" "200"}} }; diff --git a/confd/production.toml b/confd/production.toml index ac2bbf4..00456a4 100644 --- a/confd/production.toml +++ b/confd/production.toml @@ -13,5 +13,6 @@ keys = [ "/configuration/default/terrain/provider/url", "/configuration/csw/3d/url", "/configuration/extractable/manager/url", - "/configuration/show/poi/tool" + "/configuration/show/poi/tool", + "/configuration/number/of/records/per/page" ] diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 4d8d3e1..cec3747 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -22,6 +22,7 @@ CONFIGURATION_CSW_3D_URL: {{ quote .Values.env.csw3DUrl }} CONFIGURATION_EXTRACTABLE_MANAGER_URL: {{ quote .Values.env.extractableManagerUrl }} CONFIGURATION_SHOW_POI_TOOL: {{ quote .Values.env.showPOITool }} + NUMBER_OF_RECORDS_PER_PAGE: {{ quote .Values.env.numberOfRecordsPerPage }} {{- end }} {{ include "mc-chart.configmap" (dict "MAIN_OBJECT_BLOCK" $MAIN_OBJECT_BLOCK "COMPONENT_NAME" $COMPONENT_NAME "DATA" $DATA "WITH_TELEMETRY_TRACING" false "WITH_TELEMETRY_METRICS" false "context" .)}} diff --git a/helm/values.yaml b/helm/values.yaml index 6d187f6..262a970 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -113,6 +113,7 @@ env: csw3DUrl: '' extractableManagerUrl: '' showPOITool: true + numberOfRecordsPerPage: 200 route: enabled: true annotations: diff --git a/src/common/services/CatalogService.ts b/src/common/services/CatalogService.ts index 8854560..75871a5 100644 --- a/src/common/services/CatalogService.ts +++ b/src/common/services/CatalogService.ts @@ -1,41 +1,81 @@ import { createCatalogTree } from '../../components/common/Tree/TreeGroup'; import { IDENTIFIER_FIELD } from '../../components/Wizard/Wizard.types'; import appConfig from '../../utils/Config'; -import { get3DRecordsXML, parse3DQueryResults } from '../../utils/cswQueryBuilder'; +import { + get3DRecordsXML, + getNumberOfMatchedRecords, + parse3DQueryResults, +} from '../../utils/cswQueryBuilder'; import { loadingUpdater } from '../../utils/loadingUpdater'; import { execute } from '../../utils/requestHandler'; -import { ExtractableRecord } from './ExtractableService'; +import { ExtractableRecord, ExtractableResponse } from './ExtractableService'; + +const PAGE_SIZE = appConfig.numberOfRecordsPerPage; +const EXTRACTABLE_PAGE_SIZE = appConfig.numberOfExtractablesPerPage; + +const fetchAll3DRecordsParallel = async () => { + const numberOfRecordsXml = get3DRecordsXML('hits', 0); + + const resNumberOfRecords = await execute(`${appConfig.csw3dUrl}`, 'POST', { + data: numberOfRecordsXml, + }); + + const totalRecords = getNumberOfMatchedRecords(resNumberOfRecords as string); + const totalPages = Math.ceil(totalRecords / PAGE_SIZE); + + const promises = Array.from({ length: totalPages }, (_, i) => { + const startPosition = i * PAGE_SIZE + 1; + const xml = get3DRecordsXML('results', PAGE_SIZE, startPosition); + return execute(appConfig.csw3dUrl, 'POST', { data: xml }); + }); + + const responses = await Promise.all(promises); + + const allRecords = responses.flatMap( + (res) => parse3DQueryResults(res as string) as Record[] + ); + + return allRecords; +}; + +const fetchExtractable = async () => { + let extract: ExtractableRecord[] = []; + let startIndex = 1; + + while (startIndex > 0) { + const extractableResponse: ExtractableResponse = (await execute( + `${appConfig.extractableManagerUrl}/records?startPosition=${startIndex}&maxRecords=${EXTRACTABLE_PAGE_SIZE}`, + 'GET' + )) as unknown as ExtractableResponse; + if (Array.isArray(extractableResponse.records)) { + extract.push(...extractableResponse.records); + startIndex = extractableResponse.nextRecord as number; + } + } + return extract; +}; export const fetchCatalog = async (setLoading: loadingUpdater) => { - let parsed, extractables; + let records; + let extractables: ExtractableRecord[] = []; + try { setLoading(true); - const data = get3DRecordsXML(); - const records = await execute(`${appConfig.csw3dUrl}`, 'POST', { data }); - parsed = parse3DQueryResults(records as string) as Record[]; - extractables = await execute( - `${appConfig.extractableManagerUrl}/records?startPosition=1&maxRecords=1000`, - 'GET' - ); + records = await fetchAll3DRecordsParallel(); + extractables = await fetchExtractable(); } catch (error) { console.error('Failed to fetch catalog/extractable data:', error); } finally { - const catalogRecords = Array.isArray(parsed) ? parsed : []; - const extractablesPayload = extractables as { records?: ExtractableRecord[] } | undefined; - const extractablesList = extractablesPayload?.records; - const extractablesRecords: ExtractableRecord[] = Array.isArray(extractablesList) - ? extractablesList - : []; - const enriched = enrichRecords(catalogRecords, extractablesRecords); + const catalogRecords = Array.isArray(records) ? records : []; + const enriched = enrichRecords(catalogRecords, extractables); setLoading(false); return { data: createCatalogTree(enriched), sumAll: catalogRecords.length, - sumExtractable: - catalogRecords.length > 0 ? extractablesRecords.length : catalogRecords.length, + sumExtractable: catalogRecords.length > 0 ? extractables.length : catalogRecords.length, sumNotExtractable: catalogRecords.length > 0 - ? catalogRecords.length - extractablesRecords.length + ? catalogRecords.length - extractables.length : catalogRecords.length, }; } diff --git a/src/common/services/ExtractableService.ts b/src/common/services/ExtractableService.ts index 5995e38..6d65ccb 100644 --- a/src/common/services/ExtractableService.ts +++ b/src/common/services/ExtractableService.ts @@ -2,6 +2,13 @@ import appConfig from '../../utils/Config'; import { loadingUpdater } from '../../utils/loadingUpdater'; import { execute } from '../../utils/requestHandler'; +export interface ExtractableResponse { + nextRecord: number; + numberOfRecords: number; + numberOfRecordsReturned: number; + records: ExtractableRecord[]; +} + export interface ExtractableRecord { id: string; recordName: string; diff --git a/src/utils/Config.ts b/src/utils/Config.ts index ca96722..4a87da1 100644 --- a/src/utils/Config.ts +++ b/src/utils/Config.ts @@ -18,6 +18,7 @@ const DEFAULT_TERRAIN_PROVIDER_URL = (window as any)._env_.DEFAULT_TERRAIN_PROVI const CSW_3D_URL = (window as any)._env_.CSW_3D_URL; const EXTRACTABLE_MANAGER_URL = (window as any)._env_.EXTRACTABLE_MANAGER_URL; const SHOW_POI_TOOL = (window as any)._env_.SHOW_POI_TOOL; +const NUMBER_OF_RECORDS_PER_PAGE = (window as any)._env_.NUMBER_OF_RECORDS_PER_PAGE; const enrichBaseMaps = (baseMaps: IBaseMaps): IBaseMaps => { return { @@ -56,6 +57,8 @@ class Config { public csw3dUrl = CSW_3D_URL; public extractableManagerUrl = EXTRACTABLE_MANAGER_URL; public showPOITool = SHOW_POI_TOOL; + public numberOfRecordsPerPage = NUMBER_OF_RECORDS_PER_PAGE; + public numberOfExtractablesPerPage = 500; } const appConfig = new Config(); // Singleton diff --git a/src/utils/cswQueryBuilder.ts b/src/utils/cswQueryBuilder.ts index 211891a..07f552d 100644 --- a/src/utils/cswQueryBuilder.ts +++ b/src/utils/cswQueryBuilder.ts @@ -1,6 +1,18 @@ import { XMLParser } from 'fast-xml-parser'; +import appConfig from './Config'; -export const get3DRecordsXML = () => { +const extractSearchResults = (xml: string) => { + const parser = new XMLParser({ ignoreAttributes: false }); + const parsed = parser.parse(xml); + + return parsed?.['csw:GetRecordsResponse']?.['csw:SearchResults']; +}; + +export const get3DRecordsXML = ( + resultType: string = 'results', + maxRecords: number = appConfig.numberOfRecordsPerPage, + startPosition: number = 1 +) => { return ` { xmlns:dct="http://purl.org/dc/terms/" service="CSW" version="2.0.2" - resultType="results" + resultType="${resultType}" + maxRecords="${maxRecords}" + startPosition="${startPosition}" outputSchema="http://schema.mapcolonies.com/3d"> full @@ -22,14 +36,12 @@ export const get3DRecordsXML = () => { export const parse3DQueryResults = (xml: string): Record[] | null => { let retValue = null; - const parser = new XMLParser({ ignoreAttributes: false }); - const parsedQuery = parser.parse(xml); - const recordsResult = parsedQuery['csw:GetRecordsResponse']['csw:SearchResults']; - if (recordsResult['@_numberOfRecordsMatched'] === '0') { + const searchResults = extractSearchResults(xml); + if (searchResults?.['@_numberOfRecordsMatched'] === '0') { console.error(`Didn't find matched IDs!`); return retValue; } - const records = parsedQuery['csw:GetRecordsResponse']['csw:SearchResults']['mc:MC3DRecord']; + const records = searchResults['mc:MC3DRecord']; if (Array.isArray(records)) { retValue = records; } else { @@ -40,3 +52,8 @@ export const parse3DQueryResults = (xml: string): Record[] | nu ['3DPhotoRealistic', 'PointCloud'].includes(record['mc:productType']) ); }; + +export const getNumberOfMatchedRecords = (xml: string): number => { + const searchResults = extractSearchResults(xml); + return Number(searchResults?.['@_numberOfRecordsMatched'] ?? 0); +};