From 1d32e548f5046871ea575f2c899345390a1ff491 Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Tue, 7 Oct 2025 23:04:16 +0530 Subject: [PATCH 01/14] Reprocessor Changes Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../MosipVerticleManager.java | 7 +- .../status/dao/RegistrationStatusDao.java | 29 +++ .../repositary/RegistrationRepositary.java | 4 + .../service/RegistrationStatusService.java | 20 ++ .../impl/RegistrationStatusServiceImpl.java | 40 +++- .../config/ReprocessorConfigBeans.java | 2 + .../service/ReprocessorVerticalService.java | 12 ++ .../impl/ReprocessorVerticalServiceImpl.java | 28 +++ .../verticle/ReprocessorVerticle.java | 173 ++++++++++++++++-- .../src/main/resources/bootstrap.properties | 3 +- .../verticle/ReprocessorVerticleTest.java | 22 ++- 11 files changed, 320 insertions(+), 20 deletions(-) create mode 100644 registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java create mode 100644 registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java diff --git a/registration-processor/registration-processor-core/src/main/java/io/mosip/registration/processor/core/abstractverticle/MosipVerticleManager.java b/registration-processor/registration-processor-core/src/main/java/io/mosip/registration/processor/core/abstractverticle/MosipVerticleManager.java index 8763a5f2f2f..7bd9a8e81d3 100644 --- a/registration-processor/registration-processor-core/src/main/java/io/mosip/registration/processor/core/abstractverticle/MosipVerticleManager.java +++ b/registration-processor/registration-processor-core/src/main/java/io/mosip/registration/processor/core/abstractverticle/MosipVerticleManager.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.MDC; @@ -146,7 +147,7 @@ public MosipEventBus getEventBus(Object verticleName, String clusterManagerUrl, VertxOptions options = new VertxOptions().setClustered(true).setClusterManager(clusterManager) .setHAEnabled(false).setWorkerPoolSize(instanceNumber) .setEventBusOptions(new EventBusOptions().setPort(getEventBusPort()).setHost(address)) - .setMetricsOptions(micrometerMetricsOptions); + .setMetricsOptions(micrometerMetricsOptions).setMaxEventLoopExecuteTime(getIntegerPropertyForSuffix("eventbus.maxloop.exec.time.seconds", 2)).setMaxEventLoopExecuteTimeUnit(TimeUnit.SECONDS); Vertx.clusteredVertx(options, result -> { if (result.succeeded()) { result.result().deployVerticle((Verticle) verticleName, @@ -290,6 +291,10 @@ protected Integer getIntegerPropertyForSuffix(String propSuffix) { return propertiesUtil.getIntegerProperty(getPropertyPrefix(), propSuffix); } + protected Integer getIntegerPropertyForSuffix(String propSuffix, Integer defaultValue) { + return propertiesUtil.getProperty(getPropertyPrefix() + propSuffix, Integer.class, defaultValue); + } + protected Boolean getBooleanPropertyForSuffix(String propSuffix, Boolean defaultValue) { return propertiesUtil.getProperty(getPropertyPrefix() + propSuffix, Boolean.class, defaultValue); } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java index e5810589190..30bfc60240e 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java @@ -247,4 +247,33 @@ public List findByIdAndProcessAndIteration(String id, { return registrationStatusRepositary.getByIdAndProcessAndIteration(id, process, iteration); } + + /** + * Gets the un processed packets. + * + * @param fetchSize + * the fetch size + * @param elapseTime + * the elapse time + * @param reprocessCount + * the reprocess count + * @param status + * the status + * @return the un processed packets + */ + public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, + Integer reprocessCount, List status, List excludeStageNames) { + + LocalDateTime timeDifference = LocalDateTime.now().minusSeconds(elapseTime); + List statusCodes=new ArrayList<>(); + statusCodes.add(RegistrationStatusCode.PAUSED.toString()); + statusCodes.add(RegistrationStatusCode.RESUMABLE.toString()); + statusCodes.add(RegistrationStatusCode.PAUSED_FOR_ADDITIONAL_INFO.toString()); + statusCodes.add(RegistrationStatusCode.REJECTED.toString()); + statusCodes.add(RegistrationStatusCode.FAILED.toString()); + statusCodes.add(RegistrationStatusCode.PROCESSED.toString()); + + return registrationStatusRepositary.getUnProcessedPackets(processList, status, reprocessCount, timeDifference, + statusCodes, fetchSize, excludeStageNames); + } } \ No newline at end of file diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java index 5b9027a921e..50126b4c5ae 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java @@ -64,5 +64,9 @@ public List getProcessedOrProcessingRegIds(@Param("regIds") List @Query("SELECT registration FROM RegistrationStatusEntity registration WHERE registration.regId = :regId AND registration.registrationType = :registrationType AND registration.iteration = :iteration") public List getByIdAndProcessAndIteration(@Param("regId") String regId, @Param("registrationType") String process, @Param("iteration") int iteration); + + @Query(value ="SELECT * FROM registration r WHERE r.process IN :processes AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code NOT IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) + public List getUnProcessedPackets(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames); + } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java index ac9e84d9b29..3ca0e420a58 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java @@ -164,4 +164,24 @@ public Integer getUnProcessedPacketsCount(long elapseTime, Integer reprocessCoun public List getResumablePackets(Integer fetchSize); + /** + * Gets the un processed packets. + * + * @param processList + * the process List + * @param fetchSize + * the fetch size + * @param elapseTime + * the elapse time + * @param reprocessCount + * the reprocess count + * @param status + * the status + * @param excludeStageNames + * the exclude stage names + * @return the un processed packets + */ + public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, + List status, List excludeStageNames); + } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java index a6c3e1cc4cd..5b1c3f32cbb 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java @@ -964,5 +964,43 @@ public void updateRegistrationStatusForWorkflow(InternalRegistrationStatusDto re "RegistrationStatusServiceImpl::updateRegistrationStatusForWorkFlow()::exit"); } - + + /** + * Gets the un processed packets. + * + * @param processList + * the process List + * @param fetchSize + * the fetch size + * @param elapseTime + * the elapse time + * @param reprocessCount + * the reprocess count + * @param status + * the status + * @return the un processed packets + */ + public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, + Integer reprocessCount, List status, List excludeStageNames) { + + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), "", + "RegistrationStatusServiceImpl::getReprocessPacket()::entry"); + try { + List entityList = registrationStatusDao.getUnProcessedPackets(processList, fetchSize, + elapseTime, reprocessCount, status, excludeStageNames); + + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), "", + "RegistrationStatusServiceImpl::getReprocessPacket()::exit"); + + return convertEntityListToDtoList(entityList); + + } catch (DataAccessException | DataAccessLayerException e) { + + regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), + "", e.getMessage() + ExceptionUtils.getStackTrace(e)); + throw new TablenotAccessibleException( + PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage(), e); + } + } + } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/config/ReprocessorConfigBeans.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/config/ReprocessorConfigBeans.java index b7794a31d84..a4e0b90d197 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/config/ReprocessorConfigBeans.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/config/ReprocessorConfigBeans.java @@ -8,6 +8,7 @@ import io.mosip.registration.processor.packet.storage.utils.PacketManagerService; import io.mosip.registration.processor.reprocessor.verticle.ReprocessorVerticle; import io.mosip.registration.processor.rest.client.service.impl.RegistrationProcessorRestClientServiceImpl; +import org.springframework.scheduling.annotation.EnableAsync; /** * Config class to get configurations and beans for Reprocessor Verticle @@ -18,6 +19,7 @@ */ @PropertySource("classpath:bootstrap.properties") @Configuration +@EnableAsync public class ReprocessorConfigBeans { @Bean diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java new file mode 100644 index 00000000000..23d5f8ac8b3 --- /dev/null +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java @@ -0,0 +1,12 @@ +package io.mosip.registration.processor.reprocessor.service; + + +import io.mosip.registration.processor.status.dto.InternalRegistrationStatusDto; + +import java.util.List; + +public interface ReprocessorVerticalService { + + List fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, + List status, List excludeStageNames); +} diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java new file mode 100644 index 00000000000..0430b48f9b0 --- /dev/null +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java @@ -0,0 +1,28 @@ +package io.mosip.registration.processor.reprocessor.service.impl; + +import io.mosip.registration.processor.reprocessor.service.ReprocessorVerticalService; +import io.mosip.registration.processor.status.dto.InternalRegistrationStatusDto; +import io.mosip.registration.processor.status.dto.RegistrationStatusDto; +import io.mosip.registration.processor.status.service.RegistrationStatusService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import java.util.Arrays; +import java.util.List; + +@Service +public class ReprocessorVerticalServiceImpl implements ReprocessorVerticalService { + + /** The registration status service. */ + @Autowired + RegistrationStatusService registrationStatusService; + + + @Override + @Async + public List fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, List status, List excludeStageNames) { + return registrationStatusService.getUnProcessedPackets(processList, fetchSize, elapseTime, + reprocessCount, status, excludeStageNames); + } +} diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index bebd730a69a..d6b7003ab86 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -1,12 +1,9 @@ package io.mosip.registration.processor.reprocessor.verticle; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import io.mosip.registration.processor.reprocessor.service.ReprocessorVerticalService; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -96,6 +93,23 @@ public class ReprocessorVerticle extends MosipVerticleAPIManager { @Value("#{'${registration.processor.reprocess.restart-trigger-filter}'.split(',')}") private List reprocessRestartTriggerFilter; + @Value("#{${registration.processor.reprocess.process-based.fetch.enabled:false}") + private Boolean enabledProcessBasedFetch; + + @Value("#{${registration.processor.reprocess.process-based.fetch.count:{:}}}") + private LinkedHashMap processBasedFetchCountMapping; + + @Value("${registration.processor.reprocess.record.fetch.size:500}") + private int recordFetchSize; + + @Value("${registration.processor.reprocess.record.threashold:50}") + private int threasholdForFetch; + + private ConcurrentHashMap> packetCacheMap = new ConcurrentHashMap<>(); + + @Autowired + private ReprocessorVerticalService reprocessorVerticalService; + /** The is transaction successful. */ boolean isTransactionSuccessful; @@ -230,20 +244,41 @@ public MessageDTO process(MessageDTO object) { try { Map> reprocessRestartTriggerMap = intializeReprocessRestartTriggerMapping(); reprocessorDtoList = registrationStatusService.getResumablePackets(fetchSize); - if (!CollectionUtils.isEmpty(reprocessorDtoList)) { - if (reprocessorDtoList.size() < fetchSize) { - List reprocessorPacketList = registrationStatusService.getUnProcessedPackets(fetchSize - reprocessorDtoList.size(), elapseTime, - reprocessCount, statusList, reprocessExcludeStageNames); - if (!CollectionUtils.isEmpty(reprocessorPacketList)) { - reprocessorDtoList.addAll(reprocessorPacketList); - } + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + "Resumable Packets Count " + reprocessorDtoList.size() ); + + if(enabledProcessBasedFetch) { + if(reprocessorDtoList.size() < fetchSize) { + LinkedHashMap requiredCountMap = prepareRequiredCount( + fetchSize - reprocessorDtoList.size() + ); + + fetchPacketsIfBelowThreshold(requiredCountMap, statusList); + + reprocessorDtoList.addAll(fetchUnprocessedPacketsWithBalance( + requiredCountMap, + fetchSize - reprocessorDtoList.size() + )); + } } else { - reprocessorDtoList = registrationStatusService.getUnProcessedPackets(fetchSize, elapseTime, - reprocessCount, statusList, reprocessExcludeStageNames); + if (!CollectionUtils.isEmpty(reprocessorDtoList)) { + if (reprocessorDtoList.size() < fetchSize) { + List reprocessorPacketList = registrationStatusService.getUnProcessedPackets(fetchSize - reprocessorDtoList.size(), elapseTime, + reprocessCount, statusList, reprocessExcludeStageNames); + if (!CollectionUtils.isEmpty(reprocessorPacketList)) { + reprocessorDtoList.addAll(reprocessorPacketList); + } + } + } else { + reprocessorDtoList = registrationStatusService.getUnProcessedPackets(fetchSize, elapseTime, + reprocessCount, statusList, reprocessExcludeStageNames); + } } - + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + "Reprocessor Total Packets Fetched " + reprocessorDtoList.size()); + if (!CollectionUtils.isEmpty(reprocessorDtoList)) { reprocessorDtoList.forEach(dto -> { String registrationId = dto.getRegistrationId(); @@ -422,4 +457,110 @@ private void sendAndSetStatus(InternalRegistrationStatusDto dto, MessageDTO mess protected String getPropertyPrefix() { return VERTICLE_PROPERTY_PREFIX; } + + private LinkedHashMap prepareRequiredCount(int availableCount) { + LinkedHashMap requiredCountMap = new LinkedHashMap<>(); + int totalMapCount = processBasedFetchCountMapping.size(); + int index = 1; + int allocated = 0; + + for(Map.Entry entry : processBasedFetchCountMapping.entrySet()) { + int requiredCount; + + if(index == totalMapCount) { + // Last entry → allocate the remaining + requiredCount = fetchSize - allocated; + } else { + // Percentage-based allocation + requiredCount = (availableCount * entry.getValue()) /100; + } + + // Ensure at least 1 + requiredCount = Math.max(requiredCount, 1); + + // Prevent overshooting the total + if(allocated + requiredCount > fetchSize) { + requiredCount = fetchSize - allocated; + } + + requiredCountMap.put(entry.getKey(), requiredCount); + allocated += requiredCount; + index++; + } + return requiredCountMap; + } + + private void fetchPacketsIfBelowThreshold(LinkedHashMap requiredCountMap, List statusList) { + requiredCountMap.entrySet().parallelStream() + .filter(entry -> { + List cachedPackets = packetCacheMap.get(entry.getKey()); + return cachedPackets == null || cachedPackets.size() < threasholdForFetch; + }) + .forEach(entry -> { + String key = entry.getKey(); + + // Parse key into Process & Status + AbstractMap.SimpleEntry, List> entryPair = parseProcessAndStatus(key); + List processList = entryPair.getKey(); + List statusValList = entryPair.getValue(); + + // Fetch unprocessed packets + List registratiobRegistrationStatusDtos = reprocessorVerticalService.fetchUnProcessedPackets(processList, recordFetchSize, elapseTime, + reprocessCount, (!statusValList.isEmpty() ? statusValList : statusList), reprocessExcludeStageNames); + + // Thread-safe update to cache + packetCacheMap.compute(key, (k, existingList) -> { + if (existingList == null) return new ArrayList<>(registratiobRegistrationStatusDtos); + existingList.addAll(new ArrayList<>(registratiobRegistrationStatusDtos)); + return existingList; + }); + }); + } + + private AbstractMap.SimpleEntry, List> parseProcessAndStatus(String key) { + String[] parts = key.split("#", 2); + List process = Arrays.asList(parts[0].split(",")); + List status = parts.length > 1 ? Arrays.asList(parts[1].split(",")) : Collections.emptyList(); + return new AbstractMap.SimpleEntry<>(process, status); + } + + private List fetchUnprocessedPacketsWithBalance(LinkedHashMap requiredCountMap, int fetchCount) { + int previousBalanceCount = 0; + List reprocessorPacketList = new ArrayList<>(); + + for(Map.Entry entry : requiredCountMap.entrySet()) { + int remainingToFetch = fetchCount - reprocessorPacketList.size(); + if(remainingToFetch <= 0) + break; + + int requiredCount = entry.getValue() + previousBalanceCount; + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + entry.getKey() + " Packets Required Count " + requiredCount); + + List cachedPackets = packetCacheMap.getOrDefault(entry.getKey(), Collections.emptyList()); + + if(!cachedPackets.isEmpty()) { + int count = Math.min(requiredCount, remainingToFetch); + List fetchedPackets = fetchFromCache(cachedPackets, count); + reprocessorPacketList.addAll(fetchedPackets); + + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + entry.getKey() + " Packets Count " + fetchedPackets.size()); + + previousBalanceCount = Math.max(0, requiredCount-fetchedPackets.size()); + } else { + previousBalanceCount = requiredCount; + } + } + + return reprocessorPacketList; + } + + private List fetchFromCache(List cache, int count) { + int actualFetchCount = Math.min(count, cache.size()); + List fetched = new ArrayList<>(cache.subList(0, actualFetchCount)); + cache.subList(0, actualFetchCount).clear(); + return fetched; + } + } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties index 1403ff403b7..be96fea26be 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties @@ -17,4 +17,5 @@ registration.processor.identityjson=RegistrationProcessorIdentity.json registration.processor.demographic.identity=identity packet.info.storage.service=registration-processor-packet-info-storage-service config.server.file.storage.uri=${spring.cloud.config.uri}/${packet.info.storage.service}/${spring.profiles.active}/${spring.cloud.config.label}/ -mosip.regproc.registration.status.service.disable-audit=true \ No newline at end of file +mosip.regproc.registration.status.service.disable-audit=true +registration.processor.reprocess.process-based.fetch.count={"MIGRATOR":40, "RENEWAL#ON_HOLD_RENEWAL":30, "NEW,RENEWAL":30} \ No newline at end of file diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java index b028d511454..104167fba12 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java @@ -8,8 +8,12 @@ import java.lang.reflect.Field; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import io.mosip.registration.processor.reprocessor.service.ReprocessorVerticalService; +import io.mosip.registration.processor.reprocessor.service.impl.ReprocessorVerticalServiceImpl; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -104,6 +108,9 @@ public void send(MosipEventBus mosipEventBus, MessageBusAddress toAddress, Messa @Mock RegistrationStatusService registrationStatusService; + @Mock + private ReprocessorVerticalService reprocessorVerticalService; + @Mock private AuditLogRequestBuilder auditLogRequestBuilder; @@ -117,9 +124,17 @@ public void setup() throws Exception { //Mockito.doNothing().when(description).setMessage(Mockito.anyString()); //Mockito.when(description.getCode()).thenReturn("CODE"); //Mockito.when(description.getMessage()).thenReturn("MESSAGE"); - ReflectionTestUtils.setField(reprocessorVerticle, "fetchSize", 2); + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + LinkedHashMap processBasedFetchCountMapping = new LinkedHashMap<>(); + processBasedFetchCountMapping.put("MIGRATOR", 40); + processBasedFetchCountMapping.put("RENEWAL#ON_HOLD_RENEWAL", 30); + processBasedFetchCountMapping.put("NEW,RENEWAL", 30); + ReflectionTestUtils.setField(reprocessorVerticle, "processBasedFetchCountMapping", processBasedFetchCountMapping); + ReflectionTestUtils.setField(reprocessorVerticle, "fetchSize", 4); ReflectionTestUtils.setField(reprocessorVerticle, "elapseTime", 21600); ReflectionTestUtils.setField(reprocessorVerticle, "reprocessCount", 3); + ReflectionTestUtils.setField(reprocessorVerticle, "recordFetchSize", 500); + ReflectionTestUtils.setField(reprocessorVerticle, "threasholdForFetch", 50); ReflectionTestUtils.setField(reprocessorVerticle, "reprocessExcludeStageNames", new ArrayList<>()); List reprocessRestartTriggerFilterList = new ArrayList<>(); reprocessRestartTriggerFilterList.add("DemodedupStage:Success"); @@ -172,6 +187,11 @@ public void testProcessValid() throws TablenotAccessibleException, PacketManager dtolist.add(registrationStatusDto1); Mockito.when(registrationStatusService.getUnProcessedPackets(anyInt(), anyLong(), anyInt(), anyList(), anyList())) .thenReturn(dtolist); + Mockito.when(registrationStatusService.getUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList())) + .thenReturn(dtolist); + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList())) + .thenReturn(new ArrayList<>(dtolist)); + reprocessorVerticle.process(dto); } From 7518533101cc56a3a8343a4db211ad0db8c60d67 Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Wed, 8 Oct 2025 08:44:08 +0530 Subject: [PATCH 02/14] Reprocessor Changes Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../verticle/ReprocessorVerticle.java | 228 +++++++++++------- 1 file changed, 135 insertions(+), 93 deletions(-) diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index d6b7003ab86..728d893c6b6 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -1,7 +1,9 @@ package io.mosip.registration.processor.reprocessor.verticle; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import io.mosip.registration.processor.reprocessor.service.ReprocessorVerticalService; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -42,6 +44,8 @@ import io.vertx.core.eventbus.EventBus; import io.vertx.core.json.JsonObject; +import javax.annotation.PreDestroy; + /** * The Reprocessor Verticle to deploy the scheduler and implement re-processing * logic @@ -93,7 +97,7 @@ public class ReprocessorVerticle extends MosipVerticleAPIManager { @Value("#{'${registration.processor.reprocess.restart-trigger-filter}'.split(',')}") private List reprocessRestartTriggerFilter; - @Value("#{${registration.processor.reprocess.process-based.fetch.enabled:false}") + @Value("${registration.processor.reprocess.process-based.fetch.enabled:false}") private Boolean enabledProcessBasedFetch; @Value("#{${registration.processor.reprocess.process-based.fetch.count:{:}}}") @@ -105,13 +109,13 @@ public class ReprocessorVerticle extends MosipVerticleAPIManager { @Value("${registration.processor.reprocess.record.threashold:50}") private int threasholdForFetch; - private ConcurrentHashMap> packetCacheMap = new ConcurrentHashMap<>(); + private ConcurrentHashMap> packetCacheMap = new ConcurrentHashMap<>(); @Autowired private ReprocessorVerticalService reprocessorVerticalService; /** The is transaction successful. */ - boolean isTransactionSuccessful; + private final AtomicBoolean isTransactionSuccessful = new AtomicBoolean(false); /** The registration status service. */ @Autowired @@ -129,6 +133,22 @@ public class ReprocessorVerticle extends MosipVerticleAPIManager { @Value("${server.port}") private String port; + private final ExecutorService sendExecutor = new ThreadPoolExecutor( + Math.max(2, Runtime.getRuntime().availableProcessors()), + Math.max(2, Runtime.getRuntime().availableProcessors()), + 60L, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(1000), // bounded queue + new ThreadPoolExecutor.CallerRunsPolicy() + ); + + private final ExecutorService fetchExecutor = new ThreadPoolExecutor( + Math.max(2, Runtime.getRuntime().availableProcessors()), + Math.max(2, Runtime.getRuntime().availableProcessors()), + 60L, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(1000), // bounded queue + new ThreadPoolExecutor.CallerRunsPolicy() + ); + /** * Deploy verticle. */ @@ -240,7 +260,7 @@ public MessageDTO process(MessageDTO object) { statusList.add(RegistrationTransactionStatusCode.IN_PROGRESS.toString()); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "ReprocessorVerticle::process()::entry"); - StringBuffer ridSb=new StringBuffer(); + ConcurrentLinkedQueue ridSb = new ConcurrentLinkedQueue<>(); try { Map> reprocessRestartTriggerMap = intializeReprocessRestartTriggerMapping(); reprocessorDtoList = registrationStatusService.getResumablePackets(fetchSize); @@ -280,80 +300,83 @@ public MessageDTO process(MessageDTO object) { "Reprocessor Total Packets Fetched " + reprocessorDtoList.size()); if (!CollectionUtils.isEmpty(reprocessorDtoList)) { - reprocessorDtoList.forEach(dto -> { - String registrationId = dto.getRegistrationId(); - ridSb.append(registrationId); - ridSb.append(","); - MessageDTO messageDTO = new MessageDTO(); - messageDTO.setRid(registrationId); - messageDTO.setReg_type(dto.getRegistrationType()); - messageDTO.setSource(dto.getSource()); - messageDTO.setIteration(dto.getIteration()); - messageDTO.setWorkflowInstanceId(dto.getWorkflowInstanceId()); - if (reprocessCount.equals(dto.getReProcessRetryCount())) { - dto.setLatestTransactionStatusCode( - RegistrationTransactionStatusCode.REPROCESS_FAILED.toString()); - dto.setLatestTransactionTypeCode( - RegistrationTransactionTypeCode.PACKET_REPROCESS.toString()); - dto.setStatusComment(StatusUtil.RE_PROCESS_FAILED.getMessage()); - dto.setStatusCode(RegistrationStatusCode.REPROCESS_FAILED.toString()); - dto.setSubStatusCode(StatusUtil.RE_PROCESS_FAILED.getCode()); - messageDTO.setIsValid(false); - description.setMessage(PlatformSuccessMessages.RPR_RE_PROCESS_FAILED.getMessage()); - description.setCode(PlatformSuccessMessages.RPR_RE_PROCESS_FAILED.getCode()); - - } else { - messageDTO.setIsValid(true); - isTransactionSuccessful = true; - String stageName; - if (isRestartFromStageRequired(dto, reprocessRestartTriggerMap)) { - stageName = MessageBusUtil.getMessageBusAdress(reprocessRestartFromStage); - stageName = stageName.concat(ReprocessorConstants.BUS_IN); - sendAndSetStatus(dto, messageDTO, stageName); - dto.setStatusComment(StatusUtil.RE_PROCESS_RESTART_FROM_STAGE.getMessage()); - dto.setSubStatusCode(StatusUtil.RE_PROCESS_RESTART_FROM_STAGE.getCode()); - description - .setMessage( - PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_RESTART_FROM_STAGE_SUCCESS - .getMessage()); - description.setCode( - PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_RESTART_FROM_STAGE_SUCCESS - .getCode()); - - } else { - stageName = MessageBusUtil.getMessageBusAdress(dto.getRegistrationStageName()); - if (RegistrationTransactionStatusCode.SUCCESS.name() - .equalsIgnoreCase(dto.getLatestTransactionStatusCode())) { - stageName = stageName.concat(ReprocessorConstants.BUS_OUT); - } else { - stageName = stageName.concat(ReprocessorConstants.BUS_IN); - } - sendAndSetStatus(dto, messageDTO, stageName); - dto.setStatusComment(StatusUtil.RE_PROCESS_COMPLETED.getMessage()); - dto.setSubStatusCode(StatusUtil.RE_PROCESS_COMPLETED.getCode()); - description.setMessage(PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getMessage()); - description.setCode(PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getCode()); - } - } - regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), - LoggerFileConstant.REGISTRATIONID.toString(), registrationId, description.getMessage()); - - /** Module-Id can be Both Success/Error code */ - String moduleId = PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getCode(); - String moduleName = ModuleName.RE_PROCESSOR.toString(); - registrationStatusService.updateRegistrationStatusForWorkflowEngine(dto, moduleId, moduleName); - String eventId = EventId.RPR_402.toString(); - String eventName = EventName.UPDATE.toString(); - String eventType = EventType.BUSINESS.toString(); - - if (!isTransactionSuccessful) - auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, - eventType, moduleId, moduleName, registrationId); - }); - + List> sendTasks = reprocessorDtoList.stream() + .map(dto -> CompletableFuture.runAsync(() -> { + { + String registrationId = dto.getRegistrationId(); + ridSb.add(registrationId); + MessageDTO messageDTO = new MessageDTO(); + messageDTO.setRid(registrationId); + messageDTO.setReg_type(dto.getRegistrationType()); + messageDTO.setSource(dto.getSource()); + messageDTO.setIteration(dto.getIteration()); + messageDTO.setWorkflowInstanceId(dto.getWorkflowInstanceId()); + if (reprocessCount.equals(dto.getReProcessRetryCount())) { + dto.setLatestTransactionStatusCode( + RegistrationTransactionStatusCode.REPROCESS_FAILED.toString()); + dto.setLatestTransactionTypeCode( + RegistrationTransactionTypeCode.PACKET_REPROCESS.toString()); + dto.setStatusComment(StatusUtil.RE_PROCESS_FAILED.getMessage()); + dto.setStatusCode(RegistrationStatusCode.REPROCESS_FAILED.toString()); + dto.setSubStatusCode(StatusUtil.RE_PROCESS_FAILED.getCode()); + messageDTO.setIsValid(false); + description.setMessage(PlatformSuccessMessages.RPR_RE_PROCESS_FAILED.getMessage()); + description.setCode(PlatformSuccessMessages.RPR_RE_PROCESS_FAILED.getCode()); + + } else { + messageDTO.setIsValid(true); + isTransactionSuccessful.set(true); + String stageName; + if (isRestartFromStageRequired(dto, reprocessRestartTriggerMap)) { + stageName = MessageBusUtil.getMessageBusAdress(reprocessRestartFromStage); + stageName = stageName.concat(ReprocessorConstants.BUS_IN); + sendAndSetStatus(dto, messageDTO, stageName); + dto.setStatusComment(StatusUtil.RE_PROCESS_RESTART_FROM_STAGE.getMessage()); + dto.setSubStatusCode(StatusUtil.RE_PROCESS_RESTART_FROM_STAGE.getCode()); + description + .setMessage( + PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_RESTART_FROM_STAGE_SUCCESS + .getMessage()); + description.setCode( + PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_RESTART_FROM_STAGE_SUCCESS + .getCode()); + + } else { + stageName = MessageBusUtil.getMessageBusAdress(dto.getRegistrationStageName()); + if (RegistrationTransactionStatusCode.SUCCESS.name() + .equalsIgnoreCase(dto.getLatestTransactionStatusCode())) { + stageName = stageName.concat(ReprocessorConstants.BUS_OUT); + } else { + stageName = stageName.concat(ReprocessorConstants.BUS_IN); + } + sendAndSetStatus(dto, messageDTO, stageName); + dto.setStatusComment(StatusUtil.RE_PROCESS_COMPLETED.getMessage()); + dto.setSubStatusCode(StatusUtil.RE_PROCESS_COMPLETED.getCode()); + description.setMessage(PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getMessage()); + description.setCode(PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getCode()); + } + } + regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), + LoggerFileConstant.REGISTRATIONID.toString(), registrationId, description.getMessage()); + + /** Module-Id can be Both Success/Error code */ + String moduleId = PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getCode(); + String moduleName = ModuleName.RE_PROCESSOR.toString(); + registrationStatusService.updateRegistrationStatusForWorkflowEngine(dto, moduleId, moduleName); + String eventId = EventId.RPR_402.toString(); + String eventName = EventName.UPDATE.toString(); + String eventType = EventType.BUSINESS.toString(); + + if (!isTransactionSuccessful.get()) + auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, + eventType, moduleId, moduleName, registrationId); + } + },sendExecutor)).collect(Collectors.toList()); + + CompletableFuture.allOf(sendTasks.toArray(new CompletableFuture[0])).join(); } } catch (TablenotAccessibleException e) { - isTransactionSuccessful = false; + isTransactionSuccessful.set(false); object.setInternalError(Boolean.TRUE); description.setMessage(PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage()); description.setCode(PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getCode()); @@ -362,7 +385,7 @@ public MessageDTO process(MessageDTO object) { PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage(), e.toString()); }catch (Exception ex) { - isTransactionSuccessful = false; + isTransactionSuccessful.set(false); description.setMessage(PlatformErrorMessages.REPROCESSOR_VERTICLE_FAILED.getMessage()); description.setCode(PlatformErrorMessages.REPROCESSOR_VERTICLE_FAILED.getCode()); regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), @@ -374,19 +397,19 @@ public MessageDTO process(MessageDTO object) { } finally { regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), null, description.getMessage()); - if (isTransactionSuccessful) + if (isTransactionSuccessful.get()) description.setMessage(PlatformSuccessMessages.RPR_RE_PROCESS_SUCCESS.getMessage()); - String eventId = isTransactionSuccessful ? EventId.RPR_402.toString() : EventId.RPR_405.toString(); - String eventName = isTransactionSuccessful ? EventName.UPDATE.toString() : EventName.EXCEPTION.toString(); - String eventType = isTransactionSuccessful ? EventType.BUSINESS.toString() : EventType.SYSTEM.toString(); + String eventId = isTransactionSuccessful.get() ? EventId.RPR_402.toString() : EventId.RPR_405.toString(); + String eventName = isTransactionSuccessful.get() ? EventName.UPDATE.toString() : EventName.EXCEPTION.toString(); + String eventType = isTransactionSuccessful.get() ? EventType.BUSINESS.toString() : EventType.SYSTEM.toString(); /** Module-Id can be Both Success/Error code */ - String moduleId = isTransactionSuccessful ? PlatformSuccessMessages.RPR_RE_PROCESS_SUCCESS.getCode() + String moduleId = isTransactionSuccessful.get() ? PlatformSuccessMessages.RPR_RE_PROCESS_SUCCESS.getCode() : description.getCode(); String moduleName = ModuleName.RE_PROCESSOR.toString(); auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, eventType, - moduleId, moduleName, (ridSb.toString().length()>1?ridSb.substring(0,ridSb.length()-1):"")); + moduleId, moduleName, String.join(",", ridSb)); } return object; @@ -491,12 +514,12 @@ private LinkedHashMap prepareRequiredCount(int availableCount) } private void fetchPacketsIfBelowThreshold(LinkedHashMap requiredCountMap, List statusList) { - requiredCountMap.entrySet().parallelStream() + List> futures = requiredCountMap.entrySet().stream() .filter(entry -> { - List cachedPackets = packetCacheMap.get(entry.getKey()); + Deque cachedPackets = packetCacheMap.get(entry.getKey()); return cachedPackets == null || cachedPackets.size() < threasholdForFetch; }) - .forEach(entry -> { + .map(entry -> CompletableFuture.runAsync(() -> { String key = entry.getKey(); // Parse key into Process & Status @@ -510,11 +533,14 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require // Thread-safe update to cache packetCacheMap.compute(key, (k, existingList) -> { - if (existingList == null) return new ArrayList<>(registratiobRegistrationStatusDtos); + if (existingList == null) return new ConcurrentLinkedDeque<>(registratiobRegistrationStatusDtos); existingList.addAll(new ArrayList<>(registratiobRegistrationStatusDtos)); return existingList; }); - }); + }, fetchExecutor)) + .collect(Collectors.toList()); + + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); } private AbstractMap.SimpleEntry, List> parseProcessAndStatus(String key) { @@ -537,7 +563,7 @@ private List fetchUnprocessedPacketsWithBalance(L regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", entry.getKey() + " Packets Required Count " + requiredCount); - List cachedPackets = packetCacheMap.getOrDefault(entry.getKey(), Collections.emptyList()); + Deque cachedPackets = packetCacheMap.getOrDefault(entry.getKey(), new ConcurrentLinkedDeque<>()); if(!cachedPackets.isEmpty()) { int count = Math.min(requiredCount, remainingToFetch); @@ -556,11 +582,27 @@ private List fetchUnprocessedPacketsWithBalance(L return reprocessorPacketList; } - private List fetchFromCache(List cache, int count) { + private List fetchFromCache(Deque cache, int count) { int actualFetchCount = Math.min(count, cache.size()); - List fetched = new ArrayList<>(cache.subList(0, actualFetchCount)); - cache.subList(0, actualFetchCount).clear(); + List fetched = new ArrayList<>(actualFetchCount); + for (int i = 0; i < actualFetchCount; i++) { + InternalRegistrationStatusDto dto = cache.pollFirst(); + if (dto == null) break; + fetched.add(dto); + } return fetched; } + @PreDestroy + public void shutdown() { + fetchExecutor.shutdown(); + sendExecutor.shutdown(); + try { + sendExecutor.awaitTermination(30, TimeUnit.SECONDS); + fetchExecutor.awaitTermination(30, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } From ccc783db7a861d7cbe530a7e314b219e029b0f42 Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Wed, 8 Oct 2025 21:15:10 +0530 Subject: [PATCH 03/14] Reprocessor Changes Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../repositary/RegistrationRepositary.java | 2 +- .../reprocessor/ReprocessorApplication.java | 3 +- .../service/ReprocessorVerticalService.java | 12 ++++-- .../impl/ReprocessorVerticalServiceImpl.java | 10 ++--- .../verticle/ReprocessorVerticle.java | 38 ++++++++++++------- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java index 50126b4c5ae..f685bb4f69d 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java @@ -65,7 +65,7 @@ public List getProcessedOrProcessingRegIds(@Param("regIds") List @Query("SELECT registration FROM RegistrationStatusEntity registration WHERE registration.regId = :regId AND registration.registrationType = :registrationType AND registration.iteration = :iteration") public List getByIdAndProcessAndIteration(@Param("regId") String regId, @Param("registrationType") String process, @Param("iteration") int iteration); - @Query(value ="SELECT * FROM registration r WHERE r.process IN :processes AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code NOT IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) + @Query(value ="SELECT * FROM registration r WHERE r.process IN :processList AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code NOT IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) public List getUnProcessedPackets(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames); } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/ReprocessorApplication.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/ReprocessorApplication.java index 3f9a3ad585c..40b5f7b4ad7 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/ReprocessorApplication.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/ReprocessorApplication.java @@ -21,7 +21,8 @@ public static void main(String[] args) { "io.mosip.registration.processor.reprocessor.config", "io.mosip.registration.processor.status.config", "io.mosip.registration.processor.core.kernel.beans", - "io.mosip.registration.processor.packet.storage.config"); + "io.mosip.registration.processor.packet.storage.config", + "io.mosip.registration.processor.reprocessor.service.impl"); ctx.refresh(); ReprocessorVerticle reprocessorVerticle = ctx.getBean(ReprocessorVerticle.class); reprocessorVerticle.deployVerticle(); diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java index 23d5f8ac8b3..8daa49986bf 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java @@ -1,12 +1,16 @@ package io.mosip.registration.processor.reprocessor.service; - import io.mosip.registration.processor.status.dto.InternalRegistrationStatusDto; - import java.util.List; +import java.util.concurrent.CompletableFuture; public interface ReprocessorVerticalService { - List fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, - List status, List excludeStageNames); + CompletableFuture> fetchUnProcessedPackets( + List processList, + Integer fetchSize, + long elapseTime, + Integer reprocessCount, + List status, + List excludeStageNames); } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java index 0430b48f9b0..c9714e7b7cc 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java @@ -8,21 +8,21 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import java.util.Arrays; import java.util.List; +import java.util.concurrent.CompletableFuture; @Service public class ReprocessorVerticalServiceImpl implements ReprocessorVerticalService { /** The registration status service. */ @Autowired - RegistrationStatusService registrationStatusService; - + private RegistrationStatusService registrationStatusService; @Override @Async - public List fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, List status, List excludeStageNames) { - return registrationStatusService.getUnProcessedPackets(processList, fetchSize, elapseTime, + public CompletableFuture> fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, List status, List excludeStageNames) { + List result = registrationStatusService.getUnProcessedPackets(processList, fetchSize, elapseTime, reprocessCount, status, excludeStageNames); + return CompletableFuture.completedFuture(result); } } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index 728d893c6b6..a7d796f6426 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -155,7 +155,6 @@ public class ReprocessorVerticle extends MosipVerticleAPIManager { public void deployVerticle() { mosipEventBus = this.getEventBus(this, clusterManagerUrl); deployScheduler(getVertx()); - } /** @@ -371,7 +370,11 @@ public MessageDTO process(MessageDTO object) { auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, eventType, moduleId, moduleName, registrationId); } - },sendExecutor)).collect(Collectors.toList()); + },sendExecutor).exceptionally(ex -> { + regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), + description.getCode() + " -- ", + PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage(), ex.toString()); return null; + })).collect(Collectors.toList()); CompletableFuture.allOf(sendTasks.toArray(new CompletableFuture[0])).join(); } @@ -384,7 +387,7 @@ public MessageDTO process(MessageDTO object) { description.getCode() + " -- ", PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage(), e.toString()); - }catch (Exception ex) { + } catch (Exception ex) { isTransactionSuccessful.set(false); description.setMessage(PlatformErrorMessages.REPROCESSOR_VERTICLE_FAILED.getMessage()); description.setCode(PlatformErrorMessages.REPROCESSOR_VERTICLE_FAILED.getCode()); @@ -519,7 +522,7 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require Deque cachedPackets = packetCacheMap.get(entry.getKey()); return cachedPackets == null || cachedPackets.size() < threasholdForFetch; }) - .map(entry -> CompletableFuture.runAsync(() -> { + .map(entry -> { String key = entry.getKey(); // Parse key into Process & Status @@ -528,16 +531,23 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require List statusValList = entryPair.getValue(); // Fetch unprocessed packets - List registratiobRegistrationStatusDtos = reprocessorVerticalService.fetchUnProcessedPackets(processList, recordFetchSize, elapseTime, - reprocessCount, (!statusValList.isEmpty() ? statusValList : statusList), reprocessExcludeStageNames); - - // Thread-safe update to cache - packetCacheMap.compute(key, (k, existingList) -> { - if (existingList == null) return new ConcurrentLinkedDeque<>(registratiobRegistrationStatusDtos); - existingList.addAll(new ArrayList<>(registratiobRegistrationStatusDtos)); - return existingList; - }); - }, fetchExecutor)) + return reprocessorVerticalService.fetchUnProcessedPackets(processList, recordFetchSize, elapseTime, + reprocessCount, (!statusValList.isEmpty() ? statusValList : statusList), reprocessExcludeStageNames) + .thenAccept(result -> { + // Thread-safe update to cache + packetCacheMap.compute(key, (k, existingList) -> { + if (existingList == null) return new ConcurrentLinkedDeque<>(result); + existingList.addAll(new ArrayList<>(result)); + return existingList; + }); + }) + .exceptionally(ex -> { + regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), + "Error Fetching UnprocessedPackets -- ", + " Error Triggered for Process [" + String.join(",", processList) + "] and Status [" + String.join(",", (!statusValList.isEmpty() ? statusValList : statusList)) + "]", ExceptionUtils.getStackTrace(ex)); + return null; + }); + }) .collect(Collectors.toList()); CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); From eda88f86b05c3cf444661029865ee4ea253241ff Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:11:50 +0530 Subject: [PATCH 04/14] Reprocessor Changes Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../verticle/ReprocessorVerticle.java | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index a7d796f6426..e7f23610037 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -267,12 +267,17 @@ public MessageDTO process(MessageDTO object) { "Resumable Packets Count " + reprocessorDtoList.size() ); if(enabledProcessBasedFetch) { + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + "Enabled Process based fetch"); if(reprocessorDtoList.size() < fetchSize) { LinkedHashMap requiredCountMap = prepareRequiredCount( fetchSize - reprocessorDtoList.size() ); - + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + "Prepared process based required count details for fetch " + requiredCountMap.toString()); fetchPacketsIfBelowThreshold(requiredCountMap, statusList); + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + "Cacheing Packets if below Threashold method completed." + packetCacheMap.entrySet().stream().map(e -> e.getKey() + " = " + e.getValue().size()).collect(Collectors.joining(", ", "[", "]"))); reprocessorDtoList.addAll(fetchUnprocessedPacketsWithBalance( requiredCountMap, @@ -303,6 +308,8 @@ public MessageDTO process(MessageDTO object) { .map(dto -> CompletableFuture.runAsync(() -> { { String registrationId = dto.getRegistrationId(); + regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), + LoggerFileConstant.REGISTRATIONID.toString(), registrationId, "Process started"); ridSb.add(registrationId); MessageDTO messageDTO = new MessageDTO(); messageDTO.setRid(registrationId); @@ -524,16 +531,27 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require }) .map(entry -> { String key = entry.getKey(); + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + "Fetch Records from Database for Process " + key); // Parse key into Process & Status AbstractMap.SimpleEntry, List> entryPair = parseProcessAndStatus(key); List processList = entryPair.getKey(); List statusValList = entryPair.getValue(); + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + "Status used to fetch Records Process " + key + " is " + statusValList); + + int recordFetchCount = recordFetchSize - packetCacheMap.get(key).size(); + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + "Record Fetch Count for process " + key + " is " + recordFetchCount); // Fetch unprocessed packets - return reprocessorVerticalService.fetchUnProcessedPackets(processList, recordFetchSize, elapseTime, + return reprocessorVerticalService.fetchUnProcessedPackets(processList, recordFetchCount, elapseTime, reprocessCount, (!statusValList.isEmpty() ? statusValList : statusList), reprocessExcludeStageNames) .thenAccept(result -> { + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + "Total Record Fetched from database for process " + key + " is " + result.size()); + // Thread-safe update to cache packetCacheMap.compute(key, (k, existingList) -> { if (existingList == null) return new ConcurrentLinkedDeque<>(result); @@ -570,23 +588,22 @@ private List fetchUnprocessedPacketsWithBalance(L break; int requiredCount = entry.getValue() + previousBalanceCount; - regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", - entry.getKey() + " Packets Required Count " + requiredCount); - Deque cachedPackets = packetCacheMap.getOrDefault(entry.getKey(), new ConcurrentLinkedDeque<>()); if(!cachedPackets.isEmpty()) { int count = Math.min(requiredCount, remainingToFetch); + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + entry.getKey() + " Packets Required Count " + count); List fetchedPackets = fetchFromCache(cachedPackets, count); - reprocessorPacketList.addAll(fetchedPackets); - regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", - entry.getKey() + " Packets Count " + fetchedPackets.size()); - + entry.getKey() + "Total Packets Fetched from Cache " + fetchedPackets.size()); + reprocessorPacketList.addAll(fetchedPackets); previousBalanceCount = Math.max(0, requiredCount-fetchedPackets.size()); } else { previousBalanceCount = requiredCount; } + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", + entry.getKey() + "Count to be moved to next process " + previousBalanceCount); } return reprocessorPacketList; From 2a8b92e3e446cb0765a55c70e6cbfcc9cdae4a05 Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Thu, 9 Oct 2025 21:41:55 +0530 Subject: [PATCH 05/14] Reprocessor Changes Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../status/dao/RegistrationStatusDao.java | 4 +- .../repositary/RegistrationRepositary.java | 4 +- .../service/RegistrationStatusService.java | 2 +- .../impl/RegistrationStatusServiceImpl.java | 4 +- .../service/ReprocessorVerticalService.java | 3 +- .../impl/ReprocessorVerticalServiceImpl.java | 7 +- .../verticle/ReprocessorVerticle.java | 33 ++-- .../verticle/ReprocessorVerticleTest.java | 162 ++++++++++++++++-- 8 files changed, 177 insertions(+), 42 deletions(-) diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java index 30bfc60240e..b7f01ebcb49 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java @@ -262,7 +262,7 @@ public List findByIdAndProcessAndIteration(String id, * @return the un processed packets */ public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, - Integer reprocessCount, List status, List excludeStageNames) { + Integer reprocessCount, List status, List excludeStageNames, List skipRegIds) { LocalDateTime timeDifference = LocalDateTime.now().minusSeconds(elapseTime); List statusCodes=new ArrayList<>(); @@ -274,6 +274,6 @@ public List getUnProcessedPackets(List process statusCodes.add(RegistrationStatusCode.PROCESSED.toString()); return registrationStatusRepositary.getUnProcessedPackets(processList, status, reprocessCount, timeDifference, - statusCodes, fetchSize, excludeStageNames); + statusCodes, fetchSize, excludeStageNames, skipRegIds); } } \ No newline at end of file diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java index f685bb4f69d..2d68c636cbb 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java @@ -65,8 +65,8 @@ public List getProcessedOrProcessingRegIds(@Param("regIds") List @Query("SELECT registration FROM RegistrationStatusEntity registration WHERE registration.regId = :regId AND registration.registrationType = :registrationType AND registration.iteration = :iteration") public List getByIdAndProcessAndIteration(@Param("regId") String regId, @Param("registrationType") String process, @Param("iteration") int iteration); - @Query(value ="SELECT * FROM registration r WHERE r.process IN :processList AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code NOT IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) - public List getUnProcessedPackets(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames); + @Query(value ="SELECT * FROM registration r WHERE r.process IN :processList AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code NOT IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames AND r.reg_id NOT IN :skipRegIds order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) + public List getUnProcessedPackets(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames, @Param("skipRegIds") List skipRegIds); } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java index 3ca0e420a58..cbdecf92624 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java @@ -182,6 +182,6 @@ public Integer getUnProcessedPacketsCount(long elapseTime, Integer reprocessCoun * @return the un processed packets */ public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, - List status, List excludeStageNames); + List status, List excludeStageNames, List skipRegIds); } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java index 5b1c3f32cbb..4f4bb6c8a3a 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java @@ -981,13 +981,13 @@ public void updateRegistrationStatusForWorkflow(InternalRegistrationStatusDto re * @return the un processed packets */ public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, - Integer reprocessCount, List status, List excludeStageNames) { + Integer reprocessCount, List status, List excludeStageNames, List skipRegIds) { regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), "", "RegistrationStatusServiceImpl::getReprocessPacket()::entry"); try { List entityList = registrationStatusDao.getUnProcessedPackets(processList, fetchSize, - elapseTime, reprocessCount, status, excludeStageNames); + elapseTime, reprocessCount, status, excludeStageNames, skipRegIds); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), "", "RegistrationStatusServiceImpl::getReprocessPacket()::exit"); diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java index 8daa49986bf..6f22fd3a596 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java @@ -12,5 +12,6 @@ CompletableFuture> fetchUnProcessedPackets( long elapseTime, Integer reprocessCount, List status, - List excludeStageNames); + List excludeStageNames, + List skipRegIds); } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java index c9714e7b7cc..55fb55b0ad1 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java @@ -8,6 +8,7 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -20,9 +21,9 @@ public class ReprocessorVerticalServiceImpl implements ReprocessorVerticalServic @Override @Async - public CompletableFuture> fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, List status, List excludeStageNames) { + public CompletableFuture> fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, List status, List excludeStageNames, List skipRegIds) { List result = registrationStatusService.getUnProcessedPackets(processList, fetchSize, elapseTime, - reprocessCount, status, excludeStageNames); - return CompletableFuture.completedFuture(result); + reprocessCount, status, excludeStageNames, skipRegIds); + return CompletableFuture.completedFuture(result != null ? result : Collections.emptyList()); } } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index e7f23610037..a5cff24fb71 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -115,7 +115,7 @@ public class ReprocessorVerticle extends MosipVerticleAPIManager { private ReprocessorVerticalService reprocessorVerticalService; /** The is transaction successful. */ - private final AtomicBoolean isTransactionSuccessful = new AtomicBoolean(false); + private boolean isBatchSuccessful = false; /** The registration status service. */ @Autowired @@ -307,6 +307,7 @@ public MessageDTO process(MessageDTO object) { List> sendTasks = reprocessorDtoList.stream() .map(dto -> CompletableFuture.runAsync(() -> { { + boolean isTransactionSuccessful = false; String registrationId = dto.getRegistrationId(); regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), registrationId, "Process started"); @@ -331,7 +332,7 @@ public MessageDTO process(MessageDTO object) { } else { messageDTO.setIsValid(true); - isTransactionSuccessful.set(true); + isTransactionSuccessful=true; String stageName; if (isRestartFromStageRequired(dto, reprocessRestartTriggerMap)) { stageName = MessageBusUtil.getMessageBusAdress(reprocessRestartFromStage); @@ -373,20 +374,20 @@ public MessageDTO process(MessageDTO object) { String eventName = EventName.UPDATE.toString(); String eventType = EventType.BUSINESS.toString(); - if (!isTransactionSuccessful.get()) + if (!isTransactionSuccessful) auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, eventType, moduleId, moduleName, registrationId); } },sendExecutor).exceptionally(ex -> { regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), description.getCode() + " -- ", - PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage(), ex.toString()); return null; + PlatformErrorMessages.RPR_PKR_UNKNOWN_EXCEPTION.getMessage(), ex.toString()); return null; })).collect(Collectors.toList()); CompletableFuture.allOf(sendTasks.toArray(new CompletableFuture[0])).join(); } } catch (TablenotAccessibleException e) { - isTransactionSuccessful.set(false); + isBatchSuccessful = false; object.setInternalError(Boolean.TRUE); description.setMessage(PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage()); description.setCode(PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getCode()); @@ -395,7 +396,7 @@ public MessageDTO process(MessageDTO object) { PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage(), e.toString()); } catch (Exception ex) { - isTransactionSuccessful.set(false); + isBatchSuccessful = false; description.setMessage(PlatformErrorMessages.REPROCESSOR_VERTICLE_FAILED.getMessage()); description.setCode(PlatformErrorMessages.REPROCESSOR_VERTICLE_FAILED.getCode()); regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), @@ -407,15 +408,15 @@ public MessageDTO process(MessageDTO object) { } finally { regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), null, description.getMessage()); - if (isTransactionSuccessful.get()) + if (isBatchSuccessful) description.setMessage(PlatformSuccessMessages.RPR_RE_PROCESS_SUCCESS.getMessage()); - String eventId = isTransactionSuccessful.get() ? EventId.RPR_402.toString() : EventId.RPR_405.toString(); - String eventName = isTransactionSuccessful.get() ? EventName.UPDATE.toString() : EventName.EXCEPTION.toString(); - String eventType = isTransactionSuccessful.get() ? EventType.BUSINESS.toString() : EventType.SYSTEM.toString(); + String eventId = isBatchSuccessful ? EventId.RPR_402.toString() : EventId.RPR_405.toString(); + String eventName = isBatchSuccessful ? EventName.UPDATE.toString() : EventName.EXCEPTION.toString(); + String eventType = isBatchSuccessful ? EventType.BUSINESS.toString() : EventType.SYSTEM.toString(); /** Module-Id can be Both Success/Error code */ - String moduleId = isTransactionSuccessful.get() ? PlatformSuccessMessages.RPR_RE_PROCESS_SUCCESS.getCode() + String moduleId = isBatchSuccessful ? PlatformSuccessMessages.RPR_RE_PROCESS_SUCCESS.getCode() : description.getCode(); String moduleName = ModuleName.RE_PROCESSOR.toString(); auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, eventType, @@ -541,13 +542,19 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Status used to fetch Records Process " + key + " is " + statusValList); - int recordFetchCount = recordFetchSize - packetCacheMap.get(key).size(); + Deque cacheList = packetCacheMap.get(key); + int recordFetchCount = recordFetchSize - (cacheList != null ? cacheList.size() : 0); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Record Fetch Count for process " + key + " is " + recordFetchCount); // Fetch unprocessed packets + + List skipRegIdList = new ArrayList<>(cacheList != null && !cacheList.isEmpty() ? cacheList.stream().map(e -> e.getRegistrationId()).collect(Collectors.toList()) : Collections.emptyList()); + + if(skipRegIdList.isEmpty()) skipRegIdList.add("-DUMMY-"); + return reprocessorVerticalService.fetchUnProcessedPackets(processList, recordFetchCount, elapseTime, - reprocessCount, (!statusValList.isEmpty() ? statusValList : statusList), reprocessExcludeStageNames) + reprocessCount, (!statusValList.isEmpty() ? statusValList : statusList), reprocessExcludeStageNames, skipRegIdList) .thenAccept(result -> { regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Total Record Fetched from database for process " + key + " is " + result.size()); diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java index 104167fba12..1def8ccc8ac 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java @@ -7,10 +7,8 @@ import java.lang.reflect.Field; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; +import java.util.*; +import java.util.concurrent.CompletableFuture; import io.mosip.registration.processor.reprocessor.service.ReprocessorVerticalService; import io.mosip.registration.processor.reprocessor.service.impl.ReprocessorVerticalServiceImpl; @@ -124,18 +122,18 @@ public void setup() throws Exception { //Mockito.doNothing().when(description).setMessage(Mockito.anyString()); //Mockito.when(description.getCode()).thenReturn("CODE"); //Mockito.when(description.getMessage()).thenReturn("MESSAGE"); - ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", false); LinkedHashMap processBasedFetchCountMapping = new LinkedHashMap<>(); - processBasedFetchCountMapping.put("MIGRATOR", 40); - processBasedFetchCountMapping.put("RENEWAL#ON_HOLD_RENEWAL", 30); - processBasedFetchCountMapping.put("NEW,RENEWAL", 30); +// processBasedFetchCountMapping.put("MIGRATOR", 40); +// processBasedFetchCountMapping.put("RENEWAL#ON_HOLD_RENEWAL", 30); + processBasedFetchCountMapping.put("NEW,RENEWAL", 100); ReflectionTestUtils.setField(reprocessorVerticle, "processBasedFetchCountMapping", processBasedFetchCountMapping); - ReflectionTestUtils.setField(reprocessorVerticle, "fetchSize", 4); - ReflectionTestUtils.setField(reprocessorVerticle, "elapseTime", 21600); - ReflectionTestUtils.setField(reprocessorVerticle, "reprocessCount", 3); + ReflectionTestUtils.setField(reprocessorVerticle, "fetchSize", 4); + ReflectionTestUtils.setField(reprocessorVerticle, "elapseTime", 21600); + ReflectionTestUtils.setField(reprocessorVerticle, "reprocessCount", 3); ReflectionTestUtils.setField(reprocessorVerticle, "recordFetchSize", 500); ReflectionTestUtils.setField(reprocessorVerticle, "threasholdForFetch", 50); - ReflectionTestUtils.setField(reprocessorVerticle, "reprocessExcludeStageNames", new ArrayList<>()); + ReflectionTestUtils.setField(reprocessorVerticle, "reprocessExcludeStageNames", new ArrayList<>()); List reprocessRestartTriggerFilterList = new ArrayList<>(); reprocessRestartTriggerFilterList.add("DemodedupStage:Success"); reprocessRestartTriggerFilterList.add("BioDedupeStage:*"); @@ -187,15 +185,40 @@ public void testProcessValid() throws TablenotAccessibleException, PacketManager dtolist.add(registrationStatusDto1); Mockito.when(registrationStatusService.getUnProcessedPackets(anyInt(), anyLong(), anyInt(), anyList(), anyList())) .thenReturn(dtolist); - Mockito.when(registrationStatusService.getUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList())) - .thenReturn(dtolist); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList())) - .thenReturn(new ArrayList<>(dtolist)); + reprocessorVerticle.process(dto); + + } + + @Test + public void testProcessValidNew() throws TablenotAccessibleException, PacketManagerException, + ApisResourceAccessException, WorkflowActionException { + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + List dtolist = new ArrayList<>(); + InternalRegistrationStatusDto registrationStatusDto = new InternalRegistrationStatusDto(); + + registrationStatusDto.setRegistrationId("2018701130000410092018110735"); + registrationStatusDto.setRegistrationType(RegistrationType.NEW.toString()); + registrationStatusDto.setRegistrationStageName("PacketValidatorStage"); + registrationStatusDto.setDefaultResumeAction("RESUME_PROCESSING"); + registrationStatusDto.setResumeTimeStamp(LocalDateTime.now()); + registrationStatusDto.setReProcessRetryCount(0); + registrationStatusDto.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.REPROCESS.toString()); + dtolist.add(registrationStatusDto); + InternalRegistrationStatusDto registrationStatusDto1 = new InternalRegistrationStatusDto(); + + registrationStatusDto1.setRegistrationId("2018701130000410092018110734"); + registrationStatusDto1.setRegistrationStageName("PacketValidatorStage"); + registrationStatusDto1.setReProcessRetryCount(1); + registrationStatusDto1.setRegistrationType("NEW"); + registrationStatusDto1.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.SUCCESS.toString()); + dtolist.add(registrationStatusDto1); + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + .thenReturn(CompletableFuture.completedFuture(dtolist)); reprocessorVerticle.process(dto); } - + @Test public void testProcessFailure() throws TablenotAccessibleException, PacketManagerException, ApisResourceAccessException, WorkflowActionException { @@ -225,6 +248,36 @@ public void testProcessFailure() throws TablenotAccessibleException, PacketManag } + @Test + public void testProcessFailureNew() throws TablenotAccessibleException, PacketManagerException, + ApisResourceAccessException, WorkflowActionException { + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + + List dtolist = new ArrayList<>(); + InternalRegistrationStatusDto registrationStatusDto = new InternalRegistrationStatusDto(); + + registrationStatusDto.setRegistrationId("2018701130000410092018110735"); + registrationStatusDto.setRegistrationStageName("PacketValidatorStage"); + + registrationStatusDto.setDefaultResumeAction("RESUME_PROCESSING"); + registrationStatusDto.setResumeTimeStamp(LocalDateTime.now()); + registrationStatusDto.setRegistrationType("NEW"); + registrationStatusDto.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.REPROCESS.toString()); + dtolist.add(registrationStatusDto); + InternalRegistrationStatusDto registrationStatusDto1 = new InternalRegistrationStatusDto(); + + registrationStatusDto1.setRegistrationId("2018701130000410092018110734"); + registrationStatusDto1.setRegistrationStageName("PacketValidatorStage"); + registrationStatusDto1.setReProcessRetryCount(3); + registrationStatusDto1.setRegistrationType("NEW"); + registrationStatusDto1.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.SUCCESS.toString()); + dtolist.add(registrationStatusDto1); + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + .thenReturn(CompletableFuture.completedFuture(dtolist)); + reprocessorVerticle.process(dto); + + } + /** * Exception test. * @@ -239,7 +292,17 @@ public void exceptionTest() throws Exception { assertEquals(null, dto.getIsValid()); } - + + @Test + public void exceptionTestNew() throws Exception { + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + .thenReturn(CompletableFuture.completedFuture(Collections.emptyList())); + dto = reprocessorVerticle.process(dto); + assertEquals(null, dto.getIsValid()); + + } + @Test public void nullPointerExceptionTest() throws Exception { Mockito.when(registrationStatusService.getResumablePackets(anyInt())) @@ -253,7 +316,17 @@ public void TablenotAccessibleExceptionTest() throws Exception { Mockito.when(registrationStatusService.getUnProcessedPackets(anyInt(), anyLong(), anyInt(), anyList(), anyList())) .thenThrow(new TablenotAccessibleException("") { }); + dto = reprocessorVerticle.process(dto); + assertEquals(true, dto.getInternalError()); + } + + @Test + public void TablenotAccessibleExceptionTestNew() throws Exception { + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + .thenThrow(new TablenotAccessibleException("") { + }); dto = reprocessorVerticle.process(dto); assertEquals(true, dto.getInternalError()); @@ -291,6 +364,38 @@ public void testProcessValidWithResumablePackets() throws TablenotAccessibleExce } + @Test + public void testProcessValidWithResumablePacketsNew() throws TablenotAccessibleException, PacketManagerException, + ApisResourceAccessException, WorkflowActionException { + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + List dtolist = new ArrayList<>(); + InternalRegistrationStatusDto registrationStatusDto = new InternalRegistrationStatusDto(); + + registrationStatusDto.setRegistrationId("2018701130000410092018110735"); + registrationStatusDto.setRegistrationType(RegistrationType.NEW.toString()); + registrationStatusDto.setRegistrationStageName("PacketValidatorStage"); + registrationStatusDto.setDefaultResumeAction("RESUME_PROCESSING"); + registrationStatusDto.setResumeTimeStamp(LocalDateTime.now()); + registrationStatusDto.setReProcessRetryCount(0); + registrationStatusDto.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.REPROCESS.toString()); + dtolist.add(registrationStatusDto); + List reprocessorDtoList = new ArrayList<>(); + InternalRegistrationStatusDto registrationStatusDto1 = new InternalRegistrationStatusDto(); + + registrationStatusDto1.setRegistrationId("2018701130000410092018110734"); + registrationStatusDto1.setRegistrationStageName("PacketValidatorStage"); + registrationStatusDto1.setReProcessRetryCount(1); + registrationStatusDto1.setRegistrationType("NEW"); + registrationStatusDto1.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.SUCCESS.toString()); + reprocessorDtoList.add(registrationStatusDto1); + Mockito.when(registrationStatusService.getResumablePackets(anyInt())) + .thenReturn(dtolist); + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + .thenReturn(CompletableFuture.completedFuture(dtolist)); + reprocessorVerticle.process(dto); + + } + @Test public void testProcessWithRestartFromStage() throws TablenotAccessibleException, PacketManagerException, @@ -313,4 +418,25 @@ public void testProcessWithRestartFromStage() throws TablenotAccessibleException } + @Test + public void testProcessWithRestartFromStageNew() throws TablenotAccessibleException, + PacketManagerException, + ApisResourceAccessException, WorkflowActionException { + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + + List dtolist = new ArrayList<>(); + InternalRegistrationStatusDto registrationStatusDto = new InternalRegistrationStatusDto(); + + registrationStatusDto.setRegistrationId("2018701130000410092018110735"); + registrationStatusDto.setRegistrationType(RegistrationType.NEW.toString()); + registrationStatusDto.setRegistrationStageName("BioDedupeStage"); + registrationStatusDto.setReProcessRetryCount(0); + registrationStatusDto.setStatusCode(RegistrationStatusCode.PROCESSING.toString()); + registrationStatusDto.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.REPROCESS.toString()); + dtolist.add(registrationStatusDto); + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + .thenReturn(CompletableFuture.completedFuture(dtolist)); + reprocessorVerticle.process(dto); + + } } From eba4918b09277ba177ab0238c2e25f5a037bbf34 Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Fri, 10 Oct 2025 13:55:59 +0530 Subject: [PATCH 06/14] Reprocessor Changes Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../reprocessor/verticle/ReprocessorVerticle.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index a5cff24fb71..cc13ea9e932 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -384,7 +384,12 @@ public MessageDTO process(MessageDTO object) { PlatformErrorMessages.RPR_PKR_UNKNOWN_EXCEPTION.getMessage(), ex.toString()); return null; })).collect(Collectors.toList()); - CompletableFuture.allOf(sendTasks.toArray(new CompletableFuture[0])).join(); + CompletableFuture.allOf(sendTasks.toArray(new CompletableFuture[0])).whenComplete((res, ex) -> { + regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), + "Error in Packet Processing -- ", + ex.getMessage(), ExceptionUtils.getStackTrace(ex)); + + });; } } catch (TablenotAccessibleException e) { isBatchSuccessful = false; @@ -575,7 +580,12 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require }) .collect(Collectors.toList()); - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).whenComplete((res, ex) -> { + regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), + "Error in async tasks -- ", + ex.getMessage(), ExceptionUtils.getStackTrace(ex)); + + }); } private AbstractMap.SimpleEntry, List> parseProcessAndStatus(String key) { From 718533386181ba31cb4d84d3698bd785b9fd98fb Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Mon, 13 Oct 2025 13:34:11 +0530 Subject: [PATCH 07/14] Reprocessor Changes Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../reprocessor/verticle/ReprocessorVerticle.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index cc13ea9e932..d61ed823847 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -580,12 +580,7 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require }) .collect(Collectors.toList()); - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).whenComplete((res, ex) -> { - regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), - "Error in async tasks -- ", - ex.getMessage(), ExceptionUtils.getStackTrace(ex)); - - }); + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); } private AbstractMap.SimpleEntry, List> parseProcessAndStatus(String key) { From 9219e7dcfffa582743d6996b10f35d696db8382c Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Mon, 13 Oct 2025 16:36:17 +0530 Subject: [PATCH 08/14] Reprocessor Changes based on TF Discussion Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../status/dao/RegistrationStatusDao.java | 11 +++++++--- .../repositary/RegistrationRepositary.java | 3 +++ .../service/RegistrationStatusService.java | 2 +- .../impl/RegistrationStatusServiceImpl.java | 4 ++-- .../service/ReprocessorVerticalService.java | 5 +++-- .../impl/ReprocessorVerticalServiceImpl.java | 4 ++-- .../verticle/ReprocessorVerticle.java | 20 +++++++++---------- .../verticle/ReprocessorVerticleTest.java | 12 +++++------ 8 files changed, 35 insertions(+), 26 deletions(-) diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java index b7f01ebcb49..59b31d586ad 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java @@ -262,7 +262,7 @@ public List findByIdAndProcessAndIteration(String id, * @return the un processed packets */ public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, - Integer reprocessCount, List status, List excludeStageNames, List skipRegIds) { + Integer reprocessCount, List trnStatusList, List excludeStageNames, List skipRegIds, List statusList) { LocalDateTime timeDifference = LocalDateTime.now().minusSeconds(elapseTime); List statusCodes=new ArrayList<>(); @@ -273,7 +273,12 @@ public List getUnProcessedPackets(List process statusCodes.add(RegistrationStatusCode.FAILED.toString()); statusCodes.add(RegistrationStatusCode.PROCESSED.toString()); - return registrationStatusRepositary.getUnProcessedPackets(processList, status, reprocessCount, timeDifference, - statusCodes, fetchSize, excludeStageNames, skipRegIds); + if(statusList != null && !statusList.isEmpty()) { + return registrationStatusRepositary.getUnProcessedPacketsWithSpecificStatus(processList, trnStatusList, reprocessCount, timeDifference, + statusList, fetchSize, excludeStageNames, skipRegIds); + } else { + return registrationStatusRepositary.getUnProcessedPackets(processList, trnStatusList, reprocessCount, timeDifference, + statusCodes, fetchSize, excludeStageNames, skipRegIds); + } } } \ No newline at end of file diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java index 2d68c636cbb..0634478cc5a 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java @@ -68,5 +68,8 @@ public List getProcessedOrProcessingRegIds(@Param("regIds") List @Query(value ="SELECT * FROM registration r WHERE r.process IN :processList AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code NOT IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames AND r.reg_id NOT IN :skipRegIds order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) public List getUnProcessedPackets(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames, @Param("skipRegIds") List skipRegIds); + @Query(value ="SELECT * FROM registration r WHERE r.process IN :processList AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames AND r.reg_id NOT IN :skipRegIds order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) + public List getUnProcessedPacketsWithSpecificStatus(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames, @Param("skipRegIds") List skipRegIds); + } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java index cbdecf92624..5067b1372bf 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java @@ -182,6 +182,6 @@ public Integer getUnProcessedPacketsCount(long elapseTime, Integer reprocessCoun * @return the un processed packets */ public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, - List status, List excludeStageNames, List skipRegIds); + List trnStatusList, List excludeStageNames, List skipRegIds, List statusList); } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java index 4f4bb6c8a3a..5e12b698d66 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java @@ -981,13 +981,13 @@ public void updateRegistrationStatusForWorkflow(InternalRegistrationStatusDto re * @return the un processed packets */ public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, - Integer reprocessCount, List status, List excludeStageNames, List skipRegIds) { + Integer reprocessCount, List trnStatusList, List excludeStageNames, List skipRegIds, List statusList) { regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), "", "RegistrationStatusServiceImpl::getReprocessPacket()::entry"); try { List entityList = registrationStatusDao.getUnProcessedPackets(processList, fetchSize, - elapseTime, reprocessCount, status, excludeStageNames, skipRegIds); + elapseTime, reprocessCount, trnStatusList, excludeStageNames, skipRegIds, statusList); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), "", "RegistrationStatusServiceImpl::getReprocessPacket()::exit"); diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java index 6f22fd3a596..f28a701345c 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java @@ -11,7 +11,8 @@ CompletableFuture> fetchUnProcessedPackets( Integer fetchSize, long elapseTime, Integer reprocessCount, - List status, + List trnStatusList, List excludeStageNames, - List skipRegIds); + List skipRegIds, + List statusList); } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java index 55fb55b0ad1..d35997cc6fe 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java @@ -21,9 +21,9 @@ public class ReprocessorVerticalServiceImpl implements ReprocessorVerticalServic @Override @Async - public CompletableFuture> fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, List status, List excludeStageNames, List skipRegIds) { + public CompletableFuture> fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, List trnStatusList, List excludeStageNames, List skipRegIds, List statusList) { List result = registrationStatusService.getUnProcessedPackets(processList, fetchSize, elapseTime, - reprocessCount, status, excludeStageNames, skipRegIds); + reprocessCount, trnStatusList, excludeStageNames, skipRegIds, statusList); return CompletableFuture.completedFuture(result != null ? result : Collections.emptyList()); } } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index d61ed823847..baf91e4d0dc 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -253,10 +253,10 @@ public void start() { public MessageDTO process(MessageDTO object) { List reprocessorDtoList = null; LogDescription description = new LogDescription(); - List statusList = new ArrayList<>(); - statusList.add(RegistrationTransactionStatusCode.SUCCESS.toString()); - statusList.add(RegistrationTransactionStatusCode.REPROCESS.toString()); - statusList.add(RegistrationTransactionStatusCode.IN_PROGRESS.toString()); + List trnStatusList = new ArrayList<>(); + trnStatusList.add(RegistrationTransactionStatusCode.SUCCESS.toString()); + trnStatusList.add(RegistrationTransactionStatusCode.REPROCESS.toString()); + trnStatusList.add(RegistrationTransactionStatusCode.IN_PROGRESS.toString()); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "ReprocessorVerticle::process()::entry"); ConcurrentLinkedQueue ridSb = new ConcurrentLinkedQueue<>(); @@ -275,7 +275,7 @@ public MessageDTO process(MessageDTO object) { ); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Prepared process based required count details for fetch " + requiredCountMap.toString()); - fetchPacketsIfBelowThreshold(requiredCountMap, statusList); + fetchPacketsIfBelowThreshold(requiredCountMap, trnStatusList); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Cacheing Packets if below Threashold method completed." + packetCacheMap.entrySet().stream().map(e -> e.getKey() + " = " + e.getValue().size()).collect(Collectors.joining(", ", "[", "]"))); @@ -289,14 +289,14 @@ public MessageDTO process(MessageDTO object) { if (!CollectionUtils.isEmpty(reprocessorDtoList)) { if (reprocessorDtoList.size() < fetchSize) { List reprocessorPacketList = registrationStatusService.getUnProcessedPackets(fetchSize - reprocessorDtoList.size(), elapseTime, - reprocessCount, statusList, reprocessExcludeStageNames); + reprocessCount, trnStatusList, reprocessExcludeStageNames); if (!CollectionUtils.isEmpty(reprocessorPacketList)) { reprocessorDtoList.addAll(reprocessorPacketList); } } } else { reprocessorDtoList = registrationStatusService.getUnProcessedPackets(fetchSize, elapseTime, - reprocessCount, statusList, reprocessExcludeStageNames); + reprocessCount, trnStatusList, reprocessExcludeStageNames); } } @@ -529,7 +529,7 @@ private LinkedHashMap prepareRequiredCount(int availableCount) return requiredCountMap; } - private void fetchPacketsIfBelowThreshold(LinkedHashMap requiredCountMap, List statusList) { + private void fetchPacketsIfBelowThreshold(LinkedHashMap requiredCountMap, List trnStatusList) { List> futures = requiredCountMap.entrySet().stream() .filter(entry -> { Deque cachedPackets = packetCacheMap.get(entry.getKey()); @@ -559,7 +559,7 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require if(skipRegIdList.isEmpty()) skipRegIdList.add("-DUMMY-"); return reprocessorVerticalService.fetchUnProcessedPackets(processList, recordFetchCount, elapseTime, - reprocessCount, (!statusValList.isEmpty() ? statusValList : statusList), reprocessExcludeStageNames, skipRegIdList) + reprocessCount, trnStatusList, reprocessExcludeStageNames, skipRegIdList, statusValList) .thenAccept(result -> { regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Total Record Fetched from database for process " + key + " is " + result.size()); @@ -574,7 +574,7 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require .exceptionally(ex -> { regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), "Error Fetching UnprocessedPackets -- ", - " Error Triggered for Process [" + String.join(",", processList) + "] and Status [" + String.join(",", (!statusValList.isEmpty() ? statusValList : statusList)) + "]", ExceptionUtils.getStackTrace(ex)); + " Error Triggered for Process [" + String.join(",", processList) + "] and Status [" + String.join(",", statusValList) + "]", ExceptionUtils.getStackTrace(ex)); return null; }); }) diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java index 1def8ccc8ac..9baf9b113fe 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java @@ -212,7 +212,7 @@ public void testProcessValidNew() throws TablenotAccessibleException, PacketMana registrationStatusDto1.setRegistrationType("NEW"); registrationStatusDto1.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.SUCCESS.toString()); dtolist.add(registrationStatusDto1); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) .thenReturn(CompletableFuture.completedFuture(dtolist)); reprocessorVerticle.process(dto); @@ -272,7 +272,7 @@ public void testProcessFailureNew() throws TablenotAccessibleException, PacketMa registrationStatusDto1.setRegistrationType("NEW"); registrationStatusDto1.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.SUCCESS.toString()); dtolist.add(registrationStatusDto1); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) .thenReturn(CompletableFuture.completedFuture(dtolist)); reprocessorVerticle.process(dto); @@ -296,7 +296,7 @@ public void exceptionTest() throws Exception { @Test public void exceptionTestNew() throws Exception { ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) .thenReturn(CompletableFuture.completedFuture(Collections.emptyList())); dto = reprocessorVerticle.process(dto); assertEquals(null, dto.getIsValid()); @@ -324,7 +324,7 @@ public void TablenotAccessibleExceptionTest() throws Exception { @Test public void TablenotAccessibleExceptionTestNew() throws Exception { ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) .thenThrow(new TablenotAccessibleException("") { }); dto = reprocessorVerticle.process(dto); @@ -390,7 +390,7 @@ public void testProcessValidWithResumablePacketsNew() throws TablenotAccessibleE reprocessorDtoList.add(registrationStatusDto1); Mockito.when(registrationStatusService.getResumablePackets(anyInt())) .thenReturn(dtolist); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) .thenReturn(CompletableFuture.completedFuture(dtolist)); reprocessorVerticle.process(dto); @@ -434,7 +434,7 @@ public void testProcessWithRestartFromStageNew() throws TablenotAccessibleExcept registrationStatusDto.setStatusCode(RegistrationStatusCode.PROCESSING.toString()); registrationStatusDto.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.REPROCESS.toString()); dtolist.add(registrationStatusDto); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList())) + Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) .thenReturn(CompletableFuture.completedFuture(dtolist)); reprocessorVerticle.process(dto); From 5e6d5757816a22961ed4184ebaa8341d1ca51d07 Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Mon, 13 Oct 2025 20:35:37 +0530 Subject: [PATCH 09/14] Reprocessor Changes based on TF Discussion Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../status/dao/RegistrationStatusDao.java | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java index 59b31d586ad..3045e2a105a 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java @@ -1,11 +1,7 @@ package io.mosip.registration.processor.status.dao; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; @@ -71,6 +67,7 @@ public class RegistrationStatusDao { public static final String UPDATED_DATE_TIME = "updateDateTime"; + private HashMap> statusCodes=new HashMap<>(); /** * Save. * @@ -265,20 +262,37 @@ public List getUnProcessedPackets(List process Integer reprocessCount, List trnStatusList, List excludeStageNames, List skipRegIds, List statusList) { LocalDateTime timeDifference = LocalDateTime.now().minusSeconds(elapseTime); - List statusCodes=new ArrayList<>(); - statusCodes.add(RegistrationStatusCode.PAUSED.toString()); - statusCodes.add(RegistrationStatusCode.RESUMABLE.toString()); - statusCodes.add(RegistrationStatusCode.PAUSED_FOR_ADDITIONAL_INFO.toString()); - statusCodes.add(RegistrationStatusCode.REJECTED.toString()); - statusCodes.add(RegistrationStatusCode.FAILED.toString()); - statusCodes.add(RegistrationStatusCode.PROCESSED.toString()); + Set statusSet = new HashSet<>(); + statusSet.add(RegistrationStatusCode.PAUSED.toString()); + statusSet.add(RegistrationStatusCode.RESUMABLE.toString()); + statusSet.add(RegistrationStatusCode.PAUSED_FOR_ADDITIONAL_INFO.toString()); + statusSet.add(RegistrationStatusCode.REJECTED.toString()); + statusSet.add(RegistrationStatusCode.FAILED.toString()); + statusSet.add(RegistrationStatusCode.PROCESSED.toString()); if(statusList != null && !statusList.isEmpty()) { + processList.forEach(key -> { + if(statusList != null && !statusList.isEmpty()) + statusSet.addAll(statusList); + + if(statusCodes.containsKey(key)) { + statusCodes.get(key).addAll(statusSet); + } else { + statusCodes.put(key, statusSet); + } + }); + + statusSet.addAll(statusList); return registrationStatusRepositary.getUnProcessedPacketsWithSpecificStatus(processList, trnStatusList, reprocessCount, timeDifference, statusList, fetchSize, excludeStageNames, skipRegIds); } else { + Set populatedStatus = new HashSet<>(); + processList.forEach(key -> { + populatedStatus.addAll(statusCodes.get(key) != null ? statusCodes.get(key) : statusSet); + }); + List status = new ArrayList<>(populatedStatus); return registrationStatusRepositary.getUnProcessedPackets(processList, trnStatusList, reprocessCount, timeDifference, - statusCodes, fetchSize, excludeStageNames, skipRegIds); + status, fetchSize, excludeStageNames, skipRegIds); } } } \ No newline at end of file From 686978cf0c6d81afa099c6643e2a404aebecc5c2 Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Fri, 17 Oct 2025 11:59:38 +0530 Subject: [PATCH 10/14] Reprocessor Changes based on internal Review Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../MosipVerticleManager.java | 6 +- .../abstractverticle/ConsumerVerticle.java | 13 +- .../core/eventbus/KafkaMosipEventBusTest.java | 2 +- .../status/dao/RegistrationStatusDao.java | 58 +-- .../repositary/RegistrationRepositary.java | 8 +- .../service/RegistrationStatusService.java | 27 +- .../status/service/TransactionService.java | 10 + .../impl/RegistrationStatusServiceImpl.java | 96 ++++- .../service/impl/TransactionServiceImpl.java | 28 ++ .../reprocessor/ReprocessorApplication.java | 3 +- .../reprocessor/dto/ProcessAllocation.java | 26 ++ .../service/ReprocessorVerticalService.java | 18 - .../impl/ReprocessorVerticalServiceImpl.java | 29 -- .../verticle/ReprocessorVerticle.java | 343 ++++++++---------- .../src/main/resources/bootstrap.properties | 3 +- .../verticle/ReprocessorVerticleTest.java | 51 +-- 16 files changed, 397 insertions(+), 324 deletions(-) create mode 100644 registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/dto/ProcessAllocation.java delete mode 100644 registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java delete mode 100644 registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java diff --git a/registration-processor/registration-processor-core/src/main/java/io/mosip/registration/processor/core/abstractverticle/MosipVerticleManager.java b/registration-processor/registration-processor-core/src/main/java/io/mosip/registration/processor/core/abstractverticle/MosipVerticleManager.java index 7bd9a8e81d3..a58f4dd74b5 100644 --- a/registration-processor/registration-processor-core/src/main/java/io/mosip/registration/processor/core/abstractverticle/MosipVerticleManager.java +++ b/registration-processor/registration-processor-core/src/main/java/io/mosip/registration/processor/core/abstractverticle/MosipVerticleManager.java @@ -147,7 +147,7 @@ public MosipEventBus getEventBus(Object verticleName, String clusterManagerUrl, VertxOptions options = new VertxOptions().setClustered(true).setClusterManager(clusterManager) .setHAEnabled(false).setWorkerPoolSize(instanceNumber) .setEventBusOptions(new EventBusOptions().setPort(getEventBusPort()).setHost(address)) - .setMetricsOptions(micrometerMetricsOptions).setMaxEventLoopExecuteTime(getIntegerPropertyForSuffix("eventbus.maxloop.exec.time.seconds", 2)).setMaxEventLoopExecuteTimeUnit(TimeUnit.SECONDS); + .setMetricsOptions(micrometerMetricsOptions).setMaxEventLoopExecuteTime(getMaxEventLoopExecutionTime()).setMaxEventLoopExecuteTimeUnit(TimeUnit.SECONDS); Vertx.clusteredVertx(options, result -> { if (result.succeeded()) { result.result().deployVerticle((Verticle) verticleName, @@ -283,6 +283,10 @@ public Integer getEventBusPort() { return getIntegerPropertyForSuffix("eventbus.port"); } + public Integer getMaxEventLoopExecutionTime() { + return getIntegerPropertyForSuffix("eventbus.maxloop.exec.time.seconds", 2); + } + public Integer getPort() { return getIntegerPropertyForSuffix("server.port"); } diff --git a/registration-processor/registration-processor-core/src/test/java/io/mosip/registration/processor/abstractverticle/ConsumerVerticle.java b/registration-processor/registration-processor-core/src/test/java/io/mosip/registration/processor/abstractverticle/ConsumerVerticle.java index 974cf840308..c62507c9497 100644 --- a/registration-processor/registration-processor-core/src/test/java/io/mosip/registration/processor/abstractverticle/ConsumerVerticle.java +++ b/registration-processor/registration-processor-core/src/test/java/io/mosip/registration/processor/abstractverticle/ConsumerVerticle.java @@ -4,14 +4,7 @@ import java.util.ArrayList; import brave.Tracing; -import io.mosip.registration.processor.core.tracing.EventTracingHandler; -import io.vertx.core.eventbus.EventBus; import io.vertx.core.logging.SLF4JLogDelegateFactory; -import org.assertj.core.util.Objects; -import org.junit.Assert; -import org.junit.Before; -import org.mockito.Mockito; -import org.springframework.boot.test.mock.mockito.MockBean; import io.mosip.registration.processor.core.abstractverticle.MessageBusAddress; import io.mosip.registration.processor.core.abstractverticle.MessageDTO; @@ -66,7 +59,7 @@ public URL findUrl() URL url=loader.getResource("cluster.xml"); return url; } - + @Override public Integer getEventBusPort() { return 5711; @@ -82,4 +75,8 @@ protected String getPropertyPrefix() { return EMPTY_STRING; } + @Override + public Integer getMaxEventLoopExecutionTime() { + return 3; + } } \ No newline at end of file diff --git a/registration-processor/registration-processor-core/src/test/java/io/mosip/registration/processor/core/eventbus/KafkaMosipEventBusTest.java b/registration-processor/registration-processor-core/src/test/java/io/mosip/registration/processor/core/eventbus/KafkaMosipEventBusTest.java index 8c35485c68c..2e799141fcf 100644 --- a/registration-processor/registration-processor-core/src/test/java/io/mosip/registration/processor/core/eventbus/KafkaMosipEventBusTest.java +++ b/registration-processor/registration-processor-core/src/test/java/io/mosip/registration/processor/core/eventbus/KafkaMosipEventBusTest.java @@ -621,7 +621,7 @@ private KafkaConsumerRecords prepareKafkaConsumerRecords(int rec consumerRecordList.add( new ConsumerRecord( MessageBusAddress.PACKET_VALIDATOR_BUS_IN.getAddress(), 0, i, "1000"+i, - "{\"rid\":\"1000"+i+"\", \"reg_type\": \"NEW\" }")); + "{\"rid\":\"1000"+i+"\", \"reg_type\": \"NEW\", \"messageBusAddress\" : { \"address\" : \"packet-uploader-new-bus-out\"}}")); Map>> topicPartitionConsumerRecordListMap = new HashMap>>(); topicPartitionConsumerRecordListMap.put( diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java index 3045e2a105a..89c26ec25b9 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/dao/RegistrationStatusDao.java @@ -67,7 +67,6 @@ public class RegistrationStatusDao { public static final String UPDATED_DATE_TIME = "updateDateTime"; - private HashMap> statusCodes=new HashMap<>(); /** * Save. * @@ -80,6 +79,17 @@ public RegistrationStatusEntity save(RegistrationStatusEntity registrationStatus return registrationStatusRepositary.save(registrationStatusEntity); } + /** + * Save. + * + * @param registrationStatusEntities + * the registration status entity list + * @return the registration status entity + */ + public List saveAll(List registrationStatusEntities) { + return registrationStatusRepositary.saveAll(registrationStatusEntities); + } + /** * Update. * @@ -248,51 +258,41 @@ public List findByIdAndProcessAndIteration(String id, /** * Gets the un processed packets. * + * + * @param processList + * the process List * @param fetchSize * the fetch size * @param elapseTime * the elapse time * @param reprocessCount * the reprocess count - * @param status + * @param trnStatusList + * the transaction status + * @param excludeStageNames + * the stage which need to exclude + * @param statusList * the status * @return the un processed packets */ public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, - Integer reprocessCount, List trnStatusList, List excludeStageNames, List skipRegIds, List statusList) { + Integer reprocessCount, List trnStatusList, List excludeStageNames, List statusList) { LocalDateTime timeDifference = LocalDateTime.now().minusSeconds(elapseTime); - Set statusSet = new HashSet<>(); - statusSet.add(RegistrationStatusCode.PAUSED.toString()); - statusSet.add(RegistrationStatusCode.RESUMABLE.toString()); - statusSet.add(RegistrationStatusCode.PAUSED_FOR_ADDITIONAL_INFO.toString()); - statusSet.add(RegistrationStatusCode.REJECTED.toString()); - statusSet.add(RegistrationStatusCode.FAILED.toString()); - statusSet.add(RegistrationStatusCode.PROCESSED.toString()); + List statusCodes=new ArrayList<>(); + statusCodes.add(RegistrationStatusCode.PAUSED.toString()); + statusCodes.add(RegistrationStatusCode.RESUMABLE.toString()); + statusCodes.add(RegistrationStatusCode.PAUSED_FOR_ADDITIONAL_INFO.toString()); + statusCodes.add(RegistrationStatusCode.REJECTED.toString()); + statusCodes.add(RegistrationStatusCode.FAILED.toString()); + statusCodes.add(RegistrationStatusCode.PROCESSED.toString()); if(statusList != null && !statusList.isEmpty()) { - processList.forEach(key -> { - if(statusList != null && !statusList.isEmpty()) - statusSet.addAll(statusList); - - if(statusCodes.containsKey(key)) { - statusCodes.get(key).addAll(statusSet); - } else { - statusCodes.put(key, statusSet); - } - }); - - statusSet.addAll(statusList); return registrationStatusRepositary.getUnProcessedPacketsWithSpecificStatus(processList, trnStatusList, reprocessCount, timeDifference, - statusList, fetchSize, excludeStageNames, skipRegIds); + statusList, fetchSize, excludeStageNames); } else { - Set populatedStatus = new HashSet<>(); - processList.forEach(key -> { - populatedStatus.addAll(statusCodes.get(key) != null ? statusCodes.get(key) : statusSet); - }); - List status = new ArrayList<>(populatedStatus); return registrationStatusRepositary.getUnProcessedPackets(processList, trnStatusList, reprocessCount, timeDifference, - status, fetchSize, excludeStageNames, skipRegIds); + statusCodes, fetchSize, excludeStageNames); } } } \ No newline at end of file diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java index 0634478cc5a..08745b2e7ae 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/repositary/RegistrationRepositary.java @@ -65,11 +65,11 @@ public List getProcessedOrProcessingRegIds(@Param("regIds") List @Query("SELECT registration FROM RegistrationStatusEntity registration WHERE registration.regId = :regId AND registration.registrationType = :registrationType AND registration.iteration = :iteration") public List getByIdAndProcessAndIteration(@Param("regId") String regId, @Param("registrationType") String process, @Param("iteration") int iteration); - @Query(value ="SELECT * FROM registration r WHERE r.process IN :processList AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code NOT IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames AND r.reg_id NOT IN :skipRegIds order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) - public List getUnProcessedPackets(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames, @Param("skipRegIds") List skipRegIds); + @Query(value ="SELECT * FROM registration r WHERE r.process IN :processList AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code NOT IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) + public List getUnProcessedPackets(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames); - @Query(value ="SELECT * FROM registration r WHERE r.process IN :processList AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames AND r.reg_id NOT IN :skipRegIds order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) - public List getUnProcessedPacketsWithSpecificStatus(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames, @Param("skipRegIds") List skipRegIds); + @Query(value ="SELECT * FROM registration r WHERE r.process IN :processList AND r.latest_trn_status_code IN :status AND r.reg_process_retry_count<=:reprocessCount AND r.latest_trn_dtimes <:timeDifference AND r.status_code IN :statusCodes AND r.reg_stage_name NOT IN :excludeStageNames order by r.latest_trn_dtimes LIMIT :fetchSize ", nativeQuery = true) + public List getUnProcessedPacketsWithSpecificStatus(@Param("processList") List processes, @Param("status") List status,@Param("reprocessCount") Integer reprocessCount,@Param("timeDifference") LocalDateTime timeDifference,@Param("statusCodes") List statusCodes,@Param("fetchSize") Integer fetchSize,@Param("excludeStageNames") List excludeStageNames); } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java index 5067b1372bf..7fc19eb8678 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/RegistrationStatusService.java @@ -1,6 +1,7 @@ package io.mosip.registration.processor.status.service; import java.util.List; +import java.util.concurrent.CompletableFuture; import org.springframework.data.domain.Page; import org.springframework.stereotype.Service; @@ -168,20 +169,34 @@ public Integer getUnProcessedPacketsCount(long elapseTime, Integer reprocessCoun * Gets the un processed packets. * * @param processList - * the process List + * the process List * @param fetchSize * the fetch size * @param elapseTime * the elapse time * @param reprocessCount * the reprocess count - * @param status - * the status + * @param trnStatusList + * the transaction status * @param excludeStageNames - * the exclude stage names + * the stage which need to exclude + * @param statusList + * the status * @return the un processed packets */ - public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, - List trnStatusList, List excludeStageNames, List skipRegIds, List statusList); + public CompletableFuture> getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, + List trnStatusList, List excludeStageNames, List statusList); + + /** + * Update registration status for workflow Engine. + * + * @param registrationStatusDtos + * the list of registration status dto + * @param moduleId + * the module id + * @param moduleName + * the module name + */ + public void updateRegistrationStatusForWorkflowEngines(List registrationStatusDtos, String moduleId, String moduleName); } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/TransactionService.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/TransactionService.java index 431a4002fa8..586a38daaec 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/TransactionService.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/TransactionService.java @@ -27,6 +27,16 @@ public interface TransactionService { */ public TransactionEntity addRegistrationTransaction(U registrationStatusDto); + /** + * Adds the registration transaction. + * + * @param registrationStatusDtoList + * the registration status dto List + * @return the transaction entity + */ + public List addRegistrationTransactions(List registrationStatusDtoList); + + /** * Gets the transaction by reg id and status code. * diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java index 5e12b698d66..fa734ed56c9 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java @@ -7,9 +7,9 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import io.mosip.registration.processor.core.code.RegistrationTransactionTypeCode; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -17,6 +17,7 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.PageRequest; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import io.mosip.kernel.core.dataaccess.exception.DataAccessLayerException; @@ -969,31 +970,35 @@ public void updateRegistrationStatusForWorkflow(InternalRegistrationStatusDto re * Gets the un processed packets. * * @param processList - * the process List + * the process List * @param fetchSize * the fetch size * @param elapseTime * the elapse time * @param reprocessCount * the reprocess count - * @param status + * @param trnStatusList + * the transaction status + * @param excludeStageNames + * the stage which need to exclude + * @param statusList * the status * @return the un processed packets */ - public List getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, - Integer reprocessCount, List trnStatusList, List excludeStageNames, List skipRegIds, List statusList) { + @Async + public CompletableFuture> getUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, + Integer reprocessCount, List trnStatusList, List excludeStageNames, List statusList) { regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), "", "RegistrationStatusServiceImpl::getReprocessPacket()::entry"); try { List entityList = registrationStatusDao.getUnProcessedPackets(processList, fetchSize, - elapseTime, reprocessCount, trnStatusList, excludeStageNames, skipRegIds, statusList); + elapseTime, reprocessCount, trnStatusList, excludeStageNames, statusList); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), "", "RegistrationStatusServiceImpl::getReprocessPacket()::exit"); - return convertEntityListToDtoList(entityList); - + return CompletableFuture.completedFuture(convertEntityListToDtoList(entityList)); } catch (DataAccessException | DataAccessLayerException e) { regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), @@ -1003,4 +1008,79 @@ public List getUnProcessedPackets(List pr } } + @Override + public void updateRegistrationStatusForWorkflowEngines(List registrationStatusDtos, String moduleId, + String moduleName) { + List registrationStatusEntities = new ArrayList<>(); + List transactionDtoList = new ArrayList<>(); + + registrationStatusDtos.forEach(registrationStatusDto -> { + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), + registrationStatusDto.getRegistrationId(), + "RegistrationStatusServiceImpl::updateRegistrationStatus()::entry"); + boolean isTransactionSuccessful = false; + LogDescription description = new LogDescription(); + String transactionId = generateId(); + String latestTransactionId = getLatestTransactionId(registrationStatusDto.getRegistrationId(), + registrationStatusDto.getRegistrationType(), registrationStatusDto.getIteration(), registrationStatusDto.getWorkflowInstanceId()); + TransactionDto transactionDto = new TransactionDto(transactionId, registrationStatusDto.getRegistrationId(), + latestTransactionId, registrationStatusDto.getLatestTransactionTypeCode(), + "updated registration status record", registrationStatusDto.getLatestTransactionStatusCode(), + registrationStatusDto.getStatusComment(), registrationStatusDto.getSubStatusCode()); + if (registrationStatusDto.getRefId() == null) { + transactionDto.setReferenceId(registrationStatusDto.getRegistrationId()); + } else { + transactionDto.setReferenceId(registrationStatusDto.getRefId()); + } + + transactionDto.setReferenceIdType("updated registration record"); + transactionDtoList.add(transactionDto); + + registrationStatusDto.setLatestRegistrationTransactionId(transactionId); + try { + InternalRegistrationStatusDto dto = getRegistrationStatus(registrationStatusDto.getRegistrationId(), + registrationStatusDto.getRegistrationType(), registrationStatusDto.getIteration(), registrationStatusDto.getWorkflowInstanceId()); + if (dto != null) { + dto.setUpdateDateTime(LocalDateTime.now(ZoneId.of("UTC"))); + RegistrationStatusEntity entity = convertDtoToEntity(registrationStatusDto, + dto.getLastSuccessStageName(), true); + if (entity.getStatusCode() == null) { + entity.setStatusCode(dto.getStatusCode()); + } + registrationStatusEntities.add(entity); + isTransactionSuccessful = true; + description.setMessage("Updated registration status successfully"); + } + } catch (DataAccessException | DataAccessLayerException e) { + description.setMessage("DataAccessLayerException while Updating registration status for registration Id" + + registrationStatusDto.getRegistrationId() + "::" + e.getMessage()); + + regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), + registrationStatusDto.getRegistrationId(), e.getMessage() + ExceptionUtils.getStackTrace(e)); + throw new TablenotAccessibleException( + PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage(), e); + } finally { + transcationStatusService.addRegistrationTransaction(transactionDto); + + if(!registrationStatusEntities.isEmpty()) + registrationStatusDao.saveAll(registrationStatusEntities); + + String eventId = isTransactionSuccessful ? EventId.RPR_407.toString() : EventId.RPR_405.toString(); + String eventName = eventId.equalsIgnoreCase(EventId.RPR_407.toString()) ? EventName.UPDATE.toString() + : EventName.EXCEPTION.toString(); + String eventType = eventId.equalsIgnoreCase(EventId.RPR_407.toString()) ? EventType.BUSINESS.toString() + : EventType.SYSTEM.toString(); + + if(!disableAudit) + auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, eventType, + moduleId, moduleName, registrationStatusDto.getRegistrationId()); + + } + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), + registrationStatusDto.getRegistrationId(), + "RegistrationStatusServiceImpl::updateRegistrationStatus()::exit"); + + }); + } + } diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/TransactionServiceImpl.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/TransactionServiceImpl.java index 5cb736f3631..1e9b7bded57 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/TransactionServiceImpl.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/TransactionServiceImpl.java @@ -148,4 +148,32 @@ private RegistrationTransactionDto convertEntityToRegistrationTransactionDto(Tra entity.getParentid(), entity.getStatusCode(), entity.getSubStatusCode(), entity.getStatusComment(), entity.getCreateDateTime()); } + + /* + * (non-Javadoc) + * + * @see io.mosip.registration.processor.status.service.TransactionService# + * addRegistrationTransaction(java.lang.Object) + */ + @Override + public List addRegistrationTransactions(List transactionStatusDtoList) { + List entities = new ArrayList<>(); + transactionStatusDtoList.forEach(transactionStatusDto -> { + try { + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), + transactionStatusDto.getRegistrationId(), + "TransactionServiceImpl::addRegistrationTransaction()::entry"); + TransactionEntity entity = convertDtoToEntity(transactionStatusDto); + entities.add(entity); + regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.USERID.toString(), + transactionStatusDto.getRegistrationId(), + "TransactionServiceImpl::addRegistrationTransaction()::exit"); + } catch (DataAccessLayerException e) { + throw new TransactionTableNotAccessibleException( + PlatformErrorMessages.RPR_RGS_TRANSACTION_TABLE_NOT_ACCESSIBLE.getMessage(), e); + } + }); + + return transactionRepositary.saveAll(entities); + } } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/ReprocessorApplication.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/ReprocessorApplication.java index 40b5f7b4ad7..3f9a3ad585c 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/ReprocessorApplication.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/ReprocessorApplication.java @@ -21,8 +21,7 @@ public static void main(String[] args) { "io.mosip.registration.processor.reprocessor.config", "io.mosip.registration.processor.status.config", "io.mosip.registration.processor.core.kernel.beans", - "io.mosip.registration.processor.packet.storage.config", - "io.mosip.registration.processor.reprocessor.service.impl"); + "io.mosip.registration.processor.packet.storage.config"); ctx.refresh(); ReprocessorVerticle reprocessorVerticle = ctx.getBean(ReprocessorVerticle.class); reprocessorVerticle.deployVerticle(); diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/dto/ProcessAllocation.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/dto/ProcessAllocation.java new file mode 100644 index 00000000000..64e0e21755d --- /dev/null +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/dto/ProcessAllocation.java @@ -0,0 +1,26 @@ +package io.mosip.registration.processor.reprocessor.dto; + +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Data +@Setter +@Getter +public class ProcessAllocation { + private List processes; + private List statuses; + private int percentageAllocation; + + @Override + public String toString() { + return "ProcessAllocation{" + + "processes=" + processes + + ", statuses=" + statuses + + ", percentageAllocation=" + percentageAllocation + + '}'; + } +} + diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java deleted file mode 100644 index f28a701345c..00000000000 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/ReprocessorVerticalService.java +++ /dev/null @@ -1,18 +0,0 @@ -package io.mosip.registration.processor.reprocessor.service; - -import io.mosip.registration.processor.status.dto.InternalRegistrationStatusDto; -import java.util.List; -import java.util.concurrent.CompletableFuture; - -public interface ReprocessorVerticalService { - - CompletableFuture> fetchUnProcessedPackets( - List processList, - Integer fetchSize, - long elapseTime, - Integer reprocessCount, - List trnStatusList, - List excludeStageNames, - List skipRegIds, - List statusList); -} diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java deleted file mode 100644 index d35997cc6fe..00000000000 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/service/impl/ReprocessorVerticalServiceImpl.java +++ /dev/null @@ -1,29 +0,0 @@ -package io.mosip.registration.processor.reprocessor.service.impl; - -import io.mosip.registration.processor.reprocessor.service.ReprocessorVerticalService; -import io.mosip.registration.processor.status.dto.InternalRegistrationStatusDto; -import io.mosip.registration.processor.status.dto.RegistrationStatusDto; -import io.mosip.registration.processor.status.service.RegistrationStatusService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Service; - -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CompletableFuture; - -@Service -public class ReprocessorVerticalServiceImpl implements ReprocessorVerticalService { - - /** The registration status service. */ - @Autowired - private RegistrationStatusService registrationStatusService; - - @Override - @Async - public CompletableFuture> fetchUnProcessedPackets(List processList, Integer fetchSize, long elapseTime, Integer reprocessCount, List trnStatusList, List excludeStageNames, List skipRegIds, List statusList) { - List result = registrationStatusService.getUnProcessedPackets(processList, fetchSize, elapseTime, - reprocessCount, trnStatusList, excludeStageNames, skipRegIds, statusList); - return CompletableFuture.completedFuture(result != null ? result : Collections.emptyList()); - } -} diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index baf91e4d0dc..b1001944188 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -2,10 +2,12 @@ import java.util.*; import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -import io.mosip.registration.processor.reprocessor.service.ReprocessorVerticalService; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.mosip.registration.processor.reprocessor.dto.ProcessAllocation; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -21,10 +23,10 @@ import io.mosip.registration.processor.core.abstractverticle.MosipVerticleAPIManager; import io.mosip.registration.processor.core.code.EventId; import io.mosip.registration.processor.core.code.EventName; +import io.mosip.registration.processor.core.code.RegistrationTransactionTypeCode; import io.mosip.registration.processor.core.code.EventType; import io.mosip.registration.processor.core.code.ModuleName; import io.mosip.registration.processor.core.code.RegistrationTransactionStatusCode; -import io.mosip.registration.processor.core.code.RegistrationTransactionTypeCode; import io.mosip.registration.processor.core.constant.LoggerFileConstant; import io.mosip.registration.processor.core.exception.util.PlatformErrorMessages; import io.mosip.registration.processor.core.exception.util.PlatformSuccessMessages; @@ -44,7 +46,7 @@ import io.vertx.core.eventbus.EventBus; import io.vertx.core.json.JsonObject; -import javax.annotation.PreDestroy; +import javax.annotation.PostConstruct; /** * The Reprocessor Verticle to deploy the scheduler and implement re-processing @@ -97,25 +99,24 @@ public class ReprocessorVerticle extends MosipVerticleAPIManager { @Value("#{'${registration.processor.reprocess.restart-trigger-filter}'.split(',')}") private List reprocessRestartTriggerFilter; - @Value("${registration.processor.reprocess.process-based.fetch.enabled:false}") - private Boolean enabledProcessBasedFetch; + @Value("${registration.processor.reprocess.process-based.cache-enabled:false}") + private Boolean enabledProcessBasedCache; - @Value("#{${registration.processor.reprocess.process-based.fetch.count:{:}}}") - private LinkedHashMap processBasedFetchCountMapping; + @Value("${registration.processor.reprocess.process-based.fetch-count-map}") + private String processBasedFetchCountMapJson; - @Value("${registration.processor.reprocess.record.fetch.size:500}") - private int recordFetchSize; + private List processBasedFetchCountMap; - @Value("${registration.processor.reprocess.record.threashold:50}") - private int threasholdForFetch; + @Value("${registration.processor.reprocess.process-based.prefetch-limit:500}") + private int processBasedPrefetchLimit; - private ConcurrentHashMap> packetCacheMap = new ConcurrentHashMap<>(); + @Value("${registration.processor.reprocess.process-based.threshold:50}") + private int processBasedThreashold; - @Autowired - private ReprocessorVerticalService reprocessorVerticalService; + private ConcurrentHashMap> packetCacheMap = new ConcurrentHashMap<>(); /** The is transaction successful. */ - private boolean isBatchSuccessful = false; + private boolean isTransactionSuccessful = false; /** The registration status service. */ @Autowired @@ -133,21 +134,12 @@ public class ReprocessorVerticle extends MosipVerticleAPIManager { @Value("${server.port}") private String port; - private final ExecutorService sendExecutor = new ThreadPoolExecutor( - Math.max(2, Runtime.getRuntime().availableProcessors()), - Math.max(2, Runtime.getRuntime().availableProcessors()), - 60L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(1000), // bounded queue - new ThreadPoolExecutor.CallerRunsPolicy() - ); - - private final ExecutorService fetchExecutor = new ThreadPoolExecutor( - Math.max(2, Runtime.getRuntime().availableProcessors()), - Math.max(2, Runtime.getRuntime().availableProcessors()), - 60L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(1000), // bounded queue - new ThreadPoolExecutor.CallerRunsPolicy() - ); + @PostConstruct + public void init() throws JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + processBasedFetchCountMap = + mapper.readValue(processBasedFetchCountMapJson, new TypeReference>() {}); + } /** * Deploy verticle. @@ -259,27 +251,29 @@ public MessageDTO process(MessageDTO object) { trnStatusList.add(RegistrationTransactionStatusCode.IN_PROGRESS.toString()); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "ReprocessorVerticle::process()::entry"); - ConcurrentLinkedQueue ridSb = new ConcurrentLinkedQueue<>(); + StringBuffer ridSb = new StringBuffer(); try { Map> reprocessRestartTriggerMap = intializeReprocessRestartTriggerMapping(); reprocessorDtoList = registrationStatusService.getResumablePackets(fetchSize); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Resumable Packets Count " + reprocessorDtoList.size() ); - if(enabledProcessBasedFetch) { + if(enabledProcessBasedCache) { regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Enabled Process based fetch"); if(reprocessorDtoList.size() < fetchSize) { - LinkedHashMap requiredCountMap = prepareRequiredCount( + LinkedHashMap requiredCountMap = prepareRequiredCount( fetchSize - reprocessorDtoList.size() ); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Prepared process based required count details for fetch " + requiredCountMap.toString()); - fetchPacketsIfBelowThreshold(requiredCountMap, trnStatusList); - regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", - "Cacheing Packets if below Threashold method completed." + packetCacheMap.entrySet().stream().map(e -> e.getKey() + " = " + e.getValue().size()).collect(Collectors.joining(", ", "[", "]"))); + cachePacketsIfBelowThreshold(requiredCountMap, trnStatusList); + regProcLogger.debug( + "Caching Packets if below Threshold method completed. Packet sizes: {}", + formatPacketCacheMap() + ); - reprocessorDtoList.addAll(fetchUnprocessedPacketsWithBalance( + reprocessorDtoList.addAll(fetchUnprocessedPacketFromCache( requiredCountMap, fetchSize - reprocessorDtoList.size() )); @@ -304,95 +298,83 @@ public MessageDTO process(MessageDTO object) { "Reprocessor Total Packets Fetched " + reprocessorDtoList.size()); if (!CollectionUtils.isEmpty(reprocessorDtoList)) { - List> sendTasks = reprocessorDtoList.stream() - .map(dto -> CompletableFuture.runAsync(() -> { - { - boolean isTransactionSuccessful = false; - String registrationId = dto.getRegistrationId(); - regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), - LoggerFileConstant.REGISTRATIONID.toString(), registrationId, "Process started"); - ridSb.add(registrationId); - MessageDTO messageDTO = new MessageDTO(); - messageDTO.setRid(registrationId); - messageDTO.setReg_type(dto.getRegistrationType()); - messageDTO.setSource(dto.getSource()); - messageDTO.setIteration(dto.getIteration()); - messageDTO.setWorkflowInstanceId(dto.getWorkflowInstanceId()); - if (reprocessCount.equals(dto.getReProcessRetryCount())) { - dto.setLatestTransactionStatusCode( - RegistrationTransactionStatusCode.REPROCESS_FAILED.toString()); - dto.setLatestTransactionTypeCode( - RegistrationTransactionTypeCode.PACKET_REPROCESS.toString()); - dto.setStatusComment(StatusUtil.RE_PROCESS_FAILED.getMessage()); - dto.setStatusCode(RegistrationStatusCode.REPROCESS_FAILED.toString()); - dto.setSubStatusCode(StatusUtil.RE_PROCESS_FAILED.getCode()); - messageDTO.setIsValid(false); - description.setMessage(PlatformSuccessMessages.RPR_RE_PROCESS_FAILED.getMessage()); - description.setCode(PlatformSuccessMessages.RPR_RE_PROCESS_FAILED.getCode()); - - } else { - messageDTO.setIsValid(true); - isTransactionSuccessful=true; - String stageName; - if (isRestartFromStageRequired(dto, reprocessRestartTriggerMap)) { - stageName = MessageBusUtil.getMessageBusAdress(reprocessRestartFromStage); - stageName = stageName.concat(ReprocessorConstants.BUS_IN); - sendAndSetStatus(dto, messageDTO, stageName); - dto.setStatusComment(StatusUtil.RE_PROCESS_RESTART_FROM_STAGE.getMessage()); - dto.setSubStatusCode(StatusUtil.RE_PROCESS_RESTART_FROM_STAGE.getCode()); - description - .setMessage( - PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_RESTART_FROM_STAGE_SUCCESS - .getMessage()); - description.setCode( - PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_RESTART_FROM_STAGE_SUCCESS - .getCode()); - - } else { - stageName = MessageBusUtil.getMessageBusAdress(dto.getRegistrationStageName()); - if (RegistrationTransactionStatusCode.SUCCESS.name() - .equalsIgnoreCase(dto.getLatestTransactionStatusCode())) { - stageName = stageName.concat(ReprocessorConstants.BUS_OUT); - } else { - stageName = stageName.concat(ReprocessorConstants.BUS_IN); - } - sendAndSetStatus(dto, messageDTO, stageName); - dto.setStatusComment(StatusUtil.RE_PROCESS_COMPLETED.getMessage()); - dto.setSubStatusCode(StatusUtil.RE_PROCESS_COMPLETED.getCode()); - description.setMessage(PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getMessage()); - description.setCode(PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getCode()); - } - } - regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), - LoggerFileConstant.REGISTRATIONID.toString(), registrationId, description.getMessage()); - - /** Module-Id can be Both Success/Error code */ - String moduleId = PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getCode(); - String moduleName = ModuleName.RE_PROCESSOR.toString(); - registrationStatusService.updateRegistrationStatusForWorkflowEngine(dto, moduleId, moduleName); - String eventId = EventId.RPR_402.toString(); - String eventName = EventName.UPDATE.toString(); - String eventType = EventType.BUSINESS.toString(); - - if (!isTransactionSuccessful) - auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, - eventType, moduleId, moduleName, registrationId); - } - },sendExecutor).exceptionally(ex -> { - regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), - description.getCode() + " -- ", - PlatformErrorMessages.RPR_PKR_UNKNOWN_EXCEPTION.getMessage(), ex.toString()); return null; - })).collect(Collectors.toList()); - - CompletableFuture.allOf(sendTasks.toArray(new CompletableFuture[0])).whenComplete((res, ex) -> { - regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), - "Error in Packet Processing -- ", - ex.getMessage(), ExceptionUtils.getStackTrace(ex)); - - });; + List processedList = new ArrayList<>(); + /** Module-Id can be Both Success/Error code */ + String moduleId = PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getCode(); + String moduleName = ModuleName.RE_PROCESSOR.toString(); + + reprocessorDtoList.forEach(dto -> { + String registrationId = dto.getRegistrationId(); + ridSb.append(registrationId); + ridSb.append(","); + MessageDTO messageDTO = new MessageDTO(); + messageDTO.setRid(registrationId); + messageDTO.setReg_type(dto.getRegistrationType()); + messageDTO.setSource(dto.getSource()); + messageDTO.setIteration(dto.getIteration()); + messageDTO.setWorkflowInstanceId(dto.getWorkflowInstanceId()); + if (reprocessCount.equals(dto.getReProcessRetryCount())) { + dto.setLatestTransactionStatusCode( + RegistrationTransactionStatusCode.REPROCESS_FAILED.toString()); + dto.setLatestTransactionTypeCode( + RegistrationTransactionTypeCode.PACKET_REPROCESS.toString()); + dto.setStatusComment(StatusUtil.RE_PROCESS_FAILED.getMessage()); + dto.setStatusCode(RegistrationStatusCode.REPROCESS_FAILED.toString()); + dto.setSubStatusCode(StatusUtil.RE_PROCESS_FAILED.getCode()); + messageDTO.setIsValid(false); + description.setMessage(PlatformSuccessMessages.RPR_RE_PROCESS_FAILED.getMessage()); + description.setCode(PlatformSuccessMessages.RPR_RE_PROCESS_FAILED.getCode()); + + } else { + messageDTO.setIsValid(true); + isTransactionSuccessful = true; + String stageName; + if (isRestartFromStageRequired(dto, reprocessRestartTriggerMap)) { + stageName = MessageBusUtil.getMessageBusAdress(reprocessRestartFromStage); + stageName = stageName.concat(ReprocessorConstants.BUS_IN); + sendAndSetStatus(dto, messageDTO, stageName); + dto.setStatusComment(StatusUtil.RE_PROCESS_RESTART_FROM_STAGE.getMessage()); + dto.setSubStatusCode(StatusUtil.RE_PROCESS_RESTART_FROM_STAGE.getCode()); + description + .setMessage( + PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_RESTART_FROM_STAGE_SUCCESS + .getMessage()); + description.setCode( + PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_RESTART_FROM_STAGE_SUCCESS + .getCode()); + + } else { + stageName = MessageBusUtil.getMessageBusAdress(dto.getRegistrationStageName()); + if (RegistrationTransactionStatusCode.SUCCESS.name() + .equalsIgnoreCase(dto.getLatestTransactionStatusCode())) { + stageName = stageName.concat(ReprocessorConstants.BUS_OUT); + } else { + stageName = stageName.concat(ReprocessorConstants.BUS_IN); + } + sendAndSetStatus(dto, messageDTO, stageName); + dto.setStatusComment(StatusUtil.RE_PROCESS_COMPLETED.getMessage()); + dto.setSubStatusCode(StatusUtil.RE_PROCESS_COMPLETED.getCode()); + description.setMessage(PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getMessage()); + description.setCode(PlatformSuccessMessages.RPR_SENT_TO_REPROCESS_SUCCESS.getCode()); + } + } + regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), + LoggerFileConstant.REGISTRATIONID.toString(), registrationId, description.getMessage()); + + String eventId = EventId.RPR_402.toString(); + String eventName = EventName.UPDATE.toString(); + String eventType = EventType.BUSINESS.toString(); + processedList.add(dto); + + if (!isTransactionSuccessful) + auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, + eventType, moduleId, moduleName, registrationId); + }); + + registrationStatusService.updateRegistrationStatusForWorkflowEngines(processedList, moduleId, moduleName); } } catch (TablenotAccessibleException e) { - isBatchSuccessful = false; + isTransactionSuccessful = false; object.setInternalError(Boolean.TRUE); description.setMessage(PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage()); description.setCode(PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getCode()); @@ -401,7 +383,7 @@ public MessageDTO process(MessageDTO object) { PlatformErrorMessages.RPR_RGS_REGISTRATION_TABLE_NOT_ACCESSIBLE.getMessage(), e.toString()); } catch (Exception ex) { - isBatchSuccessful = false; + isTransactionSuccessful = false; description.setMessage(PlatformErrorMessages.REPROCESSOR_VERTICLE_FAILED.getMessage()); description.setCode(PlatformErrorMessages.REPROCESSOR_VERTICLE_FAILED.getCode()); regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), @@ -413,19 +395,19 @@ public MessageDTO process(MessageDTO object) { } finally { regProcLogger.info(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), null, description.getMessage()); - if (isBatchSuccessful) + if (isTransactionSuccessful) description.setMessage(PlatformSuccessMessages.RPR_RE_PROCESS_SUCCESS.getMessage()); - String eventId = isBatchSuccessful ? EventId.RPR_402.toString() : EventId.RPR_405.toString(); - String eventName = isBatchSuccessful ? EventName.UPDATE.toString() : EventName.EXCEPTION.toString(); - String eventType = isBatchSuccessful ? EventType.BUSINESS.toString() : EventType.SYSTEM.toString(); + String eventId = isTransactionSuccessful ? EventId.RPR_402.toString() : EventId.RPR_405.toString(); + String eventName = isTransactionSuccessful ? EventName.UPDATE.toString() : EventName.EXCEPTION.toString(); + String eventType = isTransactionSuccessful ? EventType.BUSINESS.toString() : EventType.SYSTEM.toString(); /** Module-Id can be Both Success/Error code */ - String moduleId = isBatchSuccessful ? PlatformSuccessMessages.RPR_RE_PROCESS_SUCCESS.getCode() + String moduleId = isTransactionSuccessful ? PlatformSuccessMessages.RPR_RE_PROCESS_SUCCESS.getCode() : description.getCode(); String moduleName = ModuleName.RE_PROCESSOR.toString(); auditLogRequestBuilder.createAuditRequestBuilder(description.getMessage(), eventId, eventName, eventType, - moduleId, moduleName, String.join(",", ridSb)); + moduleId, moduleName, ridSb.toString()); } return object; @@ -497,13 +479,13 @@ protected String getPropertyPrefix() { return VERTICLE_PROPERTY_PREFIX; } - private LinkedHashMap prepareRequiredCount(int availableCount) { - LinkedHashMap requiredCountMap = new LinkedHashMap<>(); - int totalMapCount = processBasedFetchCountMapping.size(); + private LinkedHashMap prepareRequiredCount(int availableCount) { + LinkedHashMap requiredCountMap = new LinkedHashMap<>(); + int totalMapCount = processBasedFetchCountMap.size(); int index = 1; int allocated = 0; - for(Map.Entry entry : processBasedFetchCountMapping.entrySet()) { + for(ProcessAllocation processAllocation : processBasedFetchCountMap) { int requiredCount; if(index == totalMapCount) { @@ -511,44 +493,44 @@ private LinkedHashMap prepareRequiredCount(int availableCount) requiredCount = fetchSize - allocated; } else { // Percentage-based allocation - requiredCount = (availableCount * entry.getValue()) /100; + requiredCount = (availableCount * processAllocation.getPercentageAllocation()) /100; } // Ensure at least 1 requiredCount = Math.max(requiredCount, 1); // Prevent overshooting the total - if(allocated + requiredCount > fetchSize) { - requiredCount = fetchSize - allocated; + if(allocated + requiredCount > availableCount) { + requiredCount = availableCount - allocated; } - requiredCountMap.put(entry.getKey(), requiredCount); + requiredCountMap.put(processAllocation, requiredCount); allocated += requiredCount; index++; } return requiredCountMap; } - private void fetchPacketsIfBelowThreshold(LinkedHashMap requiredCountMap, List trnStatusList) { + private void cachePacketsIfBelowThreshold(LinkedHashMap requiredCountMap, List trnStatusList) { List> futures = requiredCountMap.entrySet().stream() .filter(entry -> { - Deque cachedPackets = packetCacheMap.get(entry.getKey()); - return cachedPackets == null || cachedPackets.size() < threasholdForFetch; + String key = getKeyFromProcessAllocation(entry.getKey()); + ConcurrentLinkedQueue cachedPackets = packetCacheMap.get(key); + return cachedPackets == null || cachedPackets.size() < processBasedThreashold; }) .map(entry -> { - String key = entry.getKey(); + ProcessAllocation processAllocation = entry.getKey(); + String key = getKeyFromProcessAllocation(processAllocation); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Fetch Records from Database for Process " + key); - // Parse key into Process & Status - AbstractMap.SimpleEntry, List> entryPair = parseProcessAndStatus(key); - List processList = entryPair.getKey(); - List statusValList = entryPair.getValue(); + List processList = processAllocation.getProcesses(); + List statusValList = processAllocation.getStatuses(); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Status used to fetch Records Process " + key + " is " + statusValList); - Deque cacheList = packetCacheMap.get(key); - int recordFetchCount = recordFetchSize - (cacheList != null ? cacheList.size() : 0); + ConcurrentLinkedQueue cacheList = packetCacheMap.get(key); + int recordFetchCount = processBasedPrefetchLimit - (cacheList != null ? cacheList.size() : 0); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Record Fetch Count for process " + key + " is " + recordFetchCount); @@ -556,25 +538,27 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require List skipRegIdList = new ArrayList<>(cacheList != null && !cacheList.isEmpty() ? cacheList.stream().map(e -> e.getRegistrationId()).collect(Collectors.toList()) : Collections.emptyList()); - if(skipRegIdList.isEmpty()) skipRegIdList.add("-DUMMY-"); - - return reprocessorVerticalService.fetchUnProcessedPackets(processList, recordFetchCount, elapseTime, - reprocessCount, trnStatusList, reprocessExcludeStageNames, skipRegIdList, statusValList) + return registrationStatusService.getUnProcessedPackets(processList, recordFetchCount, elapseTime, + reprocessCount, trnStatusList, reprocessExcludeStageNames, statusValList) .thenAccept(result -> { regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", "Total Record Fetched from database for process " + key + " is " + result.size()); + List filteredList; + if(!skipRegIdList.isEmpty()) + filteredList = result.stream().filter(obj -> !skipRegIdList.contains(obj.getRegistrationId())).collect(Collectors.toList()); + else + filteredList = result; + // Thread-safe update to cache packetCacheMap.compute(key, (k, existingList) -> { - if (existingList == null) return new ConcurrentLinkedDeque<>(result); - existingList.addAll(new ArrayList<>(result)); + if (existingList == null) return new ConcurrentLinkedQueue<>(filteredList); + existingList.addAll(filteredList); return existingList; }); }) .exceptionally(ex -> { - regProcLogger.error(LoggerFileConstant.SESSIONID.toString(), - "Error Fetching UnprocessedPackets -- ", - " Error Triggered for Process [" + String.join(",", processList) + "] and Status [" + String.join(",", statusValList) + "]", ExceptionUtils.getStackTrace(ex)); + regProcLogger.error("Error Triggered for Process [{}] and Status [{}]", processList, statusValList, ex); return null; }); }) @@ -583,65 +567,58 @@ private void fetchPacketsIfBelowThreshold(LinkedHashMap require CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); } - private AbstractMap.SimpleEntry, List> parseProcessAndStatus(String key) { - String[] parts = key.split("#", 2); - List process = Arrays.asList(parts[0].split(",")); - List status = parts.length > 1 ? Arrays.asList(parts[1].split(",")) : Collections.emptyList(); - return new AbstractMap.SimpleEntry<>(process, status); + private String getKeyFromProcessAllocation(ProcessAllocation processAllocation) { + return String.join(",", processAllocation.getProcesses()) + "#" + String.join(",", processAllocation.getStatuses()); } - private List fetchUnprocessedPacketsWithBalance(LinkedHashMap requiredCountMap, int fetchCount) { + private List fetchUnprocessedPacketFromCache(LinkedHashMap requiredCountMap, int fetchCount) { int previousBalanceCount = 0; List reprocessorPacketList = new ArrayList<>(); - for(Map.Entry entry : requiredCountMap.entrySet()) { + for(Map.Entry entry : requiredCountMap.entrySet()) { + String key = getKeyFromProcessAllocation(entry.getKey()); int remainingToFetch = fetchCount - reprocessorPacketList.size(); if(remainingToFetch <= 0) break; int requiredCount = entry.getValue() + previousBalanceCount; - Deque cachedPackets = packetCacheMap.getOrDefault(entry.getKey(), new ConcurrentLinkedDeque<>()); + ConcurrentLinkedQueue cachedPackets = packetCacheMap.getOrDefault(key, new ConcurrentLinkedQueue<>()); if(!cachedPackets.isEmpty()) { int count = Math.min(requiredCount, remainingToFetch); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", - entry.getKey() + " Packets Required Count " + count); + key + " Packets Required Count " + count); List fetchedPackets = fetchFromCache(cachedPackets, count); regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", - entry.getKey() + "Total Packets Fetched from Cache " + fetchedPackets.size()); + key + "Total Packets Fetched from Cache " + fetchedPackets.size()); reprocessorPacketList.addAll(fetchedPackets); previousBalanceCount = Math.max(0, requiredCount-fetchedPackets.size()); } else { previousBalanceCount = requiredCount; } regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", - entry.getKey() + "Count to be moved to next process " + previousBalanceCount); + key + "Count to be moved to next process " + previousBalanceCount); } return reprocessorPacketList; } - private List fetchFromCache(Deque cache, int count) { + private List fetchFromCache(ConcurrentLinkedQueue cache, int count) { int actualFetchCount = Math.min(count, cache.size()); List fetched = new ArrayList<>(actualFetchCount); for (int i = 0; i < actualFetchCount; i++) { - InternalRegistrationStatusDto dto = cache.pollFirst(); + InternalRegistrationStatusDto dto = cache.poll(); if (dto == null) break; fetched.add(dto); } return fetched; } - @PreDestroy - public void shutdown() { - fetchExecutor.shutdown(); - sendExecutor.shutdown(); - try { - sendExecutor.awaitTermination(30, TimeUnit.SECONDS); - fetchExecutor.awaitTermination(30, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } + private String formatPacketCacheMap() { + return packetCacheMap.entrySet() + .stream() + .map(e -> e.getKey() + " = " + e.getValue().size()) + .collect(Collectors.joining(", ", "[", "]")); + } } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties index be96fea26be..1403ff403b7 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties @@ -17,5 +17,4 @@ registration.processor.identityjson=RegistrationProcessorIdentity.json registration.processor.demographic.identity=identity packet.info.storage.service=registration-processor-packet-info-storage-service config.server.file.storage.uri=${spring.cloud.config.uri}/${packet.info.storage.service}/${spring.profiles.active}/${spring.cloud.config.label}/ -mosip.regproc.registration.status.service.disable-audit=true -registration.processor.reprocess.process-based.fetch.count={"MIGRATOR":40, "RENEWAL#ON_HOLD_RENEWAL":30, "NEW,RENEWAL":30} \ No newline at end of file +mosip.regproc.registration.status.service.disable-audit=true \ No newline at end of file diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java index 9baf9b113fe..6a352465d1d 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/test/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticleTest.java @@ -10,8 +10,7 @@ import java.util.*; import java.util.concurrent.CompletableFuture; -import io.mosip.registration.processor.reprocessor.service.ReprocessorVerticalService; -import io.mosip.registration.processor.reprocessor.service.impl.ReprocessorVerticalServiceImpl; +import io.mosip.registration.processor.reprocessor.dto.ProcessAllocation; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -106,9 +105,6 @@ public void send(MosipEventBus mosipEventBus, MessageBusAddress toAddress, Messa @Mock RegistrationStatusService registrationStatusService; - @Mock - private ReprocessorVerticalService reprocessorVerticalService; - @Mock private AuditLogRequestBuilder auditLogRequestBuilder; @@ -122,17 +118,20 @@ public void setup() throws Exception { //Mockito.doNothing().when(description).setMessage(Mockito.anyString()); //Mockito.when(description.getCode()).thenReturn("CODE"); //Mockito.when(description.getMessage()).thenReturn("MESSAGE"); - ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", false); - LinkedHashMap processBasedFetchCountMapping = new LinkedHashMap<>(); -// processBasedFetchCountMapping.put("MIGRATOR", 40); -// processBasedFetchCountMapping.put("RENEWAL#ON_HOLD_RENEWAL", 30); - processBasedFetchCountMapping.put("NEW,RENEWAL", 100); - ReflectionTestUtils.setField(reprocessorVerticle, "processBasedFetchCountMapping", processBasedFetchCountMapping); + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedCache", false); + List list = new ArrayList<>(); + ProcessAllocation processAllocation1 = new ProcessAllocation(); + processAllocation1.setPercentageAllocation(40); + List processList = new ArrayList<>(); + processList.add("NEW"); + processAllocation1.setProcesses(processList); + list.add(processAllocation1); + ReflectionTestUtils.setField(reprocessorVerticle, "processBasedFetchCountMap", list); ReflectionTestUtils.setField(reprocessorVerticle, "fetchSize", 4); ReflectionTestUtils.setField(reprocessorVerticle, "elapseTime", 21600); ReflectionTestUtils.setField(reprocessorVerticle, "reprocessCount", 3); - ReflectionTestUtils.setField(reprocessorVerticle, "recordFetchSize", 500); - ReflectionTestUtils.setField(reprocessorVerticle, "threasholdForFetch", 50); + ReflectionTestUtils.setField(reprocessorVerticle, "processBasedPrefetchLimit", 500); + ReflectionTestUtils.setField(reprocessorVerticle, "processBasedThreashold", 50); ReflectionTestUtils.setField(reprocessorVerticle, "reprocessExcludeStageNames", new ArrayList<>()); List reprocessRestartTriggerFilterList = new ArrayList<>(); reprocessRestartTriggerFilterList.add("DemodedupStage:Success"); @@ -192,7 +191,7 @@ public void testProcessValid() throws TablenotAccessibleException, PacketManager @Test public void testProcessValidNew() throws TablenotAccessibleException, PacketManagerException, ApisResourceAccessException, WorkflowActionException { - ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedCache", true); List dtolist = new ArrayList<>(); InternalRegistrationStatusDto registrationStatusDto = new InternalRegistrationStatusDto(); @@ -212,9 +211,6 @@ public void testProcessValidNew() throws TablenotAccessibleException, PacketMana registrationStatusDto1.setRegistrationType("NEW"); registrationStatusDto1.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.SUCCESS.toString()); dtolist.add(registrationStatusDto1); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) - .thenReturn(CompletableFuture.completedFuture(dtolist)); - reprocessorVerticle.process(dto); } @@ -251,7 +247,7 @@ public void testProcessFailure() throws TablenotAccessibleException, PacketManag @Test public void testProcessFailureNew() throws TablenotAccessibleException, PacketManagerException, ApisResourceAccessException, WorkflowActionException { - ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedCache", true); List dtolist = new ArrayList<>(); InternalRegistrationStatusDto registrationStatusDto = new InternalRegistrationStatusDto(); @@ -272,8 +268,6 @@ public void testProcessFailureNew() throws TablenotAccessibleException, PacketMa registrationStatusDto1.setRegistrationType("NEW"); registrationStatusDto1.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.SUCCESS.toString()); dtolist.add(registrationStatusDto1); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) - .thenReturn(CompletableFuture.completedFuture(dtolist)); reprocessorVerticle.process(dto); } @@ -295,9 +289,7 @@ public void exceptionTest() throws Exception { @Test public void exceptionTestNew() throws Exception { - ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) - .thenReturn(CompletableFuture.completedFuture(Collections.emptyList())); + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedCache", true); dto = reprocessorVerticle.process(dto); assertEquals(null, dto.getIsValid()); @@ -323,10 +315,7 @@ public void TablenotAccessibleExceptionTest() throws Exception { @Test public void TablenotAccessibleExceptionTestNew() throws Exception { - ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) - .thenThrow(new TablenotAccessibleException("") { - }); + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedCache", true); dto = reprocessorVerticle.process(dto); assertEquals(true, dto.getInternalError()); @@ -367,7 +356,7 @@ public void testProcessValidWithResumablePackets() throws TablenotAccessibleExce @Test public void testProcessValidWithResumablePacketsNew() throws TablenotAccessibleException, PacketManagerException, ApisResourceAccessException, WorkflowActionException { - ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedCache", true); List dtolist = new ArrayList<>(); InternalRegistrationStatusDto registrationStatusDto = new InternalRegistrationStatusDto(); @@ -390,8 +379,6 @@ public void testProcessValidWithResumablePacketsNew() throws TablenotAccessibleE reprocessorDtoList.add(registrationStatusDto1); Mockito.when(registrationStatusService.getResumablePackets(anyInt())) .thenReturn(dtolist); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) - .thenReturn(CompletableFuture.completedFuture(dtolist)); reprocessorVerticle.process(dto); } @@ -422,7 +409,7 @@ public void testProcessWithRestartFromStage() throws TablenotAccessibleException public void testProcessWithRestartFromStageNew() throws TablenotAccessibleException, PacketManagerException, ApisResourceAccessException, WorkflowActionException { - ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedFetch", true); + ReflectionTestUtils.setField(reprocessorVerticle, "enabledProcessBasedCache", true); List dtolist = new ArrayList<>(); InternalRegistrationStatusDto registrationStatusDto = new InternalRegistrationStatusDto(); @@ -434,8 +421,6 @@ public void testProcessWithRestartFromStageNew() throws TablenotAccessibleExcept registrationStatusDto.setStatusCode(RegistrationStatusCode.PROCESSING.toString()); registrationStatusDto.setLatestTransactionStatusCode(RegistrationTransactionStatusCode.REPROCESS.toString()); dtolist.add(registrationStatusDto); - Mockito.when(reprocessorVerticalService.fetchUnProcessedPackets(anyList(), anyInt(), anyLong(), anyInt(), anyList(), anyList(), anyList(), anyList())) - .thenReturn(CompletableFuture.completedFuture(dtolist)); reprocessorVerticle.process(dto); } From 4b9117263e38ea3a8fe5d507b22b5f7d9d5d89e5 Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Sat, 18 Oct 2025 14:30:54 +0530 Subject: [PATCH 11/14] Reprocessor Changes based on internal Review Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../reprocessor/verticle/ReprocessorVerticle.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java index b1001944188..6b9845adc76 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/java/io/mosip/registration/processor/reprocessor/verticle/ReprocessorVerticle.java @@ -530,15 +530,11 @@ private void cachePacketsIfBelowThreshold(LinkedHashMap cacheList = packetCacheMap.get(key); - int recordFetchCount = processBasedPrefetchLimit - (cacheList != null ? cacheList.size() : 0); - regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", - "Record Fetch Count for process " + key + " is " + recordFetchCount); // Fetch unprocessed packets - List skipRegIdList = new ArrayList<>(cacheList != null && !cacheList.isEmpty() ? cacheList.stream().map(e -> e.getRegistrationId()).collect(Collectors.toList()) : Collections.emptyList()); - return registrationStatusService.getUnProcessedPackets(processList, recordFetchCount, elapseTime, + return registrationStatusService.getUnProcessedPackets(processList, processBasedPrefetchLimit, elapseTime, reprocessCount, trnStatusList, reprocessExcludeStageNames, statusValList) .thenAccept(result -> { regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", @@ -573,6 +569,7 @@ private String getKeyFromProcessAllocation(ProcessAllocation processAllocation) private List fetchUnprocessedPacketFromCache(LinkedHashMap requiredCountMap, int fetchCount) { int previousBalanceCount = 0; + int emptyCacheCount = 0; List reprocessorPacketList = new ArrayList<>(); for(Map.Entry entry : requiredCountMap.entrySet()) { @@ -594,12 +591,16 @@ private List fetchUnprocessedPacketFromCache(Link reprocessorPacketList.addAll(fetchedPackets); previousBalanceCount = Math.max(0, requiredCount-fetchedPackets.size()); } else { + emptyCacheCount++; previousBalanceCount = requiredCount; } regProcLogger.debug(LoggerFileConstant.SESSIONID.toString(), LoggerFileConstant.REGISTRATIONID.toString(), "", key + "Count to be moved to next process " + previousBalanceCount); } + if((reprocessorPacketList.size() < fetchCount) && (requiredCountMap.size() != emptyCacheCount)) + reprocessorPacketList.addAll(fetchUnprocessedPacketFromCache(requiredCountMap, fetchCount-reprocessorPacketList.size())); + return reprocessorPacketList; } From 2d8aba2683687001154ddc5775b1e5432170a12a Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Sat, 18 Oct 2025 17:06:52 +0530 Subject: [PATCH 12/14] Reprocessor Changes based on internal Review Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../impl/RegistrationStatusServiceImpl.java | 11 ++++++----- .../config/ReprocessorConfigBeans.java | 15 +++++++++++++++ .../src/main/resources/bootstrap.properties | 6 +++++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java index fa734ed56c9..45308d3c9f4 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java @@ -1060,11 +1060,6 @@ public void updateRegistrationStatusForWorkflowEngines(List getRegistrationProcessorRe public PacketManagerService getPacketManagerService() { return new PacketManagerService(); } + + // 🔹 Add this bean + @Bean(name = "taskExecutor") + public Executor taskExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(10); + executor.setMaxPoolSize(20); + executor.setQueueCapacity(100); + executor.setThreadNamePrefix("Reprocessor-Async-"); + executor.initialize(); + return executor; + } } diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties index 1403ff403b7..816b3a255ca 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties @@ -17,4 +17,8 @@ registration.processor.identityjson=RegistrationProcessorIdentity.json registration.processor.demographic.identity=identity packet.info.storage.service=registration-processor-packet-info-storage-service config.server.file.storage.uri=${spring.cloud.config.uri}/${packet.info.storage.service}/${spring.profiles.active}/${spring.cloud.config.label}/ -mosip.regproc.registration.status.service.disable-audit=true \ No newline at end of file +mosip.regproc.registration.status.service.disable-audit=true +spring.jpa.properties.hibernate.jdbc.batch_size=50 +spring.jpa.properties.hibernate.order_updates=true +spring.jpa.properties.hibernate.order_inserts=true +spring.jpa.properties.hibernate.batch_versioned_data=true From 09b44922fdd1f4beca7363bab03616dc973d5b49 Mon Sep 17 00:00:00 2001 From: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> Date: Wed, 22 Oct 2025 13:36:42 +0530 Subject: [PATCH 13/14] Reprocessor Changes based on internal Review Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../impl/RegistrationStatusServiceImpl.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java index 45308d3c9f4..4e9bcc3eb97 100644 --- a/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java +++ b/registration-processor/registration-processor-registration-status-service-impl/src/main/java/io/mosip/registration/processor/status/service/impl/RegistrationStatusServiceImpl.java @@ -1021,10 +1021,10 @@ public void updateRegistrationStatusForWorkflowEngines(List Date: Thu, 23 Oct 2025 09:25:36 +0530 Subject: [PATCH 14/14] Reprocessor Changes based on internal Review Signed-off-by: trialblazerseee <84778104+trialblazerseee@users.noreply.github.com> --- .../src/main/resources/bootstrap.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties index 816b3a255ca..38471d6e54b 100644 --- a/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties +++ b/registration-processor/workflow-engine/registration-processor-reprocessor/src/main/resources/bootstrap.properties @@ -22,3 +22,5 @@ spring.jpa.properties.hibernate.jdbc.batch_size=50 spring.jpa.properties.hibernate.order_updates=true spring.jpa.properties.hibernate.order_inserts=true spring.jpa.properties.hibernate.batch_versioned_data=true +logging.level.org.hibernate.SQL=DEBUG +logging.level.org.hibernate.type.descriptor.sql=TRACE \ No newline at end of file