From a88bc732bf242bade5c3e785c5d9bc36ada5154a Mon Sep 17 00:00:00 2001 From: benedwards Date: Mon, 16 Feb 2026 22:47:30 +0000 Subject: [PATCH 1/7] First draft --- .../arm/service/CleanUpDetsDataProcessor.java | 7 + .../impl/CleanUpDetsDataProcessorImpl.java | 127 ++++++++++++++++++ .../ExternalObjectDirectoryRepository.java | 24 ++++ .../ObjectStateRecordRepository.java | 7 + .../dets/service/impl/DetsApiServiceImpl.java | 2 +- .../darts/task/api/AutomatedTaskName.java | 3 +- .../CleanUpDetsDataAutomatedTaskConfig.java | 18 +++ .../impl/CleanUpDetsDataAutomatedTask.java | 42 ++++++ src/main/resources/application.yaml | 9 ++ ...V1_501__CleanUpDetsData_automated_task.sql | 9 ++ 10 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java create mode 100644 src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java create mode 100644 src/main/java/uk/gov/hmcts/darts/task/config/CleanUpDetsDataAutomatedTaskConfig.java create mode 100644 src/main/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTask.java create mode 100644 src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java b/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java new file mode 100644 index 00000000000..bddf5b0e4ee --- /dev/null +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java @@ -0,0 +1,7 @@ +package uk.gov.hmcts.darts.arm.service; + +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; + +public interface CleanUpDetsDataProcessor { + void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskConfig minimumStoredAge); +} diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java new file mode 100644 index 00000000000..29f494911a3 --- /dev/null +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java @@ -0,0 +1,127 @@ +package uk.gov.hmcts.darts.arm.service.impl; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.springframework.data.domain.Limit; +import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import uk.gov.hmcts.darts.arm.service.CleanUpDetsDataProcessor; +import uk.gov.hmcts.darts.common.exception.AzureDeleteBlobException; +import uk.gov.hmcts.darts.common.repository.ExternalObjectDirectoryRepository; +import uk.gov.hmcts.darts.common.repository.ObjectStateRecordRepository; +import uk.gov.hmcts.darts.common.util.EodHelper; +import uk.gov.hmcts.darts.dets.service.DetsApiService; +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +import uk.gov.hmcts.darts.util.AsyncUtil; + +import java.time.Clock; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +@Slf4j +@Service +@RequiredArgsConstructor +public class CleanUpDetsDataProcessorImpl implements CleanUpDetsDataProcessor { + + private final ExternalObjectDirectoryRepository externalObjectDirectoryRepository; + private final CleanUpDetsDataBatchProcessor cleanUpDetsDataBatchProcessor; + private final Clock clock; + + @Override + public void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskConfig config) { + log.info("Processing clean up of DETS data with batch size: {}", batchSize); + + List eodIdsToCleanUp = externalObjectDirectoryRepository.findEodsWithMatchingRecordInArm( + EodHelper.storedStatus(), + EodHelper.detsLocation(), + EodHelper.armLocation(), + OffsetDateTime.now(clock), + Limit.of(batchSize) + ); + + log.info("Found {} EOD records to clean up", eodIdsToCleanUp.size()); + if (CollectionUtils.isNotEmpty(eodIdsToCleanUp)) { + //Chunk the EOD IDs into batches and process each batch in parallel + List> batchesForArm = ListUtils.partition(eodIdsToCleanUp, config.getChunkSize()); + + + List> tasks = batchesForArm + .stream() + .map(eodsForBatch -> (Callable) () -> { + //Call another serive to ensure the batch is processed within a transaction + cleanUpDetsDataBatchProcessor.process(eodsForBatch); + return null; + }) + .toList(); + + try { + AsyncUtil.invokeAllAwaitTermination(tasks, config); + } catch (InterruptedException e) { + log.error("Clean up dets data batch processing interrupted", e); + Thread.currentThread().interrupt(); + } catch (Exception e) { + log.error("Clean up dets data unexpected exception", e); + } + } else { + log.info("No DETS EODs require clean up"); + } + } + + @Component + @RequiredArgsConstructor + public class CleanUpDetsDataBatchProcessor { + + private final ExternalObjectDirectoryRepository externalObjectDirectoryRepository; + private final ObjectStateRecordRepository objectStateRecordRepository; + private final DetsApiService detsApiService; + + @Transactional + public void process(List eodsForBatch) { + log.info("Cleaning up DETS data for batch of EOD IDs: {}", eodsForBatch); + List responses = externalObjectDirectoryRepository.cleanUpDetsDataProcedure(eodsForBatch); + log.info("Clean up DETS data procedure completed for batch. Responses: {}", responses); + + List objectStateRecordsForDetsRecordsCleanedUpSuccessfully = new ArrayList<>(); + List objectStateRecordsForDetsRecordsFailedToCleanUp = new ArrayList<>(); + for (CleanUpDetsProcedureResponse response : responses) { + try { + log.debug("Processing clean up response for EOD ID: {}, Location: {}", response.getId(), response.getLocation()); + if (detsApiService.deleteBlobDataFromContainer(response.location)) { + log.debug("Successfully deleted DETS blob for EOD ID: {}, Location: {}", response.getId(), response.getLocation()); + objectStateRecordsForDetsRecordsCleanedUpSuccessfully.add(response.getId()); + continue; + } else { + log.error("Failed to delete DETS blob for EOD ID: {}, Location: {}. Blob may not exist or deletion failed.", + response.getId(), response.getLocation()); + } + } catch (AzureDeleteBlobException azureDeleteBlobException) { + log.error("AzureDeleteBlobException while deleting DETS blob for EOD Location: {}, object state record id: {}.", + response.getLocation(), response.getId(), azureDeleteBlobException); + } + objectStateRecordsForDetsRecordsFailedToCleanUp.add(response.getId()); + } + objectStateRecordRepository.markDetsCleanupStatusAsComplete(objectStateRecordsForDetsRecordsCleanedUpSuccessfully); + log.info("Marked object state records as clean up complete for EOD IDs: {}", objectStateRecordsForDetsRecordsCleanedUpSuccessfully); + + if (CollectionUtils.isNotEmpty(objectStateRecordsForDetsRecordsFailedToCleanUp)) { + log.info("Dets clean up failed for Object state record Ids: {}", objectStateRecordsForDetsRecordsFailedToCleanUp); + } + } + } + + @AllArgsConstructor + @Getter + @Setter + public static class CleanUpDetsProcedureResponse { + private Long id; + private String location; + } +} diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java index 924624f2732..7bc69f4b3e1 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java +++ b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java @@ -7,8 +7,10 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.jpa.repository.query.Procedure; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import uk.gov.hmcts.darts.arm.service.impl.CleanUpDetsDataProcessorImpl; import uk.gov.hmcts.darts.common.entity.AnnotationDocumentEntity; import uk.gov.hmcts.darts.common.entity.CaseDocumentEntity; import uk.gov.hmcts.darts.common.entity.ExternalLocationTypeEntity; @@ -16,11 +18,13 @@ import uk.gov.hmcts.darts.common.entity.MediaEntity; import uk.gov.hmcts.darts.common.entity.ObjectRecordStatusEntity; import uk.gov.hmcts.darts.common.entity.TranscriptionDocumentEntity; +import uk.gov.hmcts.darts.task.runner.impl.CleanUpDetsDataAutomatedTask; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -278,6 +282,26 @@ List findIdsForAudioToBeDeletedFromUnstructured(ObjectRecordStatusEntity s OffsetDateTime unstructuredLastModifiedBefore, Limit limit); + @Query( + """ + SELECT eod.id FROM ExternalObjectDirectoryEntity parentEod, ExternalObjectDirectoryEntity armEod + WHERE parentEod.media = armEod.media + AND parentEod.status = :storedStatus + AND parentEod.externalLocationType = :parentEodLocation + AND parentEod.lastModifiedDateTime <= :lastModifiedBefore + AND armEod.externalLocationType = :armLocation + AND armEod.status = :storedStatus + """ + ) + List findEodsWithMatchingRecordInArm( + ObjectRecordStatusEntity storedStatus, + ExternalLocationTypeEntity parentEodLocation, + ExternalLocationTypeEntity armLocation, + OffsetDateTime lastModifiedBefore, + Limit limit); + + @Query(value = "SELECT * FROM dmp4312_del_eod_upd_osr_conf_loc(:ids)", nativeQuery = true) + List cleanUpDetsDataProcedure(@Param("ids") List ids); @Query( """ diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java index bfeb69eb228..a3535193fd8 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java +++ b/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java @@ -1,10 +1,17 @@ package uk.gov.hmcts.darts.common.repository; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; import uk.gov.hmcts.darts.common.entity.ObjectStateRecordEntity; +import java.util.List; import java.util.Optional; public interface ObjectStateRecordRepository extends JpaRepository { Optional findByArmEodId(long armEodId); + + @Modifying + @Query("UPDATE ObjectStateRecordEntity o SET o.flagFileDetsCleanupStatus = true where o.uuid in :uuids") + void markDetsCleanupStatusAsComplete(List uuids); } diff --git a/src/main/java/uk/gov/hmcts/darts/dets/service/impl/DetsApiServiceImpl.java b/src/main/java/uk/gov/hmcts/darts/dets/service/impl/DetsApiServiceImpl.java index 5474c711acb..2e441209707 100644 --- a/src/main/java/uk/gov/hmcts/darts/dets/service/impl/DetsApiServiceImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/dets/service/impl/DetsApiServiceImpl.java @@ -86,6 +86,7 @@ public String saveBlobData(BinaryData binaryData, String fileName) { return client.getBlobName(); } + @Override public boolean deleteBlobDataFromContainer(String blobId) throws AzureDeleteBlobException { try { @@ -107,7 +108,6 @@ public boolean deleteBlobDataFromContainer(String blobId) throws AzureDeleteBlob configuration.getContainerName(), blobId, httpStatus); throw new AzureDeleteBlobException(message); } - } catch (RuntimeException e) { throw new AzureDeleteBlobException( "Could not delete from storage container=" + configuration.getContainerName() + ", blobId=" + blobId, e diff --git a/src/main/java/uk/gov/hmcts/darts/task/api/AutomatedTaskName.java b/src/main/java/uk/gov/hmcts/darts/task/api/AutomatedTaskName.java index d6bc81ff9d1..cd59b92df68 100644 --- a/src/main/java/uk/gov/hmcts/darts/task/api/AutomatedTaskName.java +++ b/src/main/java/uk/gov/hmcts/darts/task/api/AutomatedTaskName.java @@ -42,7 +42,8 @@ public enum AutomatedTaskName { ARM_RPO_REPLAY_TASK_NAME("ArmRpoReplay", Constants.AUTOMATED_TASK_PROCESS_E2E_ARM_RPO_PENDING_PROCESS_E2E_ARM_RPO_FALSE), ARM_MISSING_RESPONSE_REPLY_TASK_NAME("ArmMissingResponseReplay"), DETS_CLEANUP_ARM_RESPONSE_FILES("DETSCleanupArmResponseFiles"), - MEDIA_REQUEST_CLEANUP("MediaRequestCleanUp"); + MEDIA_REQUEST_CLEANUP("MediaRequestCleanUp"), + CLEAN_UP_DETS_DATA("CleanUpDetsData"); private final String taskName; private final String conditionalOnSpEL; diff --git a/src/main/java/uk/gov/hmcts/darts/task/config/CleanUpDetsDataAutomatedTaskConfig.java b/src/main/java/uk/gov/hmcts/darts/task/config/CleanUpDetsDataAutomatedTaskConfig.java new file mode 100644 index 00000000000..7eef7f638e6 --- /dev/null +++ b/src/main/java/uk/gov/hmcts/darts/task/config/CleanUpDetsDataAutomatedTaskConfig.java @@ -0,0 +1,18 @@ +package uk.gov.hmcts.darts.task.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import java.time.Duration; + +@ConfigurationProperties("darts.automated.task.clean-up-dets-data") +@Getter +@Setter +@Configuration +public class CleanUpDetsDataAutomatedTaskConfig extends AbstractAsyncAutomatedTaskConfig { + + private Duration minimumStoredAge; + private int chunkSize; +} diff --git a/src/main/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTask.java b/src/main/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTask.java new file mode 100644 index 00000000000..44984b240cb --- /dev/null +++ b/src/main/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTask.java @@ -0,0 +1,42 @@ +package uk.gov.hmcts.darts.task.runner.impl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import uk.gov.hmcts.darts.arm.service.CleanUpDetsDataProcessor; +import uk.gov.hmcts.darts.common.repository.AutomatedTaskRepository; +import uk.gov.hmcts.darts.log.api.LogApi; +import uk.gov.hmcts.darts.task.api.AutomatedTaskName; +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +import uk.gov.hmcts.darts.task.runner.AutoloadingManualTask; +import uk.gov.hmcts.darts.task.service.LockService; + +import static uk.gov.hmcts.darts.task.api.AutomatedTaskName.CLEAN_UP_DETS_DATA; + +@Slf4j +@Component +public class CleanUpDetsDataAutomatedTask + extends AbstractLockableAutomatedTask + implements AutoloadingManualTask { + + private final CleanUpDetsDataProcessor cleanUpDetsDataProcessor; + + @Autowired + public CleanUpDetsDataAutomatedTask(AutomatedTaskRepository automatedTaskRepository, + CleanUpDetsDataAutomatedTaskConfig automatedTaskConfigurationProperties, + CleanUpDetsDataProcessor processor, + LogApi logApi, LockService lockService) { + super(automatedTaskRepository, automatedTaskConfigurationProperties, logApi, lockService); + this.cleanUpDetsDataProcessor = processor; + } + + @Override + public AutomatedTaskName getAutomatedTaskName() { + return CLEAN_UP_DETS_DATA; + } + + @Override + protected void runTask() { + cleanUpDetsDataProcessor.processCleanUpDetsData(getAutomatedTaskBatchSize(), getConfig()); + } +} \ No newline at end of file diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 1c98b3601ff..4b3c140bbb7 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -440,6 +440,15 @@ darts: lock: at-least-for: PT1M at-most-for: PT90M + clean-up-dets-data: + system-user-email: systemCleanUpDetsData@hmcts.net + lock: + at-least-for: PT1M + at-most-for: PT90M + minimum-stored-age: 7D + chunk-size: 1000 + async-timeout: 80M + threads: 20 cleanup-arm-response-files: system-user-email: system_CleanupArmResponseFiles@hmcts.net lock: diff --git a/src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql b/src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql new file mode 100644 index 00000000000..81eb8400e4a --- /dev/null +++ b/src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql @@ -0,0 +1,9 @@ +INSERT INTO darts.automated_task (aut_id, task_name, task_description, cron_expression, cron_editable, batch_size, + created_ts, created_by, last_modified_ts, last_modified_by, task_enabled) +VALUES (35, 'CleanUpDetsData', 'Cleans up Dets files that have successfully been stored in ARM', '0 24 0-6,19-23 ? * *', true, 100_000, + current_timestamp, 0, current_timestamp, 0, false); + +INSERT INTO user_account +VALUES (-40, NULL, '', 'systemCleanUpDetsData@hmcts.net', + 'systemCleanUpDetsDataAutomatedTask', '2025-02-16 00:00:00+00', '2025-02-16 00:00:00+00', NULL, 0, 0, NULL, true, + true, d 'systemCleanUpDetsDataAutomatedTask'); \ No newline at end of file From cad11b7ffc4ee6978b6ee63d1330f8b513643d6c Mon Sep 17 00:00:00 2001 From: benedwards Date: Wed, 25 Feb 2026 14:15:49 +0000 Subject: [PATCH 2/7] Updated dets clean up job logic --- .../tests/DETS_Cleanup_SP_DB_Unit_tests.sql | 587 ++++++++++++++++++ .../impl/CleanUpDetsDataProcessorImpl.java | 64 +- .../ExternalObjectDirectoryRepository.java | 22 +- src/main/resources/application.yaml | 4 +- ...V1_501__CleanUpDetsData_automated_task.sql | 155 ++++- 5 files changed, 779 insertions(+), 53 deletions(-) create mode 100644 src/integrationTest/resources/db/tests/DETS_Cleanup_SP_DB_Unit_tests.sql diff --git a/src/integrationTest/resources/db/tests/DETS_Cleanup_SP_DB_Unit_tests.sql b/src/integrationTest/resources/db/tests/DETS_Cleanup_SP_DB_Unit_tests.sql new file mode 100644 index 00000000000..2682a220d81 --- /dev/null +++ b/src/integrationTest/resources/db/tests/DETS_Cleanup_SP_DB_Unit_tests.sql @@ -0,0 +1,587 @@ +--psql -h -p 5432 -U -d darts -f DETS_Cleanup_SP_DB_Unit_tests.sql 1> DETS_Cleanup_SP_DB_Unit_tests.log 2>>&1 + +---------------------------------------------------------------------- +-- ONE-TIME SETUP +---------------------------------------------------------------------- + +CREATE SCHEMA IF NOT EXISTS test_dets_cleanup; + +CREATE TABLE IF NOT EXISTS test_dets_cleanup.external_object_directory +(LIKE darts.external_object_directory INCLUDING ALL); + +CREATE TABLE IF NOT EXISTS test_dets_cleanup.object_state_record +(LIKE darts.object_state_record INCLUDING ALL); + +---------------------------------------------------------------------- +-- TEST HARNESS +---------------------------------------------------------------------- + +-- Route unqualified table names to test_dets_cleanup tables first +SET search_path TO test_dets_cleanup, darts; + +-- Following values taken from Prod +SET max_parallel_workers_per_gather=4; +SET work_mem='4096kB'; +SET max_parallel_maintenance_workers=64; + +SELECT name, setting, unit, SOURCE, reset_val --, boot_val +FROM pg_settings +WHERE name IN ('search_path', 'max_parallel_workers_per_gather', 'work_mem', 'max_parallel_maintenance_workers') +ORDER BY name; + + +-- Helper to clear test tables between tests +CREATE OR REPLACE PROCEDURE test_dets_cleanup.reset_test_data() LANGUAGE plpgsql AS $$ +BEGIN + TRUNCATE TABLE test_dets_cleanup.object_state_record; + TRUNCATE TABLE test_dets_cleanup.external_object_directory; + + RAISE NOTICE '-------------- RESET ---------------'; +END; +$$; + +---------------------------------------------------------------------- +-- TEST 1: No matching records +-- Expect: 0 results, empty array returned +---------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- Noise ARM EODs (wrong ors_id), Includes valid records but outside of date range +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3001, 8980069, 2, 3, NULL, now(), now() - interval '1 day', -99, -99, FALSE, FALSE, FALSE), + (3002, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE), + (3003, 8980069, 1, 3, NULL, now(), now() - interval '7 days', -99, -99, FALSE, FALSE, FALSE), + (3004, 8980069, 2, 3, NULL, now(), now() - interval '5 days', -99, -99, FALSE, FALSE, FALSE), + (3005, 8980069, 1, 3, NULL, now(), now() - interval '20 days', -99, -99, FALSE, FALSE, FALSE); + +-- Noise DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4001, 8980069, 2, 4, 'LOC-N1', now(), now() - interval '1 day', -99, -99, FALSE, FALSE, TRUE), + (4002, 8980069, 2, 4, 'LOC-N2', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4003, 8980069, 2, 4, 'LOC-N3', now(), now() - interval '7 days', -99, -99, FALSE, FALSE, TRUE), + (4004, 8980069, 2, 4, 'LOC-N4', now(), now() - interval '5 days', -99, -99, FALSE, FALSE, TRUE), + (4005, 8980069, 2, 4, 'LOC-N5', now(), now() - interval '20 days', -99, -99, FALSE, FALSE, TRUE); + +-- Noise OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (10001, 4001, 3001, 'LOC-N1'), + (10002, 4002, 3002, 'LOC-N2'), + (10003, 4003, 3003, 'LOC-N3'), + (10004, 4004, 3004, 'LOC-N4'), + (10005, 4005, 3005, 'LOC-N5'); + +DO $$ + DECLARE + results darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 1'; + RAISE NOTICE '------------------------------------'; + + CALL darts.dets_cleanup_eod_osr(10, now() - interval '7 days', results); + + --ASSERT (results IS NULL OR cardinality(results) = 0), 'TEST 1 FAILED: expected empty results but got non-empty array'; + + -- Check results + IF results IS NULL OR cardinality(results) > 0 THEN + RAISE NOTICE 'TEST 1 FAILED: expected 0 results but got %', cardinality(results); + RETURN; + END IF; + + RAISE NOTICE 'TEST 1 PASSED'; + END $$; + +---------------------------------------------------------------------- +-- TEST 2: Normal case (5 valid pairs + noise) +---------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- 5 valid ARM EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3101, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3102, 8980069, 2, 3, NULL, now(), now() - interval '7 days', -99, -99, FALSE, FALSE, FALSE), + (3103, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3104, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3105, 8980069, 2, 3, NULL, now(), now() - interval '20 days', -99, -99, FALSE, FALSE, FALSE); + +-- 5 valid DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4101, 8980069, 2, 4, 'LOC-A1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4102, 8980069, 2, 4, 'LOC-A2', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4103, 8980069, 2, 4, 'LOC-A3', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4104, 8980069, 2, 4, 'LOC-A4', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4105, 8980069, 2, 4, 'LOC-A5', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- 5 valid OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (11001, 4101, 3101, 'LOC-A1'), + (11002, 4102, 3102, 'LOC-A2'), + (11003, 4103, 3103, 'LOC-A3'), + (11004, 4104, 3104, 'LOC-A4'), + (11005, 4105, 3105, 'LOC-A5'); + +-- Noise ARM (older than 7 days but invalid ors_id + within 7 days and valid ors_id + valid records apart from difference location) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3191, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE), + (3192, 8980069, 2, 3, NULL, now(), now() - interval '1 days', -99, -99, FALSE, FALSE, FALSE), + (3193, 8980069, 2, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE); + +-- Noise DETS +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4191, 8980069, 2, 4, 'LOC-Z1', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4192, 8980069, 2, 4, 'LOC-Z2', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4193, 8980069, 2, 4, 'LOC-ZZ Different', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE); + +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES (11901, 4191, 3191, 'LOC-Z1'), + (11902, 4192, 3192, 'LOC-Z2'), + (11903, 4193, 3193, 'LOC-ZZ'); + +DO $$ + DECLARE + results darts.id_location_pair[]; + expected darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 2'; + RAISE NOTICE '------------------------------------'; + + expected := ARRAY[ + ROW(11001, 'LOC-A1')::darts.id_location_pair, + ROW(11002, 'LOC-A2')::darts.id_location_pair, + ROW(11003, 'LOC-A3')::darts.id_location_pair, + ROW(11004, 'LOC-A4')::darts.id_location_pair, + ROW(11005, 'LOC-A5')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(10, now() - interval '7 days', results); + + --ASSERT cardinality(results) = 5, 'TEST 2 FAILED: expected 5 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 2 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 5 THEN + RAISE NOTICE 'TEST 2 FAILED: expected 5 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 2 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 2 PASSED'; + + END $$; + +---------------------------------------------------------------------------------------- +-- TEST 3: Limit behavior (limit = 3), more available +-- Test includes a check where the last_modified_ts is exactly the same as the parameter +---------------------------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- 5 valid ARM EODs (one exact timestamp match) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3201, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3202, 8980069, 2, 3, NULL, now(), (now()::date - 7)::timestamp, -99, -99, FALSE, FALSE, FALSE), -- exact match + (3203, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3204, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3205, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE); + +-- 5 valid DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4201, 8980069, 2, 4, 'LOC-B1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4202, 8980069, 2, 4, 'LOC-B2 - Exact TS', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4203, 8980069, 2, 4, 'LOC-B3', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4204, 8980069, 2, 4, 'LOC-B4', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4205, 8980069, 2, 4, 'LOC-B5', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- 5 valid OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (12001, 4201, 3201, 'LOC-B1'), + (12002, 4202, 3202, 'LOC-B2 - Exact TS'), + (12003, 4203, 3203, 'LOC-B3'), + (12004, 4204, 3204, 'LOC-B4'), + (12005, 4205, 3205, 'LOC-B5'); + +-- Noise ARM (older than 7 days but invalid ors_id) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3291, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE); + +-- Noise DETS +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4291, 8980069, 1, 4, 'LOC-Z2', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE); + +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES (12901, 4291, 3291, 'LOC-Z2'); + +DO $$ + DECLARE + results darts.id_location_pair[]; + expected darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 3'; + RAISE NOTICE '------------------------------------'; + + expected := ARRAY[ + ROW(12001, 'LOC-B1')::darts.id_location_pair, + ROW(12002, 'LOC-B2 - Exact TS')::darts.id_location_pair, + ROW(12003, 'LOC-B3')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, (now()::date - 7)::timestamp, results); + + --ASSERT cardinality(results) = 3, 'TEST 3 FAILED: expected 3 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 3 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 3 THEN + RAISE NOTICE 'TEST 3 FAILED: expected 3 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 3 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 3 PASSED'; + END $$; + +---------------------------------------------------------------------------------------- +-- TEST 4: Two-call behavior +-- Call 1: limit = 3 - expect first 3 valid pairs +-- Call 2: limit = 3 - expect 2 valid pairs + 3 incomplete cleanup, from call 1 +-- Test includes a check where the last_modified_ts is exactly the same as the parameter +---------------------------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- 5 valid ARM EODs (one exact timestamp match) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3301, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3302, 8980069, 2, 3, NULL, now(), (now()::date - 7)::timestamp, -99, -99, FALSE, FALSE, FALSE), -- exact match check + (3303, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3304, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3305, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE); + +-- 5 valid DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4301, 8980069, 2, 4, 'LOC-C1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4302, 8980069, 2, 4, 'LOC-C2', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4303, 8980069, 2, 4, 'LOC-C3', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4304, 8980069, 2, 4, 'LOC-C4', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4305, 8980069, 2, 4, 'LOC-C5', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- 5 valid OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (13001, 4301, 3301, 'LOC-C1'), + (13002, 4302, 3302, 'LOC-C2'), + (13003, 4303, 3303, 'LOC-C3'), + (13004, 4304, 3304, 'LOC-C4'), + (13005, 4305, 3305, 'LOC-C5'); + +-- Noise ARM (older than 7 days but invalid ors_id + not older than 7 days but valid ors_id) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3391, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE), + (3392, 8980069, 2, 3, NULL, now(), now() - interval '2 days', -99, -99, FALSE, FALSE, FALSE); + +-- DETS +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4391, 8980069, 2, 4, 'LOC-Z3', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4392, 8980069, 2, 4, 'LOC-Z4', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE); + +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES (13901, 4391, 3391, 'LOC-Z3'), + (13902, 4392, 3392, 'LOC-Z4'); + +-- 2 calls: limit = 3 +DO $$ + DECLARE + results darts.id_location_pair[]; + expected darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 4'; + RAISE NOTICE '------------------------------------'; + + --Call 1: limit 3 - Should behave normally + expected := ARRAY[ + ROW(13001, 'LOC-C1')::darts.id_location_pair, + ROW(13002, 'LOC-C2')::darts.id_location_pair, + ROW(13003, 'LOC-C3')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, (now()::date - 7)::timestamp, results); + + --ASSERT cardinality(results) = 3, 'TEST 4 CALL 1 FAILED: expected 3 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 4 CALL 1 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 3 THEN + RAISE NOTICE 'TEST 4 CALL 1 FAILED: expected 3 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 4 CALL 1 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 4 CALL 1 PASSED'; + + --Call 2: limit = 3 - Expect the other 2 valid pair and the 3 from call 1, because their flag hasn't been set) + expected := ARRAY[ + ROW(13004, 'LOC-C4')::darts.id_location_pair, + ROW(13005, 'LOC-C5')::darts.id_location_pair, + ROW(13001, 'LOC-C1')::darts.id_location_pair, + ROW(13002, 'LOC-C2')::darts.id_location_pair, + ROW(13003, 'LOC-C3')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, now() - interval '7 days', results); + + --ASSERT cardinality(results) = 5, 'TEST 4 CALL 2 FAILED: expected 5 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 4 CALL 2 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 5 THEN + RAISE NOTICE 'TEST 4 CALL 2 FAILED: expected 5 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 4 CALL 2 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 4 CALL 2 PASSED'; + END $$; + +---------------------------------------------------------------------------------------- +-- TEST 5: Two-call behavior +-- Call 1: limit = 3 - expect first 3 valid pairs +-- Update deleted flag +-- Call 2: limit = 3 - expect the remaining 2 valid pairs (0 incomplete cleanup) +-- Test includes a check where the last_modified_ts is exactly the same as the parameter +---------------------------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- 5 valid ARM EODs (one exact timestamp match) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3301, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3302, 8980069, 2, 3, NULL, now(), (now()::date - 7)::timestamp, -99, -99, FALSE, FALSE, FALSE), -- exact match check + (3303, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3304, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3305, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE); + +-- 5 valid DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4301, 8980069, 2, 4, 'LOC-C1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4302, 8980069, 2, 4, 'LOC-C2', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4303, 8980069, 2, 4, 'LOC-C3', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4304, 8980069, 2, 4, 'LOC-C4', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4305, 8980069, 2, 4, 'LOC-C5', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- 5 valid OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (13001, 4301, 3301, 'LOC-C1'), + (13002, 4302, 3302, 'LOC-C2'), + (13003, 4303, 3303, 'LOC-C3'), + (13004, 4304, 3304, 'LOC-C4'), + (13005, 4305, 3305, 'LOC-C5'); + +-- Noise ARM (older than 7 days but invalid ors_id + not older than 7 days but valid ors_id) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3391, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE), + (3392, 8980069, 2, 3, NULL, now(), now() - interval '2 days', -99, -99, FALSE, FALSE, FALSE); + +-- DETS +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4391, 8980069, 2, 4, 'LOC-Z3', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4392, 8980069, 2, 4, 'LOC-Z4', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE); + +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES (13901, 4391, 3391, 'LOC-Z3'), + (13902, 4392, 3392, 'LOC-Z4'); + +-- 2 calls: limit = 3 +DO $$ + DECLARE + results darts.id_location_pair[]; + expected darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 5'; + RAISE NOTICE '------------------------------------'; + + --Call 1: limit 3 - Should behave normally + expected := ARRAY[ + ROW(13001, 'LOC-C1')::darts.id_location_pair, + ROW(13002, 'LOC-C2')::darts.id_location_pair, + ROW(13003, 'LOC-C3')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, (now()::date - 7)::timestamp, results); + + --ASSERT cardinality(results) = 3, 'TEST 5 CALL 1 FAILED: expected 3 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 5 CALL 1 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 3 THEN + RAISE NOTICE 'TEST 5 CALL 1 FAILED: expected 3 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 5 CALL 1 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 5 CALL 1 PASSED'; + + --Update the flag_file_dets_cleanup_status flag to simulate final cleanup has completed + UPDATE object_state_record + SET flag_file_dets_cleanup_status = TRUE + WHERE osr_uuid IN (13001, 13002, 13003); + + --Call 2: limit = 3 - Expect the other 2 valid pairs ONLY) + expected := ARRAY[ + ROW(13004, 'LOC-C4')::darts.id_location_pair, + ROW(13005, 'LOC-C5')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, now() - interval '7 days', results); + + --ASSERT cardinality(results) = 2, 'TEST 5 CALL 2 FAILED: expected 2 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 5 CALL 2 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 2 THEN + RAISE NOTICE 'TEST 5 CALL 2 FAILED: expected 2 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 5 CALL 2 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 5 CALL 2 PASSED'; + END $$; + +--------------------------------------------------------------------------- +-- TEST 6: ARM record linked to multiple OSR/DETS records, expect exception +--------------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- Valid ARM EOD +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3501, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE); + +-- Valid DETS EOD +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4501, 8980069, 2, 4, 'LOC-D1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4502, 8980069, 2, 4, 'LOC-D2', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- OSR referencing both DETS EODs but ARM EOD is associated with both +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (15001, 4501, 3501, 'LOC-D1'), + (15002, 4502, 3501, 'LOC-D2'); + +DO $$ + DECLARE + results darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 6'; + RAISE NOTICE '------------------------------------'; + + CALL darts.dets_cleanup_eod_osr(10, now() - interval '7 days', results); + RAISE NOTICE 'TEST 6 FAILED: expected exception but procedure completed successfully.'; + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'TEST 6 PASSED (exception raised as expected): %', SQLERRM; + END $$; + +---------------------------------------------------------------------- +-- END OF TESTS +---------------------------------------------------------------------- + +DO $$ + BEGIN + RAISE NOTICE 'ALL TESTS COMPLETED'; + RAISE NOTICE 'Cleaning up...'; + END $$; + +---------------------------------------------------------------------- +-- CLEAN UP +---------------------------------------------------------------------- +RESET ALL; +--Drop schema avoiding using CASCADE +DROP PROCEDURE IF EXISTS test_dets_cleanup.reset_test_data; +DROP TABLE IF EXISTS test_dets_cleanup.external_object_directory; +DROP TABLE IF EXISTS test_dets_cleanup.object_state_record; +DROP SCHEMA IF EXISTS test_dets_cleanup; + +SELECT name, setting, unit, SOURCE +FROM pg_settings +WHERE name IN ('search_path', 'max_parallel_workers_per_gather', 'work_mem', 'max_parallel_maintenance_workers') +ORDER BY name; \ No newline at end of file diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java index 29f494911a3..4dd97adc9aa 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java @@ -39,21 +39,22 @@ public class CleanUpDetsDataProcessorImpl implements CleanUpDetsDataProcessor { public void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskConfig config) { log.info("Processing clean up of DETS data with batch size: {}", batchSize); - List eodIdsToCleanUp = externalObjectDirectoryRepository.findEodsWithMatchingRecordInArm( - EodHelper.storedStatus(), - EodHelper.detsLocation(), - EodHelper.armLocation(), - OffsetDateTime.now(clock), - Limit.of(batchSize) - ); + OffsetDateTime minimumStoredAge = OffsetDateTime.now(clock).minus(config.getMinimumStoredAge()); + int chunkSize = config.getChunkSize(); - log.info("Found {} EOD records to clean up", eodIdsToCleanUp.size()); - if (CollectionUtils.isNotEmpty(eodIdsToCleanUp)) { - //Chunk the EOD IDs into batches and process each batch in parallel - List> batchesForArm = ListUtils.partition(eodIdsToCleanUp, config.getChunkSize()); + int totalProcessed = 0; + while (totalProcessed < batchSize && chunkSize > 0) { + log.info("Processing clean up of DETS data with chunk size: {}", chunkSize); + List eodIdsToCleanUp = externalObjectDirectoryRepository.cleanUpDetsDataProcedure( + chunkSize, + minimumStoredAge); - List> tasks = batchesForArm + + List> batchesToDeleteBlobStoreRecordFor = ListUtils.partition(eodIdsToCleanUp, + config.getChunkSize() / config.getThreads()); + + List> tasks = batchesToDeleteBlobStoreRecordFor .stream() .map(eodsForBatch -> (Callable) () -> { //Call another serive to ensure the batch is processed within a transaction @@ -70,9 +71,16 @@ public void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskCo } catch (Exception e) { log.error("Clean up dets data unexpected exception", e); } - } else { - log.info("No DETS EODs require clean up"); + //Update total processed count and adjust chunk size for next iteration if needed + totalProcessed += eodIdsToCleanUp.size(); + //Ensure we do not exceed the batch size in the next iteration + //Takes into account the possibility that the procedure may return more records than requested + if (totalProcessed + chunkSize > batchSize) { + chunkSize = batchSize - totalProcessed; + } + log.info("Processed batch of DETS data clean up. Total processed so far: {}. Batch size: {}", totalProcessed, eodIdsToCleanUp.size()); } + log.info("Completed processing clean up of DETS data. Total processed: {}", totalProcessed); } @Component @@ -83,30 +91,26 @@ public class CleanUpDetsDataBatchProcessor { private final ObjectStateRecordRepository objectStateRecordRepository; private final DetsApiService detsApiService; - @Transactional - public void process(List eodsForBatch) { - log.info("Cleaning up DETS data for batch of EOD IDs: {}", eodsForBatch); - List responses = externalObjectDirectoryRepository.cleanUpDetsDataProcedure(eodsForBatch); - log.info("Clean up DETS data procedure completed for batch. Responses: {}", responses); - + public void process(List eodsCleanedUp) { List objectStateRecordsForDetsRecordsCleanedUpSuccessfully = new ArrayList<>(); List objectStateRecordsForDetsRecordsFailedToCleanUp = new ArrayList<>(); - for (CleanUpDetsProcedureResponse response : responses) { + + for (CleanUpDetsProcedureResponse response : eodsCleanedUp) { try { - log.debug("Processing clean up response for EOD ID: {}, Location: {}", response.getId(), response.getLocation()); - if (detsApiService.deleteBlobDataFromContainer(response.location)) { - log.debug("Successfully deleted DETS blob for EOD ID: {}, Location: {}", response.getId(), response.getLocation()); - objectStateRecordsForDetsRecordsCleanedUpSuccessfully.add(response.getId()); + log.debug("Processing clean up response for EOD ID: {}, Location: {}", response.getOsrUuid(), response.getDetsLocation()); + if (detsApiService.deleteBlobDataFromContainer(response.getDetsLocation())) { + log.debug("Successfully deleted DETS blob for EOD ID: {}, Location: {}", response.getOsrUuid(), response.getDetsLocation()); + objectStateRecordsForDetsRecordsCleanedUpSuccessfully.add(response.getOsrUuid()); continue; } else { log.error("Failed to delete DETS blob for EOD ID: {}, Location: {}. Blob may not exist or deletion failed.", - response.getId(), response.getLocation()); + response.getOsrUuid(), response.getDetsLocation()); } } catch (AzureDeleteBlobException azureDeleteBlobException) { log.error("AzureDeleteBlobException while deleting DETS blob for EOD Location: {}, object state record id: {}.", - response.getLocation(), response.getId(), azureDeleteBlobException); + response.getDetsLocation(), response.getOsrUuid(), azureDeleteBlobException); } - objectStateRecordsForDetsRecordsFailedToCleanUp.add(response.getId()); + objectStateRecordsForDetsRecordsFailedToCleanUp.add(response.getOsrUuid()); } objectStateRecordRepository.markDetsCleanupStatusAsComplete(objectStateRecordsForDetsRecordsCleanedUpSuccessfully); log.info("Marked object state records as clean up complete for EOD IDs: {}", objectStateRecordsForDetsRecordsCleanedUpSuccessfully); @@ -121,7 +125,7 @@ public void process(List eodsForBatch) { @Getter @Setter public static class CleanUpDetsProcedureResponse { - private Long id; - private String location; + private Long osrUuid; + private String detsLocation; } } diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java index 7bc69f4b3e1..edb50eaeac2 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java +++ b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java @@ -282,26 +282,8 @@ List findIdsForAudioToBeDeletedFromUnstructured(ObjectRecordStatusEntity s OffsetDateTime unstructuredLastModifiedBefore, Limit limit); - @Query( - """ - SELECT eod.id FROM ExternalObjectDirectoryEntity parentEod, ExternalObjectDirectoryEntity armEod - WHERE parentEod.media = armEod.media - AND parentEod.status = :storedStatus - AND parentEod.externalLocationType = :parentEodLocation - AND parentEod.lastModifiedDateTime <= :lastModifiedBefore - AND armEod.externalLocationType = :armLocation - AND armEod.status = :storedStatus - """ - ) - List findEodsWithMatchingRecordInArm( - ObjectRecordStatusEntity storedStatus, - ExternalLocationTypeEntity parentEodLocation, - ExternalLocationTypeEntity armLocation, - OffsetDateTime lastModifiedBefore, - Limit limit); - - @Query(value = "SELECT * FROM dmp4312_del_eod_upd_osr_conf_loc(:ids)", nativeQuery = true) - List cleanUpDetsDataProcedure(@Param("ids") List ids); + @Query(value = "SELECT * FROM dets_cleanup_eod_osr(:pi_limit, :pi_last_modified_ts)", nativeQuery = true) + List cleanUpDetsDataProcedure(@Param("pi_limit") Integer limit, @Param("pi_last_modified_ts") OffsetDateTime lastModifiedBefore); @Query( """ diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 4b3c140bbb7..acc1366cbe1 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -446,8 +446,8 @@ darts: at-least-for: PT1M at-most-for: PT90M minimum-stored-age: 7D - chunk-size: 1000 - async-timeout: 80M + chunk-size: 10000 + async-timeout: 20M threads: 20 cleanup-arm-response-files: system-user-email: system_CleanupArmResponseFiles@hmcts.net diff --git a/src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql b/src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql index 81eb8400e4a..c5d8b8ab7e3 100644 --- a/src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql +++ b/src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql @@ -6,4 +6,157 @@ VALUES (35, 'CleanUpDetsData', 'Cleans up Dets files that have successfully been INSERT INTO user_account VALUES (-40, NULL, '', 'systemCleanUpDetsData@hmcts.net', 'systemCleanUpDetsDataAutomatedTask', '2025-02-16 00:00:00+00', '2025-02-16 00:00:00+00', NULL, 0, 0, NULL, true, - true, d 'systemCleanUpDetsDataAutomatedTask'); \ No newline at end of file + true, d 'systemCleanUpDetsDataAutomatedTask'); + +SET SEARCH_PATH TO darts; + +DROP PROCEDURE IF EXISTS dets_cleanup_eod_osr; +DROP TYPE IF EXISTS id_location_pair; + +CREATE TYPE id_location_pair AS ( + osr_uuid BIGINT, + dets_location TEXT + ); + +CREATE INDEX IF NOT EXISTS osr_file_dets_cleanup_idx ON object_state_record (date_file_dets_cleanup, flag_file_dets_cleanup_status); + +CREATE OR REPLACE PROCEDURE dets_cleanup_eod_osr( + IN pi_limit INTEGER, + IN pi_last_modified_ts TIMESTAMPTZ, + OUT po_results_array id_location_pair[] +) + LANGUAGE 'plpgsql' +AS $$ +DECLARE + c_msg_prefix CONSTANT TEXT := 'dets_cleanup_eod_osr:'; + v_dets_eod_ids BIGINT[]; + v_arm_eod_ids BIGINT[]; + v_incomplete_cleanup id_location_pair[]; + + v_dets_eod_count INTEGER; + v_arm_eod_count INTEGER; + v_results_count INTEGER; + v_incomplete_cleanup_count INTEGER; + + v_dup_count INTEGER; + v_deleted_count INTEGER; + v_updated_count INTEGER; +BEGIN + RAISE NOTICE '% Started at % [pi_limit = %, pi_last_modified_ts = %]', c_msg_prefix, clock_timestamp(), pi_limit, pi_last_modified_ts; + + BEGIN + --Get DETS records to be deleted, limited by pi_limit + -- DETS record: OSR and EOD records must match on eod_id and locations, where eod.elt_id = 4 (DETS) + -- ARM record : OSR and EOD records must match where OSR.arm_eod_id = EOD.eod_id, where eod.elt_id = 3 (ARM) + -- ARM record must be STORED (i.e. ors_id = 2) and it's last_modified_ts is before or equal to pi_last_modified_ts + SELECT array_agg(eod_id) + , array_agg(arm_eod_id) + , array_agg((osr_uuid, dets_location)::id_location_pair) + INTO v_dets_eod_ids + , v_arm_eod_ids + , po_results_array + FROM ( + SELECT osr.osr_uuid, osr.dets_location, eod_dets.eod_id, eod_arm.eod_id AS arm_eod_id + FROM object_state_record osr + JOIN external_object_directory eod_dets + ON eod_dets.eod_id = osr.eod_id + AND eod_dets.external_location = osr.dets_location + JOIN external_object_directory eod_arm + ON eod_arm.eod_id = osr.arm_eod_id + WHERE eod_dets.elt_id = 4 --DETS + AND eod_arm.elt_id = 3 --ARM + AND eod_arm.ors_id = 2 --STORED + AND eod_arm.last_modified_ts <= pi_last_modified_ts + ORDER BY osr.osr_uuid + LIMIT pi_limit + ) t; + + v_dets_eod_count := COALESCE(cardinality(v_dets_eod_ids), 0); + v_arm_eod_count := COALESCE(cardinality(v_arm_eod_ids), 0); + v_results_count := COALESCE(cardinality(po_results_array), 0); + + RAISE NOTICE '% Number of records in DETS EOD id array = %', c_msg_prefix, v_dets_eod_count; + RAISE NOTICE '% Number of records in ARM EOD id array = %', c_msg_prefix, v_arm_eod_count; + RAISE NOTICE '% Number of records in results array = %', c_msg_prefix, v_results_count; + + -- Check to ensure duplicate records were not returned. Indicates that there is more than one ARM or DETS record, which should not happen + SELECT COUNT(*) + INTO v_dup_count + FROM ( + SELECT elem + FROM unnest(v_arm_eod_ids || v_dets_eod_ids) AS elem + GROUP BY elem + HAVING COUNT(*) > 1 + ) AS t; + + RAISE NOTICE '% Duplicate count = %', c_msg_prefix, v_dup_count; + + IF v_dup_count > 0 THEN + RAISE EXCEPTION 'Validation failed: Duplicate records [%] found.', v_dup_count; + END IF; + + --Before updating OSR, retrieve records from OSR that have a date_file_dets_cleanup value but flag_file_dets_cleanup_status has not been set. + -- append the results to the OUT parameter + SELECT array_agg((osr_uuid, dets_location)::id_location_pair ORDER BY osr_uuid) AS id_location_pairs + INTO v_incomplete_cleanup + FROM object_state_record + WHERE date_file_dets_cleanup IS NOT NULL + AND (flag_file_dets_cleanup_status IS NULL + OR flag_file_dets_cleanup_status = FALSE); + + v_incomplete_cleanup_count := COALESCE(cardinality(v_incomplete_cleanup), 0); + + RAISE NOTICE '% Number of records in incomplete cleanup array = %', c_msg_prefix, v_incomplete_cleanup_count; + + po_results_array := COALESCE( CASE WHEN v_results_count > 0 + AND v_incomplete_cleanup_count > 0 + THEN po_results_array || v_incomplete_cleanup + WHEN v_results_count > 0 + THEN po_results_array + WHEN v_incomplete_cleanup_count > 0 + THEN v_incomplete_cleanup + END, + '{}'::id_location_pair[] + ); + + RAISE NOTICE '% Number of records in combined results array = %', c_msg_prefix, COALESCE(cardinality(po_results_array), 0); + + --Only perform DELETE and UPDATE if there are values to process + IF v_dets_eod_count > 0 THEN + + --Delete records from EOD + DELETE FROM external_object_directory + WHERE eod_id = ANY(v_dets_eod_ids); + + GET DIAGNOSTICS v_deleted_count = ROW_COUNT; + RAISE NOTICE '% Number of EOD records deleted = %', c_msg_prefix, v_deleted_count; + + --Update OSR records + UPDATE object_state_record + SET eod_id = NULL + , date_file_dets_cleanup = CURRENT_TIMESTAMP + WHERE eod_id = ANY(v_dets_eod_ids); + + GET DIAGNOSTICS v_updated_count = ROW_COUNT; + RAISE NOTICE '% Number of OSR records updated = %', c_msg_prefix, v_updated_count; + + ELSE + RAISE NOTICE '% There is nothing to process. No EOD Deletes or OSR Updates were executed.', c_msg_prefix; + END IF; + + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE '% Error in dets_cleanup_eod_osr: % - %', c_msg_prefix, SQLSTATE, SQLERRM; + RAISE; + END; + + --Only COMMIT if there were values to process + IF v_dets_eod_count > 0 THEN + RAISE NOTICE '% Commit started at %', c_msg_prefix, clock_timestamp(); + COMMIT; + RAISE NOTICE '% Commit finished at %', c_msg_prefix, clock_timestamp(); + END IF; + + RAISE NOTICE '% Finished at %', c_msg_prefix, clock_timestamp(); +END; +$$; \ No newline at end of file From d02df8dc786b8d252597cf59ac4665d13f7a649e Mon Sep 17 00:00:00 2001 From: benedwards Date: Fri, 27 Feb 2026 21:29:33 +0000 Subject: [PATCH 3/7] Added unit tests --- .../impl/CleanUpDetsDataProcessorImpl.java | 25 +- .../CleanUpDetsDataProcessorImplTest.java | 254 ++++++++++++++++++ .../hmcts/darts/testutils/AsyncTestUtil.java | 30 +++ 3 files changed, 300 insertions(+), 9 deletions(-) create mode 100644 src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java create mode 100644 src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java index 4dd97adc9aa..3a01c97de77 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java @@ -46,13 +46,16 @@ public void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskCo while (totalProcessed < batchSize && chunkSize > 0) { log.info("Processing clean up of DETS data with chunk size: {}", chunkSize); - List eodIdsToCleanUp = externalObjectDirectoryRepository.cleanUpDetsDataProcedure( - chunkSize, - minimumStoredAge); + List eodIdsToCleanUp = externalObjectDirectoryRepository + .cleanUpDetsDataProcedure(chunkSize, minimumStoredAge); + if (eodIdsToCleanUp.isEmpty()) { + log.info("No more DETS data to clean up. Ending process."); + break; + } - List> batchesToDeleteBlobStoreRecordFor = ListUtils.partition(eodIdsToCleanUp, - config.getChunkSize() / config.getThreads()); + List> batchesToDeleteBlobStoreRecordFor = + ListUtils.partition(eodIdsToCleanUp, config.getChunkSize() / config.getThreads()); List> tasks = batchesToDeleteBlobStoreRecordFor .stream() @@ -95,6 +98,10 @@ public void process(List objectStateRecordsForDetsRecordsCleanedUpSuccessfully = new ArrayList<>(); List objectStateRecordsForDetsRecordsFailedToCleanUp = new ArrayList<>(); + if (eodsCleanedUp.isEmpty()) { + return; + } + for (CleanUpDetsProcedureResponse response : eodsCleanedUp) { try { log.debug("Processing clean up response for EOD ID: {}, Location: {}", response.getOsrUuid(), response.getDetsLocation()); @@ -106,9 +113,9 @@ public void process(List processor.processCleanUpDetsData(10, config)); + + verify(cleanUpDetsDataBatchProcessor, never()).process(anyList()); + verify(externalObjectDirectoryRepository, times(1)) + .cleanUpDetsDataProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); + } + + + @Test + @DisplayName("Partitions responses and delegates to batch processor per chunk") + void shouldPartitionResponsesIntoBatches() { + configureTaskConfig(4, 2); + List responses = + List.of(response(1L), response(2L), response(3L), response(4L)); + + when(externalObjectDirectoryRepository.cleanUpDetsDataProcedure(eq(4), any())) + .thenReturn(responses)//First call returns 4 records to process + .thenReturn(Collections.emptyList()); //Second call returns empty list to end processing + + AsyncTestUtil.processTasksSynchronously(() -> processor.processCleanUpDetsData(8, config)); + + ArgumentCaptor> captor = ArgumentCaptor.captor(); + //Call twice because with chunk size of 2 and 4 records, we expect 2 batches to be processed (Chunk size 4 / threads = 2 = batch size for processor) + verify(cleanUpDetsDataBatchProcessor, times(2)).process(captor.capture()); + List> batches = captor.getAllValues(); + + assertEquals(2, batches.size()); + assertEquals(List.of(responses.get(0), responses.get(1)), batches.get(0)); + assertEquals(List.of(responses.get(2), responses.get(3)), batches.get(1)); + } + + + @Test + @DisplayName("Continues looping until repository reports there is no more data") + void shouldContinueProcessingUntilNoMoreData() { + configureTaskConfig(4, 2); + List firstBatch = List.of( + response(1L), response(2L), response(3L), response(4L) + ); + List secondBatch = List.of( + response(5L), response(6L), response(7L), response(8L) + ); + + when(externalObjectDirectoryRepository.cleanUpDetsDataProcedure(eq(4), any())) + .thenReturn(firstBatch) + .thenReturn(secondBatch) + .thenReturn(Collections.emptyList()); + + AsyncTestUtil.processTasksSynchronously(() -> processor.processCleanUpDetsData(12, config)); + + //Called 3 times - first batch, second batch, then empty list to end processing + verify(externalObjectDirectoryRepository, times(3)) + .cleanUpDetsDataProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); + verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(0), firstBatch.get(1))); + verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(2), firstBatch.get(3))); + verify(cleanUpDetsDataBatchProcessor).process(List.of(secondBatch.get(0), secondBatch.get(1))); + verify(cleanUpDetsDataBatchProcessor).process(List.of(secondBatch.get(2), secondBatch.get(3))); + } + + @Test + @DisplayName("Stops processing once batch size limit is reached, even if repository returns more data") + void shouldStopProcessingWhenBatchSizeLimitReached() { + configureTaskConfig(4, 2); + + List firstBatch = List.of( + response(1L), response(2L), response(3L), response(4L) + ); + //Second batch should be ignored as batch size limit of 4 will have been reached after processing first batch, even though repository returns more data + List secondBatch = List.of( + response(5L), response(6L), response(7L), response(8L) + ); + + when(externalObjectDirectoryRepository.cleanUpDetsDataProcedure(eq(4), any())) + .thenReturn(firstBatch) + .thenReturn(secondBatch) + .thenReturn(Collections.emptyList()); + + AsyncTestUtil.processTasksSynchronously(() -> processor.processCleanUpDetsData(4, config)); + + verify(externalObjectDirectoryRepository, times(1)) + .cleanUpDetsDataProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); + verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(0), firstBatch.get(1))); + verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(2), firstBatch.get(3))); + } + + + private OffsetDateTime expectedMinimumStoredAge(Duration minimumStoredAge) { + return OffsetDateTime.ofInstant(FIXED_INSTANT, ZoneOffset.UTC).minus(minimumStoredAge); + } + } + + + @DisplayName("CleanUpDetsDataBatchProcessor tests") + @Nested + class CleanUpDetsDataBatchProcessorTests { + + @Test + @DisplayName("Deletes each blob and marks the OSRs as complete when everything succeeds") + void shouldDeleteAllRecordsSuccessfully() throws AzureDeleteBlobException { + CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor batchProcessor = createBatchProcessor(); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response1 = response(1L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response2 = response(2L); + + when(detsApiService.deleteBlobDataFromContainer(response1.getDetsLocation())).thenReturn(true); + when(detsApiService.deleteBlobDataFromContainer(response2.getDetsLocation())).thenReturn(true); + + batchProcessor.process(List.of(response1, response2)); + + verify(detsApiService).deleteBlobDataFromContainer(response1.getDetsLocation()); + verify(detsApiService).deleteBlobDataFromContainer(response2.getDetsLocation()); + verify(objectStateRecordRepository).markDetsCleanupStatusAsComplete(List.of(1L, 2L)); + } + + @Test + @DisplayName("Logs failures but continues when deleteBlobDataFromContainer returns false") + void shouldContinueProcessingWhenDeleteReturnsFalse() throws AzureDeleteBlobException { + CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor batchProcessor = createBatchProcessor(); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response1 = response(1L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response2 = response(2L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response3 = response(3L); + + when(detsApiService.deleteBlobDataFromContainer(response1.getDetsLocation())).thenReturn(true); + when(detsApiService.deleteBlobDataFromContainer(response2.getDetsLocation())).thenReturn(false); + when(detsApiService.deleteBlobDataFromContainer(response3.getDetsLocation())).thenReturn(true); + + assertDoesNotThrow(() -> batchProcessor.process(List.of(response1, response2, response3))); + + verify(detsApiService).deleteBlobDataFromContainer(response1.getDetsLocation()); + verify(detsApiService).deleteBlobDataFromContainer(response2.getDetsLocation()); + verify(detsApiService).deleteBlobDataFromContainer(response3.getDetsLocation()); + verify(objectStateRecordRepository).markDetsCleanupStatusAsComplete(List.of(1L, 3L)); + } + + @Test + @DisplayName("Handles exceptions thrown while deleting blobs and continues with remaining records") + void shouldHandleExceptionsAndContinueProcessing() throws AzureDeleteBlobException { + CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor batchProcessor = createBatchProcessor(); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response1 = response(1L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response2 = response(2L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response3 = response(3L); + + when(detsApiService.deleteBlobDataFromContainer(response1.getDetsLocation())).thenReturn(true); + when(detsApiService.deleteBlobDataFromContainer(response2.getDetsLocation())) + .thenThrow(new AzureDeleteBlobException("failure")); + when(detsApiService.deleteBlobDataFromContainer(response3.getDetsLocation())).thenReturn(true); + + assertDoesNotThrow(() -> batchProcessor.process(List.of(response1, response2, response3))); + + InOrder order = inOrder(detsApiService); + order.verify(detsApiService).deleteBlobDataFromContainer(response1.getDetsLocation()); + order.verify(detsApiService).deleteBlobDataFromContainer(response2.getDetsLocation()); + order.verify(detsApiService).deleteBlobDataFromContainer(response3.getDetsLocation()); + + verify(objectStateRecordRepository).markDetsCleanupStatusAsComplete(List.of(1L, 3L)); + } + } + + private void configureTaskConfig(int chunkSize, int threads) { + when(config.getMinimumStoredAge()).thenReturn(DEFAULT_MINIMUM_STORED_AGE); + when(config.getChunkSize()).thenReturn(chunkSize); + lenient().when(config.getThreads()).thenReturn(threads); + } + + private CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response(long osrUuid) { + return new CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse(osrUuid, "location-" + osrUuid); + } + + private CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor createBatchProcessor() { + return processor.new CleanUpDetsDataBatchProcessor(externalObjectDirectoryRepository, objectStateRecordRepository, detsApiService); + } +} diff --git a/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java b/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java new file mode 100644 index 00000000000..2bf23b8c91d --- /dev/null +++ b/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java @@ -0,0 +1,30 @@ +package uk.gov.hmcts.darts.testutils; + +import org.mockito.MockedStatic; +import uk.gov.hmcts.darts.util.AsyncUtil; + +import java.util.List; +import java.util.concurrent.Callable; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mockStatic; + +public class AsyncTestUtil { + + public static void processTasksSynchronously(Runnable runnable) { + try (MockedStatic asyncUtilMock = mockStatic(AsyncUtil.class)) { + //Mock AsyncUtil to synchronously execute the tasks so we can verify the batch processor calls within the test + asyncUtilMock.when(() -> AsyncUtil.invokeAllAwaitTermination(anyList(), any())) + .thenAnswer(invocation -> { + List> tasks = invocation.getArgument(0); + for (Callable task : tasks) { + task.call(); + } + return null; + }); + runnable.run(); + } + } +} + From beb8099b7f8589095c2a3d8c4b730eeabcb71d06 Mon Sep 17 00:00:00 2001 From: benedwards Date: Sat, 28 Feb 2026 01:26:13 +0000 Subject: [PATCH 4/7] Added tests --- .../CleanUpDetsDataProcessorImplIntTest.java | 344 ++++++++++++++++++ .../testutils/stubs/DartsDatabaseStub.java | 1 + .../testutils/stubs/DartsPersistence.java | 10 + .../impl/CleanUpDetsDataProcessorImpl.java | 19 +- .../ExternalObjectDirectoryRepository.java | 7 - .../ObjectStateRecordRepository.java | 8 + ...V1_501__CleanUpDetsData_automated_task.sql | 30 +- .../CleanUpDetsDataProcessorImplTest.java | 25 +- .../CleanUpDetsDataAutomatedTaskTest.java | 54 +++ 9 files changed, 457 insertions(+), 41 deletions(-) create mode 100644 src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java rename src/main/resources/db/migration/{common => postgres}/V1_501__CleanUpDetsData_automated_task.sql (88%) create mode 100644 src/test/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTaskTest.java diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java new file mode 100644 index 00000000000..3e53d5d28f1 --- /dev/null +++ b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java @@ -0,0 +1,344 @@ +package uk.gov.hmcts.darts.arm.service.impl; + +import lombok.Getter; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Isolated; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import uk.gov.hmcts.darts.common.entity.AnnotationDocumentEntity; +import uk.gov.hmcts.darts.common.entity.CaseDocumentEntity; +import uk.gov.hmcts.darts.common.entity.ExternalObjectDirectoryEntity; +import uk.gov.hmcts.darts.common.entity.MediaEntity; +import uk.gov.hmcts.darts.common.entity.ObjectStateRecordEntity; +import uk.gov.hmcts.darts.common.entity.TranscriptionDocumentEntity; +import uk.gov.hmcts.darts.common.enums.ExternalLocationTypeEnum; +import uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum; +import uk.gov.hmcts.darts.dets.service.DetsApiService; +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +import uk.gov.hmcts.darts.testutils.PostgresIntegrationBase; + +import java.time.Clock; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +import static java.time.temporal.ChronoUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static uk.gov.hmcts.darts.test.common.data.ExternalLocationTypeTestData.locationTypeOf; +import static uk.gov.hmcts.darts.test.common.data.ObjectRecordStatusTestData.statusOf; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getAnnotationDocumentTestData; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getCaseDocumentTestData; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getExternalObjectDirectoryTestData; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getMediaTestData; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getTranscriptionDocument; + +@Isolated +class CleanUpDetsDataProcessorImplIntTest extends PostgresIntegrationBase { + private static final Duration DEFAULT_TEST_DATA_AGE = Duration.ofHours(10); + + @Autowired + private CleanUpDetsDataProcessorImpl cleanUpDetsDataProcessor; + + @Autowired + private Clock clock; + + @MockitoBean + private DetsApiService detsApiService; + + + @Test + void cleanUpDetsData_typical() { + //Valid data that should be cleaned up + List> validData = List.of( + new MediaTestData().createDataThatShouldBeCleanedUp(), + new TranscriptionDocumentTestData().createDataThatShouldBeCleanedUp(), + new AnnotationDocumentTestData().createDataThatShouldBeCleanedUp(), + new CaseDocumentTestData().createDataThatShouldBeCleanedUp() + ); + //Inject some data that should not be picked up + List> inelibileData = List.of( + new MediaTestData().createDetsRecordButWithoutArmRecord(), + new MediaTestData().createDetsRecordButWithArmRecordButNotInStoredStatus(), + new MediaTestData().createDetsRecordButWithArmStoredButWithinMiniumStoredTime(), + new AnnotationDocumentTestData().createDetsRecordButOsrAndDetsEodLocationsDoNotAlign() + ); + + cleanUpDetsDataProcessor.processCleanUpDetsData(20, getCleanUpDetsDataAutomatedTaskConfig()); + + assertSuccessfull(validData); + assertNoChange(inelibileData); + } + + @Test + void cleanUpDetsData_shouldPickUpAnyUnprocessedDeletes() { + //Valid data that should be cleaned up + TestData fullCleanUpData1 = new MediaTestData().createDataThatShouldBeCleanedUp(true); + TestData fullCleanUpData2 = new MediaTestData().createDataThatShouldBeCleanedUp(true); + TestData partialCleanUpData1 = new MediaTestData().createDataThatShouldBeCleanedUp(false); + + + cleanUpDetsDataProcessor.processCleanUpDetsData(3, getCleanUpDetsDataAutomatedTaskConfig()); + + assertSuccessfull(fullCleanUpData1); + assertPartialSuccess(partialCleanUpData1); + assertSuccessfull(fullCleanUpData2); + + //Reset mock + //Check to ensure that the failed blob delete is attempted again but the successful database deletes are not attempted again + Mockito.clearInvocations(detsApiService); + partialCleanUpData1.stubBlobStoreDelete(true); + TestData fullCleanUpData3 = new MediaTestData().createDataThatShouldBeCleanedUp(true); + + cleanUpDetsDataProcessor.processCleanUpDetsData(20, getCleanUpDetsDataAutomatedTaskConfig()); + assertSuccessfull(partialCleanUpData1); + assertSuccessfull(fullCleanUpData3); + } + + + private void assertNoChange(List> testDataList) { + testDataList.forEach(this::assertNoChange); + } + + @SneakyThrows + private void assertNoChange(TestData testData) { + assertCommon(testData); + //Check that the EOD records still exist + assertThat(dartsPersistence.getExternalObjectDirectoryRepository().existsById(testData.getDetsEod().getId())) + .as("DETS EOD record should still exist").isTrue(); + //Check that the Object State Record still exists and has the same dets eod id and arm eod id + ObjectStateRecordEntity objectStateRecordEntity = dartsPersistence.getObjectStateRecordRepository() + .findById(testData.getObjectStateRecordEntity().getUuid()).orElseThrow(); + assertThat(objectStateRecordEntity.getEodId()).isEqualTo(testData.getDetsEod().getId()); + assertThat(objectStateRecordEntity.getDateFileDetsCleanup()).isNull(); + assertThat(objectStateRecordEntity.getFlagFileDetsCleanupStatus()).isNull(); + verify(detsApiService, never()).deleteBlobDataFromContainer(testData.getDetsEod().getLocation()); + } + + private void assertSuccessfull(List> testDataList) { + testDataList.forEach(this::assertSuccessfull); + } + + @SneakyThrows + private void assertSuccessfull(TestData testData) { + assertCommon(testData); + //Check that the Dets EOD records has been deleted + assertThat(dartsPersistence.getExternalObjectDirectoryRepository().existsById(testData.getDetsEod().getId())) + .as("DETS EOD record should not exist").isFalse(); + //Check that the Object State Record has been updated + ObjectStateRecordEntity objectStateRecordEntity = dartsPersistence.getObjectStateRecordRepository() + .findById(testData.getObjectStateRecordEntity().getUuid()).orElseThrow(); + assertThat(objectStateRecordEntity.getEodId()).isNull(); + assertThat(objectStateRecordEntity.getDateFileDetsCleanup()) + .isCloseTo(OffsetDateTime.now(), within(1, SECONDS)); + assertThat(objectStateRecordEntity.getFlagFileDetsCleanupStatus()).isTrue(); + verify(detsApiService).deleteBlobDataFromContainer(testData.getDetsEod().getLocation()); + } + + @SneakyThrows + private void assertPartialSuccess(TestData testData) { + assertCommon(testData); + //Check that the Dets EOD records has been deleted + assertThat(dartsPersistence.getExternalObjectDirectoryRepository().existsById(testData.getDetsEod().getId())) + .as("DETS EOD record should not exist").isFalse(); + //Check that the Object State Record has been updated + ObjectStateRecordEntity objectStateRecordEntity = dartsPersistence.getObjectStateRecordRepository() + .findById(testData.getObjectStateRecordEntity().getUuid()).orElseThrow(); + assertThat(objectStateRecordEntity.getEodId()).isNull(); + assertThat(objectStateRecordEntity.getDateFileDetsCleanup()) + .isCloseTo(OffsetDateTime.now(), within(1, SECONDS)); + verify(detsApiService).deleteBlobDataFromContainer(testData.getDetsEod().getLocation()); + + //If database delete was successful but blob store was not this flag should not be set + assertThat(objectStateRecordEntity.getFlagFileDetsCleanupStatus()).isNull(); + } + + private void assertCommon(TestData testData) { + //Check that the ARM EOD record still exists + if (testData.getArmEod() != null) { + assertThat(dartsPersistence.getExternalObjectDirectoryRepository().existsById(testData.getArmEod().getId())) + .as("ARM EOD record should still exist").isTrue(); + } + } + + private CleanUpDetsDataAutomatedTaskConfig getCleanUpDetsDataAutomatedTaskConfig() { + CleanUpDetsDataAutomatedTaskConfig config = new CleanUpDetsDataAutomatedTaskConfig(); + config.setMinimumStoredAge(DEFAULT_TEST_DATA_AGE); + config.setChunkSize(10); + config.setThreads(2); + config.setAsyncTimeout(Duration.ofMinutes(5)); + return config; + } + + //Test Data + public class MediaTestData extends TestData { + @Override + protected MediaEntity createConfidenceAware() { + return dartsPersistence.save(getMediaTestData().someMinimal()); + } + + @Override + protected void assignContextAwareToexternalObjectDirectoryEntity(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, + MediaEntity confidenceAware) { + externalObjectDirectoryEntity.setMedia(confidenceAware); + } + } + + public class TranscriptionDocumentTestData extends TestData { + @Override + protected TranscriptionDocumentEntity createConfidenceAware() { + return dartsPersistence.save(getTranscriptionDocument().someMinimal()); + } + + @Override + protected void assignContextAwareToexternalObjectDirectoryEntity(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, + TranscriptionDocumentEntity confidenceAware) { + externalObjectDirectoryEntity.setTranscriptionDocumentEntity(confidenceAware); + } + } + + public class AnnotationDocumentTestData extends TestData { + @Override + protected AnnotationDocumentEntity createConfidenceAware() { + return dartsPersistence.save(getAnnotationDocumentTestData().someMinimal()); + } + + @Override + protected void assignContextAwareToexternalObjectDirectoryEntity(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, + AnnotationDocumentEntity confidenceAware) { + externalObjectDirectoryEntity.setAnnotationDocumentEntity(confidenceAware); + } + } + + public class CaseDocumentTestData extends TestData { + @Override + protected CaseDocumentEntity createConfidenceAware() { + return dartsPersistence.save(getCaseDocumentTestData().someMinimal()); + } + + @Override + protected void assignContextAwareToexternalObjectDirectoryEntity(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, + CaseDocumentEntity confidenceAware) { + externalObjectDirectoryEntity.setCaseDocument(confidenceAware); + } + } + + + @Getter + public abstract class TestData { + public static long uniqueCounter = 0; + ExternalObjectDirectoryEntity detsEod; + ExternalObjectDirectoryEntity armEod; + ObjectStateRecordEntity objectStateRecordEntity; + T confidenceAware; + + public TestData() { + } + + public TestData createDataThatShouldBeCleanedUp() { + return createDataThatShouldBeCleanedUp(true); + } + + public TestData createDataThatShouldBeCleanedUp(boolean shouldBlobDeleteSucceed) { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + armEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.ARM, ObjectRecordStatusEnum.STORED, confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, armEod); + stubBlobStoreDelete(shouldBlobDeleteSucceed); + return this; + } + + public TestData createDetsRecordButWithoutArmRecord() { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, null); + stubBlobStoreDelete(true); + return this; + } + + public TestData createDetsRecordButWithArmRecordButNotInStoredStatus() { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + armEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.ARM, ObjectRecordStatusEnum.ARM_DROP_ZONE, confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, armEod); + stubBlobStoreDelete(true); + return this; + } + + public TestData createDetsRecordButWithArmStoredButWithinMiniumStoredTime() { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + armEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.ARM, ObjectRecordStatusEnum.ARM_DROP_ZONE, OffsetDateTime.now(), + confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, armEod); + stubBlobStoreDelete(true); + return this; + } + + public TestData createDetsRecordButOsrAndDetsEodLocationsDoNotAlign() { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + armEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.ARM, ObjectRecordStatusEnum.ARM_DROP_ZONE, confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, armEod); + objectStateRecordEntity.setDetsLocation("Some random location that does not align with the dets eod location"); + dartsPersistence.getObjectStateRecordRepository().save(objectStateRecordEntity); + stubBlobStoreDelete(true); + return this; + } + + @SneakyThrows + public void stubBlobStoreDelete(boolean shouldBlobDeleteSucceed) { + lenient().when(detsApiService.deleteBlobDataFromContainer(detsEod.getLocation())) + .thenReturn(shouldBlobDeleteSucceed); + } + + + private ObjectStateRecordEntity createObjectStateRecordEntity(ExternalObjectDirectoryEntity detsEod, ExternalObjectDirectoryEntity armEod) { + ObjectStateRecordEntity osrEntity = new ObjectStateRecordEntity(); + osrEntity.setUuid(uniqueCounter++); + osrEntity.setEodId(detsEod.getId()); + if (armEod != null) { + osrEntity.setArmEodId(armEod.getId()); + } + osrEntity.setDetsLocation(detsEod.getLocation()); + return dartsPersistence.getObjectStateRecordRepository().save(osrEntity); + } + + public ExternalObjectDirectoryEntity createExternalObjectDirectoryEntity(ExternalLocationTypeEnum externalLocationTypeEnum, + ObjectRecordStatusEnum objectRecordStatusEnum, + T confidenceAware) { + return createExternalObjectDirectoryEntity(externalLocationTypeEnum, + objectRecordStatusEnum, + OffsetDateTime.now().minus(DEFAULT_TEST_DATA_AGE).minusHours(1), + confidenceAware); + } + + public ExternalObjectDirectoryEntity createExternalObjectDirectoryEntity(ExternalLocationTypeEnum externalLocationTypeEnum, + ObjectRecordStatusEnum objectRecordStatusEnum, + OffsetDateTime lastModifiedDateTime, + T confidenceAware) { + ExternalObjectDirectoryEntity externalObjectDirectoryEntity = getExternalObjectDirectoryTestData().someMinimal(); + externalObjectDirectoryEntity.setStatus(statusOf(objectRecordStatusEnum)); + externalObjectDirectoryEntity.setExternalLocationType(locationTypeOf(externalLocationTypeEnum)); + externalObjectDirectoryEntity.setExternalLocation(UUID.randomUUID().toString()); + externalObjectDirectoryEntity.setLastModifiedDateTime(lastModifiedDateTime); + assignContextAwareToexternalObjectDirectoryEntity(externalObjectDirectoryEntity, confidenceAware); + + externalObjectDirectoryEntity = dartsPersistence.save(externalObjectDirectoryEntity); + dartsPersistence.overrideLastModifiedBy(externalObjectDirectoryEntity, lastModifiedDateTime); + return externalObjectDirectoryEntity; + } + + protected abstract T createConfidenceAware(); + + protected abstract void assignContextAwareToexternalObjectDirectoryEntity( + ExternalObjectDirectoryEntity externalObjectDirectoryEntity, T confidenceAware); + } +} diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsDatabaseStub.java b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsDatabaseStub.java index 2992bc06e4f..47937fc3e33 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsDatabaseStub.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsDatabaseStub.java @@ -335,6 +335,7 @@ public void clearDatabaseInThisOrder() { transcriptionRepository.deleteAll(); transcriptionWorkflowRepository.deleteAll(); retentionConfidenceCategoryMapperRepository.deleteAll(); + objectStateRecordRepository.deleteAll(); }); } diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java index f6b2806bafc..9349ce17a32 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java @@ -9,6 +9,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Service; import uk.gov.hmcts.darts.audio.entity.MediaRequestEntity; @@ -79,6 +80,7 @@ import uk.gov.hmcts.darts.common.repository.ObjectAdminActionRepository; import uk.gov.hmcts.darts.common.repository.ObjectHiddenReasonRepository; import uk.gov.hmcts.darts.common.repository.ObjectRecordStatusRepository; +import uk.gov.hmcts.darts.common.repository.ObjectStateRecordRepository; import uk.gov.hmcts.darts.common.repository.ProsecutorRepository; import uk.gov.hmcts.darts.common.repository.RegionRepository; import uk.gov.hmcts.darts.common.repository.RetentionConfidenceCategoryMapperRepository; @@ -170,6 +172,7 @@ public class DartsPersistence { private final ObjectAdminActionRepository objectAdminActionRepository; private final EventLinkedCaseRepository eventLinkedCaseRepository; private final RetentionConfidenceCategoryMapperRepository retentionConfidenceCategoryMapperRepository; + private final ObjectStateRecordRepository objectStateRecordRepository; private final EntityManager entityManager; private final CurrentTimeHelper currentTimeHelper; @@ -910,6 +913,13 @@ public void overrideLastModifiedBy(MediaRequestEntity mediaRequestEntity, Offset .setParameter("id", mediaRequestEntity.getId()) .executeUpdate(); } + @Transactional + public void overrideLastModifiedBy(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, OffsetDateTime lastModifiedDate) { + entityManager.createNativeQuery("UPDATE external_object_directory SET last_modified_ts = :lastModifiedDate WHERE eod_id = :id") + .setParameter("lastModifiedDate", lastModifiedDate) + .setParameter("id", externalObjectDirectoryEntity.getId()) + .executeUpdate(); + } @Transactional public & CreatedBy> void updateCreatedBy(T object, OffsetDateTime createdDateTime) { diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java index 3a01c97de77..786ab91f77e 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java @@ -7,15 +7,13 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; -import org.springframework.data.domain.Limit; import org.springframework.stereotype.Component; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import uk.gov.hmcts.darts.arm.service.CleanUpDetsDataProcessor; -import uk.gov.hmcts.darts.common.exception.AzureDeleteBlobException; import uk.gov.hmcts.darts.common.repository.ExternalObjectDirectoryRepository; import uk.gov.hmcts.darts.common.repository.ObjectStateRecordRepository; -import uk.gov.hmcts.darts.common.util.EodHelper; import uk.gov.hmcts.darts.dets.service.DetsApiService; import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; import uk.gov.hmcts.darts.util.AsyncUtil; @@ -31,7 +29,6 @@ @RequiredArgsConstructor public class CleanUpDetsDataProcessorImpl implements CleanUpDetsDataProcessor { - private final ExternalObjectDirectoryRepository externalObjectDirectoryRepository; private final CleanUpDetsDataBatchProcessor cleanUpDetsDataBatchProcessor; private final Clock clock; @@ -46,8 +43,9 @@ public void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskCo while (totalProcessed < batchSize && chunkSize > 0) { log.info("Processing clean up of DETS data with chunk size: {}", chunkSize); - List eodIdsToCleanUp = externalObjectDirectoryRepository - .cleanUpDetsDataProcedure(chunkSize, minimumStoredAge); + + List eodIdsToCleanUp = + cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(chunkSize, minimumStoredAge); if (eodIdsToCleanUp.isEmpty()) { log.info("No more DETS data to clean up. Ending process."); @@ -60,7 +58,6 @@ public void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskCo List> tasks = batchesToDeleteBlobStoreRecordFor .stream() .map(eodsForBatch -> (Callable) () -> { - //Call another serive to ensure the batch is processed within a transaction cleanUpDetsDataBatchProcessor.process(eodsForBatch); return null; }) @@ -88,12 +85,13 @@ public void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskCo @Component @RequiredArgsConstructor - public class CleanUpDetsDataBatchProcessor { + public static class CleanUpDetsDataBatchProcessor { private final ExternalObjectDirectoryRepository externalObjectDirectoryRepository; private final ObjectStateRecordRepository objectStateRecordRepository; private final DetsApiService detsApiService; + @Transactional(propagation = Propagation.REQUIRES_NEW) public void process(List eodsCleanedUp) { List objectStateRecordsForDetsRecordsCleanedUpSuccessfully = new ArrayList<>(); List objectStateRecordsForDetsRecordsFailedToCleanUp = new ArrayList<>(); @@ -126,6 +124,11 @@ public void process(List callDetsCleanUpStoredProcedure(int chunkSize, OffsetDateTime minimumStoredAge) { + return objectStateRecordRepository.cleanUpDetsDataProcedure(chunkSize, minimumStoredAge); + } } @AllArgsConstructor diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java index edb50eaeac2..b4bbf39e071 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java +++ b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java @@ -7,10 +7,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; -import org.springframework.data.jpa.repository.query.Procedure; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; -import uk.gov.hmcts.darts.arm.service.impl.CleanUpDetsDataProcessorImpl; import uk.gov.hmcts.darts.common.entity.AnnotationDocumentEntity; import uk.gov.hmcts.darts.common.entity.CaseDocumentEntity; import uk.gov.hmcts.darts.common.entity.ExternalLocationTypeEntity; @@ -18,13 +16,11 @@ import uk.gov.hmcts.darts.common.entity.MediaEntity; import uk.gov.hmcts.darts.common.entity.ObjectRecordStatusEntity; import uk.gov.hmcts.darts.common.entity.TranscriptionDocumentEntity; -import uk.gov.hmcts.darts.task.runner.impl.CleanUpDetsDataAutomatedTask; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; @@ -282,9 +278,6 @@ List findIdsForAudioToBeDeletedFromUnstructured(ObjectRecordStatusEntity s OffsetDateTime unstructuredLastModifiedBefore, Limit limit); - @Query(value = "SELECT * FROM dets_cleanup_eod_osr(:pi_limit, :pi_last_modified_ts)", nativeQuery = true) - List cleanUpDetsDataProcedure(@Param("pi_limit") Integer limit, @Param("pi_last_modified_ts") OffsetDateTime lastModifiedBefore); - @Query( """ SELECT eod FROM ExternalObjectDirectoryEntity eod diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java index a3535193fd8..def019f5603 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java +++ b/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java @@ -3,8 +3,11 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import uk.gov.hmcts.darts.arm.service.impl.CleanUpDetsDataProcessorImpl; import uk.gov.hmcts.darts.common.entity.ObjectStateRecordEntity; +import java.time.OffsetDateTime; import java.util.List; import java.util.Optional; @@ -14,4 +17,9 @@ public interface ObjectStateRecordRepository extends JpaRepository uuids); + + + @Query(value = "SELECT osr_uuid AS osrUuid, dets_location AS detsLocation FROM dets_cleanup_eod_osr_rows(:limit, :last_modified_before_ts)", nativeQuery = true) + List cleanUpDetsDataProcedure(@Param("limit") Integer limit, @Param("last_modified_before_ts") OffsetDateTime lastModifiedBefore); + } diff --git a/src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql b/src/main/resources/db/migration/postgres/V1_501__CleanUpDetsData_automated_task.sql similarity index 88% rename from src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql rename to src/main/resources/db/migration/postgres/V1_501__CleanUpDetsData_automated_task.sql index c5d8b8ab7e3..3473aca989c 100644 --- a/src/main/resources/db/migration/common/V1_501__CleanUpDetsData_automated_task.sql +++ b/src/main/resources/db/migration/postgres/V1_501__CleanUpDetsData_automated_task.sql @@ -3,13 +3,15 @@ INSERT INTO darts.automated_task (aut_id, task_name, task_description, cron_expr VALUES (35, 'CleanUpDetsData', 'Cleans up Dets files that have successfully been stored in ARM', '0 24 0-6,19-23 ? * *', true, 100_000, current_timestamp, 0, current_timestamp, 0, false); -INSERT INTO user_account -VALUES (-40, NULL, '', 'systemCleanUpDetsData@hmcts.net', - 'systemCleanUpDetsDataAutomatedTask', '2025-02-16 00:00:00+00', '2025-02-16 00:00:00+00', NULL, 0, 0, NULL, true, - true, d 'systemCleanUpDetsDataAutomatedTask'); +INSERT INTO user_account (usr_id, user_name, user_email_address, description, created_ts, last_modified_ts, last_modified_by, created_by, is_system_user, + is_active, user_full_name) + +VALUES (-40, 'systemCleanUpDetsDataAutomatedTask', 'systemCleanUpDetsDataAutomatedTask@hmcts.net', 'systemCleanUpDetsDataAutomatedTask', + '2024-01-01 00:00:00+00', '2024-01-01 00:00:00+00', 0, 0, true, true, 'systemCleanUpDetsDataAutomatedTask'); SET SEARCH_PATH TO darts; +DROP FUNCTION IF EXISTS dets_cleanup_eod_osr_rows; DROP PROCEDURE IF EXISTS dets_cleanup_eod_osr; DROP TYPE IF EXISTS id_location_pair; @@ -149,14 +151,16 @@ BEGIN RAISE NOTICE '% Error in dets_cleanup_eod_osr: % - %', c_msg_prefix, SQLSTATE, SQLERRM; RAISE; END; - - --Only COMMIT if there were values to process - IF v_dets_eod_count > 0 THEN - RAISE NOTICE '% Commit started at %', c_msg_prefix, clock_timestamp(); - COMMIT; - RAISE NOTICE '% Commit finished at %', c_msg_prefix, clock_timestamp(); - END IF; - RAISE NOTICE '% Finished at %', c_msg_prefix, clock_timestamp(); END; -$$; \ No newline at end of file +$$; + +CREATE OR REPLACE FUNCTION dets_cleanup_eod_osr_rows(pi_limit int, pi_last_modified_ts timestamptz) + RETURNS TABLE (osr_uuid bigint, dets_location text) + LANGUAGE plpgsql AS $$ +DECLARE v_arr id_location_pair[]; +BEGIN + CALL dets_cleanup_eod_osr(pi_limit, pi_last_modified_ts, v_arr); + RETURN QUERY SELECT (x).osr_uuid, (x).dets_location + FROM unnest(COALESCE(v_arr, '{}'::id_location_pair[])) x; +END $$; \ No newline at end of file diff --git a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java index e7b405c9000..af663f0a763 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java @@ -65,8 +65,7 @@ class CleanUpDetsDataProcessorImplTest { @BeforeEach void setUp() { - processor = new CleanUpDetsDataProcessorImpl(externalObjectDirectoryRepository, cleanUpDetsDataBatchProcessor, - Clock.fixed(FIXED_INSTANT, ZoneOffset.UTC)); + processor = new CleanUpDetsDataProcessorImpl(cleanUpDetsDataBatchProcessor, Clock.fixed(FIXED_INSTANT, ZoneOffset.UTC)); } @@ -78,14 +77,14 @@ class ProcessCleanUpDetsDataTests { @DisplayName("Skips processing when no EOD rows are returned") void shouldSkipProcessingWhenRepositoryReturnsEmptyList() { configureTaskConfig(4, 2); - when(externalObjectDirectoryRepository.cleanUpDetsDataProcedure(eq(4), any())) + when(cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(eq(4), any())) .thenReturn(Collections.emptyList()); assertDoesNotThrow(() -> processor.processCleanUpDetsData(10, config)); verify(cleanUpDetsDataBatchProcessor, never()).process(anyList()); - verify(externalObjectDirectoryRepository, times(1)) - .cleanUpDetsDataProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); + verify(cleanUpDetsDataBatchProcessor, times(1)) + .callDetsCleanUpStoredProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); } @@ -96,7 +95,7 @@ void shouldPartitionResponsesIntoBatches() { List responses = List.of(response(1L), response(2L), response(3L), response(4L)); - when(externalObjectDirectoryRepository.cleanUpDetsDataProcedure(eq(4), any())) + when(cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(eq(4), any())) .thenReturn(responses)//First call returns 4 records to process .thenReturn(Collections.emptyList()); //Second call returns empty list to end processing @@ -124,7 +123,7 @@ void shouldContinueProcessingUntilNoMoreData() { response(5L), response(6L), response(7L), response(8L) ); - when(externalObjectDirectoryRepository.cleanUpDetsDataProcedure(eq(4), any())) + when(cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(eq(4), any())) .thenReturn(firstBatch) .thenReturn(secondBatch) .thenReturn(Collections.emptyList()); @@ -132,8 +131,8 @@ void shouldContinueProcessingUntilNoMoreData() { AsyncTestUtil.processTasksSynchronously(() -> processor.processCleanUpDetsData(12, config)); //Called 3 times - first batch, second batch, then empty list to end processing - verify(externalObjectDirectoryRepository, times(3)) - .cleanUpDetsDataProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); + verify(cleanUpDetsDataBatchProcessor, times(3)) + .callDetsCleanUpStoredProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(0), firstBatch.get(1))); verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(2), firstBatch.get(3))); verify(cleanUpDetsDataBatchProcessor).process(List.of(secondBatch.get(0), secondBatch.get(1))); @@ -153,15 +152,15 @@ void shouldStopProcessingWhenBatchSizeLimitReached() { response(5L), response(6L), response(7L), response(8L) ); - when(externalObjectDirectoryRepository.cleanUpDetsDataProcedure(eq(4), any())) + when(cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(eq(4), any())) .thenReturn(firstBatch) .thenReturn(secondBatch) .thenReturn(Collections.emptyList()); AsyncTestUtil.processTasksSynchronously(() -> processor.processCleanUpDetsData(4, config)); - verify(externalObjectDirectoryRepository, times(1)) - .cleanUpDetsDataProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); + verify(cleanUpDetsDataBatchProcessor, times(1)) + .callDetsCleanUpStoredProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(0), firstBatch.get(1))); verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(2), firstBatch.get(3))); } @@ -249,6 +248,6 @@ private CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response(long } private CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor createBatchProcessor() { - return processor.new CleanUpDetsDataBatchProcessor(externalObjectDirectoryRepository, objectStateRecordRepository, detsApiService); + return new CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor(externalObjectDirectoryRepository, objectStateRecordRepository, detsApiService); } } diff --git a/src/test/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTaskTest.java b/src/test/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTaskTest.java new file mode 100644 index 00000000000..8c99c7658b2 --- /dev/null +++ b/src/test/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTaskTest.java @@ -0,0 +1,54 @@ +package uk.gov.hmcts.darts.task.runner.impl; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import uk.gov.hmcts.darts.arm.service.CleanUpDetsDataProcessor; +import uk.gov.hmcts.darts.common.repository.AutomatedTaskRepository; +import uk.gov.hmcts.darts.log.api.LogApi; +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +import uk.gov.hmcts.darts.task.service.LockService; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static uk.gov.hmcts.darts.task.api.AutomatedTaskName.CLEAN_UP_DETS_DATA; + +@ExtendWith(MockitoExtension.class) +class CleanUpDetsDataAutomatedTaskTest { + + @Mock + private AutomatedTaskRepository automatedTaskRepository; + @Mock + private CleanUpDetsDataAutomatedTaskConfig automatedTaskConfigurationProperties; + @Mock + private CleanUpDetsDataProcessor cleanUpDetsDataProcessor; + @Mock + private LogApi logApi; + @Mock + private LockService lockService; + + @InjectMocks + private CleanUpDetsDataAutomatedTask cleanUpDetsDataAutomatedTask; + + @Test + void positiveGetAutomatedTaskName() { + assertEquals(CLEAN_UP_DETS_DATA, cleanUpDetsDataAutomatedTask.getAutomatedTaskName()); + } + + @Test + void positiveRunTask() { + CleanUpDetsDataAutomatedTask automatedTask = spy(cleanUpDetsDataAutomatedTask); + int batchSize = 25; + doReturn(batchSize).when(automatedTask).getAutomatedTaskBatchSize(); + + automatedTask.runTask(); + + verify(automatedTask).getAutomatedTaskBatchSize(); + verify(cleanUpDetsDataProcessor).processCleanUpDetsData(batchSize, automatedTaskConfigurationProperties); + } +} + From 70977160664ba7243163e4d41a63bd53b759e0dd Mon Sep 17 00:00:00 2001 From: benedwards Date: Sat, 28 Feb 2026 01:36:06 +0000 Subject: [PATCH 5/7] Fixed styles --- .../service/impl/CleanUpDetsDataProcessorImplIntTest.java | 6 +----- .../gov/hmcts/darts/testutils/stubs/DartsPersistence.java | 2 +- .../hmcts/darts/arm/service/CleanUpDetsDataProcessor.java | 1 + .../arm/service/impl/CleanUpDetsDataProcessorImpl.java | 2 ++ .../common/repository/ObjectStateRecordRepository.java | 6 ++++-- .../arm/service/impl/CleanUpDetsDataProcessorImplTest.java | 1 - .../java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java | 6 +++++- 7 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java index 3e53d5d28f1..f269bbc9cee 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java @@ -31,7 +31,6 @@ import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import static uk.gov.hmcts.darts.test.common.data.ExternalLocationTypeTestData.locationTypeOf; import static uk.gov.hmcts.darts.test.common.data.ObjectRecordStatusTestData.statusOf; import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getAnnotationDocumentTestData; @@ -233,15 +232,12 @@ protected void assignContextAwareToexternalObjectDirectoryEntity(ExternalObjectD @Getter public abstract class TestData { - public static long uniqueCounter = 0; + private static long uniqueCounter; ExternalObjectDirectoryEntity detsEod; ExternalObjectDirectoryEntity armEod; ObjectStateRecordEntity objectStateRecordEntity; T confidenceAware; - public TestData() { - } - public TestData createDataThatShouldBeCleanedUp() { return createDataThatShouldBeCleanedUp(true); } diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java index 9349ce17a32..5384f0a9a4b 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java @@ -9,7 +9,6 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Service; import uk.gov.hmcts.darts.audio.entity.MediaRequestEntity; @@ -913,6 +912,7 @@ public void overrideLastModifiedBy(MediaRequestEntity mediaRequestEntity, Offset .setParameter("id", mediaRequestEntity.getId()) .executeUpdate(); } + @Transactional public void overrideLastModifiedBy(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, OffsetDateTime lastModifiedDate) { entityManager.createNativeQuery("UPDATE external_object_directory SET last_modified_ts = :lastModifiedDate WHERE eod_id = :id") diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java b/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java index bddf5b0e4ee..700613c8597 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java @@ -2,6 +2,7 @@ import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +@FunctionalInterface public interface CleanUpDetsDataProcessor { void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskConfig minimumStoredAge); } diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java index 786ab91f77e..ffc6871b4af 100644 --- a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java @@ -33,6 +33,8 @@ public class CleanUpDetsDataProcessorImpl implements CleanUpDetsDataProcessor { private final Clock clock; @Override + //Required for async batch processing + @SuppressWarnings({"PMD.DoNotUseThreads"}) public void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskConfig config) { log.info("Processing clean up of DETS data with batch size: {}", batchSize); diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java index def019f5603..28e4cb6ec68 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java +++ b/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java @@ -19,7 +19,9 @@ public interface ObjectStateRecordRepository extends JpaRepository uuids); - @Query(value = "SELECT osr_uuid AS osrUuid, dets_location AS detsLocation FROM dets_cleanup_eod_osr_rows(:limit, :last_modified_before_ts)", nativeQuery = true) - List cleanUpDetsDataProcedure(@Param("limit") Integer limit, @Param("last_modified_before_ts") OffsetDateTime lastModifiedBefore); + @Query(value = "SELECT osr_uuid AS osrUuid, dets_location AS detsLocation " + + "FROM dets_cleanup_eod_osr_rows(:limit, :last_modified_before_ts)", nativeQuery = true) + List cleanUpDetsDataProcedure( + @Param("limit") Integer limit, @Param("last_modified_before_ts") OffsetDateTime lastModifiedBefore); } diff --git a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java index af663f0a763..3dbe1af266b 100644 --- a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java +++ b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java @@ -147,7 +147,6 @@ void shouldStopProcessingWhenBatchSizeLimitReached() { List firstBatch = List.of( response(1L), response(2L), response(3L), response(4L) ); - //Second batch should be ignored as batch size limit of 4 will have been reached after processing first batch, even though repository returns more data List secondBatch = List.of( response(5L), response(6L), response(7L), response(8L) ); diff --git a/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java b/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java index 2bf23b8c91d..8ed4892816b 100644 --- a/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java +++ b/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java @@ -10,7 +10,11 @@ import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.mockStatic; -public class AsyncTestUtil { +public final class AsyncTestUtil { + + private AsyncTestUtil() { + + } public static void processTasksSynchronously(Runnable runnable) { try (MockedStatic asyncUtilMock = mockStatic(AsyncUtil.class)) { From fd5930dd5af2e4c045c9fb747c65136346e4bd40 Mon Sep 17 00:00:00 2001 From: benedwards Date: Mon, 2 Mar 2026 09:44:46 +0000 Subject: [PATCH 6/7] Migration name update --- ...ated_task.sql => V1_503__CleanUpDetsData_automated_task.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/main/resources/db/migration/postgres/{V1_501__CleanUpDetsData_automated_task.sql => V1_503__CleanUpDetsData_automated_task.sql} (99%) diff --git a/src/main/resources/db/migration/postgres/V1_501__CleanUpDetsData_automated_task.sql b/src/main/resources/db/migration/postgres/V1_503__CleanUpDetsData_automated_task.sql similarity index 99% rename from src/main/resources/db/migration/postgres/V1_501__CleanUpDetsData_automated_task.sql rename to src/main/resources/db/migration/postgres/V1_503__CleanUpDetsData_automated_task.sql index 3473aca989c..805a6c1f2c4 100644 --- a/src/main/resources/db/migration/postgres/V1_501__CleanUpDetsData_automated_task.sql +++ b/src/main/resources/db/migration/postgres/V1_503__CleanUpDetsData_automated_task.sql @@ -1,6 +1,6 @@ INSERT INTO darts.automated_task (aut_id, task_name, task_description, cron_expression, cron_editable, batch_size, created_ts, created_by, last_modified_ts, last_modified_by, task_enabled) -VALUES (35, 'CleanUpDetsData', 'Cleans up Dets files that have successfully been stored in ARM', '0 24 0-6,19-23 ? * *', true, 100_000, +VALUES (36, 'CleanUpDetsData', 'Cleans up Dets files that have successfully been stored in ARM', '0 24 0-6,19-23 ? * *', true, 100_000, current_timestamp, 0, current_timestamp, 0, false); INSERT INTO user_account (usr_id, user_name, user_email_address, description, created_ts, last_modified_ts, last_modified_by, created_by, is_system_user, From 7b169b12c4edb54f2fe31dca84a57cf22564f23b Mon Sep 17 00:00:00 2001 From: benedwards Date: Mon, 2 Mar 2026 09:55:55 +0000 Subject: [PATCH 7/7] Migration update --- .../common/V1_505__CleanUpDetsData_automated_task.sql | 10 ++++++++++ ...l => V1_506__CleanUpDetsData_stored_procedure.sql} | 11 ----------- 2 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 src/main/resources/db/migration/common/V1_505__CleanUpDetsData_automated_task.sql rename src/main/resources/db/migration/postgres/{V1_503__CleanUpDetsData_automated_task.sql => V1_506__CleanUpDetsData_stored_procedure.sql} (88%) diff --git a/src/main/resources/db/migration/common/V1_505__CleanUpDetsData_automated_task.sql b/src/main/resources/db/migration/common/V1_505__CleanUpDetsData_automated_task.sql new file mode 100644 index 00000000000..f8a6872e152 --- /dev/null +++ b/src/main/resources/db/migration/common/V1_505__CleanUpDetsData_automated_task.sql @@ -0,0 +1,10 @@ +INSERT INTO darts.automated_task (aut_id, task_name, task_description, cron_expression, cron_editable, batch_size, + created_ts, created_by, last_modified_ts, last_modified_by, task_enabled) +VALUES (36, 'CleanUpDetsData', 'Cleans up Dets files that have successfully been stored in ARM', '0 24 0-6,19-23 ? * *', true, 100_000, + current_timestamp, 0, current_timestamp, 0, false); + +INSERT INTO user_account (usr_id, user_name, user_email_address, description, created_ts, last_modified_ts, last_modified_by, created_by, is_system_user, + is_active, user_full_name) + +VALUES (-51, 'systemCleanUpDetsDataAutomatedTask', 'systemCleanUpDetsDataAutomatedTask@hmcts.net', 'systemCleanUpDetsDataAutomatedTask', + '2024-01-01 00:00:00+00', '2024-01-01 00:00:00+00', 0, 0, true, true, 'systemCleanUpDetsDataAutomatedTask'); \ No newline at end of file diff --git a/src/main/resources/db/migration/postgres/V1_503__CleanUpDetsData_automated_task.sql b/src/main/resources/db/migration/postgres/V1_506__CleanUpDetsData_stored_procedure.sql similarity index 88% rename from src/main/resources/db/migration/postgres/V1_503__CleanUpDetsData_automated_task.sql rename to src/main/resources/db/migration/postgres/V1_506__CleanUpDetsData_stored_procedure.sql index 805a6c1f2c4..110339d1262 100644 --- a/src/main/resources/db/migration/postgres/V1_503__CleanUpDetsData_automated_task.sql +++ b/src/main/resources/db/migration/postgres/V1_506__CleanUpDetsData_stored_procedure.sql @@ -1,14 +1,3 @@ -INSERT INTO darts.automated_task (aut_id, task_name, task_description, cron_expression, cron_editable, batch_size, - created_ts, created_by, last_modified_ts, last_modified_by, task_enabled) -VALUES (36, 'CleanUpDetsData', 'Cleans up Dets files that have successfully been stored in ARM', '0 24 0-6,19-23 ? * *', true, 100_000, - current_timestamp, 0, current_timestamp, 0, false); - -INSERT INTO user_account (usr_id, user_name, user_email_address, description, created_ts, last_modified_ts, last_modified_by, created_by, is_system_user, - is_active, user_full_name) - -VALUES (-40, 'systemCleanUpDetsDataAutomatedTask', 'systemCleanUpDetsDataAutomatedTask@hmcts.net', 'systemCleanUpDetsDataAutomatedTask', - '2024-01-01 00:00:00+00', '2024-01-01 00:00:00+00', 0, 0, true, true, 'systemCleanUpDetsDataAutomatedTask'); - SET SEARCH_PATH TO darts; DROP FUNCTION IF EXISTS dets_cleanup_eod_osr_rows;