diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java new file mode 100644 index 00000000000..f269bbc9cee --- /dev/null +++ b/src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplIntTest.java @@ -0,0 +1,340 @@ +package uk.gov.hmcts.darts.arm.service.impl; + +import lombok.Getter; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Isolated; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import uk.gov.hmcts.darts.common.entity.AnnotationDocumentEntity; +import uk.gov.hmcts.darts.common.entity.CaseDocumentEntity; +import uk.gov.hmcts.darts.common.entity.ExternalObjectDirectoryEntity; +import uk.gov.hmcts.darts.common.entity.MediaEntity; +import uk.gov.hmcts.darts.common.entity.ObjectStateRecordEntity; +import uk.gov.hmcts.darts.common.entity.TranscriptionDocumentEntity; +import uk.gov.hmcts.darts.common.enums.ExternalLocationTypeEnum; +import uk.gov.hmcts.darts.common.enums.ObjectRecordStatusEnum; +import uk.gov.hmcts.darts.dets.service.DetsApiService; +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +import uk.gov.hmcts.darts.testutils.PostgresIntegrationBase; + +import java.time.Clock; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +import static java.time.temporal.ChronoUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static uk.gov.hmcts.darts.test.common.data.ExternalLocationTypeTestData.locationTypeOf; +import static uk.gov.hmcts.darts.test.common.data.ObjectRecordStatusTestData.statusOf; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getAnnotationDocumentTestData; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getCaseDocumentTestData; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getExternalObjectDirectoryTestData; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getMediaTestData; +import static uk.gov.hmcts.darts.test.common.data.PersistableFactory.getTranscriptionDocument; + +@Isolated +class CleanUpDetsDataProcessorImplIntTest extends PostgresIntegrationBase { + private static final Duration DEFAULT_TEST_DATA_AGE = Duration.ofHours(10); + + @Autowired + private CleanUpDetsDataProcessorImpl cleanUpDetsDataProcessor; + + @Autowired + private Clock clock; + + @MockitoBean + private DetsApiService detsApiService; + + + @Test + void cleanUpDetsData_typical() { + //Valid data that should be cleaned up + List> validData = List.of( + new MediaTestData().createDataThatShouldBeCleanedUp(), + new TranscriptionDocumentTestData().createDataThatShouldBeCleanedUp(), + new AnnotationDocumentTestData().createDataThatShouldBeCleanedUp(), + new CaseDocumentTestData().createDataThatShouldBeCleanedUp() + ); + //Inject some data that should not be picked up + List> inelibileData = List.of( + new MediaTestData().createDetsRecordButWithoutArmRecord(), + new MediaTestData().createDetsRecordButWithArmRecordButNotInStoredStatus(), + new MediaTestData().createDetsRecordButWithArmStoredButWithinMiniumStoredTime(), + new AnnotationDocumentTestData().createDetsRecordButOsrAndDetsEodLocationsDoNotAlign() + ); + + cleanUpDetsDataProcessor.processCleanUpDetsData(20, getCleanUpDetsDataAutomatedTaskConfig()); + + assertSuccessfull(validData); + assertNoChange(inelibileData); + } + + @Test + void cleanUpDetsData_shouldPickUpAnyUnprocessedDeletes() { + //Valid data that should be cleaned up + TestData fullCleanUpData1 = new MediaTestData().createDataThatShouldBeCleanedUp(true); + TestData fullCleanUpData2 = new MediaTestData().createDataThatShouldBeCleanedUp(true); + TestData partialCleanUpData1 = new MediaTestData().createDataThatShouldBeCleanedUp(false); + + + cleanUpDetsDataProcessor.processCleanUpDetsData(3, getCleanUpDetsDataAutomatedTaskConfig()); + + assertSuccessfull(fullCleanUpData1); + assertPartialSuccess(partialCleanUpData1); + assertSuccessfull(fullCleanUpData2); + + //Reset mock + //Check to ensure that the failed blob delete is attempted again but the successful database deletes are not attempted again + Mockito.clearInvocations(detsApiService); + partialCleanUpData1.stubBlobStoreDelete(true); + TestData fullCleanUpData3 = new MediaTestData().createDataThatShouldBeCleanedUp(true); + + cleanUpDetsDataProcessor.processCleanUpDetsData(20, getCleanUpDetsDataAutomatedTaskConfig()); + assertSuccessfull(partialCleanUpData1); + assertSuccessfull(fullCleanUpData3); + } + + + private void assertNoChange(List> testDataList) { + testDataList.forEach(this::assertNoChange); + } + + @SneakyThrows + private void assertNoChange(TestData testData) { + assertCommon(testData); + //Check that the EOD records still exist + assertThat(dartsPersistence.getExternalObjectDirectoryRepository().existsById(testData.getDetsEod().getId())) + .as("DETS EOD record should still exist").isTrue(); + //Check that the Object State Record still exists and has the same dets eod id and arm eod id + ObjectStateRecordEntity objectStateRecordEntity = dartsPersistence.getObjectStateRecordRepository() + .findById(testData.getObjectStateRecordEntity().getUuid()).orElseThrow(); + assertThat(objectStateRecordEntity.getEodId()).isEqualTo(testData.getDetsEod().getId()); + assertThat(objectStateRecordEntity.getDateFileDetsCleanup()).isNull(); + assertThat(objectStateRecordEntity.getFlagFileDetsCleanupStatus()).isNull(); + verify(detsApiService, never()).deleteBlobDataFromContainer(testData.getDetsEod().getLocation()); + } + + private void assertSuccessfull(List> testDataList) { + testDataList.forEach(this::assertSuccessfull); + } + + @SneakyThrows + private void assertSuccessfull(TestData testData) { + assertCommon(testData); + //Check that the Dets EOD records has been deleted + assertThat(dartsPersistence.getExternalObjectDirectoryRepository().existsById(testData.getDetsEod().getId())) + .as("DETS EOD record should not exist").isFalse(); + //Check that the Object State Record has been updated + ObjectStateRecordEntity objectStateRecordEntity = dartsPersistence.getObjectStateRecordRepository() + .findById(testData.getObjectStateRecordEntity().getUuid()).orElseThrow(); + assertThat(objectStateRecordEntity.getEodId()).isNull(); + assertThat(objectStateRecordEntity.getDateFileDetsCleanup()) + .isCloseTo(OffsetDateTime.now(), within(1, SECONDS)); + assertThat(objectStateRecordEntity.getFlagFileDetsCleanupStatus()).isTrue(); + verify(detsApiService).deleteBlobDataFromContainer(testData.getDetsEod().getLocation()); + } + + @SneakyThrows + private void assertPartialSuccess(TestData testData) { + assertCommon(testData); + //Check that the Dets EOD records has been deleted + assertThat(dartsPersistence.getExternalObjectDirectoryRepository().existsById(testData.getDetsEod().getId())) + .as("DETS EOD record should not exist").isFalse(); + //Check that the Object State Record has been updated + ObjectStateRecordEntity objectStateRecordEntity = dartsPersistence.getObjectStateRecordRepository() + .findById(testData.getObjectStateRecordEntity().getUuid()).orElseThrow(); + assertThat(objectStateRecordEntity.getEodId()).isNull(); + assertThat(objectStateRecordEntity.getDateFileDetsCleanup()) + .isCloseTo(OffsetDateTime.now(), within(1, SECONDS)); + verify(detsApiService).deleteBlobDataFromContainer(testData.getDetsEod().getLocation()); + + //If database delete was successful but blob store was not this flag should not be set + assertThat(objectStateRecordEntity.getFlagFileDetsCleanupStatus()).isNull(); + } + + private void assertCommon(TestData testData) { + //Check that the ARM EOD record still exists + if (testData.getArmEod() != null) { + assertThat(dartsPersistence.getExternalObjectDirectoryRepository().existsById(testData.getArmEod().getId())) + .as("ARM EOD record should still exist").isTrue(); + } + } + + private CleanUpDetsDataAutomatedTaskConfig getCleanUpDetsDataAutomatedTaskConfig() { + CleanUpDetsDataAutomatedTaskConfig config = new CleanUpDetsDataAutomatedTaskConfig(); + config.setMinimumStoredAge(DEFAULT_TEST_DATA_AGE); + config.setChunkSize(10); + config.setThreads(2); + config.setAsyncTimeout(Duration.ofMinutes(5)); + return config; + } + + //Test Data + public class MediaTestData extends TestData { + @Override + protected MediaEntity createConfidenceAware() { + return dartsPersistence.save(getMediaTestData().someMinimal()); + } + + @Override + protected void assignContextAwareToexternalObjectDirectoryEntity(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, + MediaEntity confidenceAware) { + externalObjectDirectoryEntity.setMedia(confidenceAware); + } + } + + public class TranscriptionDocumentTestData extends TestData { + @Override + protected TranscriptionDocumentEntity createConfidenceAware() { + return dartsPersistence.save(getTranscriptionDocument().someMinimal()); + } + + @Override + protected void assignContextAwareToexternalObjectDirectoryEntity(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, + TranscriptionDocumentEntity confidenceAware) { + externalObjectDirectoryEntity.setTranscriptionDocumentEntity(confidenceAware); + } + } + + public class AnnotationDocumentTestData extends TestData { + @Override + protected AnnotationDocumentEntity createConfidenceAware() { + return dartsPersistence.save(getAnnotationDocumentTestData().someMinimal()); + } + + @Override + protected void assignContextAwareToexternalObjectDirectoryEntity(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, + AnnotationDocumentEntity confidenceAware) { + externalObjectDirectoryEntity.setAnnotationDocumentEntity(confidenceAware); + } + } + + public class CaseDocumentTestData extends TestData { + @Override + protected CaseDocumentEntity createConfidenceAware() { + return dartsPersistence.save(getCaseDocumentTestData().someMinimal()); + } + + @Override + protected void assignContextAwareToexternalObjectDirectoryEntity(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, + CaseDocumentEntity confidenceAware) { + externalObjectDirectoryEntity.setCaseDocument(confidenceAware); + } + } + + + @Getter + public abstract class TestData { + private static long uniqueCounter; + ExternalObjectDirectoryEntity detsEod; + ExternalObjectDirectoryEntity armEod; + ObjectStateRecordEntity objectStateRecordEntity; + T confidenceAware; + + public TestData createDataThatShouldBeCleanedUp() { + return createDataThatShouldBeCleanedUp(true); + } + + public TestData createDataThatShouldBeCleanedUp(boolean shouldBlobDeleteSucceed) { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + armEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.ARM, ObjectRecordStatusEnum.STORED, confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, armEod); + stubBlobStoreDelete(shouldBlobDeleteSucceed); + return this; + } + + public TestData createDetsRecordButWithoutArmRecord() { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, null); + stubBlobStoreDelete(true); + return this; + } + + public TestData createDetsRecordButWithArmRecordButNotInStoredStatus() { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + armEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.ARM, ObjectRecordStatusEnum.ARM_DROP_ZONE, confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, armEod); + stubBlobStoreDelete(true); + return this; + } + + public TestData createDetsRecordButWithArmStoredButWithinMiniumStoredTime() { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + armEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.ARM, ObjectRecordStatusEnum.ARM_DROP_ZONE, OffsetDateTime.now(), + confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, armEod); + stubBlobStoreDelete(true); + return this; + } + + public TestData createDetsRecordButOsrAndDetsEodLocationsDoNotAlign() { + confidenceAware = createConfidenceAware(); + detsEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.DETS, ObjectRecordStatusEnum.STORED, confidenceAware); + armEod = createExternalObjectDirectoryEntity(ExternalLocationTypeEnum.ARM, ObjectRecordStatusEnum.ARM_DROP_ZONE, confidenceAware); + objectStateRecordEntity = createObjectStateRecordEntity(detsEod, armEod); + objectStateRecordEntity.setDetsLocation("Some random location that does not align with the dets eod location"); + dartsPersistence.getObjectStateRecordRepository().save(objectStateRecordEntity); + stubBlobStoreDelete(true); + return this; + } + + @SneakyThrows + public void stubBlobStoreDelete(boolean shouldBlobDeleteSucceed) { + lenient().when(detsApiService.deleteBlobDataFromContainer(detsEod.getLocation())) + .thenReturn(shouldBlobDeleteSucceed); + } + + + private ObjectStateRecordEntity createObjectStateRecordEntity(ExternalObjectDirectoryEntity detsEod, ExternalObjectDirectoryEntity armEod) { + ObjectStateRecordEntity osrEntity = new ObjectStateRecordEntity(); + osrEntity.setUuid(uniqueCounter++); + osrEntity.setEodId(detsEod.getId()); + if (armEod != null) { + osrEntity.setArmEodId(armEod.getId()); + } + osrEntity.setDetsLocation(detsEod.getLocation()); + return dartsPersistence.getObjectStateRecordRepository().save(osrEntity); + } + + public ExternalObjectDirectoryEntity createExternalObjectDirectoryEntity(ExternalLocationTypeEnum externalLocationTypeEnum, + ObjectRecordStatusEnum objectRecordStatusEnum, + T confidenceAware) { + return createExternalObjectDirectoryEntity(externalLocationTypeEnum, + objectRecordStatusEnum, + OffsetDateTime.now().minus(DEFAULT_TEST_DATA_AGE).minusHours(1), + confidenceAware); + } + + public ExternalObjectDirectoryEntity createExternalObjectDirectoryEntity(ExternalLocationTypeEnum externalLocationTypeEnum, + ObjectRecordStatusEnum objectRecordStatusEnum, + OffsetDateTime lastModifiedDateTime, + T confidenceAware) { + ExternalObjectDirectoryEntity externalObjectDirectoryEntity = getExternalObjectDirectoryTestData().someMinimal(); + externalObjectDirectoryEntity.setStatus(statusOf(objectRecordStatusEnum)); + externalObjectDirectoryEntity.setExternalLocationType(locationTypeOf(externalLocationTypeEnum)); + externalObjectDirectoryEntity.setExternalLocation(UUID.randomUUID().toString()); + externalObjectDirectoryEntity.setLastModifiedDateTime(lastModifiedDateTime); + assignContextAwareToexternalObjectDirectoryEntity(externalObjectDirectoryEntity, confidenceAware); + + externalObjectDirectoryEntity = dartsPersistence.save(externalObjectDirectoryEntity); + dartsPersistence.overrideLastModifiedBy(externalObjectDirectoryEntity, lastModifiedDateTime); + return externalObjectDirectoryEntity; + } + + protected abstract T createConfidenceAware(); + + protected abstract void assignContextAwareToexternalObjectDirectoryEntity( + ExternalObjectDirectoryEntity externalObjectDirectoryEntity, T confidenceAware); + } +} diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsDatabaseStub.java b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsDatabaseStub.java index cd840b1e7c9..0d73196687c 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsDatabaseStub.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsDatabaseStub.java @@ -336,6 +336,7 @@ public void clearDatabaseInThisOrder() { transcriptionRepository.deleteAllInBatch(); transcriptionWorkflowRepository.deleteAllInBatch(); retentionConfidenceCategoryMapperRepository.deleteAllInBatch(); + objectStateRecordRepository.deleteAllInBatch(); }); } diff --git a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java index f6b2806bafc..5384f0a9a4b 100644 --- a/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java +++ b/src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/DartsPersistence.java @@ -79,6 +79,7 @@ import uk.gov.hmcts.darts.common.repository.ObjectAdminActionRepository; import uk.gov.hmcts.darts.common.repository.ObjectHiddenReasonRepository; import uk.gov.hmcts.darts.common.repository.ObjectRecordStatusRepository; +import uk.gov.hmcts.darts.common.repository.ObjectStateRecordRepository; import uk.gov.hmcts.darts.common.repository.ProsecutorRepository; import uk.gov.hmcts.darts.common.repository.RegionRepository; import uk.gov.hmcts.darts.common.repository.RetentionConfidenceCategoryMapperRepository; @@ -170,6 +171,7 @@ public class DartsPersistence { private final ObjectAdminActionRepository objectAdminActionRepository; private final EventLinkedCaseRepository eventLinkedCaseRepository; private final RetentionConfidenceCategoryMapperRepository retentionConfidenceCategoryMapperRepository; + private final ObjectStateRecordRepository objectStateRecordRepository; private final EntityManager entityManager; private final CurrentTimeHelper currentTimeHelper; @@ -911,6 +913,14 @@ public void overrideLastModifiedBy(MediaRequestEntity mediaRequestEntity, Offset .executeUpdate(); } + @Transactional + public void overrideLastModifiedBy(ExternalObjectDirectoryEntity externalObjectDirectoryEntity, OffsetDateTime lastModifiedDate) { + entityManager.createNativeQuery("UPDATE external_object_directory SET last_modified_ts = :lastModifiedDate WHERE eod_id = :id") + .setParameter("lastModifiedDate", lastModifiedDate) + .setParameter("id", externalObjectDirectoryEntity.getId()) + .executeUpdate(); + } + @Transactional public & CreatedBy> void updateCreatedBy(T object, OffsetDateTime createdDateTime) { Table table = object.getClass().getAnnotation(Table.class); diff --git a/src/integrationTest/resources/db/tests/DETS_Cleanup_SP_DB_Unit_tests.sql b/src/integrationTest/resources/db/tests/DETS_Cleanup_SP_DB_Unit_tests.sql new file mode 100644 index 00000000000..2682a220d81 --- /dev/null +++ b/src/integrationTest/resources/db/tests/DETS_Cleanup_SP_DB_Unit_tests.sql @@ -0,0 +1,587 @@ +--psql -h -p 5432 -U -d darts -f DETS_Cleanup_SP_DB_Unit_tests.sql 1> DETS_Cleanup_SP_DB_Unit_tests.log 2>>&1 + +---------------------------------------------------------------------- +-- ONE-TIME SETUP +---------------------------------------------------------------------- + +CREATE SCHEMA IF NOT EXISTS test_dets_cleanup; + +CREATE TABLE IF NOT EXISTS test_dets_cleanup.external_object_directory +(LIKE darts.external_object_directory INCLUDING ALL); + +CREATE TABLE IF NOT EXISTS test_dets_cleanup.object_state_record +(LIKE darts.object_state_record INCLUDING ALL); + +---------------------------------------------------------------------- +-- TEST HARNESS +---------------------------------------------------------------------- + +-- Route unqualified table names to test_dets_cleanup tables first +SET search_path TO test_dets_cleanup, darts; + +-- Following values taken from Prod +SET max_parallel_workers_per_gather=4; +SET work_mem='4096kB'; +SET max_parallel_maintenance_workers=64; + +SELECT name, setting, unit, SOURCE, reset_val --, boot_val +FROM pg_settings +WHERE name IN ('search_path', 'max_parallel_workers_per_gather', 'work_mem', 'max_parallel_maintenance_workers') +ORDER BY name; + + +-- Helper to clear test tables between tests +CREATE OR REPLACE PROCEDURE test_dets_cleanup.reset_test_data() LANGUAGE plpgsql AS $$ +BEGIN + TRUNCATE TABLE test_dets_cleanup.object_state_record; + TRUNCATE TABLE test_dets_cleanup.external_object_directory; + + RAISE NOTICE '-------------- RESET ---------------'; +END; +$$; + +---------------------------------------------------------------------- +-- TEST 1: No matching records +-- Expect: 0 results, empty array returned +---------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- Noise ARM EODs (wrong ors_id), Includes valid records but outside of date range +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3001, 8980069, 2, 3, NULL, now(), now() - interval '1 day', -99, -99, FALSE, FALSE, FALSE), + (3002, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE), + (3003, 8980069, 1, 3, NULL, now(), now() - interval '7 days', -99, -99, FALSE, FALSE, FALSE), + (3004, 8980069, 2, 3, NULL, now(), now() - interval '5 days', -99, -99, FALSE, FALSE, FALSE), + (3005, 8980069, 1, 3, NULL, now(), now() - interval '20 days', -99, -99, FALSE, FALSE, FALSE); + +-- Noise DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4001, 8980069, 2, 4, 'LOC-N1', now(), now() - interval '1 day', -99, -99, FALSE, FALSE, TRUE), + (4002, 8980069, 2, 4, 'LOC-N2', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4003, 8980069, 2, 4, 'LOC-N3', now(), now() - interval '7 days', -99, -99, FALSE, FALSE, TRUE), + (4004, 8980069, 2, 4, 'LOC-N4', now(), now() - interval '5 days', -99, -99, FALSE, FALSE, TRUE), + (4005, 8980069, 2, 4, 'LOC-N5', now(), now() - interval '20 days', -99, -99, FALSE, FALSE, TRUE); + +-- Noise OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (10001, 4001, 3001, 'LOC-N1'), + (10002, 4002, 3002, 'LOC-N2'), + (10003, 4003, 3003, 'LOC-N3'), + (10004, 4004, 3004, 'LOC-N4'), + (10005, 4005, 3005, 'LOC-N5'); + +DO $$ + DECLARE + results darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 1'; + RAISE NOTICE '------------------------------------'; + + CALL darts.dets_cleanup_eod_osr(10, now() - interval '7 days', results); + + --ASSERT (results IS NULL OR cardinality(results) = 0), 'TEST 1 FAILED: expected empty results but got non-empty array'; + + -- Check results + IF results IS NULL OR cardinality(results) > 0 THEN + RAISE NOTICE 'TEST 1 FAILED: expected 0 results but got %', cardinality(results); + RETURN; + END IF; + + RAISE NOTICE 'TEST 1 PASSED'; + END $$; + +---------------------------------------------------------------------- +-- TEST 2: Normal case (5 valid pairs + noise) +---------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- 5 valid ARM EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3101, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3102, 8980069, 2, 3, NULL, now(), now() - interval '7 days', -99, -99, FALSE, FALSE, FALSE), + (3103, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3104, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3105, 8980069, 2, 3, NULL, now(), now() - interval '20 days', -99, -99, FALSE, FALSE, FALSE); + +-- 5 valid DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4101, 8980069, 2, 4, 'LOC-A1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4102, 8980069, 2, 4, 'LOC-A2', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4103, 8980069, 2, 4, 'LOC-A3', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4104, 8980069, 2, 4, 'LOC-A4', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4105, 8980069, 2, 4, 'LOC-A5', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- 5 valid OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (11001, 4101, 3101, 'LOC-A1'), + (11002, 4102, 3102, 'LOC-A2'), + (11003, 4103, 3103, 'LOC-A3'), + (11004, 4104, 3104, 'LOC-A4'), + (11005, 4105, 3105, 'LOC-A5'); + +-- Noise ARM (older than 7 days but invalid ors_id + within 7 days and valid ors_id + valid records apart from difference location) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3191, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE), + (3192, 8980069, 2, 3, NULL, now(), now() - interval '1 days', -99, -99, FALSE, FALSE, FALSE), + (3193, 8980069, 2, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE); + +-- Noise DETS +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4191, 8980069, 2, 4, 'LOC-Z1', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4192, 8980069, 2, 4, 'LOC-Z2', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4193, 8980069, 2, 4, 'LOC-ZZ Different', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE); + +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES (11901, 4191, 3191, 'LOC-Z1'), + (11902, 4192, 3192, 'LOC-Z2'), + (11903, 4193, 3193, 'LOC-ZZ'); + +DO $$ + DECLARE + results darts.id_location_pair[]; + expected darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 2'; + RAISE NOTICE '------------------------------------'; + + expected := ARRAY[ + ROW(11001, 'LOC-A1')::darts.id_location_pair, + ROW(11002, 'LOC-A2')::darts.id_location_pair, + ROW(11003, 'LOC-A3')::darts.id_location_pair, + ROW(11004, 'LOC-A4')::darts.id_location_pair, + ROW(11005, 'LOC-A5')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(10, now() - interval '7 days', results); + + --ASSERT cardinality(results) = 5, 'TEST 2 FAILED: expected 5 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 2 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 5 THEN + RAISE NOTICE 'TEST 2 FAILED: expected 5 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 2 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 2 PASSED'; + + END $$; + +---------------------------------------------------------------------------------------- +-- TEST 3: Limit behavior (limit = 3), more available +-- Test includes a check where the last_modified_ts is exactly the same as the parameter +---------------------------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- 5 valid ARM EODs (one exact timestamp match) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3201, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3202, 8980069, 2, 3, NULL, now(), (now()::date - 7)::timestamp, -99, -99, FALSE, FALSE, FALSE), -- exact match + (3203, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3204, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3205, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE); + +-- 5 valid DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4201, 8980069, 2, 4, 'LOC-B1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4202, 8980069, 2, 4, 'LOC-B2 - Exact TS', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4203, 8980069, 2, 4, 'LOC-B3', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4204, 8980069, 2, 4, 'LOC-B4', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4205, 8980069, 2, 4, 'LOC-B5', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- 5 valid OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (12001, 4201, 3201, 'LOC-B1'), + (12002, 4202, 3202, 'LOC-B2 - Exact TS'), + (12003, 4203, 3203, 'LOC-B3'), + (12004, 4204, 3204, 'LOC-B4'), + (12005, 4205, 3205, 'LOC-B5'); + +-- Noise ARM (older than 7 days but invalid ors_id) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3291, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE); + +-- Noise DETS +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4291, 8980069, 1, 4, 'LOC-Z2', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE); + +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES (12901, 4291, 3291, 'LOC-Z2'); + +DO $$ + DECLARE + results darts.id_location_pair[]; + expected darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 3'; + RAISE NOTICE '------------------------------------'; + + expected := ARRAY[ + ROW(12001, 'LOC-B1')::darts.id_location_pair, + ROW(12002, 'LOC-B2 - Exact TS')::darts.id_location_pair, + ROW(12003, 'LOC-B3')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, (now()::date - 7)::timestamp, results); + + --ASSERT cardinality(results) = 3, 'TEST 3 FAILED: expected 3 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 3 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 3 THEN + RAISE NOTICE 'TEST 3 FAILED: expected 3 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 3 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 3 PASSED'; + END $$; + +---------------------------------------------------------------------------------------- +-- TEST 4: Two-call behavior +-- Call 1: limit = 3 - expect first 3 valid pairs +-- Call 2: limit = 3 - expect 2 valid pairs + 3 incomplete cleanup, from call 1 +-- Test includes a check where the last_modified_ts is exactly the same as the parameter +---------------------------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- 5 valid ARM EODs (one exact timestamp match) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3301, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3302, 8980069, 2, 3, NULL, now(), (now()::date - 7)::timestamp, -99, -99, FALSE, FALSE, FALSE), -- exact match check + (3303, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3304, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3305, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE); + +-- 5 valid DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4301, 8980069, 2, 4, 'LOC-C1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4302, 8980069, 2, 4, 'LOC-C2', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4303, 8980069, 2, 4, 'LOC-C3', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4304, 8980069, 2, 4, 'LOC-C4', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4305, 8980069, 2, 4, 'LOC-C5', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- 5 valid OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (13001, 4301, 3301, 'LOC-C1'), + (13002, 4302, 3302, 'LOC-C2'), + (13003, 4303, 3303, 'LOC-C3'), + (13004, 4304, 3304, 'LOC-C4'), + (13005, 4305, 3305, 'LOC-C5'); + +-- Noise ARM (older than 7 days but invalid ors_id + not older than 7 days but valid ors_id) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3391, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE), + (3392, 8980069, 2, 3, NULL, now(), now() - interval '2 days', -99, -99, FALSE, FALSE, FALSE); + +-- DETS +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4391, 8980069, 2, 4, 'LOC-Z3', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4392, 8980069, 2, 4, 'LOC-Z4', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE); + +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES (13901, 4391, 3391, 'LOC-Z3'), + (13902, 4392, 3392, 'LOC-Z4'); + +-- 2 calls: limit = 3 +DO $$ + DECLARE + results darts.id_location_pair[]; + expected darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 4'; + RAISE NOTICE '------------------------------------'; + + --Call 1: limit 3 - Should behave normally + expected := ARRAY[ + ROW(13001, 'LOC-C1')::darts.id_location_pair, + ROW(13002, 'LOC-C2')::darts.id_location_pair, + ROW(13003, 'LOC-C3')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, (now()::date - 7)::timestamp, results); + + --ASSERT cardinality(results) = 3, 'TEST 4 CALL 1 FAILED: expected 3 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 4 CALL 1 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 3 THEN + RAISE NOTICE 'TEST 4 CALL 1 FAILED: expected 3 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 4 CALL 1 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 4 CALL 1 PASSED'; + + --Call 2: limit = 3 - Expect the other 2 valid pair and the 3 from call 1, because their flag hasn't been set) + expected := ARRAY[ + ROW(13004, 'LOC-C4')::darts.id_location_pair, + ROW(13005, 'LOC-C5')::darts.id_location_pair, + ROW(13001, 'LOC-C1')::darts.id_location_pair, + ROW(13002, 'LOC-C2')::darts.id_location_pair, + ROW(13003, 'LOC-C3')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, now() - interval '7 days', results); + + --ASSERT cardinality(results) = 5, 'TEST 4 CALL 2 FAILED: expected 5 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 4 CALL 2 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 5 THEN + RAISE NOTICE 'TEST 4 CALL 2 FAILED: expected 5 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 4 CALL 2 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 4 CALL 2 PASSED'; + END $$; + +---------------------------------------------------------------------------------------- +-- TEST 5: Two-call behavior +-- Call 1: limit = 3 - expect first 3 valid pairs +-- Update deleted flag +-- Call 2: limit = 3 - expect the remaining 2 valid pairs (0 incomplete cleanup) +-- Test includes a check where the last_modified_ts is exactly the same as the parameter +---------------------------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- 5 valid ARM EODs (one exact timestamp match) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3301, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3302, 8980069, 2, 3, NULL, now(), (now()::date - 7)::timestamp, -99, -99, FALSE, FALSE, FALSE), -- exact match check + (3303, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3304, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE), + (3305, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE); + +-- 5 valid DETS EODs +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4301, 8980069, 2, 4, 'LOC-C1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4302, 8980069, 2, 4, 'LOC-C2', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4303, 8980069, 2, 4, 'LOC-C3', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4304, 8980069, 2, 4, 'LOC-C4', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4305, 8980069, 2, 4, 'LOC-C5', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- 5 valid OSR rows +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (13001, 4301, 3301, 'LOC-C1'), + (13002, 4302, 3302, 'LOC-C2'), + (13003, 4303, 3303, 'LOC-C3'), + (13004, 4304, 3304, 'LOC-C4'), + (13005, 4305, 3305, 'LOC-C5'); + +-- Noise ARM (older than 7 days but invalid ors_id + not older than 7 days but valid ors_id) +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3391, 8980069, 1, 3, NULL, now(), now() - interval '10 days', -99, -99, FALSE, FALSE, FALSE), + (3392, 8980069, 2, 3, NULL, now(), now() - interval '2 days', -99, -99, FALSE, FALSE, FALSE); + +-- DETS +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4391, 8980069, 2, 4, 'LOC-Z3', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE), + (4392, 8980069, 2, 4, 'LOC-Z4', now(), now() - interval '10 days', -99, -99, FALSE, FALSE, TRUE); + +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES (13901, 4391, 3391, 'LOC-Z3'), + (13902, 4392, 3392, 'LOC-Z4'); + +-- 2 calls: limit = 3 +DO $$ + DECLARE + results darts.id_location_pair[]; + expected darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 5'; + RAISE NOTICE '------------------------------------'; + + --Call 1: limit 3 - Should behave normally + expected := ARRAY[ + ROW(13001, 'LOC-C1')::darts.id_location_pair, + ROW(13002, 'LOC-C2')::darts.id_location_pair, + ROW(13003, 'LOC-C3')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, (now()::date - 7)::timestamp, results); + + --ASSERT cardinality(results) = 3, 'TEST 5 CALL 1 FAILED: expected 3 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 5 CALL 1 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 3 THEN + RAISE NOTICE 'TEST 5 CALL 1 FAILED: expected 3 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 5 CALL 1 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 5 CALL 1 PASSED'; + + --Update the flag_file_dets_cleanup_status flag to simulate final cleanup has completed + UPDATE object_state_record + SET flag_file_dets_cleanup_status = TRUE + WHERE osr_uuid IN (13001, 13002, 13003); + + --Call 2: limit = 3 - Expect the other 2 valid pairs ONLY) + expected := ARRAY[ + ROW(13004, 'LOC-C4')::darts.id_location_pair, + ROW(13005, 'LOC-C5')::darts.id_location_pair + ]; + + CALL darts.dets_cleanup_eod_osr(3, now() - interval '7 days', results); + + --ASSERT cardinality(results) = 2, 'TEST 5 CALL 2 FAILED: expected 2 results but got ' || cardinality(results); + --ASSERT results = expected, 'TEST 5 CALL 2 FAILED: results array does not match expected values'; + + -- Check count + IF cardinality(results) <> 2 THEN + RAISE NOTICE 'TEST 5 CALL 2 FAILED: expected 2 results but got %', cardinality(results); + RETURN; + END IF; + + -- Check content + IF results IS DISTINCT FROM expected THEN + RAISE NOTICE 'TEST 5 CALL 2 FAILED: results array does not match expected values'; + RETURN; + END IF; + + RAISE NOTICE 'TEST 5 CALL 2 PASSED'; + END $$; + +--------------------------------------------------------------------------- +-- TEST 6: ARM record linked to multiple OSR/DETS records, expect exception +--------------------------------------------------------------------------- + +CALL test_dets_cleanup.reset_test_data(); + +-- Valid ARM EOD +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (3501, 8980069, 2, 3, NULL, now(), now() - interval '8 days', -99, -99, FALSE, FALSE, FALSE); + +-- Valid DETS EOD +INSERT INTO external_object_directory ( eod_id, med_id, ors_id, elt_id, external_location, created_ts, last_modified_ts, created_by, last_modified_by, + update_retention, is_response_cleaned, is_dets) +VALUES + (4501, 8980069, 2, 4, 'LOC-D1', now(), now(), -99, -99, FALSE, FALSE, TRUE), + (4502, 8980069, 2, 4, 'LOC-D2', now(), now(), -99, -99, FALSE, FALSE, TRUE); + +-- OSR referencing both DETS EODs but ARM EOD is associated with both +INSERT INTO object_state_record (osr_uuid, eod_id, arm_eod_id, dets_location) +VALUES + (15001, 4501, 3501, 'LOC-D1'), + (15002, 4502, 3501, 'LOC-D2'); + +DO $$ + DECLARE + results darts.id_location_pair[]; + BEGIN + RAISE NOTICE '------------------------------------'; + RAISE NOTICE ' TEST 6'; + RAISE NOTICE '------------------------------------'; + + CALL darts.dets_cleanup_eod_osr(10, now() - interval '7 days', results); + RAISE NOTICE 'TEST 6 FAILED: expected exception but procedure completed successfully.'; + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'TEST 6 PASSED (exception raised as expected): %', SQLERRM; + END $$; + +---------------------------------------------------------------------- +-- END OF TESTS +---------------------------------------------------------------------- + +DO $$ + BEGIN + RAISE NOTICE 'ALL TESTS COMPLETED'; + RAISE NOTICE 'Cleaning up...'; + END $$; + +---------------------------------------------------------------------- +-- CLEAN UP +---------------------------------------------------------------------- +RESET ALL; +--Drop schema avoiding using CASCADE +DROP PROCEDURE IF EXISTS test_dets_cleanup.reset_test_data; +DROP TABLE IF EXISTS test_dets_cleanup.external_object_directory; +DROP TABLE IF EXISTS test_dets_cleanup.object_state_record; +DROP SCHEMA IF EXISTS test_dets_cleanup; + +SELECT name, setting, unit, SOURCE +FROM pg_settings +WHERE name IN ('search_path', 'max_parallel_workers_per_gather', 'work_mem', 'max_parallel_maintenance_workers') +ORDER BY name; \ No newline at end of file diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java b/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java new file mode 100644 index 00000000000..700613c8597 --- /dev/null +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/CleanUpDetsDataProcessor.java @@ -0,0 +1,8 @@ +package uk.gov.hmcts.darts.arm.service; + +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; + +@FunctionalInterface +public interface CleanUpDetsDataProcessor { + void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskConfig minimumStoredAge); +} diff --git a/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java new file mode 100644 index 00000000000..ffc6871b4af --- /dev/null +++ b/src/main/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImpl.java @@ -0,0 +1,143 @@ +package uk.gov.hmcts.darts.arm.service.impl; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import uk.gov.hmcts.darts.arm.service.CleanUpDetsDataProcessor; +import uk.gov.hmcts.darts.common.repository.ExternalObjectDirectoryRepository; +import uk.gov.hmcts.darts.common.repository.ObjectStateRecordRepository; +import uk.gov.hmcts.darts.dets.service.DetsApiService; +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +import uk.gov.hmcts.darts.util.AsyncUtil; + +import java.time.Clock; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +@Slf4j +@Service +@RequiredArgsConstructor +public class CleanUpDetsDataProcessorImpl implements CleanUpDetsDataProcessor { + + private final CleanUpDetsDataBatchProcessor cleanUpDetsDataBatchProcessor; + private final Clock clock; + + @Override + //Required for async batch processing + @SuppressWarnings({"PMD.DoNotUseThreads"}) + public void processCleanUpDetsData(int batchSize, CleanUpDetsDataAutomatedTaskConfig config) { + log.info("Processing clean up of DETS data with batch size: {}", batchSize); + + OffsetDateTime minimumStoredAge = OffsetDateTime.now(clock).minus(config.getMinimumStoredAge()); + int chunkSize = config.getChunkSize(); + + int totalProcessed = 0; + + while (totalProcessed < batchSize && chunkSize > 0) { + log.info("Processing clean up of DETS data with chunk size: {}", chunkSize); + + List eodIdsToCleanUp = + cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(chunkSize, minimumStoredAge); + + if (eodIdsToCleanUp.isEmpty()) { + log.info("No more DETS data to clean up. Ending process."); + break; + } + + List> batchesToDeleteBlobStoreRecordFor = + ListUtils.partition(eodIdsToCleanUp, config.getChunkSize() / config.getThreads()); + + List> tasks = batchesToDeleteBlobStoreRecordFor + .stream() + .map(eodsForBatch -> (Callable) () -> { + cleanUpDetsDataBatchProcessor.process(eodsForBatch); + return null; + }) + .toList(); + + try { + AsyncUtil.invokeAllAwaitTermination(tasks, config); + } catch (InterruptedException e) { + log.error("Clean up dets data batch processing interrupted", e); + Thread.currentThread().interrupt(); + } catch (Exception e) { + log.error("Clean up dets data unexpected exception", e); + } + //Update total processed count and adjust chunk size for next iteration if needed + totalProcessed += eodIdsToCleanUp.size(); + //Ensure we do not exceed the batch size in the next iteration + //Takes into account the possibility that the procedure may return more records than requested + if (totalProcessed + chunkSize > batchSize) { + chunkSize = batchSize - totalProcessed; + } + log.info("Processed batch of DETS data clean up. Total processed so far: {}. Batch size: {}", totalProcessed, eodIdsToCleanUp.size()); + } + log.info("Completed processing clean up of DETS data. Total processed: {}", totalProcessed); + } + + @Component + @RequiredArgsConstructor + public static class CleanUpDetsDataBatchProcessor { + + private final ExternalObjectDirectoryRepository externalObjectDirectoryRepository; + private final ObjectStateRecordRepository objectStateRecordRepository; + private final DetsApiService detsApiService; + + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void process(List eodsCleanedUp) { + List objectStateRecordsForDetsRecordsCleanedUpSuccessfully = new ArrayList<>(); + List objectStateRecordsForDetsRecordsFailedToCleanUp = new ArrayList<>(); + + if (eodsCleanedUp.isEmpty()) { + return; + } + + for (CleanUpDetsProcedureResponse response : eodsCleanedUp) { + try { + log.debug("Processing clean up response for EOD ID: {}, Location: {}", response.getOsrUuid(), response.getDetsLocation()); + if (detsApiService.deleteBlobDataFromContainer(response.getDetsLocation())) { + log.debug("Successfully deleted DETS blob for EOD ID: {}, Location: {}", response.getOsrUuid(), response.getDetsLocation()); + objectStateRecordsForDetsRecordsCleanedUpSuccessfully.add(response.getOsrUuid()); + continue; + } else { + log.error("Failed to delete DETS blob for EOD ID: {}, Location: {}. Blob may not exist or deletion failed.", + response.getOsrUuid(), response.getDetsLocation()); + } + } catch (Exception exception) { + log.error("Failed to delete DETS blob for EOD Location: {}, object state record id: {}.", + response.getDetsLocation(), response.getOsrUuid(), exception); + } + objectStateRecordsForDetsRecordsFailedToCleanUp.add(response.getOsrUuid()); + } + objectStateRecordRepository.markDetsCleanupStatusAsComplete(objectStateRecordsForDetsRecordsCleanedUpSuccessfully); + log.info("Marked object state records as clean up complete for EOD IDs: {}", objectStateRecordsForDetsRecordsCleanedUpSuccessfully); + + if (CollectionUtils.isNotEmpty(objectStateRecordsForDetsRecordsFailedToCleanUp)) { + log.error("Dets clean up failed for Object state record Ids: {}", objectStateRecordsForDetsRecordsFailedToCleanUp); + } + } + + @Transactional(propagation = Propagation.REQUIRES_NEW) + public List callDetsCleanUpStoredProcedure(int chunkSize, OffsetDateTime minimumStoredAge) { + return objectStateRecordRepository.cleanUpDetsDataProcedure(chunkSize, minimumStoredAge); + } + } + + @AllArgsConstructor + @Getter + @Setter + public static class CleanUpDetsProcedureResponse { + private Long osrUuid; + private String detsLocation; + } +} diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java index 4ed00d18832..38b27020bd5 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java +++ b/src/main/java/uk/gov/hmcts/darts/common/repository/ExternalObjectDirectoryRepository.java @@ -278,7 +278,6 @@ List findIdsForAudioToBeDeletedFromUnstructured(ObjectRecordStatusEntity s OffsetDateTime unstructuredLastModifiedBefore, Limit limit); - @Query( """ SELECT eod FROM ExternalObjectDirectoryEntity eod diff --git a/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java b/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java index bfeb69eb228..28e4cb6ec68 100644 --- a/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java +++ b/src/main/java/uk/gov/hmcts/darts/common/repository/ObjectStateRecordRepository.java @@ -1,10 +1,27 @@ package uk.gov.hmcts.darts.common.repository; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import uk.gov.hmcts.darts.arm.service.impl.CleanUpDetsDataProcessorImpl; import uk.gov.hmcts.darts.common.entity.ObjectStateRecordEntity; +import java.time.OffsetDateTime; +import java.util.List; import java.util.Optional; public interface ObjectStateRecordRepository extends JpaRepository { Optional findByArmEodId(long armEodId); + + @Modifying + @Query("UPDATE ObjectStateRecordEntity o SET o.flagFileDetsCleanupStatus = true where o.uuid in :uuids") + void markDetsCleanupStatusAsComplete(List uuids); + + + @Query(value = "SELECT osr_uuid AS osrUuid, dets_location AS detsLocation " + + "FROM dets_cleanup_eod_osr_rows(:limit, :last_modified_before_ts)", nativeQuery = true) + List cleanUpDetsDataProcedure( + @Param("limit") Integer limit, @Param("last_modified_before_ts") OffsetDateTime lastModifiedBefore); + } diff --git a/src/main/java/uk/gov/hmcts/darts/dets/service/impl/DetsApiServiceImpl.java b/src/main/java/uk/gov/hmcts/darts/dets/service/impl/DetsApiServiceImpl.java index 5474c711acb..2e441209707 100644 --- a/src/main/java/uk/gov/hmcts/darts/dets/service/impl/DetsApiServiceImpl.java +++ b/src/main/java/uk/gov/hmcts/darts/dets/service/impl/DetsApiServiceImpl.java @@ -86,6 +86,7 @@ public String saveBlobData(BinaryData binaryData, String fileName) { return client.getBlobName(); } + @Override public boolean deleteBlobDataFromContainer(String blobId) throws AzureDeleteBlobException { try { @@ -107,7 +108,6 @@ public boolean deleteBlobDataFromContainer(String blobId) throws AzureDeleteBlob configuration.getContainerName(), blobId, httpStatus); throw new AzureDeleteBlobException(message); } - } catch (RuntimeException e) { throw new AzureDeleteBlobException( "Could not delete from storage container=" + configuration.getContainerName() + ", blobId=" + blobId, e diff --git a/src/main/java/uk/gov/hmcts/darts/task/api/AutomatedTaskName.java b/src/main/java/uk/gov/hmcts/darts/task/api/AutomatedTaskName.java index fa9f4ebbad4..8ac226db077 100644 --- a/src/main/java/uk/gov/hmcts/darts/task/api/AutomatedTaskName.java +++ b/src/main/java/uk/gov/hmcts/darts/task/api/AutomatedTaskName.java @@ -43,7 +43,8 @@ public enum AutomatedTaskName { ARM_MISSING_RESPONSE_REPLY_TASK_NAME("ArmMissingResponseReplay"), DETS_CLEANUP_ARM_RESPONSE_FILES("DETSCleanupArmResponseFiles"), MEDIA_REQUEST_CLEANUP("MediaRequestCleanUp"), - ARM_RPO_BACKLOG_CATCHUP("ArmRpoBacklogCatchup", Constants.AUTOMATED_TASK_PROCESS_E2E_ARM_RPO_PENDING_PROCESS_E2E_ARM_RPO_FALSE); + ARM_RPO_BACKLOG_CATCHUP("ArmRpoBacklogCatchup", Constants.AUTOMATED_TASK_PROCESS_E2E_ARM_RPO_PENDING_PROCESS_E2E_ARM_RPO_FALSE), + CLEAN_UP_DETS_DATA("CleanUpDetsData"); private final String taskName; private final String conditionalOnSpEL; diff --git a/src/main/java/uk/gov/hmcts/darts/task/config/CleanUpDetsDataAutomatedTaskConfig.java b/src/main/java/uk/gov/hmcts/darts/task/config/CleanUpDetsDataAutomatedTaskConfig.java new file mode 100644 index 00000000000..7eef7f638e6 --- /dev/null +++ b/src/main/java/uk/gov/hmcts/darts/task/config/CleanUpDetsDataAutomatedTaskConfig.java @@ -0,0 +1,18 @@ +package uk.gov.hmcts.darts.task.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import java.time.Duration; + +@ConfigurationProperties("darts.automated.task.clean-up-dets-data") +@Getter +@Setter +@Configuration +public class CleanUpDetsDataAutomatedTaskConfig extends AbstractAsyncAutomatedTaskConfig { + + private Duration minimumStoredAge; + private int chunkSize; +} diff --git a/src/main/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTask.java b/src/main/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTask.java new file mode 100644 index 00000000000..44984b240cb --- /dev/null +++ b/src/main/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTask.java @@ -0,0 +1,42 @@ +package uk.gov.hmcts.darts.task.runner.impl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import uk.gov.hmcts.darts.arm.service.CleanUpDetsDataProcessor; +import uk.gov.hmcts.darts.common.repository.AutomatedTaskRepository; +import uk.gov.hmcts.darts.log.api.LogApi; +import uk.gov.hmcts.darts.task.api.AutomatedTaskName; +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +import uk.gov.hmcts.darts.task.runner.AutoloadingManualTask; +import uk.gov.hmcts.darts.task.service.LockService; + +import static uk.gov.hmcts.darts.task.api.AutomatedTaskName.CLEAN_UP_DETS_DATA; + +@Slf4j +@Component +public class CleanUpDetsDataAutomatedTask + extends AbstractLockableAutomatedTask + implements AutoloadingManualTask { + + private final CleanUpDetsDataProcessor cleanUpDetsDataProcessor; + + @Autowired + public CleanUpDetsDataAutomatedTask(AutomatedTaskRepository automatedTaskRepository, + CleanUpDetsDataAutomatedTaskConfig automatedTaskConfigurationProperties, + CleanUpDetsDataProcessor processor, + LogApi logApi, LockService lockService) { + super(automatedTaskRepository, automatedTaskConfigurationProperties, logApi, lockService); + this.cleanUpDetsDataProcessor = processor; + } + + @Override + public AutomatedTaskName getAutomatedTaskName() { + return CLEAN_UP_DETS_DATA; + } + + @Override + protected void runTask() { + cleanUpDetsDataProcessor.processCleanUpDetsData(getAutomatedTaskBatchSize(), getConfig()); + } +} \ No newline at end of file diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 5cb3f9e92ec..0d58ceb0d94 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -440,6 +440,15 @@ darts: lock: at-least-for: PT1M at-most-for: PT90M + clean-up-dets-data: + system-user-email: systemCleanUpDetsData@hmcts.net + lock: + at-least-for: PT1M + at-most-for: PT90M + minimum-stored-age: 7D + chunk-size: 10000 + async-timeout: 20M + threads: 20 cleanup-arm-response-files: system-user-email: system_CleanupArmResponseFiles@hmcts.net lock: diff --git a/src/main/resources/db/migration/common/V1_505__CleanUpDetsData_automated_task.sql b/src/main/resources/db/migration/common/V1_505__CleanUpDetsData_automated_task.sql new file mode 100644 index 00000000000..f8a6872e152 --- /dev/null +++ b/src/main/resources/db/migration/common/V1_505__CleanUpDetsData_automated_task.sql @@ -0,0 +1,10 @@ +INSERT INTO darts.automated_task (aut_id, task_name, task_description, cron_expression, cron_editable, batch_size, + created_ts, created_by, last_modified_ts, last_modified_by, task_enabled) +VALUES (36, 'CleanUpDetsData', 'Cleans up Dets files that have successfully been stored in ARM', '0 24 0-6,19-23 ? * *', true, 100_000, + current_timestamp, 0, current_timestamp, 0, false); + +INSERT INTO user_account (usr_id, user_name, user_email_address, description, created_ts, last_modified_ts, last_modified_by, created_by, is_system_user, + is_active, user_full_name) + +VALUES (-51, 'systemCleanUpDetsDataAutomatedTask', 'systemCleanUpDetsDataAutomatedTask@hmcts.net', 'systemCleanUpDetsDataAutomatedTask', + '2024-01-01 00:00:00+00', '2024-01-01 00:00:00+00', 0, 0, true, true, 'systemCleanUpDetsDataAutomatedTask'); \ No newline at end of file diff --git a/src/main/resources/db/migration/postgres/V1_506__CleanUpDetsData_stored_procedure.sql b/src/main/resources/db/migration/postgres/V1_506__CleanUpDetsData_stored_procedure.sql new file mode 100644 index 00000000000..110339d1262 --- /dev/null +++ b/src/main/resources/db/migration/postgres/V1_506__CleanUpDetsData_stored_procedure.sql @@ -0,0 +1,155 @@ +SET SEARCH_PATH TO darts; + +DROP FUNCTION IF EXISTS dets_cleanup_eod_osr_rows; +DROP PROCEDURE IF EXISTS dets_cleanup_eod_osr; +DROP TYPE IF EXISTS id_location_pair; + +CREATE TYPE id_location_pair AS ( + osr_uuid BIGINT, + dets_location TEXT + ); + +CREATE INDEX IF NOT EXISTS osr_file_dets_cleanup_idx ON object_state_record (date_file_dets_cleanup, flag_file_dets_cleanup_status); + +CREATE OR REPLACE PROCEDURE dets_cleanup_eod_osr( + IN pi_limit INTEGER, + IN pi_last_modified_ts TIMESTAMPTZ, + OUT po_results_array id_location_pair[] +) + LANGUAGE 'plpgsql' +AS $$ +DECLARE + c_msg_prefix CONSTANT TEXT := 'dets_cleanup_eod_osr:'; + v_dets_eod_ids BIGINT[]; + v_arm_eod_ids BIGINT[]; + v_incomplete_cleanup id_location_pair[]; + + v_dets_eod_count INTEGER; + v_arm_eod_count INTEGER; + v_results_count INTEGER; + v_incomplete_cleanup_count INTEGER; + + v_dup_count INTEGER; + v_deleted_count INTEGER; + v_updated_count INTEGER; +BEGIN + RAISE NOTICE '% Started at % [pi_limit = %, pi_last_modified_ts = %]', c_msg_prefix, clock_timestamp(), pi_limit, pi_last_modified_ts; + + BEGIN + --Get DETS records to be deleted, limited by pi_limit + -- DETS record: OSR and EOD records must match on eod_id and locations, where eod.elt_id = 4 (DETS) + -- ARM record : OSR and EOD records must match where OSR.arm_eod_id = EOD.eod_id, where eod.elt_id = 3 (ARM) + -- ARM record must be STORED (i.e. ors_id = 2) and it's last_modified_ts is before or equal to pi_last_modified_ts + SELECT array_agg(eod_id) + , array_agg(arm_eod_id) + , array_agg((osr_uuid, dets_location)::id_location_pair) + INTO v_dets_eod_ids + , v_arm_eod_ids + , po_results_array + FROM ( + SELECT osr.osr_uuid, osr.dets_location, eod_dets.eod_id, eod_arm.eod_id AS arm_eod_id + FROM object_state_record osr + JOIN external_object_directory eod_dets + ON eod_dets.eod_id = osr.eod_id + AND eod_dets.external_location = osr.dets_location + JOIN external_object_directory eod_arm + ON eod_arm.eod_id = osr.arm_eod_id + WHERE eod_dets.elt_id = 4 --DETS + AND eod_arm.elt_id = 3 --ARM + AND eod_arm.ors_id = 2 --STORED + AND eod_arm.last_modified_ts <= pi_last_modified_ts + ORDER BY osr.osr_uuid + LIMIT pi_limit + ) t; + + v_dets_eod_count := COALESCE(cardinality(v_dets_eod_ids), 0); + v_arm_eod_count := COALESCE(cardinality(v_arm_eod_ids), 0); + v_results_count := COALESCE(cardinality(po_results_array), 0); + + RAISE NOTICE '% Number of records in DETS EOD id array = %', c_msg_prefix, v_dets_eod_count; + RAISE NOTICE '% Number of records in ARM EOD id array = %', c_msg_prefix, v_arm_eod_count; + RAISE NOTICE '% Number of records in results array = %', c_msg_prefix, v_results_count; + + -- Check to ensure duplicate records were not returned. Indicates that there is more than one ARM or DETS record, which should not happen + SELECT COUNT(*) + INTO v_dup_count + FROM ( + SELECT elem + FROM unnest(v_arm_eod_ids || v_dets_eod_ids) AS elem + GROUP BY elem + HAVING COUNT(*) > 1 + ) AS t; + + RAISE NOTICE '% Duplicate count = %', c_msg_prefix, v_dup_count; + + IF v_dup_count > 0 THEN + RAISE EXCEPTION 'Validation failed: Duplicate records [%] found.', v_dup_count; + END IF; + + --Before updating OSR, retrieve records from OSR that have a date_file_dets_cleanup value but flag_file_dets_cleanup_status has not been set. + -- append the results to the OUT parameter + SELECT array_agg((osr_uuid, dets_location)::id_location_pair ORDER BY osr_uuid) AS id_location_pairs + INTO v_incomplete_cleanup + FROM object_state_record + WHERE date_file_dets_cleanup IS NOT NULL + AND (flag_file_dets_cleanup_status IS NULL + OR flag_file_dets_cleanup_status = FALSE); + + v_incomplete_cleanup_count := COALESCE(cardinality(v_incomplete_cleanup), 0); + + RAISE NOTICE '% Number of records in incomplete cleanup array = %', c_msg_prefix, v_incomplete_cleanup_count; + + po_results_array := COALESCE( CASE WHEN v_results_count > 0 + AND v_incomplete_cleanup_count > 0 + THEN po_results_array || v_incomplete_cleanup + WHEN v_results_count > 0 + THEN po_results_array + WHEN v_incomplete_cleanup_count > 0 + THEN v_incomplete_cleanup + END, + '{}'::id_location_pair[] + ); + + RAISE NOTICE '% Number of records in combined results array = %', c_msg_prefix, COALESCE(cardinality(po_results_array), 0); + + --Only perform DELETE and UPDATE if there are values to process + IF v_dets_eod_count > 0 THEN + + --Delete records from EOD + DELETE FROM external_object_directory + WHERE eod_id = ANY(v_dets_eod_ids); + + GET DIAGNOSTICS v_deleted_count = ROW_COUNT; + RAISE NOTICE '% Number of EOD records deleted = %', c_msg_prefix, v_deleted_count; + + --Update OSR records + UPDATE object_state_record + SET eod_id = NULL + , date_file_dets_cleanup = CURRENT_TIMESTAMP + WHERE eod_id = ANY(v_dets_eod_ids); + + GET DIAGNOSTICS v_updated_count = ROW_COUNT; + RAISE NOTICE '% Number of OSR records updated = %', c_msg_prefix, v_updated_count; + + ELSE + RAISE NOTICE '% There is nothing to process. No EOD Deletes or OSR Updates were executed.', c_msg_prefix; + END IF; + + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE '% Error in dets_cleanup_eod_osr: % - %', c_msg_prefix, SQLSTATE, SQLERRM; + RAISE; + END; + RAISE NOTICE '% Finished at %', c_msg_prefix, clock_timestamp(); +END; +$$; + +CREATE OR REPLACE FUNCTION dets_cleanup_eod_osr_rows(pi_limit int, pi_last_modified_ts timestamptz) + RETURNS TABLE (osr_uuid bigint, dets_location text) + LANGUAGE plpgsql AS $$ +DECLARE v_arr id_location_pair[]; +BEGIN + CALL dets_cleanup_eod_osr(pi_limit, pi_last_modified_ts, v_arr); + RETURN QUERY SELECT (x).osr_uuid, (x).dets_location + FROM unnest(COALESCE(v_arr, '{}'::id_location_pair[])) x; +END $$; \ No newline at end of file diff --git a/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java new file mode 100644 index 00000000000..3dbe1af266b --- /dev/null +++ b/src/test/java/uk/gov/hmcts/darts/arm/service/impl/CleanUpDetsDataProcessorImplTest.java @@ -0,0 +1,252 @@ +package uk.gov.hmcts.darts.arm.service.impl; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import uk.gov.hmcts.darts.common.exception.AzureDeleteBlobException; +import uk.gov.hmcts.darts.common.repository.ExternalObjectDirectoryRepository; +import uk.gov.hmcts.darts.common.repository.ObjectStateRecordRepository; +import uk.gov.hmcts.darts.dets.service.DetsApiService; +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +import uk.gov.hmcts.darts.testutils.AsyncTestUtil; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +@DisplayName("CleanUpDetsDataProcessorImpl Tests") +class CleanUpDetsDataProcessorImplTest { + + private static final Instant FIXED_INSTANT = Instant.parse("2024-01-01T00:00:00Z"); + private static final Duration DEFAULT_MINIMUM_STORED_AGE = Duration.ofDays(30); + + @Mock + private ExternalObjectDirectoryRepository externalObjectDirectoryRepository; + + @Mock + private CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor cleanUpDetsDataBatchProcessor; + + + @Mock + private CleanUpDetsDataAutomatedTaskConfig config; + + @Mock + private ObjectStateRecordRepository objectStateRecordRepository; + + @Mock + private DetsApiService detsApiService; + + @InjectMocks + private CleanUpDetsDataProcessorImpl processor; + + @BeforeEach + void setUp() { + processor = new CleanUpDetsDataProcessorImpl(cleanUpDetsDataBatchProcessor, Clock.fixed(FIXED_INSTANT, ZoneOffset.UTC)); + } + + + @DisplayName("Mehod: processCleanUpDetsData Tests") + @Nested + class ProcessCleanUpDetsDataTests { + + @Test + @DisplayName("Skips processing when no EOD rows are returned") + void shouldSkipProcessingWhenRepositoryReturnsEmptyList() { + configureTaskConfig(4, 2); + when(cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(eq(4), any())) + .thenReturn(Collections.emptyList()); + + assertDoesNotThrow(() -> processor.processCleanUpDetsData(10, config)); + + verify(cleanUpDetsDataBatchProcessor, never()).process(anyList()); + verify(cleanUpDetsDataBatchProcessor, times(1)) + .callDetsCleanUpStoredProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); + } + + + @Test + @DisplayName("Partitions responses and delegates to batch processor per chunk") + void shouldPartitionResponsesIntoBatches() { + configureTaskConfig(4, 2); + List responses = + List.of(response(1L), response(2L), response(3L), response(4L)); + + when(cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(eq(4), any())) + .thenReturn(responses)//First call returns 4 records to process + .thenReturn(Collections.emptyList()); //Second call returns empty list to end processing + + AsyncTestUtil.processTasksSynchronously(() -> processor.processCleanUpDetsData(8, config)); + + ArgumentCaptor> captor = ArgumentCaptor.captor(); + //Call twice because with chunk size of 2 and 4 records, we expect 2 batches to be processed (Chunk size 4 / threads = 2 = batch size for processor) + verify(cleanUpDetsDataBatchProcessor, times(2)).process(captor.capture()); + List> batches = captor.getAllValues(); + + assertEquals(2, batches.size()); + assertEquals(List.of(responses.get(0), responses.get(1)), batches.get(0)); + assertEquals(List.of(responses.get(2), responses.get(3)), batches.get(1)); + } + + + @Test + @DisplayName("Continues looping until repository reports there is no more data") + void shouldContinueProcessingUntilNoMoreData() { + configureTaskConfig(4, 2); + List firstBatch = List.of( + response(1L), response(2L), response(3L), response(4L) + ); + List secondBatch = List.of( + response(5L), response(6L), response(7L), response(8L) + ); + + when(cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(eq(4), any())) + .thenReturn(firstBatch) + .thenReturn(secondBatch) + .thenReturn(Collections.emptyList()); + + AsyncTestUtil.processTasksSynchronously(() -> processor.processCleanUpDetsData(12, config)); + + //Called 3 times - first batch, second batch, then empty list to end processing + verify(cleanUpDetsDataBatchProcessor, times(3)) + .callDetsCleanUpStoredProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); + verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(0), firstBatch.get(1))); + verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(2), firstBatch.get(3))); + verify(cleanUpDetsDataBatchProcessor).process(List.of(secondBatch.get(0), secondBatch.get(1))); + verify(cleanUpDetsDataBatchProcessor).process(List.of(secondBatch.get(2), secondBatch.get(3))); + } + + @Test + @DisplayName("Stops processing once batch size limit is reached, even if repository returns more data") + void shouldStopProcessingWhenBatchSizeLimitReached() { + configureTaskConfig(4, 2); + + List firstBatch = List.of( + response(1L), response(2L), response(3L), response(4L) + ); + List secondBatch = List.of( + response(5L), response(6L), response(7L), response(8L) + ); + + when(cleanUpDetsDataBatchProcessor.callDetsCleanUpStoredProcedure(eq(4), any())) + .thenReturn(firstBatch) + .thenReturn(secondBatch) + .thenReturn(Collections.emptyList()); + + AsyncTestUtil.processTasksSynchronously(() -> processor.processCleanUpDetsData(4, config)); + + verify(cleanUpDetsDataBatchProcessor, times(1)) + .callDetsCleanUpStoredProcedure(4, expectedMinimumStoredAge(DEFAULT_MINIMUM_STORED_AGE)); + verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(0), firstBatch.get(1))); + verify(cleanUpDetsDataBatchProcessor).process(List.of(firstBatch.get(2), firstBatch.get(3))); + } + + + private OffsetDateTime expectedMinimumStoredAge(Duration minimumStoredAge) { + return OffsetDateTime.ofInstant(FIXED_INSTANT, ZoneOffset.UTC).minus(minimumStoredAge); + } + } + + + @DisplayName("CleanUpDetsDataBatchProcessor tests") + @Nested + class CleanUpDetsDataBatchProcessorTests { + + @Test + @DisplayName("Deletes each blob and marks the OSRs as complete when everything succeeds") + void shouldDeleteAllRecordsSuccessfully() throws AzureDeleteBlobException { + CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor batchProcessor = createBatchProcessor(); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response1 = response(1L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response2 = response(2L); + + when(detsApiService.deleteBlobDataFromContainer(response1.getDetsLocation())).thenReturn(true); + when(detsApiService.deleteBlobDataFromContainer(response2.getDetsLocation())).thenReturn(true); + + batchProcessor.process(List.of(response1, response2)); + + verify(detsApiService).deleteBlobDataFromContainer(response1.getDetsLocation()); + verify(detsApiService).deleteBlobDataFromContainer(response2.getDetsLocation()); + verify(objectStateRecordRepository).markDetsCleanupStatusAsComplete(List.of(1L, 2L)); + } + + @Test + @DisplayName("Logs failures but continues when deleteBlobDataFromContainer returns false") + void shouldContinueProcessingWhenDeleteReturnsFalse() throws AzureDeleteBlobException { + CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor batchProcessor = createBatchProcessor(); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response1 = response(1L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response2 = response(2L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response3 = response(3L); + + when(detsApiService.deleteBlobDataFromContainer(response1.getDetsLocation())).thenReturn(true); + when(detsApiService.deleteBlobDataFromContainer(response2.getDetsLocation())).thenReturn(false); + when(detsApiService.deleteBlobDataFromContainer(response3.getDetsLocation())).thenReturn(true); + + assertDoesNotThrow(() -> batchProcessor.process(List.of(response1, response2, response3))); + + verify(detsApiService).deleteBlobDataFromContainer(response1.getDetsLocation()); + verify(detsApiService).deleteBlobDataFromContainer(response2.getDetsLocation()); + verify(detsApiService).deleteBlobDataFromContainer(response3.getDetsLocation()); + verify(objectStateRecordRepository).markDetsCleanupStatusAsComplete(List.of(1L, 3L)); + } + + @Test + @DisplayName("Handles exceptions thrown while deleting blobs and continues with remaining records") + void shouldHandleExceptionsAndContinueProcessing() throws AzureDeleteBlobException { + CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor batchProcessor = createBatchProcessor(); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response1 = response(1L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response2 = response(2L); + CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response3 = response(3L); + + when(detsApiService.deleteBlobDataFromContainer(response1.getDetsLocation())).thenReturn(true); + when(detsApiService.deleteBlobDataFromContainer(response2.getDetsLocation())) + .thenThrow(new AzureDeleteBlobException("failure")); + when(detsApiService.deleteBlobDataFromContainer(response3.getDetsLocation())).thenReturn(true); + + assertDoesNotThrow(() -> batchProcessor.process(List.of(response1, response2, response3))); + + InOrder order = inOrder(detsApiService); + order.verify(detsApiService).deleteBlobDataFromContainer(response1.getDetsLocation()); + order.verify(detsApiService).deleteBlobDataFromContainer(response2.getDetsLocation()); + order.verify(detsApiService).deleteBlobDataFromContainer(response3.getDetsLocation()); + + verify(objectStateRecordRepository).markDetsCleanupStatusAsComplete(List.of(1L, 3L)); + } + } + + private void configureTaskConfig(int chunkSize, int threads) { + when(config.getMinimumStoredAge()).thenReturn(DEFAULT_MINIMUM_STORED_AGE); + when(config.getChunkSize()).thenReturn(chunkSize); + lenient().when(config.getThreads()).thenReturn(threads); + } + + private CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse response(long osrUuid) { + return new CleanUpDetsDataProcessorImpl.CleanUpDetsProcedureResponse(osrUuid, "location-" + osrUuid); + } + + private CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor createBatchProcessor() { + return new CleanUpDetsDataProcessorImpl.CleanUpDetsDataBatchProcessor(externalObjectDirectoryRepository, objectStateRecordRepository, detsApiService); + } +} diff --git a/src/test/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTaskTest.java b/src/test/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTaskTest.java new file mode 100644 index 00000000000..8c99c7658b2 --- /dev/null +++ b/src/test/java/uk/gov/hmcts/darts/task/runner/impl/CleanUpDetsDataAutomatedTaskTest.java @@ -0,0 +1,54 @@ +package uk.gov.hmcts.darts.task.runner.impl; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import uk.gov.hmcts.darts.arm.service.CleanUpDetsDataProcessor; +import uk.gov.hmcts.darts.common.repository.AutomatedTaskRepository; +import uk.gov.hmcts.darts.log.api.LogApi; +import uk.gov.hmcts.darts.task.config.CleanUpDetsDataAutomatedTaskConfig; +import uk.gov.hmcts.darts.task.service.LockService; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static uk.gov.hmcts.darts.task.api.AutomatedTaskName.CLEAN_UP_DETS_DATA; + +@ExtendWith(MockitoExtension.class) +class CleanUpDetsDataAutomatedTaskTest { + + @Mock + private AutomatedTaskRepository automatedTaskRepository; + @Mock + private CleanUpDetsDataAutomatedTaskConfig automatedTaskConfigurationProperties; + @Mock + private CleanUpDetsDataProcessor cleanUpDetsDataProcessor; + @Mock + private LogApi logApi; + @Mock + private LockService lockService; + + @InjectMocks + private CleanUpDetsDataAutomatedTask cleanUpDetsDataAutomatedTask; + + @Test + void positiveGetAutomatedTaskName() { + assertEquals(CLEAN_UP_DETS_DATA, cleanUpDetsDataAutomatedTask.getAutomatedTaskName()); + } + + @Test + void positiveRunTask() { + CleanUpDetsDataAutomatedTask automatedTask = spy(cleanUpDetsDataAutomatedTask); + int batchSize = 25; + doReturn(batchSize).when(automatedTask).getAutomatedTaskBatchSize(); + + automatedTask.runTask(); + + verify(automatedTask).getAutomatedTaskBatchSize(); + verify(cleanUpDetsDataProcessor).processCleanUpDetsData(batchSize, automatedTaskConfigurationProperties); + } +} + diff --git a/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java b/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java new file mode 100644 index 00000000000..8ed4892816b --- /dev/null +++ b/src/test/java/uk/gov/hmcts/darts/testutils/AsyncTestUtil.java @@ -0,0 +1,34 @@ +package uk.gov.hmcts.darts.testutils; + +import org.mockito.MockedStatic; +import uk.gov.hmcts.darts.util.AsyncUtil; + +import java.util.List; +import java.util.concurrent.Callable; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mockStatic; + +public final class AsyncTestUtil { + + private AsyncTestUtil() { + + } + + public static void processTasksSynchronously(Runnable runnable) { + try (MockedStatic asyncUtilMock = mockStatic(AsyncUtil.class)) { + //Mock AsyncUtil to synchronously execute the tasks so we can verify the batch processor calls within the test + asyncUtilMock.when(() -> AsyncUtil.invokeAllAwaitTermination(anyList(), any())) + .thenAnswer(invocation -> { + List> tasks = invocation.getArgument(0); + for (Callable task : tasks) { + task.call(); + } + return null; + }); + runnable.run(); + } + } +} +