DMP-5415 Interruption of deleting CR and UF files

build.gradle

@@ -458,6 +458,7 @@ dependencies {
 
   implementation platform('com.azure:azure-sdk-bom:1.3.4')
   implementation 'com.azure:azure-storage-blob'
+  implementation 'com.azure:azure-storage-blob-batch'
 
   implementation group: 'io.rest-assured', name: 'rest-assured'
   implementation group: 'org.flywaydb', name: 'flyway-core', version: '11.20.3'

src/functionalTest/java/uk/gov/hmcts/darts/arm/ArmServiceFunctionalTest.java

@@ -172,6 +172,31 @@ void listBlobsUsingMarkerTestContinuationToken() {
 
     }
 
+    @Test
+    void saveAndDeleteMultipleBlobs() {
+        log.info("------------------  saveAndDeleteMultipleBlobs test");
+
+        byte[] testStringInBytes = TEST_BINARY_STRING.getBytes(StandardCharsets.UTF_8);
+        BinaryData data = BinaryData.fromBytes(testStringInBytes);
+        String filename1 = String.format("%s_functional_test", UUID.randomUUID());
+        String actualResult1 = armService.saveBlobData(armContainerName, filename1, data);
+        armSubmissionBlobsToBeDeleted.add(actualResult1);
+        assertNotNull(actualResult1);
+        log.info("Blob filename1: {}", actualResult1);
+
+        String filename2 = String.format("%s_functional_test2", UUID.randomUUID());
+        String actualResult2 = armService.saveBlobData(armContainerName, filename2, data);
+        armSubmissionBlobsToBeDeleted.add(actualResult2);
+        assertNotNull(actualResult2);
+        log.info("Blob filename2: {}", actualResult2);
+
+        armService.deleteMultipleBlobs(armContainerName, List.of(
+            actualResult1,
+            actualResult2
+        ));
+        cleanupArmBlobData();
+    }
+
     private void uploadBatchedSubmissionBlobs(BinaryData data) {
         for (int counter = 0; counter < 11; counter++) {
             String filename = String.format("functional_test_%s", UUID.randomUUID());

src/functionalTest/java/uk/gov/hmcts/darts/datamanagement/DataManagementServiceFunctionalTest.java

@@ -12,7 +12,6 @@
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.ActiveProfiles;
 import uk.gov.hmcts.darts.FunctionalTest;
-import uk.gov.hmcts.darts.arm.config.ArmDataManagementConfiguration;
 import uk.gov.hmcts.darts.common.datamanagement.component.impl.DownloadResponseMetaData;
 import uk.gov.hmcts.darts.common.datamanagement.enums.DatastoreContainerType;
 import uk.gov.hmcts.darts.common.exception.AzureDeleteBlobException;
@@ -43,14 +42,12 @@ class DataManagementServiceFunctionalTest extends FunctionalTest {
     private static final String TEST_BLOB_ID = "b0f23c62-8dd3-4e4e-ae6a-321ff6eb61d8";
 
     @Value("${darts.storage.blob.container-name.unstructured}")
-    String unstructuredStorageContainerName;
+    private String unstructuredStorageContainerName;
 
     @Autowired
-    DataManagementService dataManagementService;
+    private DataManagementService dataManagementService;
     @Autowired
-    DataManagementConfiguration dataManagementConfiguration;
-    @Autowired
-    ArmDataManagementConfiguration armDataManagementConfiguration;
+    private DataManagementConfiguration dataManagementConfiguration;
 
     @Test
     void saveBinaryDataToBlobStorage() {

src/integrationTest/java/uk/gov/hmcts/darts/arm/service/impl/DeleteArmResponseFilesHelperIntTest.java

@@ -6,15 +6,13 @@
 import org.springframework.test.context.bean.override.mockito.MockitoBean;
 import uk.gov.hmcts.darts.arm.api.ArmDataManagementApi;
 import uk.gov.hmcts.darts.arm.model.blobs.ArmResponseBatchData;
-import uk.gov.hmcts.darts.arm.service.ExternalObjectDirectoryService;
 import uk.gov.hmcts.darts.arm.util.files.BatchInputUploadFileFilenameProcessor;
 import uk.gov.hmcts.darts.arm.util.files.CreateRecordFilenameProcessor;
 import uk.gov.hmcts.darts.arm.util.files.UploadFileFilenameProcessor;
 import uk.gov.hmcts.darts.authorisation.component.UserIdentity;
 import uk.gov.hmcts.darts.common.entity.ExternalObjectDirectoryEntity;
 import uk.gov.hmcts.darts.common.entity.MediaEntity;
 import uk.gov.hmcts.darts.common.entity.UserAccountEntity;
-import uk.gov.hmcts.darts.common.repository.ExternalObjectDirectoryRepository;
 import uk.gov.hmcts.darts.testutils.PostgresIntegrationBase;
 
 import java.util.List;
@@ -36,18 +34,12 @@ class DeleteArmResponseFilesHelperIntTest extends PostgresIntegrationBase {
     private static final String MANIFEST_PREFIX = "DARTS_6a374f19a9ce7dc9cc480ea8d4eca0fb";
     private static final String RESPONSE_FILE_PREFIX = "04e6bc3b-952a-79b6-8362-13259aae1895";
 
-    @Autowired
-    private ExternalObjectDirectoryRepository externalObjectDirectoryRepository;
-
     @MockitoBean
     private ArmDataManagementApi armDataManagementApi;
 
     @MockitoBean
     private UserIdentity userIdentity;
 
-    @Autowired
-    private ExternalObjectDirectoryService externalObjectDirectoryService;
-
     @Autowired
     private DeleteArmResponseFilesHelperImpl deleteArmResponseFilesHelper;
 
@@ -66,7 +58,7 @@ void setUp() {
     }
 
     @Test
-    void deleteResponseBlobsByManifestName_shouldDeleteBlobsWhenAllResponsesAreCompletedAndCleaned() {
+    void deleteResponseBlobsByManifestName_shouldDeleteBlobsIndividuallyWhenAllResponsesAreCompletedAndCleaned() {
         // given
         String manifestName = MANIFEST_PREFIX + ".a360";
         eodRpoPending.setManifestFile(manifestName);
@@ -92,33 +84,36 @@ void deleteResponseBlobsByManifestName_shouldDeleteBlobsWhenAllResponsesAreCompl
     void deleteDanglingResponses_shouldDeleteDanglingResponses() {
         // given
         BatchInputUploadFileFilenameProcessor batchInputUploadFileFilenameProcessor = new BatchInputUploadFileFilenameProcessor(DARTS_INPUT_UPLOAD_FILE);
-        String responseFile = "dropzone/DARTS/response/" + RESPONSE_FILE_PREFIX + "_ABC_1_rsp";
-        when(armDataManagementApi.listResponseBlobs(any())).thenReturn(List.of(responseFile));
+        String otherResponseFile = "dropzone/DARTS/response/" + RESPONSE_FILE_PREFIX + "_ABC_1_rsp";
+        String crResponseFile = "dropzone/DARTS/response/" + RESPONSE_FILE_PREFIX + "_b17b9015-e6ad-77c5-8d1e-13259aae1896_0_cr.rsp";
+        String ilResponseFile = "dropzone/DARTS/response/" + RESPONSE_FILE_PREFIX + "_c17b9015-e6ad-77c5-8d1e-13259aae1896_1_il.rsp";
+
+        when(armDataManagementApi.listResponseBlobs(any())).thenReturn(List.of(otherResponseFile, crResponseFile, ilResponseFile));
+        when(armDataManagementApi.deleteMultipleBlobs(any())).thenReturn(true);
         when(armDataManagementApi.deleteBlobData(anyString())).thenReturn(true);
 
         // when
         deleteArmResponseFilesHelper.deleteDanglingResponses(batchInputUploadFileFilenameProcessor);
 
         // then
-        verify(armDataManagementApi).deleteBlobData(responseFile);
+        verify(armDataManagementApi).deleteMultipleBlobs(List.of(otherResponseFile, crResponseFile, ilResponseFile));
         verify(armDataManagementApi).deleteBlobData(DARTS_INPUT_UPLOAD_FILE);
         verify(armDataManagementApi).listResponseBlobs(batchInputUploadFileFilenameProcessor.getHashcode());
         verifyNoMoreInteractions(armDataManagementApi);
     }
 
     @Test
-    void deleteResponseBlobs_shouldDeleteAllResponseBlobs() {
+    void deleteResponseBlobsIndividually_shouldDeleteAllResponseBlobIndividually() {
         // given
-        List<String> responseBlobs = List.of("blob1", "blob2");
+        String responseBlob = "blob1";
         when(armDataManagementApi.deleteBlobData(anyString())).thenReturn(true);
 
         // when
-        List<Boolean> result = deleteArmResponseFilesHelper.deleteResponseBlobs(responseBlobs);
+        Boolean result = deleteArmResponseFilesHelper.deleteResponseBlobIndividually(responseBlob);
 
         // then
-        assertTrue(result.stream().allMatch(Boolean::booleanValue));
+        assertTrue(result);
         verify(armDataManagementApi).deleteBlobData("blob1");
-        verify(armDataManagementApi).deleteBlobData("blob2");
         verifyNoMoreInteractions(armDataManagementApi);
     }
 
@@ -134,14 +129,13 @@ void deleteResponseBlobs_shouldUpdateEodWhenResponsesAreDeleted() {
             .uploadFileFilenameProcessor(new UploadFileFilenameProcessor(uploadFileFilename1))
             .build();
 
-        when(armDataManagementApi.deleteBlobData(anyString())).thenReturn(true);
+        when(armDataManagementApi.deleteMultipleBlobs(any())).thenReturn(true);
 
         // when
         deleteArmResponseFilesHelper.deleteResponseBlobs(batchData);
 
         // then
-        verify(armDataManagementApi).deleteBlobData(createRecordFilename1);
-        verify(armDataManagementApi).deleteBlobData(uploadFileFilename1);
+        verify(armDataManagementApi).deleteMultipleBlobs(List.of(createRecordFilename1, uploadFileFilename1));
         verifyNoMoreInteractions(armDataManagementApi);
     }
 

src/integrationTest/java/uk/gov/hmcts/darts/testutils/stubs/ArmServiceStubImpl.java

@@ -76,6 +76,12 @@ public boolean deleteBlobData(String containerName, String blobPathAndName) {
         return true;
     }
 
+    @Override
+    public boolean deleteMultipleBlobs(String containerName, List<String> blobPathAndName) {
+        logStubUsageWarning();
+        return true;
+    }
+
     private void logStubUsageWarning() {
         log.warn("### This implementation is intended only for integration tests. If you see this log message elsewhere"
                      + " you should ask questions! ###");

src/main/java/uk/gov/hmcts/darts/arm/api/ArmDataManagementApi.java

@@ -27,6 +27,8 @@ public interface ArmDataManagementApi extends BlobContainerDownloadable {
 
     boolean deleteBlobData(String blobPathAndName);
 
+    boolean deleteMultipleBlobs(List<String> blobPathAndNames);
+
     UpdateMetadataResponse updateMetadata(String externalRecordId,
                                           OffsetDateTime eventTimestamp,
                                           RetentionConfidenceScoreEnum retConfScore,

src/main/java/uk/gov/hmcts/darts/arm/api/impl/ArmDataManagementApiImpl.java

@@ -26,6 +26,7 @@
 
 @Service
 @RequiredArgsConstructor
+@SuppressWarnings({"PMD.TooManyMethods"})
 public class ArmDataManagementApiImpl implements ArmDataManagementApi {
 
     private final ArmService armService;
@@ -88,6 +89,11 @@ public boolean deleteBlobData(String blobPathAndName) {
         return armService.deleteBlobData(armDataManagementConfiguration.getContainerName(), blobPathAndName);
     }
 
+    @Override
+    public boolean deleteMultipleBlobs(List<String> blobPathAndNames) {
+        return armService.deleteMultipleBlobs(armDataManagementConfiguration.getContainerName(), blobPathAndNames);
+    }
+
     @Override
     public UpdateMetadataResponse updateMetadata(String externalRecordId,
                                                  OffsetDateTime eventTimestamp,

src/main/java/uk/gov/hmcts/darts/arm/service/ArmService.java

@@ -34,4 +34,6 @@ public interface ArmService {
      */
     boolean deleteBlobData(String containerName, String blobPathAndName);
 
+    boolean deleteMultipleBlobs(String containerName, List<String> blobPathAndName);
+
 }

src/main/java/uk/gov/hmcts/darts/arm/service/DeleteArmResponseFilesHelper.java

@@ -11,10 +11,12 @@ public interface DeleteArmResponseFilesHelper {
 
     void deleteDanglingResponses(BatchInputUploadFileFilenameProcessor batchUploadFileFilenameProcessor);
 
-    List<Boolean> deleteResponseBlobs(List<String> responseBlobsToBeDeleted);
+    Boolean deleteResponseBlobIndividually(String responseBlobsToBeDeleted);
+
+    Boolean deleteResponseBlobs(List<String> responseBlobsToBeDeleted);
 
     void deleteResponseBlobs(ArmResponseBatchData armResponseBatchData);
 
     List<String> getResponseBlobsToBeDeleted(ArmResponseBatchData armResponseBatchData);
-    
+
 }

src/main/java/uk/gov/hmcts/darts/arm/service/impl/AbstractArmBatchProcessResponseFiles.java

@@ -575,7 +575,7 @@ private ArmResponseCreateRecord getResponseCreateRecordOrDelete(String createRec
             return objectMapper.readValue(createRecordBinary.toString(), ArmResponseCreateRecord.class);
         } catch (Exception e) {
             log.error("Unable to read ARM response create record file {} - About to delete ", createRecordFilenameAndPath, e);
-            deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(createRecordFilenameAndPath));
+            deleteArmResponseFilesHelper.deleteResponseBlobIndividually(createRecordFilenameAndPath);
             throw e;
         }
     }
@@ -628,7 +628,7 @@ private ArmResponseUploadFileRecord getResponseUploadFileRecordOrDelete(String u
             return objectMapper.readValue(uploadFileBinary.toString(), ArmResponseUploadFileRecord.class);
         } catch (Exception e) {
             log.error("Unable to read ARM response upload file {} - About to delete ", uploadFileFilenameAndPath, e);
-            deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(uploadFileFilenameAndPath));
+            deleteArmResponseFilesHelper.deleteResponseBlobIndividually(uploadFileFilenameAndPath);
             throw e;
         }
     }
@@ -759,7 +759,7 @@ private ResponseFilenames getArmResponseFilenames(List<String> responseFiles, St
             } catch (IllegalArgumentException e) {
                 // This occurs when the filename is not parsable
                 log.error("Invalid ARM response filename: {} for manifest {}", responseFile, manifestName);
-                deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(responseFile));
+                deleteArmResponseFilesHelper.deleteResponseBlobIndividually(responseFile);
             }
         }
         return responseFilenames;
@@ -798,7 +798,7 @@ private void readInvalidLineFile(BinaryData invalidLineFileBinary, InvalidLineFi
                 } else {
                     log.warn("Failed to obtain EOD id (relation id) from invalid line record {} from file {}", input,
                              invalidLineFileFilenameProcessor.getInvalidLineFilename());
-                    deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(invalidLineFileFilenameAndPath));
+                    deleteArmResponseFilesHelper.deleteResponseBlobIndividually(invalidLineFileFilenameAndPath);
                 }
 
             } catch (Exception e) {
@@ -815,7 +815,7 @@ private ArmResponseInvalidLineRecord getResponseInvalidLineRecordOrDelete(String
             return objectMapper.readValue(invalidLineFileBinary.toString(), ArmResponseInvalidLineRecord.class);
         } catch (Exception e) {
             log.error("Unable to read ARM response {} - About to delete ", invalidLineFileFilenameAndPath, e);
-            deleteArmResponseFilesHelper.deleteResponseBlobs(List.of(invalidLineFileFilenameAndPath));
+            deleteArmResponseFilesHelper.deleteResponseBlobIndividually(invalidLineFileFilenameAndPath);
             throw e;
         }
     }

src/main/java/uk/gov/hmcts/darts/arm/service/impl/ArmServiceImpl.java

@@ -6,6 +6,9 @@
 import com.azure.core.util.BinaryData;
 import com.azure.storage.blob.BlobClient;
 import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.batch.BlobBatchClient;
+import com.azure.storage.blob.batch.BlobBatchClientBuilder;
 import com.azure.storage.blob.models.BlobItem;
 import com.azure.storage.blob.models.BlobListDetails;
 import com.azure.storage.blob.models.DeleteSnapshotsOptionType;
@@ -36,7 +39,7 @@
 
 @Service
 @Slf4j
-@SuppressWarnings("PMD.TooManyMethods")//TODO - refactor to reduce methods when this class is next edited
+@SuppressWarnings({"PMD.TooManyMethods", "PMD.CouplingBetweenObjects"})
 public class ArmServiceImpl implements ArmService {
 
     private static final String FILE_PATH_DELIMITER = "/";
@@ -280,4 +283,52 @@ public boolean deleteBlobData(String containerName, String blobPathAndName) {
         }
     }
 
+    @Override
+    public boolean deleteMultipleBlobs(String containerName, List<String> blobPathAndName) {
+        if (blobPathAndName == null || blobPathAndName.isEmpty()) {
+            log.info("No blobs provided to delete for containerName={}", containerName);
+            return false;
+        }
+
+        try {
+            BlobContainerClient containerClient = armDataManagementDao.getBlobContainerClient(containerName);
+
+            // Delete *all* blobs in one go using Azure Storage Blob Batch.
+            BlobServiceClient blobServiceClient = containerClient.getServiceClient();
+            BlobBatchClient batchClient = new BlobBatchClientBuilder(blobServiceClient).buildClient();
+
+            boolean allSuccessful = true;
+            PagedIterable<Response<Void>> responses = batchClient.deleteBlobs(
+                blobPathAndName,
+                DeleteSnapshotsOptionType.INCLUDE,
+                Duration.of(TIMEOUT, ChronoUnit.SECONDS),
+                null
+            );
+
+            int index = 0;
+            for (Response<Void> response : responses) {
+                int statusCode = response.getStatusCode();
+                HttpStatus httpStatus = valueOf(statusCode);
+                String blobName = index < blobPathAndName.size() ? blobPathAndName.get(index) : "<unknown>";
+                index++;
+
+                if (!(httpStatus.is2xxSuccessful() || NOT_FOUND.equals(httpStatus))) {
+                    allSuccessful = false;
+                    log.warn("Failed to delete blob in batch containerName={}, blobPathAndName={}, statusCode={}, httpStatus={}",
+                             containerName, blobName, statusCode, httpStatus);
+                }
+            }
+
+            if (allSuccessful) {
+                log.info("Successfully deleted {} blobs from containerName={} using batch", blobPathAndName.size(), containerName);
+            } else {
+                log.error("Batch deletion completed with one or more failures for containerName={}", containerName);
+            }
+            return allSuccessful;
+        } catch (Exception e) {
+            log.error("Could not delete multiple blobs from storage container={}", containerName, e);
+            return false;
+        }
+    }
+
 }