small clean ups

Signed-off-by: Neil South <neil.south@answerdigital.com>

Author: neildsouth

src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs

@@ -77,7 +77,6 @@ public async Task MoveFilesAsync(Payload payload, ActionBlock<Payload> moveQueue
             var stopwatch = Stopwatch.StartNew();
             try
             {
-                await Move(payload, cancellationToken).ConfigureAwait(false);
                 await NotifyIfCompleted(payload, notificationQueue, cancellationToken).ConfigureAwait(false);
             }
             catch (Exception ex)
@@ -127,149 +126,6 @@ private async Task NotifyIfCompleted(Payload payload, ActionBlock<Payload> notif
             }
         }
 
-        private async Task Move(Payload payload, CancellationToken cancellationToken)
-        {
-            Guard.Against.Null(payload);
-
-            _logger.MovingFIlesInPayload(payload.PayloadId, _options.Value.Storage.StorageServiceBucketName);
-
-            var options = new ParallelOptions
-            {
-                CancellationToken = cancellationToken,
-                MaxDegreeOfParallelism = _options.Value.Storage.ConcurrentUploads
-            };
-
-            var exceptions = new List<Exception>();
-            await Parallel.ForEachAsync(payload.Files, options, async (file, cancellationToke) =>
-            {
-                try
-                {
-                    switch (file)
-                    {
-                        case DicomFileStorageMetadata dicom:
-                            if (!string.IsNullOrWhiteSpace(dicom.JsonFile.TemporaryPath))
-                            {
-                                await MoveFile(payload.PayloadId, dicom.Id, dicom.JsonFile, cancellationToken).ConfigureAwait(false);
-                            }
-                            break;
-                    }
-
-                    await MoveFile(payload.PayloadId, file.Id, file.File, cancellationToken).ConfigureAwait(false);
-                }
-                catch (Exception ex)
-                {
-                    exceptions.Add(ex);
-                }
-            }).ConfigureAwait(false);
-
-            if (exceptions.Any())
-            {
-                throw new AggregateException(exceptions);
-            }
-        }
-
-        private async Task MoveFile(Guid payloadId, string identity, StorageObjectMetadata file, CancellationToken cancellationToken)
-        {
-            Guard.Against.NullOrWhiteSpace(identity);
-            Guard.Against.Null(file);
-
-            if (file.IsMoveCompleted)
-            {
-                _logger.AlreadyMoved(payloadId, file.UploadPath);
-                return;
-            }
-
-            _logger.MovingFileToPayloadDirectory(payloadId, file.UploadPath);
-
-            try
-            {
-                //await _storageService.CopyObjectAsync(
-                //    file.TemporaryBucketName,
-                //    file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath),
-                //    _options.Value.Storage.StorageServiceBucketName,
-                //    file.GetPayloadPath(payloadId),
-                //    cancellationToken).ConfigureAwait(false);
-
-                await VerifyFileExists(payloadId, file, cancellationToken).ConfigureAwait(false);
-            }
-            catch (StorageObjectNotFoundException ex) when (ex.Message.Contains("Not found", StringComparison.OrdinalIgnoreCase)) // TODO: StorageLib shall not throw any errors from MINIO
-            {
-                // when file cannot be found on the Storage Service, we assume file has been moved previously by verifying the file exists on destination.
-                _logger.FileMissingInPayload(payloadId, file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), ex);
-                await VerifyFileExists(payloadId, file, cancellationToken).ConfigureAwait(false);
-            }
-            catch (StorageConnectionException ex)
-            {
-                _logger.StorageServiceConnectionError(ex);
-                throw new PayloadNotifyException(PayloadNotifyException.FailureReason.ServiceUnavailable);
-            }
-            catch (Exception ex)
-            {
-                _logger.PayloadMoveException(ex);
-                await LogFilesInMinIo(file.TemporaryBucketName, cancellationToken).ConfigureAwait(false);
-                throw new FileMoveException(file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), file.UploadPath, ex);
-            }
-
-            try
-            {
-                //_logger.DeletingFileFromTemporaryBbucket(file.TemporaryBucketName, identity, file.TemporaryPath);
-                //await _storageService.RemoveObjectAsync(file.TemporaryBucketName, file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), cancellationToken).ConfigureAwait(false);
-            }
-            catch (Exception)
-            {
-                _logger.ErrorDeletingFileAfterMoveComplete(file.TemporaryBucketName, identity, file.TemporaryPath);
-            }
-            finally
-            {
-                file.SetMoved(_options.Value.Storage.StorageServiceBucketName);
-            }
-        }
-
-        private async Task VerifyFileExists(Guid payloadId, StorageObjectMetadata file, CancellationToken cancellationToken)
-        {
-            await Policy
-               .Handle<VerifyObjectsException>()
-               .WaitAndRetryAsync(
-                   _options.Value.Storage.Retries.RetryDelays,
-                   (exception, timeSpan, retryCount, context) =>
-                   {
-                       _logger.ErrorUploadingFileToTemporaryStore(timeSpan, retryCount, exception);
-                   })
-               .ExecuteAsync(async () =>
-               {
-                   var internalCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-                   internalCancellationTokenSource.CancelAfter(_options.Value.Storage.StorageServiceListTimeout);
-                   var exists = await _storageService.VerifyObjectExistsAsync(_options.Value.Storage.StorageServiceBucketName, file.GetPayloadPath(payloadId), cancellationToken).ConfigureAwait(false);
-                   if (!exists)
-                   {
-                       _logger.FileMovedVerificationFailure(payloadId, file.UploadPath);
-                       throw new PayloadNotifyException(PayloadNotifyException.FailureReason.MoveFailure, false);
-                   }
-               })
-               .ConfigureAwait(false);
-        }
-
-        private async Task LogFilesInMinIo(string bucketName, CancellationToken cancellationToken)
-        {
-            try
-            {
-                var internalCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-                internalCancellationTokenSource.CancelAfter(_options.Value.Storage.StorageServiceListTimeout);
-                var listingResults = await _storageService.ListObjectsAsync(bucketName, recursive: true, cancellationToken: internalCancellationTokenSource.Token).ConfigureAwait(false);
-                _logger.FilesFounddOnStorageService(bucketName, listingResults.Count);
-                var files = new List<string>();
-                foreach (var item in listingResults)
-                {
-                    files.Add(item.FilePath);
-                }
-                _logger.FileFounddOnStorageService(bucketName, string.Join(Environment.NewLine, files));
-            }
-            catch (Exception ex)
-            {
-                _logger.ErrorListingFilesOnStorageService(ex);
-            }
-        }
-
         private async Task<PayloadAction> UpdatePayloadState(Payload payload, Exception ex, CancellationToken cancellationToken = default)
         {
             Guard.Against.Null(payload);

src/InformaticsGateway/Services/Storage/ObjectUploadService.cs

@@ -198,7 +198,6 @@ private async Task UploadFileAndConfirm(string identifier, StorageObjectMetadata
                     throw new FileUploadException($"Failed to upload file after retries {identifier}.");
                 }
             } while (!(
-            //await VerifyExists(storageObjectMetadata.GetTempStoragPath(_configuration.Value.Storage.RemoteTemporaryStoragePath), cancellationToken).ConfigureAwait(false)
             await VerifyExists(storageObjectMetadata.GetPayloadPath(Guid.Parse(payloadId)), cancellationToken).ConfigureAwait(false)
             ));
         }
@@ -217,7 +216,6 @@ private async Task<bool> VerifyExists(string path, CancellationToken cancellatio
                {
                    var internalCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
                    internalCancellationTokenSource.CancelAfter(_configuration.Value.Storage.StorageServiceListTimeout);
-                   //var exists = await _storageService.VerifyObjectExistsAsync(_configuration.Value.Storage.TemporaryStorageBucket, path).ConfigureAwait(false);
                    var exists = await _storageService.VerifyObjectExistsAsync(_configuration.Value.Storage.StorageServiceBucketName, path).ConfigureAwait(false);
                    _logger.VerifyFileExists(path, exists);
                    return exists;
@@ -246,25 +244,21 @@ await Policy
                {
                    if (storageObjectMetadata.IsUploaded) { return; }
 
-                   ////////////////////
                    var bucket = _configuration.Value.Storage.StorageServiceBucketName;
                    var path = storageObjectMetadata.GetPayloadPath(Guid.Parse(payloadId));
-                   //////////////////
 
                    storageObjectMetadata.Data.Seek(0, System.IO.SeekOrigin.Begin);
                    await _storageService.PutObjectAsync(
-                       //_configuration.Value.Storage.TemporaryStorageBucket,
                        bucket,
-                       //storageObjectMetadata.GetTempStoragPath(_configuration.Value.Storage.RemoteTemporaryStoragePath),
                        path,
                        storageObjectMetadata.Data,
                        storageObjectMetadata.Data.Length,
                        storageObjectMetadata.ContentType,
                        metadata,
                        cancellationToken).ConfigureAwait(false);
-                   storageObjectMetadata.SetUploaded(_configuration.Value.Storage.TemporaryStorageBucket); // deletes local file
+                   storageObjectMetadata.SetUploaded(_configuration.Value.Storage.TemporaryStorageBucket); // deletes local file and sets uploaded to true
                    _logger.UploadedFileToTemporaryStore(storageObjectMetadata.TemporaryPath);
-                   storageObjectMetadata.SetMoved(_configuration.Value.Storage.StorageServiceBucketName);
+                   storageObjectMetadata.SetMoved(_configuration.Value.Storage.StorageServiceBucketName); // set bucket, date moved, and move complete
                })
                .ConfigureAwait(false);
         }