gh-419 Implement data output plug-in engine and integration with SCU/DICOMWeb export services (#428)

* Update Ardalis.GuardClauses
* Update coverlet.collector
* Update Docker.DotNet
* Update xunit
* Update NLog
* Update MongoDB
* Update Polly
* Update Monai.Deploy libraries
* Update Microsoft .NET 6 libraries
* Update fo-dicom
* Remove fo-dicom logging support and use .NET logging
* Implement OutputDataPluginEngine  for export service (SCU, DICOMWeb)
* Add WorkflowInstanceId and TaskId properties for plugins to use
* Fix package hash
* Fix JSON property name of TaskId
* Update Moq, NLog, NLog.Web.AspNetCore and dependencie decisions
* Update dependency decisions: Devlooped.SponsorLink
* Fix comments
Signed-off-by: Victor Chang <vicchang@nvidia.com>

Author: mocsharp

doc/dependency_decisions.yml

@@ -795,14 +795,14 @@
   - :who: neilsouth
     :why: Apache-2.0 (https://github.com/Project-MONAI/monai-deploy-messaging/raw/main/LICENSE)
     :versions:
-    - 0.1.23
+    - 0.1.24
     :when: 2022-08-16 23:06:21.051573547 Z
 - - :approve
   - Monai.Deploy.Messaging.RabbitMQ
   - :who: neilsouth
     :why: Apache-2.0 (https://github.com/Project-MONAI/monai-deploy-messaging/raw/main/LICENSE)
     :versions:
-    - 0.1.23
+    - 0.1.24
     :when: 2022-08-16 23:06:21.511789690 Z
 - - :approve
   - Monai.Deploy.Storage
@@ -838,7 +838,7 @@
     :why: BSD 3-Clause License ( https://raw.githubusercontent.com/moq/moq4/main/License.txt)
     :versions:
     - 4.18.1
-    - 4.18.4
+    - 4.20.1
     :when: 2022-08-16 23:06:23.359197359 Z
 - - :approve
   - NETStandard.Library
@@ -897,7 +897,7 @@
   - :who: mocsharp
     :why: Apache-2.0 (https://github.com/rabbitmq/rabbitmq-dotnet-client/raw/main/LICENSE-APACHE2)
     :versions:
-    - 6.4.0
+    - 6.5.0
     :when: 2022-08-16 23:06:28.818109746 Z
 - - :approve
   - SQLitePCLRaw.bundle_e_sqlite3
@@ -1299,6 +1299,7 @@
     :why: MIT (https://github.com/dotnet/corefx/raw/master/LICENSE.TXT)
     :versions:
     - 4.5.4
+    - 4.5.5
     :when: 2022-08-16 23:06:55.672403035 Z
 - - :approve
   - System.Net.Http
@@ -1694,6 +1695,7 @@
     :why: MIT (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
     :versions:
     - 6.0.0
+    - 7.0.0
     :when: 2022-08-16 23:07:22.653576384 Z
 - - :approve
   - System.Threading.Overlapped
@@ -2295,21 +2297,21 @@
   - :who: mocsharp
     :why: BSD 3-Clause License (https://github.com/NLog/NLog/raw/dev/LICENSE.txt)
     :versions:
-    - 5.2.2
+    - 5.2.3
     :when: 2022-10-12 03:14:06.538744982 Z
 - - :approve
   - NLog.Extensions.Logging
   - :who: mocsharp
     :why: BSD 2-Clause Simplified License (https://github.com/NLog/NLog.Extensions.Logging/raw/master/LICENSE)
     :versions:
-    - 5.3.2
+    - 5.3.3
     :when: 2022-10-12 03:14:06.964203977 Z
 - - :approve
   - NLog.Web.AspNetCore
   - :who: mocsharp
     :why: BSD 3-Clause License (https://github.com/NLog/NLog.Web/raw/master/LICENSE)
     :versions:
-    - 5.3.2
+    - 5.3.3
     :when: 2022-10-12 03:14:07.396706995 Z
 - - :approve
   - fo-dicom.NLog
@@ -2458,5 +2460,15 @@
     :versions:
     - 1.1.1
     :when: 2023-08-04 0:02:30.206982078 Z
+- - :approve
+  - Devlooped.SponsorLink
+  - :who: mocsharp
+    :why: MIT (https://licenses.nuget.org/MIT)
+    :versions:
+    - 1.0.0
+    :when: 2023-08-08 0:08:05.206982078 Z
+
+
+
 
 

docs/compliance/third-party-licenses.md

@@ -7304,14 +7304,14 @@ Apache License
 
 
 <details>
-<summary>Monai.Deploy.Messaging 0.1.21</summary>
+<summary>Monai.Deploy.Messaging 0.1.24</summary>
 
 ## Monai.Deploy.Messaging
 
-- Version: 0.1.21
+- Version: 0.1.24
 - Authors: MONAI Consortium
 - Project URL: https://github.com/Project-MONAI/monai-deploy-messaging
-- Source: [NuGet](https://www.nuget.org/packages/Monai.Deploy.Messaging/0.1.21)
+- Source: [NuGet](https://www.nuget.org/packages/Monai.Deploy.Messaging/0.1.24)
 - License: [Apache-2.0](https://github.com/Project-MONAI/monai-deploy-messaging/raw/main/LICENSE)
 
 
@@ -7532,14 +7532,14 @@ By downloading this software, you agree to the license terms & all licenses list
 
 
 <details>
-<summary>Monai.Deploy.Messaging.RabbitMQ 0.1.21</summary>
+<summary>Monai.Deploy.Messaging.RabbitMQ 0.1.24</summary>
 
 ## Monai.Deploy.Messaging.RabbitMQ
 
-- Version: 0.1.21
+- Version: 0.1.24
 - Authors: MONAI Consortium
 - Project URL: https://github.com/Project-MONAI/monai-deploy-messaging
-- Source: [NuGet](https://www.nuget.org/packages/Monai.Deploy.Messaging.RabbitMQ/0.1.21)
+- Source: [NuGet](https://www.nuget.org/packages/Monai.Deploy.Messaging.RabbitMQ/0.1.24)
 - License: [Apache-2.0](https://github.com/Project-MONAI/monai-deploy-messaging/raw/main/LICENSE)
 
 
@@ -8847,14 +8847,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 <details>
-<summary>Moq 4.18.4</summary>
+<summary>Moq 4.20.1</summary>
 
 ## Moq
 
-- Version: 4.18.4
+- Version: 4.20.1
 - Authors: Daniel Cazzulino, kzu
 - Project URL: https://github.com/moq/moq4
-- Source: [NuGet](https://www.nuget.org/packages/Moq/4.18.4)
+- Source: [NuGet](https://www.nuget.org/packages/Moq/4.20.1)
 - License: [BSD 3-Clause License]( https://raw.githubusercontent.com/moq/moq4/main/License.txt)
 
 
@@ -9182,14 +9182,14 @@ THE POSSIBILITY OF SUCH DAMAGE.
 
 
 <details>
-<summary>NLog.Extensions.Logging 5.2.2</summary>
+<summary>NLog.Extensions.Logging 5.2.3</summary>
 
 ## NLog.Extensions.Logging
 
-- Version: 5.2.2
+- Version: 5.2.3
 - Authors: Microsoft,Julian Verdurmen
 - Project URL: https://github.com/NLog/NLog.Extensions.Logging
-- Source: [NuGet](https://www.nuget.org/packages/NLog.Extensions.Logging/5.2.2)
+- Source: [NuGet](https://www.nuget.org/packages/NLog.Extensions.Logging/5.2.3)
 - License: [BSD 2-Clause Simplified License](https://github.com/NLog/NLog.Extensions.Logging/raw/master/LICENSE)
 
 
@@ -9223,14 +9223,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 <details>
-<summary>NLog.Web.AspNetCore 5.2.2</summary>
+<summary>NLog.Web.AspNetCore 5.2.3</summary>
 
 ## NLog.Web.AspNetCore
 
-- Version: 5.2.2
+- Version: 5.2.3
 - Authors: Julian Verdurmen
 - Project URL: https://github.com/NLog/NLog.Web
-- Source: [NuGet](https://www.nuget.org/packages/NLog.Web.AspNetCore/5.2.2)
+- Source: [NuGet](https://www.nuget.org/packages/NLog.Web.AspNetCore/5.2.3)
 - License: [BSD 3-Clause License](https://github.com/NLog/NLog.Web/raw/master/LICENSE)
 
 
@@ -9571,14 +9571,14 @@ C# is a registered trademark of Microsoft ®.
 
 
 <details>
-<summary>RabbitMQ.Client 6.4.0</summary>
+<summary>RabbitMQ.Client 6.5.0</summary>
 
 ## RabbitMQ.Client
 
-- Version: 6.4.0
+- Version: 6.5.0
 - Authors: VMware
 - Project URL: https://www.rabbitmq.com/dotnet.html
-- Source: [NuGet](https://www.nuget.org/packages/RabbitMQ.Client/6.4.0)
+- Source: [NuGet](https://www.nuget.org/packages/RabbitMQ.Client/6.5.0)
 - License: [Apache-2.0](https://github.com/rabbitmq/rabbitmq-dotnet-client/raw/main/LICENSE-APACHE2)
 
 
@@ -17176,15 +17176,15 @@ SOFTWARE.
 
 
 <details>
-<summary>System.Memory 4.5.4</summary>
+<summary>System.Memory 4.5.5</summary>
 
 ## System.Memory
 
-- Version: 4.5.4
+- Version: 4.5.5
 - Authors: Microsoft
 - Owners: microsoft,dotnetframework
 - Project URL: https://dot.net/
-- Source: [NuGet](https://www.nuget.org/packages/System.Memory/4.5.4)
+- Source: [NuGet](https://www.nuget.org/packages/System.Memory/4.5.5)
 - License: [MIT](https://github.com/dotnet/corefx/raw/master/LICENSE.TXT)
 
 
@@ -26326,14 +26326,14 @@ SOFTWARE.
 
 
 <details>
-<summary>System.Threading.Channels 6.0.0</summary>
+<summary>System.Threading.Channels 7.0.0</summary>
 
 ## System.Threading.Channels
 
-- Version: 6.0.0
+- Version: 7.0.0
 - Authors: Microsoft
 - Project URL: https://dot.net/
-- Source: [NuGet](https://www.nuget.org/packages/System.Threading.Channels/6.0.0)
+- Source: [NuGet](https://www.nuget.org/packages/System.Threading.Channels/7.0.0)
 - License: [MIT](https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
 
 

src/InformaticsGateway/Services/Export/ExportRequestDataMessage.cs renamed to src/Api/ExportRequestDataMessage.cs

@@ -19,18 +19,29 @@
 using Ardalis.GuardClauses;
 using Monai.Deploy.Messaging.Events;
 
-namespace Monai.Deploy.InformaticsGateway.Services.Export
+namespace Monai.Deploy.InformaticsGateway.Api
 {
     public class ExportRequestDataMessage
     {
         private readonly ExportRequestEvent _exportRequest;
 
-        public byte[] FileContent { get; private set; }
+        public byte[] FileContent { get; private set; } = default!;
         public bool IsFailed { get; private set; }
         public IList<string> Messages { get; init; }
         public FileExportStatus ExportStatus { get; private set; }
         public string Filename { get; }
 
+        /// <summary>
+        /// Optional list of data output plug-in type names to be executed by the <see cref="IOutputDataPluginEngine"/>.
+        /// </summary>
+        public List<string> PluginAssemblies
+        {
+            get
+            {
+                return _exportRequest.PluginAssemblies;
+            }
+        }
+
         public string ExportTaskId
         {
             get { return _exportRequest.ExportTaskId; }
@@ -46,6 +57,7 @@ public string[] Destinations
             get { return _exportRequest.Destinations; }
         }
 
+
         public ExportRequestDataMessage(ExportRequestEvent exportRequest, string filename)
         {
             IsFailed = false;

src/Api/IInputDataPluginEngine.cs

@@ -26,9 +26,10 @@ namespace Monai.Deploy.InformaticsGateway.Api
     /// a list of plug-ins based on <see cref="IInputDataPlugin"/>.
     /// Rules:
     /// <list type="bullet">
-    /// <item>A list of plug-ins can be included with each export request, and each plug-in is executed in the order stored, processing one file at a time, enabling piping of the data before each file is exported.</item>
-    /// <item>Plugins MUST be lightweight and not hinder the export process</item>
-    /// <item>Plugins SHALL not accumulate files in memory or storage for bulk processing</item>
+    /// <item>SCP: A list of plug-ins can be configured with each AET, and each plug-in is executed in the order stored, enabling piping of the incoming data before each file is uploaded to the storage service.</item>
+    /// <item>Incoming data is processed one file at a time and SHALL not wait for the entire study to arrive.</item>
+    /// <item>Plugins MUST be lightweight and not hinder the upload process.</item>
+    /// <item>Plugins SHALL not accumulate files in memory or storage for bulk processing.</item>
     /// </list>
     /// </summary>
     public interface IInputDataPluginEngine

src/Api/IOutputDataPlugin.cs

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2023 MONAI Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System.Threading.Tasks;
+using FellowOakDicom;
+
+namespace Monai.Deploy.InformaticsGateway.Api
+{
+    /// <summary>
+    /// <c>IOutputDataPlugin</c> enables lightweight data processing over incoming data received from supported data ingestion
+    /// services.
+    /// Refer to <see cref="IOutputDataPluginEngine" /> for additional details.
+    /// </summary>
+    public interface IOutputDataPlugin
+    {
+        Task<(DicomFile dicomFile, ExportRequestDataMessage exportRequestDataMessage)> Execute(DicomFile dicomFile, ExportRequestDataMessage exportRequestDataMessage);
+    }
+}

src/Api/IOutputDataPluginEngine.cs

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 MONAI Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Monai.Deploy.InformaticsGateway.Api
+{
+    /// <summary>
+    /// <c>IOutputDataPluginEngine</c> processes each file before exporting to its destination
+    /// through a list of plug-ins based on <see cref="IOutputDataPlugin"/>.
+    /// Rules:
+    /// <list type="bullet">
+    /// <item>A list of plug-ins can be included with each export request, and each plug-in is executed in the order stored, processing one file at a time, enabling piping of the data before each file is exported.</item>
+    /// <item>Plugins MUST be lightweight and not hinder the export process.</item>
+    /// <item>Plugins SHALL not accumulate files in memory or storage for bulk processing.</item>
+    /// </list>
+    /// </summary>
+    public interface IOutputDataPluginEngine
+    {
+        void Configure(IReadOnlyList<string> pluginAssemblies);
+
+        Task<ExportRequestDataMessage> ExecutePlugins(ExportRequestDataMessage exportRequestDataMessage);
+    }
+}

src/Api/Monai.Deploy.InformaticsGateway.Api.csproj

@@ -30,7 +30,7 @@
   <ItemGroup>
     <PackageReference Include="Macross.Json.Extensions" Version="3.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Abstractions" Version="6.0.20" />
-    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.23" />
+    <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.24" />
     <PackageReference Include="Monai.Deploy.Storage" Version="0.2.16" />
   </ItemGroup>
 

src/Api/Storage/FileStorageMetadata.cs

@@ -81,6 +81,20 @@ public abstract record FileStorageMetadata
         [JsonPropertyName("dateReceived")]
         public DateTime DateReceived { get; init; } = default!;
 
+        /// <summary>
+        /// Gets or sets the workflow instance ID for the workflow manager to resume a workflow.
+        /// </summary>
+        /// <value></value>
+        [JsonPropertyName("WorkflowInstanceId")]
+        public string? WorkflowInstanceId { get; set; }
+
+        /// <summary>
+        /// Gets or sets the task ID for the workflow manager to resume a workflow.
+        /// </summary>
+        /// <value></value>
+        [JsonPropertyName("taskId")]
+        public string? TaskId { get; set; }
+
         /// <summary>
         /// DO NOT USE
         /// This constructor is intended for JSON serializer.