Enh/5943 DropIns - Prevention of File Locks and Unloading (#5944)

* fix - supressed warning IL2072
Changed DropInDescriptor from Record to Class to be able to add the suggested DynamicallyAccesedMembers decorator.

* Add Unconfigure method to DropIn class and interface

The `DropIn` class in `DropIn.cs` now includes a new `Unconfigure` method. This method handles the unconfiguration process when a drop-in is deleted. It retrieves the `ILogger<DropIn>` and `IActivityRegistry` services from the `IServiceProvider`, removes the `SampleActivity` from the activity registry, and logs the outcome.

The `IDropIn` interface in `IDropIn.cs` has been updated to include the new `Unconfigure` method, ensuring that it is called when the drop-in is deleted.

Additional using directives have been added to `DropIn.cs` for `Elsa.Workflows`, and `Microsoft.Extensions.Logging`.

* Refactor AssemblyLoader to use memory stream for loading

Modified the `LoadPath` method in the `AssemblyLoader` class to load assemblies from a memory stream instead of directly from the file. This change prevents file locking issues by copying the file contents to a memory stream and then loading the assembly from this stream.

* Dispose packageReader after loading assembly

Ensure the packageReader is disposed of after loading the assembly from the memory stream. This change adds a call to `packageReader.Dispose()` to close the package reader and the associated `.nupkg` file, which helps in releasing resources and preventing potential file locks or memory leaks.

* Add drop-in unloading on file deletion

Updated DropInDirectoryMonitorHostedService to handle unloading of drop-ins when files are deleted. Added `_debouncedUnloader` and `_installedDropIns` fields. Modified constructor to initialize these fields. Updated `ExecuteAsync` to call `LoadDropInAssemblyAsync` initially and handle file deletion events. Added `OnDeleted` method to manage file deletions and invoke the debounced unloader. Enhanced `LoadDropInAssemblyAsync` to track loaded drop-ins. Introduced `UnloadDropInAssemblyAsync` to remove drop-ins on file deletion.

* Load assembly from MemoryStream instead of FileStream

Modified the assembly loading process to use a MemoryStream
instead of a FileStream. This change ensures that the
FileStream is properly closed before the assembly is loaded,
improving resource management and preventing potential file
access issues.

* Improve logging and error handling in DropIn services

Removed logging from DropIn.Unconfigure method and added
ILogger dependency to DropInDirectoryMonitorHostedService.
Updated csproj to include Microsoft.Extensions.Logging.Abstractions.
Enhanced error handling in UnloadDropInAssemblyAsync method.
Minor formatting improvements for readability.

samples/drop-ins/SampleDropIn/DropIn.cs

@@ -1,9 +1,11 @@
 using Elsa.DropIns.Core;
 using Elsa.Extensions;
 using Elsa.Features.Services;
+using Elsa.Workflows;
 using Elsa.Workflows.Contracts;
 using JetBrains.Annotations;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using SampleDropIn.Activities;
 
 namespace SampleDropIn;
@@ -21,4 +23,10 @@ public class DropIn : IDropIn
         var activityRegistry = serviceProvider.GetRequiredService<IActivityRegistry>();
         await activityRegistry.RegisterAsync<SampleActivity>(cancellationToken: cancellationToken);
     }
+
+    public void Unconfigure(IServiceProvider serviceProvider)
+    {
+        var activityRegistry = serviceProvider.GetRequiredService<IActivityRegistry>();
+        activityRegistry.Remove(typeof(ActivityRegistry), activityRegistry.Find<SampleActivity>()!);
+    }
 }

src/common/Elsa.DropIns.Core/IDropIn.cs

@@ -17,4 +17,9 @@ public interface IDropIn
     /// Called when the drop-in is being configured.
     /// </summary>
     ValueTask ConfigureAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken);
+
+    /// <summary>
+    /// Unconfigure the drop-in when it's deleted.
+    /// </summary>
+    void Unconfigure(IServiceProvider serviceProvider);
 }

src/common/Elsa.DropIns/Contexts/NuGetPackageAssemblyLoadContext.cs

@@ -22,6 +22,8 @@ internal sealed class NuGetPackageAssemblyLoadContext : AssemblyLoadContext
 
             _loadedAssemblies[assembly.FullName!] = assembly;
         }
+        //Closes the package reader, closing the .nupkg file
+        packageReader.Dispose();
     }
 
     protected override Assembly? Load(AssemblyName assemblyName)

src/common/Elsa.DropIns/Elsa.DropIns.csproj

@@ -10,13 +10,15 @@
     <ItemGroup>
         <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
         <PackageReference Include="Microsoft.Extensions.Options" />
+        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
         <PackageReference Include="NuGet.Packaging" />
         <PackageReference Include="NuGet.Protocol" />
         <PackageReference Include="System.Formats.Asn1" />
         <PackageReference Include="ThrottleDebounce" />
     </ItemGroup>
 
-    <!--Overridden for vulnerability reasons with dependencies referencing older versions.-->
+    <!--Overridden
+    for vulnerability reasons with dependencies referencing older versions.-->
     <ItemGroup>
     </ItemGroup>
 

src/common/Elsa.DropIns/Helpers/AssemblyLoader.cs

@@ -10,7 +10,14 @@ public static class AssemblyLoader
     {
         var loadContext = new DirectoryAssemblyLoadContext(path);
         var assemblyName = AssemblyLoadContext.GetAssemblyName(path);
-        var assembly = loadContext.LoadFromAssemblyName(assemblyName);
+        
+        // Copy drop in to memory stream to avoid file locking
+        using var fileStream = File.OpenRead(path);
+        using var memoryStream = new MemoryStream();
+        fileStream.CopyTo(memoryStream);
+        memoryStream.Seek(0, SeekOrigin.Begin);
+        fileStream.Close();
+        var assembly = loadContext.LoadFromStream(memoryStream);
 
         return assembly;
     }

src/common/Elsa.DropIns/HostedServices/DropInDirectoryMonitorHostedService.cs

@@ -2,6 +2,7 @@ using Elsa.DropIns.Catalogs;
 using Elsa.DropIns.Core;
 using Elsa.DropIns.Options;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using ThrottleDebounce;
 
@@ -13,16 +14,29 @@ namespace Elsa.DropIns.HostedServices;
 public class DropInDirectoryMonitorHostedService : BackgroundService
 {
     private readonly IOptions<DropInOptions> _options;
+    private readonly ILogger<DropInDirectoryMonitorHostedService> _logger;
     private readonly IServiceProvider _serviceProvider;
     private readonly RateLimitedFunc<string, Task> _debouncedLoader;
+    private readonly RateLimitedFunc<string, Task> _debouncedUnloader;
     private readonly FileSystemWatcher _watcher;
 
+    private readonly Dictionary<string, List<IDropIn>> _installedDropIns;
+
+
     /// <inheritdoc />
-    public DropInDirectoryMonitorHostedService(IOptions<DropInOptions> options, IServiceProvider serviceProvider)
+    public DropInDirectoryMonitorHostedService(
+        IOptions<DropInOptions> options,
+        IServiceProvider serviceProvider,
+        ILogger<DropInDirectoryMonitorHostedService> logger
+        )
     {
         _options = options;
+        _logger = logger;
         _serviceProvider = serviceProvider;
         _debouncedLoader = Debouncer.Debounce<string, Task>(LoadDropInAssemblyAsync, TimeSpan.FromSeconds(2));
+        _debouncedUnloader = Debouncer.Debounce<string, Task>(UnloadDropInAssemblyAsync, TimeSpan.FromSeconds(2));
+
+        _installedDropIns = [];
 
         var rootDirectoryPath = _options.Value.DropInRootDirectory;
 
@@ -38,10 +52,12 @@ public class DropInDirectoryMonitorHostedService : BackgroundService
     }
 
     /// <inheritdoc />
-    protected override Task ExecuteAsync(CancellationToken stoppingToken)
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
+        await LoadDropInAssemblyAsync(_options.Value.DropInRootDirectory);
+
         _watcher.Changed += OnChanged;
-        return Task.CompletedTask;
+        _watcher.Deleted += OnDeleted;
     }
 
     private async void OnChanged(object sender, FileSystemEventArgs e)
@@ -54,6 +70,16 @@ public class DropInDirectoryMonitorHostedService : BackgroundService
         await task;
     }
 
+    private async void OnDeleted(object sender, FileSystemEventArgs e)
+    {
+        var task = _debouncedUnloader.Invoke(e.FullPath);
+
+        if (task == null)
+            return;
+
+        await task;
+    }
+
     private async Task LoadDropInAssemblyAsync(string fullPath)
     {
         var directory = Path.GetDirectoryName(fullPath)!;
@@ -63,7 +89,35 @@ public class DropInDirectoryMonitorHostedService : BackgroundService
         foreach (var dropInDescriptor in dropInDescriptors)
         {
             var dropIn = (IDropIn)Activator.CreateInstance(dropInDescriptor.Type)!;
+            if (_installedDropIns.TryGetValue(fullPath, out var installedDropIns))
+            {
+                installedDropIns.Add(dropIn);
+            }
+            else
+            {
+                _installedDropIns[fullPath] = [dropIn];
+            }
             await dropIn.ConfigureAsync(_serviceProvider, CancellationToken.None);
         }
     }
+
+    private Task UnloadDropInAssemblyAsync(string fullPath)
+    {
+        if (_installedDropIns.TryGetValue(fullPath, out var installedDropIn))
+        {
+            installedDropIn.ForEach(dropIn =>
+            {
+                try
+                {
+                    dropIn.Unconfigure(_serviceProvider);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "Error unconfiguring drop-in {DropIn}", dropIn.GetType().Name);
+                }
+            });
+            _installedDropIns.Remove(fullPath);
+        }
+        return Task.CompletedTask;
+    }
 }

src/common/Elsa.DropIns/Models/DropInDescriptor.cs

@@ -1,3 +1,13 @@
+using System.Diagnostics.CodeAnalysis;
+
 namespace Elsa.DropIns.Models;
 
-public record DropInDescriptor(Type Type);
+public class DropInDescriptor([DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicParameterlessConstructor)] Type type)
+{
+    /// <summary>
+    /// Gets or sets the type of the drop-in. 
+    /// The DynamicallyAccessedMembers attribute ensures that the IL2072 warning is suppressed.
+    /// </summary>
+    [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicParameterlessConstructor)]
+    public Type Type { get; set; } = type;
+}