LogRecord storage at differents levels (#4911)

* Adding PersistenceStrategy and filter mapping for ActivityExecutionRecord

* Add default PersistenceStrategy provider/service

update ActivityMapper Implementation

* Fix Forgot to use the  Default Persistence from the Server configuration

* refactor the configuration of persistence in WorkflowManagementFeature

* rename PersistenceStrategy to LogPersistenceMode

* use const to defined LogPersistence Key in json and rename the key to logPersistenceMode

* Refactor PersistenceTab and update project references

Updated various aspects of the PersistenceTab class and its functionality to improve code quality and readability. Simplified the handling of persistence configurations and simplified the use of properties. Transitioned project reference for Elsa.Api.Client from package reference to direct project reference for better development experience in Elsa.Studio.Core.

* - Change how to get the Default Persistence Mode for an Activity.

/**
Because the entire workflow is considered as an activity, the schema must be the same
ie with
"logPersistenceMode": {
               "default": "default",
}
**/

- fix logic to get the default persistence mode working for the whole activity.

* fix default change value

---------

Co-authored-by: Jérémie DEVILLARD <jdevillard@users.noreply.github.com>
Co-authored-by: Sipke Schoorstra <sipkeschoorstra@outlook.com>

src/clients/Elsa.Api.Client/Resources/WorkflowDefinitions/Enums/LogPersistenceMode.cs

@@ -0,0 +1,23 @@
+namespace Elsa.Api.Client.Resources.WorkflowDefinitions.Enums;
+
+/// <summary>
+/// Define the Log Persistence mode to store information
+/// </summary>
+public enum LogPersistenceMode
+{
+    /// <summary>
+    /// Persist using the Parent mode
+    /// </summary>
+    Default,
+
+    /// <summary>
+    /// Include property to store
+    /// </summary>
+    Include,
+
+    /// <summary>
+    /// Exclude Property to store
+    /// </summary>
+    Exclude
+
+}

src/modules/Elsa.MassTransit.RabbitMq/Features/RabbitMqServiceBusFeature.cs

@@ -29,7 +29,9 @@ public class RabbitMqServiceBusFeature : FeatureBase
     /// A RabbitMQ connection string.
     public string? ConnectionString { get; set; }
 
+    /// <summary>
     /// Configures the RabbitMQ transport options.
+    /// </summary>
     public Action<RabbitMqTransportOptions>? TransportOptions { get; set; }
 
     /// <summary>

src/modules/Elsa.Workflows.Core/Enums/LogPersistenceMode.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Elsa.Workflows.Enums;
+/// <summary>
+/// Define the Log Persistence mode to store information
+/// </summary>
+public enum LogPersistenceMode
+{
+    /// <summary>
+    /// Persist using the Parent mode
+    /// </summary>
+    Default,
+
+    /// <summary>
+    /// Include property to store
+    /// </summary>
+    Include,
+
+    /// <summary>
+    /// Exclude Property to store
+    /// </summary>
+    Exclude
+
+}

src/modules/Elsa.Workflows.Management/Features/WorkflowManagementFeature.cs

@@ -9,6 +9,7 @@ using Elsa.Features.Abstractions;
 using Elsa.Features.Attributes;
 using Elsa.Features.Services;
 using Elsa.Workflows.Contracts;
+using Elsa.Workflows.Enums;
 using Elsa.Workflows.Features;
 using Elsa.Workflows.Management.Activities.WorkflowDefinitionActivity;
 using Elsa.Workflows.Management.Compression;
@@ -42,7 +43,7 @@ public class WorkflowManagementFeature : FeatureBase
     private const string DynamicCategory = "Dynamic";
 
     private string CompressionAlgorithm { get; set; } = nameof(None);
-
+    private LogPersistenceMode LogPersistenceMode { get; set; } = LogPersistenceMode.Include;
     /// <inheritdoc />
     public WorkflowManagementFeature(IModule module) : base(module)
     {
@@ -161,6 +162,16 @@ public class WorkflowManagementFeature : FeatureBase
         return this;
     }
 
+    /// <summary>
+    /// Set the default Log Persistence mode to use for worflow state (default is Include)
+    /// </summary>
+    /// <param name="logPersistenceMode">The mode persistence value</param>
+    public WorkflowManagementFeature SetDefaultLogPersistenceMode(LogPersistenceMode logPersistenceMode)
+    {
+        LogPersistenceMode = logPersistenceMode;
+        return this;
+    }
+
     /// <inheritdoc />
     [RequiresUnreferencedCode("The assembly containing the specified marker type will be scanned for activity types.")]
     public override void Configure()
@@ -211,6 +222,7 @@ public class WorkflowManagementFeature : FeatureBase
                 options.VariableDescriptors.Add(descriptor);
 
             options.CompressionAlgorithm = CompressionAlgorithm;
+            options.LogPersistenceMode = LogPersistenceMode;
         });
     }
 }

src/modules/Elsa.Workflows.Management/Options/ManagementOptions.cs

@@ -1,3 +1,4 @@
+using Elsa.Workflows.Enums;
 using Elsa.Workflows.Management.Models;
 
 namespace Elsa.Workflows.Management.Options;
@@ -20,5 +21,10 @@ public class ManagementOptions
     /// <summary>
     /// The format to use for compressing workflow state.
     /// </summary>
-    public string? CompressionAlgorithm { get; set; }
+    public string? CompressionAlgorithm { get; set; }
+
+    /// <summary>
+    /// The default Log Persistence Mode to use for all the system
+    /// </summary>
+    public LogPersistenceMode LogPersistenceMode { get; set; }
 }

src/modules/Elsa.Workflows.Runtime/Features/WorkflowRuntimeFeature.cs

@@ -215,7 +215,7 @@ public class WorkflowRuntimeFeature : FeatureBase
             .AddScoped<IBookmarkManager, DefaultBookmarkManager>()
             .AddScoped<IActivityExecutionManager, DefaultActivityExecutionManager>()
             .AddScoped<IActivityExecutionStatsService, ActivityExecutionStatsService>()
-            .AddScoped<IActivityExecutionMapper, DefaultActivityExecutionMapper>()
+            .AddScoped<IActivityExecutionMapper, DefaultActivityExecutionMapper>()
             .AddScoped<IWorkflowDefinitionStorePopulator, DefaultWorkflowDefinitionStorePopulator>()
             .AddScoped<IRegistriesPopulator, DefaultRegistriesPopulator>()
             .AddScoped<IWorkflowRegistry, DefaultWorkflowRegistry>()

src/modules/Elsa.Workflows.Runtime/Services/DefaultActivityExecutionMapper.cs

@@ -1,17 +1,49 @@
 using Elsa.Extensions;
+using Elsa.Workflows.Enums;
+using Elsa.Workflows.Management.Options;
 using Elsa.Workflows.Models;
 using Elsa.Workflows.Runtime.Contracts;
 using Elsa.Workflows.Runtime.Entities;
 using Elsa.Workflows.State;
+using Humanizer;
+using Microsoft.Extensions.Options;
 
 namespace Elsa.Workflows.Runtime.Services;
 
 /// <inheritdoc />
 public class DefaultActivityExecutionMapper : IActivityExecutionMapper
-{
+{
+    private LogPersistenceMode _serverLogPersistenceMode;
+    private const string LogPersistenceModeKey = "logPersistenceMode";
+
+    public DefaultActivityExecutionMapper(IOptions<ManagementOptions> options)
+    {
+        _serverLogPersistenceMode = options.Value.LogPersistenceMode;
+    }
+
     /// <inheritdoc />
     public ActivityExecutionRecord Map(ActivityExecutionContext source)
-    {
+    {
+        /*
+        * { 
+        *      "logPersistenceMode": {
+        *          "default": "default",
+        *          "inputs": { k : v }, 
+        *          "outputs": { k: v }
+        *          }
+        *  }
+        */
+
+        var workflowPersistenceProperty =
+            GetDefaultPersistenceMode(source.WorkflowExecutionContext.Workflow.CustomProperties, () => _serverLogPersistenceMode);
+            
+       
+        var activityPersistenceProperties = source.Activity.CustomProperties
+            .GetValueOrDefault<IDictionary<string, object?>>(LogPersistenceModeKey, () => new Dictionary<string, object?>());
+        var activityPersistencePropertyDefault =
+            GetDefaultPersistenceMode(source.Activity.CustomProperties, () => workflowPersistenceProperty);
+            
+ 
         // Get any outcomes that were added to the activity execution context.
         var outcomes = source.JournalData.TryGetValue("Outcomes", out var resultValue) ? resultValue as string[] : default;
         var payload = new Dictionary<string, object>();
@@ -40,7 +72,10 @@ public class DefaultActivityExecutionMapper : IActivityExecutionMapper
 
             return default;
         });
-
+
+        outputs = StorePropertyUsingPersistanceMode(outputs, activityPersistenceProperties!.GetValueOrDefault("outputs", () => new Dictionary<string, object>())!, activityPersistencePropertyDefault);
+        var activityState = StorePropertyUsingPersistanceMode(source.ActivityState, activityPersistenceProperties!.GetValueOrDefault("inputs", () => new Dictionary<string, object>())!, activityPersistencePropertyDefault );
+
         return new ActivityExecutionRecord
         {
             Id = source.Id,
@@ -49,7 +84,7 @@ public class DefaultActivityExecutionMapper : IActivityExecutionMapper
             WorkflowInstanceId = source.WorkflowExecutionContext.Id,
             ActivityType = source.Activity.Type,
             ActivityName = source.Activity.Name,
-            ActivityState = source.ActivityState,
+            ActivityState = activityState,
             Outputs = outputs,
             Properties = source.Properties,
             Payload = payload,
@@ -60,8 +95,39 @@ public class DefaultActivityExecutionMapper : IActivityExecutionMapper
             Status = GetAggregateStatus(source),
             CompletedAt = source.CompletedAt
         };
-    }
-
+    }
+
+    private static LogPersistenceMode GetDefaultPersistenceMode(IDictionary<string, object> customProperties,Func<LogPersistenceMode> defaultFactory)
+    {
+        var properties = customProperties
+            .GetValueOrDefault<IDictionary<string, object?>>(LogPersistenceModeKey, () => new Dictionary<string, object?>());
+        var persistencePropertyDefault = properties!
+            .GetValueOrDefault("default", defaultFactory);
+
+        if(persistencePropertyDefault == LogPersistenceMode.Default) 
+            return defaultFactory();
+        return persistencePropertyDefault; 
+    }
+
+    private static Dictionary<string,object?> StorePropertyUsingPersistanceMode(IDictionary<string,object?> inputs
+        , IDictionary<string,object> persistenceModeConfiguration
+        , LogPersistenceMode defaultLogPersistenceMode = LogPersistenceMode.Exclude)
+    {
+        var result = new Dictionary<string, object?>();
+
+        foreach (var input in inputs)
+        {
+            var persistence = persistenceModeConfiguration.GetValueOrDefault(input.Key.Camelize(), () => defaultLogPersistenceMode);
+            if (persistence.Equals(LogPersistenceMode.Include) 
+                || (persistence.Equals(LogPersistenceMode.Default) && defaultLogPersistenceMode == LogPersistenceMode.Include)
+                )
+                result.Add(input.Key, input.Value);
+            
+        }
+
+        return result;
+    }
+
     private ActivityStatus GetAggregateStatus(ActivityExecutionContext context)
     {
         // If any child activity is faulted, the aggregate status is faulted.