From 2195e437091d4c3119dec67eca04e9ff9712c28a Mon Sep 17 00:00:00 2001 From: jdevillard Date: Sat, 30 Mar 2024 21:22:35 +0100 Subject: [PATCH] LogRecord storage at differents levels (#4911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adding PersistenceStrategy and filter mapping for ActivityExecutionRecord * Add default PersistenceStrategy provider/service update ActivityMapper Implementation * Fix Forgot to use the Default Persistence from the Server configuration * refactor the configuration of persistence in WorkflowManagementFeature * rename PersistenceStrategy to LogPersistenceMode * use const to defined LogPersistence Key in json and rename the key to logPersistenceMode * Refactor PersistenceTab and update project references Updated various aspects of the PersistenceTab class and its functionality to improve code quality and readability. Simplified the handling of persistence configurations and simplified the use of properties. Transitioned project reference for Elsa.Api.Client from package reference to direct project reference for better development experience in Elsa.Studio.Core. * - Change how to get the Default Persistence Mode for an Activity. /** Because the entire workflow is considered as an activity, the schema must be the same ie with "logPersistenceMode": { "default": "default", } **/ - fix logic to get the default persistence mode working for the whole activity. * fix default change value --------- Co-authored-by: Jérémie DEVILLARD Co-authored-by: Sipke Schoorstra --- .../Enums/LogPersistenceMode.cs | 23 ++++++ .../Features/RabbitMqServiceBusFeature.cs | 2 + .../Enums/LogPersistenceMode.cs | 28 +++++++ .../Features/WorkflowManagementFeature.cs | 14 +++- .../Options/ManagementOptions.cs | 8 +- .../Features/WorkflowRuntimeFeature.cs | 2 +- .../DefaultActivityExecutionMapper.cs | 78 +++++++++++++++++-- 7 files changed, 146 insertions(+), 9 deletions(-) create mode 100644 src/clients/Elsa.Api.Client/Resources/WorkflowDefinitions/Enums/LogPersistenceMode.cs create mode 100644 src/modules/Elsa.Workflows.Core/Enums/LogPersistenceMode.cs diff --git a/src/clients/Elsa.Api.Client/Resources/WorkflowDefinitions/Enums/LogPersistenceMode.cs b/src/clients/Elsa.Api.Client/Resources/WorkflowDefinitions/Enums/LogPersistenceMode.cs new file mode 100644 index 000000000..01765ba40 --- /dev/null +++ b/src/clients/Elsa.Api.Client/Resources/WorkflowDefinitions/Enums/LogPersistenceMode.cs @@ -0,0 +1,23 @@ +namespace Elsa.Api.Client.Resources.WorkflowDefinitions.Enums; + +/// +/// Define the Log Persistence mode to store information +/// +public enum LogPersistenceMode +{ + /// + /// Persist using the Parent mode + /// + Default, + + /// + /// Include property to store + /// + Include, + + /// + /// Exclude Property to store + /// + Exclude + +} \ No newline at end of file diff --git a/src/modules/Elsa.MassTransit.RabbitMq/Features/RabbitMqServiceBusFeature.cs b/src/modules/Elsa.MassTransit.RabbitMq/Features/RabbitMqServiceBusFeature.cs index 12d38d83a..4a49012f9 100644 --- a/src/modules/Elsa.MassTransit.RabbitMq/Features/RabbitMqServiceBusFeature.cs +++ b/src/modules/Elsa.MassTransit.RabbitMq/Features/RabbitMqServiceBusFeature.cs @@ -29,7 +29,9 @@ public class RabbitMqServiceBusFeature : FeatureBase /// A RabbitMQ connection string. public string? ConnectionString { get; set; } + /// /// Configures the RabbitMQ transport options. + /// public Action? TransportOptions { get; set; } /// diff --git a/src/modules/Elsa.Workflows.Core/Enums/LogPersistenceMode.cs b/src/modules/Elsa.Workflows.Core/Enums/LogPersistenceMode.cs new file mode 100644 index 000000000..26ec3c88a --- /dev/null +++ b/src/modules/Elsa.Workflows.Core/Enums/LogPersistenceMode.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Elsa.Workflows.Enums; +/// +/// Define the Log Persistence mode to store information +/// +public enum LogPersistenceMode +{ + /// + /// Persist using the Parent mode + /// + Default, + + /// + /// Include property to store + /// + Include, + + /// + /// Exclude Property to store + /// + Exclude + +} diff --git a/src/modules/Elsa.Workflows.Management/Features/WorkflowManagementFeature.cs b/src/modules/Elsa.Workflows.Management/Features/WorkflowManagementFeature.cs index e96d0212d..a7a58fb2c 100644 --- a/src/modules/Elsa.Workflows.Management/Features/WorkflowManagementFeature.cs +++ b/src/modules/Elsa.Workflows.Management/Features/WorkflowManagementFeature.cs @@ -9,6 +9,7 @@ using Elsa.Features.Abstractions; using Elsa.Features.Attributes; using Elsa.Features.Services; using Elsa.Workflows.Contracts; +using Elsa.Workflows.Enums; using Elsa.Workflows.Features; using Elsa.Workflows.Management.Activities.WorkflowDefinitionActivity; using Elsa.Workflows.Management.Compression; @@ -42,7 +43,7 @@ public class WorkflowManagementFeature : FeatureBase private const string DynamicCategory = "Dynamic"; private string CompressionAlgorithm { get; set; } = nameof(None); - + private LogPersistenceMode LogPersistenceMode { get; set; } = LogPersistenceMode.Include; /// public WorkflowManagementFeature(IModule module) : base(module) { @@ -161,6 +162,16 @@ public class WorkflowManagementFeature : FeatureBase return this; } + /// + /// Set the default Log Persistence mode to use for worflow state (default is Include) + /// + /// The mode persistence value + public WorkflowManagementFeature SetDefaultLogPersistenceMode(LogPersistenceMode logPersistenceMode) + { + LogPersistenceMode = logPersistenceMode; + return this; + } + /// [RequiresUnreferencedCode("The assembly containing the specified marker type will be scanned for activity types.")] public override void Configure() @@ -211,6 +222,7 @@ public class WorkflowManagementFeature : FeatureBase options.VariableDescriptors.Add(descriptor); options.CompressionAlgorithm = CompressionAlgorithm; + options.LogPersistenceMode = LogPersistenceMode; }); } } \ No newline at end of file diff --git a/src/modules/Elsa.Workflows.Management/Options/ManagementOptions.cs b/src/modules/Elsa.Workflows.Management/Options/ManagementOptions.cs index 200c5557d..9063efa8d 100644 --- a/src/modules/Elsa.Workflows.Management/Options/ManagementOptions.cs +++ b/src/modules/Elsa.Workflows.Management/Options/ManagementOptions.cs @@ -1,3 +1,4 @@ +using Elsa.Workflows.Enums; using Elsa.Workflows.Management.Models; namespace Elsa.Workflows.Management.Options; @@ -20,5 +21,10 @@ public class ManagementOptions /// /// The format to use for compressing workflow state. /// - public string? CompressionAlgorithm { get; set; } + public string? CompressionAlgorithm { get; set; } + + /// + /// The default Log Persistence Mode to use for all the system + /// + public LogPersistenceMode LogPersistenceMode { get; set; } } \ No newline at end of file diff --git a/src/modules/Elsa.Workflows.Runtime/Features/WorkflowRuntimeFeature.cs b/src/modules/Elsa.Workflows.Runtime/Features/WorkflowRuntimeFeature.cs index bc08b314c..fd29a5ae4 100644 --- a/src/modules/Elsa.Workflows.Runtime/Features/WorkflowRuntimeFeature.cs +++ b/src/modules/Elsa.Workflows.Runtime/Features/WorkflowRuntimeFeature.cs @@ -215,7 +215,7 @@ public class WorkflowRuntimeFeature : FeatureBase .AddScoped() .AddScoped() .AddScoped() - .AddScoped() + .AddScoped() .AddScoped() .AddScoped() .AddScoped() diff --git a/src/modules/Elsa.Workflows.Runtime/Services/DefaultActivityExecutionMapper.cs b/src/modules/Elsa.Workflows.Runtime/Services/DefaultActivityExecutionMapper.cs index 0c54a2108..bf012ab59 100644 --- a/src/modules/Elsa.Workflows.Runtime/Services/DefaultActivityExecutionMapper.cs +++ b/src/modules/Elsa.Workflows.Runtime/Services/DefaultActivityExecutionMapper.cs @@ -1,17 +1,49 @@ using Elsa.Extensions; +using Elsa.Workflows.Enums; +using Elsa.Workflows.Management.Options; using Elsa.Workflows.Models; using Elsa.Workflows.Runtime.Contracts; using Elsa.Workflows.Runtime.Entities; using Elsa.Workflows.State; +using Humanizer; +using Microsoft.Extensions.Options; namespace Elsa.Workflows.Runtime.Services; /// public class DefaultActivityExecutionMapper : IActivityExecutionMapper -{ +{ + private LogPersistenceMode _serverLogPersistenceMode; + private const string LogPersistenceModeKey = "logPersistenceMode"; + + public DefaultActivityExecutionMapper(IOptions options) + { + _serverLogPersistenceMode = options.Value.LogPersistenceMode; + } + /// public ActivityExecutionRecord Map(ActivityExecutionContext source) - { + { + /* + * { + * "logPersistenceMode": { + * "default": "default", + * "inputs": { k : v }, + * "outputs": { k: v } + * } + * } + */ + + var workflowPersistenceProperty = + GetDefaultPersistenceMode(source.WorkflowExecutionContext.Workflow.CustomProperties, () => _serverLogPersistenceMode); + + + var activityPersistenceProperties = source.Activity.CustomProperties + .GetValueOrDefault>(LogPersistenceModeKey, () => new Dictionary()); + var activityPersistencePropertyDefault = + GetDefaultPersistenceMode(source.Activity.CustomProperties, () => workflowPersistenceProperty); + + // Get any outcomes that were added to the activity execution context. var outcomes = source.JournalData.TryGetValue("Outcomes", out var resultValue) ? resultValue as string[] : default; var payload = new Dictionary(); @@ -40,7 +72,10 @@ public class DefaultActivityExecutionMapper : IActivityExecutionMapper return default; }); - + + outputs = StorePropertyUsingPersistanceMode(outputs, activityPersistenceProperties!.GetValueOrDefault("outputs", () => new Dictionary())!, activityPersistencePropertyDefault); + var activityState = StorePropertyUsingPersistanceMode(source.ActivityState, activityPersistenceProperties!.GetValueOrDefault("inputs", () => new Dictionary())!, activityPersistencePropertyDefault ); + return new ActivityExecutionRecord { Id = source.Id, @@ -49,7 +84,7 @@ public class DefaultActivityExecutionMapper : IActivityExecutionMapper WorkflowInstanceId = source.WorkflowExecutionContext.Id, ActivityType = source.Activity.Type, ActivityName = source.Activity.Name, - ActivityState = source.ActivityState, + ActivityState = activityState, Outputs = outputs, Properties = source.Properties, Payload = payload, @@ -60,8 +95,39 @@ public class DefaultActivityExecutionMapper : IActivityExecutionMapper Status = GetAggregateStatus(source), CompletedAt = source.CompletedAt }; - } - + } + + private static LogPersistenceMode GetDefaultPersistenceMode(IDictionary customProperties,Func defaultFactory) + { + var properties = customProperties + .GetValueOrDefault>(LogPersistenceModeKey, () => new Dictionary()); + var persistencePropertyDefault = properties! + .GetValueOrDefault("default", defaultFactory); + + if(persistencePropertyDefault == LogPersistenceMode.Default) + return defaultFactory(); + return persistencePropertyDefault; + } + + private static Dictionary StorePropertyUsingPersistanceMode(IDictionary inputs + , IDictionary persistenceModeConfiguration + , LogPersistenceMode defaultLogPersistenceMode = LogPersistenceMode.Exclude) + { + var result = new Dictionary(); + + foreach (var input in inputs) + { + var persistence = persistenceModeConfiguration.GetValueOrDefault(input.Key.Camelize(), () => defaultLogPersistenceMode); + if (persistence.Equals(LogPersistenceMode.Include) + || (persistence.Equals(LogPersistenceMode.Default) && defaultLogPersistenceMode == LogPersistenceMode.Include) + ) + result.Add(input.Key, input.Value); + + } + + return result; + } + private ActivityStatus GetAggregateStatus(ActivityExecutionContext context) { // If any child activity is faulted, the aggregate status is faulted.