Fix variable serialization (#5974)

* Improve dispatched workflow input handling

Addressed input handling in dispatch messages by adding `SerializedInput` property. Also removed initialization logic and moved input deserialization to a helper method, ensuring compatibility with both new and deprecated input property formats.

* Update workflow Docker images and version tags

Changed Docker image tags from v3-2-0-rc3 to v3-2-1-preview across multiple GitHub workflows. Updated the VERSION environment variable in packages.yml to reflect the new versioning scheme. These changes ensure consistency with the new preview release.

* Update versioning to include 'preview' in package workflow

Modified the workflow to append 'preview' to the version number for non-tagged builds. This ensures clearer differentiation between stable and non-stable versions in the CI pipeline.

* Add WorkflowInstanceStorageDriver for workflow variable storage

Introduced a new storage driver, WorkflowInstanceStorageDriver, to store workflow variables directly in the workflow state. Updated relevant classes and methods to incorporate this new storage driver, ensuring seamless read/write/delete operations and extending support for it throughout the codebase.

* Refactor object conversion and update variable retrieval.

Switched from JsonObject to JsonNode for object conversion and corrected a typo in the summary comment. Changed the return type of GetVariablesDictionary method and updated its implementation to use VariablesDictionary.

* Rename 'input' to 'serializedInput' in DispatchWorkflowDefinition.

This change clarifies that the input provided to the workflow should be serialized. It enhances the readability and accuracy of the code documentation, ensuring that developers understand the expected format of the input parameter.

* Add priority and deprecation attributes to storage drivers

Introduced a priority attribute to the `IStorageDriver` interface and implemented it in various storage drivers. Additionally, marked `WorkflowStorageDriver` as deprecated and reordered storage driver listing based on priority.

* Switch MassTransit broker to in-memory and refactor converter

Changed MassTransit broker from AzureServiceBus to in-memory for improved performance in development environment. Simplified PolymorphicObjectConverterFactory by removing redundant constructor and dependencies. Removed unused folder from the project file.

.github/workflows/elsa-server-and-studio.yml

@@ -29,7 +29,7 @@ jobs:
         with:
           # list of Docker images to use as base name for tags
           images: |
-            elsaworkflows/elsa-server-and-studio-v3-2-0-rc3
+            elsaworkflows/elsa-server-and-studio-v3-2-1-preview
           flavor: |
             latest=true
           # generate Docker tags based on the following events/attributes

.github/workflows/elsa-server.yml

@@ -29,7 +29,7 @@ jobs:
         with:
           # list of Docker images to use as base name for tags
           images: |
-            elsaworkflows/elsa-server-v3-2-0-rc3
+            elsaworkflows/elsa-server-v3-2-1-preview
           flavor: |
             latest=true
           # generate Docker tags based on the following events/attributes

.github/workflows/elsa-studio.yml

@@ -29,7 +29,7 @@ jobs:
         with:
           # list of Docker images to use as base name for tags
           images: |
-            elsaworkflows/elsa-studio-v3-2-0-rc3
+            elsaworkflows/elsa-studio-v3-2-1-preview
           flavor: |
             latest=true
           # generate Docker tags based on the following events/attributes

.github/workflows/packages.yml

@@ -62,7 +62,7 @@ jobs:
             TAG_NAME=${TAG_NAME#refs/tags/} # remove the refs/tags/ prefix
             echo "VERSION=${TAG_NAME}" >> $GITHUB_ENV
           else
-            echo "VERSION=3.2.0-rc6.${{github.run_number}}" >> $GITHUB_ENV
+            echo "VERSION=3.2.1-preview.${{github.run_number}}" >> $GITHUB_ENV
           fi
       - name: Set up JDK 17
         uses: actions/setup-java@v2

Elsa.sln.DotSettings

@@ -14,6 +14,7 @@
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=initializable/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=materializer/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=materializers/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=persistable/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Persister/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Populator/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Postgre/@EntryIndexedValue">True</s:Boolean>

src/clients/Elsa.Api.Client/Resources/StorageDrivers/Models/StorageDriverDescriptor.cs

@@ -5,4 +5,4 @@ namespace Elsa.Api.Client.Resources.StorageDrivers.Models;
 /// </summary>
 /// <param name="TypeName">The type name of the storage driver.</param>
 /// <param name="DisplayName">The display name of the storage driver.</param>
-public record StorageDriverDescriptor(string TypeName, string DisplayName);
+public record StorageDriverDescriptor(string TypeName, string DisplayName, double Priority = 0, bool Deprecated = false);

src/modules/Elsa.Expressions/Helpers/ObjectConverter.cs

@@ -99,13 +99,13 @@ public static class ObjectConverter
             return jsonElement.Deserialize(targetType, serializerOptions);
         }
 
-        if (value is JsonObject jsonObject)
+        if (value is JsonNode jsonObject)
         {
             return underlyingTargetType switch
             {
                 { } t when t == typeof(string) => jsonObject.ToString(),
                 { } t when t != typeof(object) => jsonObject.Deserialize(targetType, serializerOptions),
-                _ => jsonObject,
+                _ => jsonObject
             };
         }
 
@@ -240,7 +240,7 @@ public static class ObjectConverter
     }
 
     /// <summary>
-    /// Returns true if the specified type is date-like type, false otherwise.
+    /// Returns true if the specified type is a date-like type, false otherwise.
     /// </summary>
     private static bool IsDateType(Type type)
     {

src/modules/Elsa.MassTransit.AzureServiceBus/Features/AzureServiceBusFeature.cs

@@ -1,4 +1,5 @@
 using Azure.Messaging.ServiceBus.Administration;
+using Elsa.Common.Contracts;
 using Elsa.Extensions;
 using Elsa.Features.Abstractions;
 using Elsa.Features.Attributes;
@@ -121,6 +122,13 @@ public class AzureServiceBusFeature : FeatureBase
                     }
 
                     configurator.ConfigureEndpoints(context, new KebabCaseEndpointNameFormatter("Elsa", false));
+                    
+                    configurator.ConfigureJsonSerializerOptions(serializerOptions =>
+                    {
+                        var serializer = context.GetRequiredService<IJsonSerializer>();
+                        serializer.ApplyOptions(serializerOptions);
+                        return serializerOptions;
+                    });
                 });
             };
         });

src/modules/Elsa.MassTransit.RabbitMq/Features/RabbitMqServiceBusFeature.cs

@@ -1,10 +1,10 @@
+using Elsa.Common.Contracts;
 using Elsa.Extensions;
 using Elsa.Features.Abstractions;
 using Elsa.Features.Attributes;
 using Elsa.Features.Services;
 using Elsa.Hosting.Management.Contracts;
 using Elsa.Hosting.Management.Features;
-using Elsa.MassTransit.Consumers;
 using Elsa.MassTransit.Extensions;
 using Elsa.MassTransit.Features;
 using Elsa.MassTransit.Options;
@@ -88,6 +88,13 @@ public class RabbitMqServiceBusFeature : FeatureBase
                     }
 
                     configurator.ConfigureEndpoints(context, new KebabCaseEndpointNameFormatter("Elsa", false));
+                    
+                    configurator.ConfigureJsonSerializerOptions(serializerOptions =>
+                    {
+                        var serializer = context.GetRequiredService<IJsonSerializer>();
+                        serializer.ApplyOptions(serializerOptions);
+                        return serializerOptions;
+                    });
                 });
             };
         });

src/modules/Elsa.MassTransit/Consumers/DispatchWorkflowRequestConsumer.cs

@@ -1,5 +1,5 @@
 using Elsa.MassTransit.Messages;
-using Elsa.Workflows.Management.Contracts;
+using Elsa.Workflows.Contracts;
 using Elsa.Workflows.Runtime.Contracts;
 using Elsa.Workflows.Runtime.Options;
 using Elsa.Workflows.Runtime.Parameters;
@@ -12,22 +12,12 @@ namespace Elsa.MassTransit.Consumers;
 /// A consumer of various dispatch message types to asynchronously execute workflows.
 /// </summary>
 [UsedImplicitly]
-public class DispatchWorkflowRequestConsumer :
+public class DispatchWorkflowRequestConsumer(IWorkflowRuntime workflowRuntime, IPayloadSerializer jsonSerializer) :
     IConsumer<DispatchWorkflowDefinition>,
     IConsumer<DispatchWorkflowInstance>,
     IConsumer<DispatchTriggerWorkflows>,
     IConsumer<DispatchResumeWorkflows>
 {
-    private readonly IWorkflowRuntime _workflowRuntime;
-
-    /// <summary>
-    /// Initializes a new instance of the <see cref="DispatchWorkflowRequestConsumer"/> class.
-    /// </summary>
-    public DispatchWorkflowRequestConsumer(IWorkflowRuntime workflowRuntime, IWorkflowInstanceManager workflowInstanceManager)
-    {
-        _workflowRuntime = workflowRuntime;
-    }
-
     /// <inheritdoc />
     public async Task Consume(ConsumeContext<DispatchWorkflowDefinition> context)
     {
@@ -42,6 +32,7 @@ public class DispatchWorkflowRequestConsumer :
     {
         var message = context.Message;
         var cancellationToken = context.CancellationToken;
+        var input = message.Input ?? DeserializeInput(message.SerializedInput);
 
         var options = new ResumeWorkflowRuntimeParams
         {
@@ -51,12 +42,12 @@ public class DispatchWorkflowRequestConsumer :
             ActivityNodeId = message.ActivityNodeId,
             ActivityInstanceId = message.ActivityInstanceId,
             ActivityHash = message.ActivityHash,
-            Input = message.Input,
+            Input = input,
             Properties = message.Properties,
             CancellationTokens = cancellationToken
         };
 
-        await _workflowRuntime.ResumeWorkflowAsync(message.InstanceId, options);
+        await workflowRuntime.ResumeWorkflowAsync(message.InstanceId, options);
     }
 
     /// <inheritdoc />
@@ -64,16 +55,17 @@ public class DispatchWorkflowRequestConsumer :
     {
         var message = context.Message;
         var cancellationToken = context.CancellationToken;
+        var input = message.Input ?? DeserializeInput(message.SerializedInput);
         var options = new TriggerWorkflowsOptions
         {
             CorrelationId = message.CorrelationId,
             WorkflowInstanceId = message.WorkflowInstanceId,
             ActivityInstanceId = message.ActivityInstanceId,
-            Input = message.Input,
+            Input = input,
             Properties = message.Properties,
             CancellationTokens = cancellationToken
         };
-        await _workflowRuntime.TriggerWorkflowsAsync(message.ActivityTypeName, message.BookmarkPayload, options);
+        await workflowRuntime.TriggerWorkflowsAsync(message.ActivityTypeName, message.BookmarkPayload, options);
     }
 
     /// <inheritdoc />
@@ -81,29 +73,31 @@ public class DispatchWorkflowRequestConsumer :
     {
         var message = context.Message;
         var cancellationToken = context.CancellationToken;
+        var input = message.Input ?? DeserializeInput(message.SerializedInput);
 
         var options = new TriggerWorkflowsOptions
         {
             CorrelationId = message.CorrelationId,
             WorkflowInstanceId = message.WorkflowInstanceId,
-            Input = message.Input,
+            Input = input,
             Properties = message.Properties,
             CancellationTokens = cancellationToken
         };
 
-        await _workflowRuntime.ResumeWorkflowsAsync(message.ActivityTypeName, message.BookmarkPayload, options);
+        await workflowRuntime.ResumeWorkflowsAsync(message.ActivityTypeName, message.BookmarkPayload, options);
     }
-    
+
     private async Task DispatchNewWorkflowInstanceAsync(DispatchWorkflowDefinition message, CancellationToken cancellationToken)
     {
         if (string.IsNullOrWhiteSpace(message.DefinitionId)) throw new ArgumentException("The definition ID is required when dispatching a workflow definition.");
         if (message.VersionOptions == null) throw new ArgumentException("The version options are required when dispatching a workflow definition.");
+        var input = message.Input ?? DeserializeInput(message.SerializedInput);
 
         var options = new StartWorkflowRuntimeParams
         {
             ParentWorkflowInstanceId = message.ParentWorkflowInstanceId,
             CorrelationId = message.CorrelationId,
-            Input = message.Input,
+            Input = input,
             Properties = message.Properties,
             VersionOptions = message.VersionOptions.Value,
             TriggerActivityId = message.TriggerActivityId,
@@ -111,7 +105,7 @@ public class DispatchWorkflowRequestConsumer :
             CancellationTokens = cancellationToken
         };
 
-        await _workflowRuntime.TryStartWorkflowAsync(message.DefinitionId, options);
+        await workflowRuntime.TryStartWorkflowAsync(message.DefinitionId, options);
     }
 
     private async Task DispatchExistingWorkflowInstanceAsync(DispatchWorkflowDefinition message, CancellationToken cancellationToken)
@@ -126,6 +120,14 @@ public class DispatchWorkflowRequestConsumer :
             CancellationTokens = cancellationToken
         };
 
-        await _workflowRuntime.StartWorkflowAsync(message.InstanceId, options);
+        await workflowRuntime.StartWorkflowAsync(message.InstanceId, options);
+    }
+
+    private IDictionary<string, object>? DeserializeInput(string? json)
+    {
+        if (string.IsNullOrWhiteSpace(json))
+            return null;
+
+        return jsonSerializer.Deserialize<IDictionary<string, object>>(json);
     }
 }

src/modules/Elsa.MassTransit/Messages/DispatchResumeWorkflows.cs

@@ -14,6 +14,10 @@ public class DispatchResumeWorkflows(string activityTypeName, object bookmarkPay
     public string? CorrelationId { get; set; }
     public string? WorkflowInstanceId { get; set; }
     public string? ActivityInstanceId { get; set; }
+
+    [Obsolete("This property is no longer used and will be removed in a future version. Use the SerializedInput property instead.")]
     public IDictionary<string, object>? Input { get; set; }
+
+    public string? SerializedInput { get; set; }
     public IDictionary<string, object>? Properties { get; set; }
 }

src/modules/Elsa.MassTransit/Messages/DispatchTriggerWorkflowsRequest.cs

@@ -14,6 +14,10 @@ public class DispatchTriggerWorkflows(string activityTypeName, object bookmarkPa
     public string? CorrelationId { get; set; }
     public string? WorkflowInstanceId { get; set; }
     public string? ActivityInstanceId { get; set; }
+    
+    [Obsolete("This property is no longer used and will be removed in a future version. Use the SerializedInput property instead.")]
     public IDictionary<string, object>? Input { get; set; }
+
+    public string? SerializedInput { get; set; }
     public IDictionary<string, object>? Properties { get; set; }
 }

src/modules/Elsa.MassTransit/Messages/DispatchWorkflowDefinition.cs

@@ -1,4 +1,3 @@
-using System.Text.Json.Serialization;
 using Elsa.Common.Models;
 
 namespace Elsa.MassTransit.Messages;
@@ -27,7 +26,7 @@ public record DispatchWorkflowDefinition
     /// <param name="definitionId">The ID of the workflow definition to dispatch.</param>
     /// <param name="versionOptions">The version options to use when dispatching the workflow definition.</param>
     /// <param name="parentWorkflowInstanceId">The ID of the parent workflow instance.</param>
-    /// <param name="input">Any input to pass to the workflow.</param>
+    /// <param name="serializedInput">Any input to pass to the workflow.</param>
     /// <param name="properties">Any properties to attach to the workflow.</param>
     /// <param name="correlationId">A correlation ID to associate the workflow with.</param>
     /// <param name="instanceId">The ID to use when creating an instance of the workflow to dispatch.</param>
@@ -36,7 +35,7 @@ public record DispatchWorkflowDefinition
         string? definitionId,
         VersionOptions? versionOptions,
         string? parentWorkflowInstanceId,
-        IDictionary<string, object>? input,
+        string? serializedInput,
         IDictionary<string, object>? properties,
         string? correlationId,
         string? instanceId,
@@ -47,7 +46,7 @@ public record DispatchWorkflowDefinition
             DefinitionId = definitionId,
             VersionOptions = versionOptions,
             ParentWorkflowInstanceId = parentWorkflowInstanceId,
-            Input = input,
+            SerializedInput = serializedInput,
             Properties = properties,
             CorrelationId = correlationId,
             InstanceId = instanceId,
@@ -64,8 +63,14 @@ public record DispatchWorkflowDefinition
     /// The ID of the parent workflow instance.
     public string? ParentWorkflowInstanceId { get; init; }
 
+    /// Deprecated. Use the <see cref="SerializedInput"/> property instead.
+    [Obsolete("This property is no longer used and will be removed in a future version. Use the SerializedInput property instead.")]
+    public IDictionary<string, object>? Input { get; set; }
+
+    /// <summary>
     /// Any input to pass to the workflow.
-    public IDictionary<string, object>? Input { get; init; }
+    /// </summary>
+    public string? SerializedInput { get; set; }
 
     /// Any properties to attach to the workflow.
     public IDictionary<string, object>? Properties { get; init; }

src/modules/Elsa.MassTransit/Messages/DispatchWorkflowInstance.cs

@@ -8,7 +8,11 @@ public class DispatchWorkflowInstance(string instanceId)
     public string? ActivityNodeId { get; set; }
     public string? ActivityInstanceId { get; set; }
     public string? ActivityHash { get; set; }
+
+    [Obsolete("This property is no longer used and will be removed in a future version. Use the SerializedInput property instead.")]
     public IDictionary<string, object>? Input { get; set; }
+
+    public string? SerializedInput { get; set; }
     public IDictionary<string, object>? Properties { get; set; }
     public string? CorrelationId { get; set; }
 }

src/modules/Elsa.MassTransit/Services/MassTransitWorkflowDispatcher.cs

@@ -11,7 +11,6 @@ using Elsa.Workflows.Runtime.Models;
 using Elsa.Workflows.Runtime.Requests;
 using Elsa.Workflows.Runtime.Responses;
 using MassTransit;
-using Medallion.Threading;
 using Microsoft.Extensions.Logging;
 
 namespace Elsa.MassTransit.Services;
@@ -27,6 +26,7 @@ public class MassTransitWorkflowDispatcher(
     IBookmarkHasher bookmarkHasher,
     ITriggerStore triggerStore,
     IBookmarkStore bookmarkStore,
+    IPayloadSerializer jsonSerializer,
     ILogger<MassTransitWorkflowDispatcher> logger)
     : IWorkflowDispatcher
 {
@@ -57,6 +57,7 @@ public class MassTransitWorkflowDispatcher(
     public async Task<DispatchWorkflowResponse> DispatchAsync(DispatchWorkflowInstanceRequest request, DispatchWorkflowOptions? options = default, CancellationToken cancellationToken = default)
     {
         var sendEndpoint = await GetSendEndpointAsync(options);
+        var serializedInput = SerializeInput(request.Input);
 
         await sendEndpoint.Send(new DispatchWorkflowInstance(request.InstanceId)
         {
@@ -66,7 +67,7 @@ public class MassTransitWorkflowDispatcher(
             ActivityInstanceId = request.ActivityInstanceId,
             ActivityHash = request.ActivityHash,
             CorrelationId = request.CorrelationId,
-            Input = request.Input
+            SerializedInput = serializedInput,
         }, cancellationToken);
         return DispatchWorkflowResponse.Success();
     }
@@ -180,4 +181,9 @@ public class MassTransitWorkflowDispatcher(
         var sendEndpoint = await bus.GetSendEndpoint(new Uri($"queue:{endpointName}"));
         return sendEndpoint;
     }
+    
+    private string? SerializeInput(object? input)
+    {
+        return input != null ? jsonSerializer.Serialize(input) : null;
+    }
 }

src/modules/Elsa.Workflows.Api/Endpoints/StorageDrivers/List/Endpoint.cs

@@ -31,7 +31,7 @@ public class List : ElsaEndpointWithoutRequest<Response>
     public override Task<Response> ExecuteAsync(CancellationToken ct)
     {
         var drivers = _registry.List();
-        var descriptors = drivers.Select(FromDriver).ToList();
+        var descriptors = drivers.Select(FromDriver).OrderByDescending(x => x.Priority).ToList();
         var response = new Response(descriptors);
 
         return Task.FromResult(response);
@@ -40,7 +40,9 @@ public class List : ElsaEndpointWithoutRequest<Response>
     private static StorageDriverDescriptor FromDriver(IStorageDriver driver)
     {
         var type = driver.GetType();
+        var deprecated = type.GetCustomAttribute<ObsoleteAttribute>() != null;
         var displayName = type.GetCustomAttribute<DisplayAttribute>()?.Name ?? type.GetCustomAttribute<DisplayNameAttribute>()?.DisplayName ?? type.Name.Replace("StorageDriver", "");
-        return new StorageDriverDescriptor(type.GetSimpleAssemblyQualifiedName(), displayName);
+        var priority = driver.Priority;
+        return new StorageDriverDescriptor(type.GetSimpleAssemblyQualifiedName(), displayName, priority, deprecated);
     }
 }

src/modules/Elsa.Workflows.Api/Endpoints/StorageDrivers/List/Models.cs

@@ -5,4 +5,4 @@ public class Response(ICollection<StorageDriverDescriptor> items)
     public ICollection<StorageDriverDescriptor> Items  { get; set; } = items;
 }
 
-public record StorageDriverDescriptor(string TypeName, string DisplayName);
+public record StorageDriverDescriptor(string TypeName, string DisplayName, double Priority = 0, bool Deprecated = false);

src/modules/Elsa.Workflows.Core/Activities/ParallelForEachT.cs

@@ -53,10 +53,10 @@ public class ParallelForEach<T> : Activity
             var currentValueVariable = new Variable<T>("CurrentValue", item)
             {
                 // TODO: This should be configurable, because this won't work for e.g. file streams and other non-serializable types.
-                StorageDriverType = typeof(WorkflowStorageDriver)
+                StorageDriverType = typeof(WorkflowInstanceStorageDriver)
             };
 
-            var currentIndexVariable = new Variable<int>("CurrentIndex", currentIndex++) { StorageDriverType = typeof(WorkflowStorageDriver) };
+            var currentIndexVariable = new Variable<int>("CurrentIndex", currentIndex++) { StorageDriverType = typeof(WorkflowInstanceStorageDriver) };
             var variables = new List<Variable> { currentValueVariable, currentIndexVariable };
 
             // Schedule a body of work for each item.

src/modules/Elsa.Workflows.Core/Contracts/IStorageDriver.cs

@@ -5,6 +5,11 @@ namespace Elsa.Workflows.Contracts;
 /// </summary>
 public interface IStorageDriver
 {
+    /// <summary>
+    /// The priority of the storage driver. Drivers with higher priority are used before drivers with lower priority.
+    /// </summary>
+    double Priority { get; }
+    
     /// <summary>
     /// Writes a value to the storage driver.
     /// </summary>

src/modules/Elsa.Workflows.Core/Extensions/ExpressionExecutionContextExtensions.cs

@@ -141,7 +141,7 @@ public static class ExpressionExecutionContextExtensions
 
         var variable = new Variable(name, value)
         {
-            StorageDriverType = storageDriverType ?? typeof(WorkflowStorageDriver)
+            StorageDriverType = storageDriverType ?? typeof(WorkflowInstanceStorageDriver)
         };
 
         // Find the first parent context that has a variable container.

src/modules/Elsa.Workflows.Core/Extensions/VariableExtensions.cs

@@ -31,14 +31,14 @@ public static class VariableExtensions
             new ExpandoObjectConverterFactory());
 
     /// <summary>
-    /// Configures the variable to use the <see cref="WorkflowStorageDriver"/>.
+    /// Configures the variable to use the <see cref="WorkflowInstanceStorageDriver"/>.
     /// </summary>
-    public static Variable WithWorkflowStorage(this Variable variable) => variable.WithStorage<WorkflowStorageDriver>();
+    public static Variable WithWorkflowStorage(this Variable variable) => variable.WithStorage<WorkflowInstanceStorageDriver>();
 
     /// <summary>
-    /// Configures the variable to use the <see cref="WorkflowStorageDriver"/>.
+    /// Configures the variable to use the <see cref="WorkflowInstanceStorageDriver"/>.
     /// </summary>
-    public static Variable<T> WithWorkflowStorage<T>(this Variable<T> variable) => (Variable<T>)variable.WithStorage<WorkflowStorageDriver>();
+    public static Variable<T> WithWorkflowStorage<T>(this Variable<T> variable) => (Variable<T>)variable.WithStorage<WorkflowInstanceStorageDriver>();
 
     /// <summary>
     /// Configures the variable to use the <see cref="MemoryStorageDriver"/>.

src/modules/Elsa.Workflows.Core/Features/WorkflowsFeature.cs

@@ -171,6 +171,7 @@ public class WorkflowsFeature : FeatureBase
             // Storage drivers.
             .AddScoped<IStorageDriverManager, StorageDriverManager>()
             .AddStorageDriver<WorkflowStorageDriver>()
+            .AddStorageDriver<WorkflowInstanceStorageDriver>()
             .AddStorageDriver<MemoryStorageDriver>()
 
             // Serialization.

src/modules/Elsa.Workflows.Core/Serialization/Converters/PolymorphicObjectConverterFactory.cs

@@ -2,6 +2,7 @@ using System.Dynamic;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using Elsa.Expressions.Contracts;
+using Elsa.Expressions.Services;
 
 namespace Elsa.Workflows.Serialization.Converters;
 
@@ -16,7 +17,8 @@ public class PolymorphicObjectConverterFactory(IWellKnownTypeRegistry wellKnownT
         var canConvert = typeToConvert.IsClass
                && typeToConvert == typeof(object)
                || typeToConvert == typeof(ExpandoObject)
-               || typeToConvert == typeof(Dictionary<string, object>);
+               || typeToConvert == typeof(Dictionary<string, object>)
+               || typeToConvert == typeof(IDictionary<string, object>);
         
         return canConvert;
     }

src/modules/Elsa.Workflows.Core/Services/MemoryStorageDriver.cs

@@ -11,6 +11,8 @@ public class MemoryStorageDriver : IStorageDriver
 {
     private readonly IDictionary<string, object> _dictionary = new Dictionary<string, object>();
 
+    public double Priority => 0;
+
     /// <inheritdoc />
     public ValueTask WriteAsync(string id, object value, StorageDriverContext context)
     {

src/modules/Elsa.Workflows.Core/Services/WorkflowInstanceStorageDriver.cs

@@ -0,0 +1,58 @@
+using System.ComponentModel.DataAnnotations;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Elsa.Extensions;
+using Elsa.Workflows.Contracts;
+using JetBrains.Annotations;
+
+namespace Elsa.Workflows.Services;
+
+/// A storage driver that stores objects in the workflow state itself.
+[Display(Name = "Workflow Instance")]
+[UsedImplicitly]
+public class WorkflowInstanceStorageDriver : IStorageDriver
+{
+    /// The key used to store the variables in the workflow state.
+    public const string VariablesDictionaryStateKey = "Variables";
+
+    /// <inheritdoc />
+    public double Priority => 1;
+
+    /// <inheritdoc />
+    public ValueTask WriteAsync(string id, object value, StorageDriverContext context)
+    {
+        UpdateVariablesDictionary(context, dictionary =>
+        {
+            var node = JsonSerializer.SerializeToNode(value);
+            dictionary[id] = node;
+        });
+        return ValueTask.CompletedTask;
+    }
+
+    /// <inheritdoc />
+    public ValueTask<object?> ReadAsync(string id, StorageDriverContext context)
+    {
+        var dictionary = GetVariablesDictionary(context);
+        var node = dictionary.GetValueOrDefault(id);
+        return new(node);
+    }
+
+    /// <inheritdoc />
+    public ValueTask DeleteAsync(string id, StorageDriverContext context)
+    {
+        UpdateVariablesDictionary(context, dictionary => dictionary.Remove(id));
+        return ValueTask.CompletedTask;
+    }
+
+    private VariablesDictionary GetVariablesDictionary(StorageDriverContext context) => context.ExecutionContext.Properties.GetOrAdd(VariablesDictionaryStateKey, () => new VariablesDictionary());
+    private void SetVariablesDictionary(StorageDriverContext context, VariablesDictionary dictionary) => context.ExecutionContext.Properties[VariablesDictionaryStateKey] = dictionary;
+
+    private void UpdateVariablesDictionary(StorageDriverContext context, Action<VariablesDictionary> update)
+    {
+        var dictionary = GetVariablesDictionary(context);
+        update(dictionary);
+        SetVariablesDictionary(context, dictionary);
+    }
+}
+
+public class VariablesDictionary : Dictionary<string, JsonNode>;

src/modules/Elsa.Workflows.Core/Services/WorkflowStateExtractor.cs

@@ -71,7 +71,7 @@ public class WorkflowStateExtractor : IWorkflowStateExtractor
     private IDictionary<string, object> GetPersistableInput(WorkflowExecutionContext workflowExecutionContext)
     {
         // TODO: This is a temporary solution. We need to find a better way to handle this.
-        var persistableInput = workflowExecutionContext.Workflow.Inputs.Where(x => x.StorageDriverType == typeof(WorkflowStorageDriver)).ToList();
+        var persistableInput = workflowExecutionContext.Workflow.Inputs.Where(x => x.StorageDriverType == typeof(WorkflowStorageDriver) || x.StorageDriverType == typeof(WorkflowInstanceStorageDriver)).ToList();
         var input = workflowExecutionContext.Input;
         var filteredInput = new Dictionary<string, object>();
 

src/modules/Elsa.Workflows.Core/Services/WorkflowStorageDriver.cs

@@ -8,6 +8,7 @@ namespace Elsa.Workflows.Services;
 /// A storage driver that stores objects in the workflow state itself.
 /// </summary>
 [Display(Name = "Workflow")]
+[Obsolete("This is no longer used and will be removed in a future version. Use the WorkflowInstanceStorageDriver instead.")]
 public class WorkflowStorageDriver : IStorageDriver
 {
     /// <summary>
@@ -15,6 +16,9 @@ public class WorkflowStorageDriver : IStorageDriver
     /// </summary>
     public const string VariablesDictionaryStateKey = "PersistentVariablesDictionary";
 
+    /// <inheritdoc />
+    public double Priority => -1;
+
     /// <inheritdoc />
     public ValueTask WriteAsync(string id, object value, StorageDriverContext context)
     {

src/modules/Elsa.Workflows.Runtime/Activities/BulkDispatchWorkflows.cs

@@ -237,7 +237,7 @@ public class BulkDispatchWorkflows : Activity
 
         var childInstanceId = new Variable<string>("ChildInstanceId", workflowInstanceId)
         {
-            StorageDriverType = typeof(WorkflowStorageDriver)
+            StorageDriverType = typeof(WorkflowInstanceStorageDriver)
         };
 
         var variables = new List<Variable>

src/modules/Elsa.Workflows.Runtime/Services/DefaultBackgroundActivityInvoker.cs

@@ -97,7 +97,7 @@ public class DefaultBackgroundActivityInvoker : IBackgroundActivityInvoker
             var driver = variableMetadata?.StorageDriverType;
 
             // We only capture output written to the workflow itself. Other drivers like blob storage, etc. will be ignored since the foreground context will be loading those.
-            if (driver != typeof(WorkflowStorageDriver))
+            if (driver != typeof(WorkflowStorageDriver) && driver != typeof(WorkflowInstanceStorageDriver))
                 continue;
 
             var outputValue = activityExecutionContext.Get(memoryBlockReference);

test/component/Elsa.Workflows.ComponentTests/Scenarios/Variables/CountdownWorkflowTests.cs

@@ -48,6 +48,8 @@ public class CountdownWorkflowTests(App app) : AppComponentTest(app)
         }
     }
 
-    private IDictionary<string, object> GetVariablesDictionary(ActivityExecutionContextState context) =>
-        context.Properties.GetOrAdd(WorkflowStorageDriver.VariablesDictionaryStateKey, () => new Dictionary<string, object>());
+    private VariablesDictionary GetVariablesDictionary(ActivityExecutionContextState context)
+    {
+        return context.Properties.GetOrAdd(WorkflowInstanceStorageDriver.VariablesDictionaryStateKey, () => new VariablesDictionary());
+    }
 }