SemianiakaVY/elsa-core
1
0
Fork 0
Code Issues Pull requests Actions 1 Packages Projects Releases Wiki Activity

Add data compression and format fields to WorkflowInstances

Browse source 
This commit introduces two new fields to the WorkflowInstances table: "DataCompressionAlgorithm" and "DataFormat". These changes allow storing additional contextual information about the payloads of workflow instances and enhance future handling and processing of this data.
This commit is contained in:
Sipke Schoorstra 2024-02-04 00:06:41 +01:00
parent 22f72080f3
commit 383eeab751
22 changed files with 242 additions and 149 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
migrations/efcore-3.1.sh
View file

@ -28,11 +28,6 @@ for module in "${mods[@]}"; do
echo "Provider path: ${providerPath:?}/${migrationsPath}"
echo "Migrations path: $migrationsPath"
echo "Connection string: ${connStrings[$provider]}"
# 1. Delete the existing migration
echo "Deleting existing migrations: ${providerPath:?}/${migrationsPath}/*V3_1.cs"
find "${providerPath:?}/${migrationsPath}/" -type f -name "*V3_1*" -exec rm -f {} \;
# rm -rf "${providerPath:?}/${migrationsPath}/*V3_1.cs"
# 2. Run the migrations command
dotnet ef migrations add V3_1 -c "$module"ElsaDbContext -p "$providerPath" -o "$migrationsPath" -- --connectionString "${connStrings[$provider]}"

5
src/bundles/Elsa.Server.Web/Program.cs
View file

@ -15,6 +15,7 @@ using Elsa.MongoDb.Extensions;
using Elsa.MongoDb.Modules.Identity;
using Elsa.MongoDb.Modules.Management;
using Elsa.MongoDb.Modules.Runtime;
using Elsa.Workflows.Management.Compression;
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Options;
using Proto.Persistence.Sqlite;
@ -29,6 +30,7 @@ const bool useQuartz = true;
const bool useMassTransit = false;
const bool useMassTransitAzureServiceBus = true;
const bool useMassTransitRabbitMq = false;
const bool useZipCompression = false;
var builder = WebApplication.CreateBuilder(args);
var services = builder.Services;
@ -106,6 +108,9 @@ services
else
ef.UseSqlite(sqliteConnectionString);
});
if(useZipCompression)
management.SetCompressionAlgorithm(nameof(GZip));
})
.UseWorkflowRuntime(runtime =>
{

22
src/modules/Elsa.EntityFrameworkCore.MySql/Migrations/Management/20240203213856_V3_1.cs
View file

@ -1,22 +0,0 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Elsa.EntityFrameworkCore.MySql.Migrations.Management
{
/// <inheritdoc />
public partial class V31 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
}
}
}

14
src/modules/Elsa.EntityFrameworkCore.MySql/Migrations/Management/20240203213856_V3_1.Designer.cs → src/modules/Elsa.EntityFrameworkCore.MySql/Migrations/Management/20240203224836_V3_1.Designer.cs generated
View file

@ -11,7 +11,7 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace Elsa.EntityFrameworkCore.MySql.Migrations.Management
{
[DbContext(typeof(ManagementElsaDbContext))]
[Migration("20240203213856_V3_1")]
[Migration("20240203224836_V3_1")]
partial class V31
{
/// <inheritdoc />
@ -37,12 +37,6 @@ namespace Elsa.EntityFrameworkCore.MySql.Migrations.Management
b.Property<string>("Data")
.HasColumnType("longtext");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("longtext");
b.Property<string>("DataFormat")
.HasColumnType("longtext");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("varchar(255)");
@ -122,6 +116,12 @@ namespace Elsa.EntityFrameworkCore.MySql.Migrations.Management
b.Property<string>("Data")
.HasColumnType("longtext");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("longtext");
b.Property<string>("DataFormat")
.HasColumnType("longtext");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("varchar(255)");

44
src/modules/Elsa.EntityFrameworkCore.MySql/Migrations/Management/20240203224836_V3_1.cs Normal file
View file

@ -0,0 +1,44 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Elsa.EntityFrameworkCore.MySql.Migrations.Management
{
/// <inheritdoc />
public partial class V31 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "DataCompressionAlgorithm",
schema: "Elsa",
table: "WorkflowInstances",
type: "longtext",
nullable: true)
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AddColumn<string>(
name: "DataFormat",
schema: "Elsa",
table: "WorkflowInstances",
type: "longtext",
nullable: true)
.Annotation("MySql:CharSet", "utf8mb4");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "DataCompressionAlgorithm",
schema: "Elsa",
table: "WorkflowInstances");
migrationBuilder.DropColumn(
name: "DataFormat",
schema: "Elsa",
table: "WorkflowInstances");
}
}
}

12
src/modules/Elsa.EntityFrameworkCore.MySql/Migrations/Management/ManagementElsaDbContextModelSnapshot.cs
View file

@ -34,12 +34,6 @@ namespace Elsa.EntityFrameworkCore.MySql.Migrations.Management
b.Property<string>("Data")
.HasColumnType("longtext");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("longtext");
b.Property<string>("DataFormat")
.HasColumnType("longtext");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("varchar(255)");
@ -119,6 +113,12 @@ namespace Elsa.EntityFrameworkCore.MySql.Migrations.Management
b.Property<string>("Data")
.HasColumnType("longtext");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("longtext");
b.Property<string>("DataFormat")
.HasColumnType("longtext");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("varchar(255)");

22
src/modules/Elsa.EntityFrameworkCore.PostgreSql/Migrations/Management/20240203213917_V3_1.cs
View file

@ -1,22 +0,0 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Elsa.EntityFrameworkCore.PostgreSql.Migrations.Management
{
/// <inheritdoc />
public partial class V3_1 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
}
}
}

14
src/modules/Elsa.EntityFrameworkCore.PostgreSql/Migrations/Management/20240203213917_V3_1.Designer.cs → src/modules/Elsa.EntityFrameworkCore.PostgreSql/Migrations/Management/20240203224851_V3_1.Designer.cs generated
View file

@ -12,7 +12,7 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
namespace Elsa.EntityFrameworkCore.PostgreSql.Migrations.Management
{
[DbContext(typeof(ManagementElsaDbContext))]
[Migration("20240203213917_V3_1")]
[Migration("20240203224851_V3_1")]
partial class V3_1
{
/// <inheritdoc />
@ -40,12 +40,6 @@ namespace Elsa.EntityFrameworkCore.PostgreSql.Migrations.Management
b.Property<string>("Data")
.HasColumnType("text");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("text");
b.Property<string>("DataFormat")
.HasColumnType("text");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("text");
@ -125,6 +119,12 @@ namespace Elsa.EntityFrameworkCore.PostgreSql.Migrations.Management
b.Property<string>("Data")
.HasColumnType("text");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("text");
b.Property<string>("DataFormat")
.HasColumnType("text");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("text");

42
src/modules/Elsa.EntityFrameworkCore.PostgreSql/Migrations/Management/20240203224851_V3_1.cs Normal file
View file

@ -0,0 +1,42 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Elsa.EntityFrameworkCore.PostgreSql.Migrations.Management
{
/// <inheritdoc />
public partial class V3_1 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "DataCompressionAlgorithm",
schema: "Elsa",
table: "WorkflowInstances",
type: "text",
nullable: true);
migrationBuilder.AddColumn<string>(
name: "DataFormat",
schema: "Elsa",
table: "WorkflowInstances",
type: "text",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "DataCompressionAlgorithm",
schema: "Elsa",
table: "WorkflowInstances");
migrationBuilder.DropColumn(
name: "DataFormat",
schema: "Elsa",
table: "WorkflowInstances");
}
}
}

12
src/modules/Elsa.EntityFrameworkCore.PostgreSql/Migrations/Management/ManagementElsaDbContextModelSnapshot.cs
View file

@ -37,12 +37,6 @@ namespace Elsa.EntityFrameworkCore.PostgreSql.Migrations.Management
b.Property<string>("Data")
.HasColumnType("text");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("text");
b.Property<string>("DataFormat")
.HasColumnType("text");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("text");
@ -122,6 +116,12 @@ namespace Elsa.EntityFrameworkCore.PostgreSql.Migrations.Management
b.Property<string>("Data")
.HasColumnType("text");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("text");
b.Property<string>("DataFormat")
.HasColumnType("text");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("text");

22
src/modules/Elsa.EntityFrameworkCore.SqlServer/Migrations/Management/20240203213902_V3_1.cs
View file

@ -1,22 +0,0 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Elsa.EntityFrameworkCore.SqlServer.Migrations.Management
{
/// <inheritdoc />
public partial class V3_1 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
}
}
}

14
src/modules/Elsa.EntityFrameworkCore.SqlServer/Migrations/Management/20240203213902_V3_1.Designer.cs → src/modules/Elsa.EntityFrameworkCore.SqlServer/Migrations/Management/20240203224841_V3_1.Designer.cs generated
View file

@ -12,7 +12,7 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace Elsa.EntityFrameworkCore.SqlServer.Migrations.Management
{
[DbContext(typeof(ManagementElsaDbContext))]
[Migration("20240203213902_V3_1")]
[Migration("20240203224841_V3_1")]
partial class V3_1
{
/// <inheritdoc />
@ -40,12 +40,6 @@ namespace Elsa.EntityFrameworkCore.SqlServer.Migrations.Management
b.Property<string>("Data")
.HasColumnType("nvarchar(max)");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("nvarchar(max)");
b.Property<string>("DataFormat")
.HasColumnType("nvarchar(max)");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("nvarchar(450)");
@ -125,6 +119,12 @@ namespace Elsa.EntityFrameworkCore.SqlServer.Migrations.Management
b.Property<string>("Data")
.HasColumnType("nvarchar(max)");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("nvarchar(max)");
b.Property<string>("DataFormat")
.HasColumnType("nvarchar(max)");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("nvarchar(450)");

42
src/modules/Elsa.EntityFrameworkCore.SqlServer/Migrations/Management/20240203224841_V3_1.cs Normal file
View file

@ -0,0 +1,42 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Elsa.EntityFrameworkCore.SqlServer.Migrations.Management
{
/// <inheritdoc />
public partial class V3_1 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "DataCompressionAlgorithm",
schema: "Elsa",
table: "WorkflowInstances",
type: "nvarchar(max)",
nullable: true);
migrationBuilder.AddColumn<string>(
name: "DataFormat",
schema: "Elsa",
table: "WorkflowInstances",
type: "nvarchar(max)",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "DataCompressionAlgorithm",
schema: "Elsa",
table: "WorkflowInstances");
migrationBuilder.DropColumn(
name: "DataFormat",
schema: "Elsa",
table: "WorkflowInstances");
}
}
}

12
src/modules/Elsa.EntityFrameworkCore.SqlServer/Migrations/Management/ManagementElsaDbContextModelSnapshot.cs
View file

@ -37,12 +37,6 @@ namespace Elsa.EntityFrameworkCore.SqlServer.Migrations.Management
b.Property<string>("Data")
.HasColumnType("nvarchar(max)");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("nvarchar(max)");
b.Property<string>("DataFormat")
.HasColumnType("nvarchar(max)");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("nvarchar(450)");
@ -122,6 +116,12 @@ namespace Elsa.EntityFrameworkCore.SqlServer.Migrations.Management
b.Property<string>("Data")
.HasColumnType("nvarchar(max)");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("nvarchar(max)");
b.Property<string>("DataFormat")
.HasColumnType("nvarchar(max)");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("nvarchar(450)");

22
src/modules/Elsa.EntityFrameworkCore.Sqlite/Migrations/Management/20240203213909_V3_1.cs
View file

@ -1,22 +0,0 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Elsa.EntityFrameworkCore.Sqlite.Migrations.Management
{
/// <inheritdoc />
public partial class V3_1 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
}
}
}

14
src/modules/Elsa.EntityFrameworkCore.Sqlite/Migrations/Management/20240203213909_V3_1.Designer.cs → src/modules/Elsa.EntityFrameworkCore.Sqlite/Migrations/Management/20240203224846_V3_1.Designer.cs generated
View file

@ -11,7 +11,7 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace Elsa.EntityFrameworkCore.Sqlite.Migrations.Management
{
[DbContext(typeof(ManagementElsaDbContext))]
[Migration("20240203213909_V3_1")]
[Migration("20240203224846_V3_1")]
partial class V3_1
{
/// <inheritdoc />
@ -35,12 +35,6 @@ namespace Elsa.EntityFrameworkCore.Sqlite.Migrations.Management
b.Property<string>("Data")
.HasColumnType("TEXT");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("TEXT");
b.Property<string>("DataFormat")
.HasColumnType("TEXT");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("TEXT");
@ -121,6 +115,12 @@ namespace Elsa.EntityFrameworkCore.Sqlite.Migrations.Management
b.Property<string>("Data")
.HasColumnType("TEXT");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("TEXT");
b.Property<string>("DataFormat")
.HasColumnType("TEXT");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("TEXT");

38
src/modules/Elsa.EntityFrameworkCore.Sqlite/Migrations/Management/20240203224846_V3_1.cs Normal file
View file

@ -0,0 +1,38 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Elsa.EntityFrameworkCore.Sqlite.Migrations.Management
{
/// <inheritdoc />
public partial class V3_1 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "DataCompressionAlgorithm",
table: "WorkflowInstances",
type: "TEXT",
nullable: true);
migrationBuilder.AddColumn<string>(
name: "DataFormat",
table: "WorkflowInstances",
type: "TEXT",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "DataCompressionAlgorithm",
table: "WorkflowInstances");
migrationBuilder.DropColumn(
name: "DataFormat",
table: "WorkflowInstances");
}
}
}

12
src/modules/Elsa.EntityFrameworkCore.Sqlite/Migrations/Management/ManagementElsaDbContextModelSnapshot.cs
View file

@ -32,12 +32,6 @@ namespace Elsa.EntityFrameworkCore.Sqlite.Migrations.Management
b.Property<string>("Data")
.HasColumnType("TEXT");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("TEXT");
b.Property<string>("DataFormat")
.HasColumnType("TEXT");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("TEXT");
@ -118,6 +112,12 @@ namespace Elsa.EntityFrameworkCore.Sqlite.Migrations.Management
b.Property<string>("Data")
.HasColumnType("TEXT");
b.Property<string>("DataCompressionAlgorithm")
.HasColumnType("TEXT");
b.Property<string>("DataFormat")
.HasColumnType("TEXT");
b.Property<string>("DefinitionId")
.IsRequired()
.HasColumnType("TEXT");

4
src/modules/Elsa.EntityFrameworkCore/Modules/Management/Configurations.cs
View file

@ -19,8 +19,6 @@ internal class Configurations : IEntityTypeConfiguration<WorkflowDefinition>, IE
builder.Ignore(x => x.CustomProperties);
builder.Ignore(x => x.Options);
builder.Property<string>("Data");
builder.Property<string>("DataFormat");
builder.Property<string>("DataCompressionAlgorithm");
builder.Property<bool?>("UsableAsActivity");
builder.Property(x => x.ToolVersion).HasConversion(VersionToStringConverter, StringToVersionConverter);
@ -36,6 +34,8 @@ internal class Configurations : IEntityTypeConfiguration<WorkflowDefinition>, IE
{
builder.Ignore(x => x.WorkflowState);
builder.Property<string>("Data");
builder.Property<string>("DataFormat");
builder.Property<string>("DataCompressionAlgorithm");
builder.Property(x => x.Status).HasConversion<string>();
builder.Property(x => x.SubStatus).HasConversion<string>();
builder.HasIndex(x => new { x.Status, x.SubStatus, x.DefinitionId, x.Version }).HasDatabaseName($"IX_{nameof(WorkflowInstance)}_{nameof(WorkflowInstance.Status)}_{nameof(WorkflowInstance.SubStatus)}_{nameof(WorkflowInstance.DefinitionId)}_{nameof(WorkflowInstance.Version)}");

1
src/modules/Elsa.EntityFrameworkCore/Modules/Management/WorkflowInstanceStore.cs
View file

@ -151,6 +151,7 @@ public class EFCoreWorkflowInstanceStore : IWorkflowInstanceStore
return;
var data = entity.WorkflowState;
var dataFormat = (string?)managementElsaDbContext.Entry(entity).Property("DataFormat").CurrentValue ?? "Json";
var json = (string?)managementElsaDbContext.Entry(entity).Property("Data").CurrentValue;
var compressionAlgorithm = (string?)managementElsaDbContext.Entry(entity).Property("DataCompressionAlgorithm").CurrentValue ?? nameof(None);
var compressionStrategy = _compressionStrategyResolver.Resolve(compressionAlgorithm);

2
src/modules/Elsa.Workflows.Management/Compression/GZip.cs
View file

@ -16,6 +16,7 @@ public class GZip : ICompressionStrategy
using var output = new MemoryStream();
await using var compressionStream = new GZipStream(output, CompressionMode.Compress);
await compressionStream.WriteAsync(inputBytes, 0, inputBytes.Length, cancellationToken);
await compressionStream.FlushAsync(cancellationToken);
return Convert.ToBase64String(output.ToArray());
}
@ -27,7 +28,6 @@ public class GZip : ICompressionStrategy
using var inputMemoryStream = new MemoryStream(inputBytes);
await using var decompressionStream = new GZipStream(inputMemoryStream, CompressionMode.Decompress);
using var outputMemoryStream = new MemoryStream();
await decompressionStream.CopyToAsync(outputMemoryStream, cancellationToken);
var decompressedBytes = outputMemoryStream.ToArray();

16
src/modules/Elsa.Workflows.Management/Features/WorkflowManagementFeature.cs
View file

@ -42,6 +42,8 @@ public class WorkflowManagementFeature : FeatureBase
private const string LookupsCategory = "Lookups";
private const string DynamicCategory = "Dynamic";
private string CompressionAlgorithm { get; set; } = nameof(None);
/// <inheritdoc />
public WorkflowManagementFeature(IModule module) : base(module)
{
@ -144,6 +146,15 @@ public class WorkflowManagementFeature : FeatureBase
VariableDescriptors.AddRange(descriptors);
return this;
}
/// <summary>
/// Sets the compression algorithm to use for compressing workflow state.
/// </summary>
public WorkflowManagementFeature SetCompressionAlgorithm(string algorithm)
{
CompressionAlgorithm = algorithm;
return this;
}
/// <inheritdoc />
public override void Configure()
@ -189,7 +200,10 @@ public class WorkflowManagementFeature : FeatureBase
foreach (var activityType in ActivityTypes)
options.ActivityTypes.Add(activityType);
foreach (var descriptor in VariableDescriptors) options.VariableDescriptors.Add(descriptor);
foreach (var descriptor in VariableDescriptors)
options.VariableDescriptors.Add(descriptor);
options.CompressionAlgorithm = CompressionAlgorithm;
});
}
}