worfklow integration per tenant support

This commit is contained in:
Vitali sharp8n 2026-09-01 22:31:27 +03:00
parent b0e0e8733d
commit 6a3f702e09
8 changed files with 515 additions and 23 deletions

3
.gitignore vendored
View file

@ -23,3 +23,6 @@ x86/
# Docker
.env
# Per-tenant workflow copies (WorkflowSource:CopiesRoot)
.data/

View file

@ -55,6 +55,11 @@ builder.Services.AddOpenApi();
builder.Services.AddHttpClient();
builder.Services.AddHttpContextAccessor();
// Lite mode flag: when true, the engine uses SQLite + in-process channels
// instead of PostgreSQL + Redis. Intended for self-hosted single-container
// deployments where external infrastructure is not available.
var liteMode = builder.Configuration.GetValue<bool>("UseLiteMode");
// A transient failure in ONE background service (worker loop, trigger scheduler,
// run lifecycle, handler consumer) must never take the whole control plane down.
// Without this, HostOptions.BackgroundServiceExceptionBehavior defaults to
@ -62,28 +67,66 @@ builder.Services.AddHttpContextAccessor();
builder.Services.Configure<HostOptions>(options =>
options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.Ignore);
builder.Services.AddDbContext<WorkflowsDbContext>(options =>
options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));
if (liteMode)
{
// Lite mode: SQLite. The connection string should be something like
// "Data Source=/app/data/workflows.db".
var sqliteCs = builder.Configuration.GetConnectionString("DefaultConnection")
?? "Data Source=workflows.db";
builder.Services.AddDbContext<WorkflowsDbContext>(options =>
options.UseSqlite(sqliteCs));
Log.Logger.Information("Lite mode: using SQLite ({ConnectionString})", sqliteCs);
}
else
{
builder.Services.AddDbContext<WorkflowsDbContext>(options =>
options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));
builder.Services.AddSingleton<IConnectionMultiplexer>(_ =>
ConnectionMultiplexer.Connect(
builder.Configuration["Redis:ConnectionString"] ?? "localhost:6379,abortConnect=false"));
builder.Services.AddSingleton<IConnectionMultiplexer>(_ =>
ConnectionMultiplexer.Connect(
builder.Configuration["Redis:ConnectionString"] ?? "localhost:6379,abortConnect=false"));
}
builder.Services.AddScoped<ApiKeyService>();
// Transport + lease: Redis Streams now, RabbitMQ later behind the same interfaces.
builder.Services.AddSingleton<RedisStreamsTransport>();
builder.Services.AddSingleton<IJobQueue>(sp => sp.GetRequiredService<RedisStreamsTransport>());
builder.Services.AddSingleton<IEventBus>(sp => sp.GetRequiredService<RedisStreamsTransport>());
builder.Services.AddSingleton<ILeaseService, LeaseService>();
// Transport + lease.
if (liteMode)
{
// Lite mode: in-process channels replace Redis Streams.
builder.Services.AddSingleton<InMemoryTransport>();
builder.Services.AddSingleton<IJobQueue>(sp => sp.GetRequiredService<InMemoryTransport>());
builder.Services.AddSingleton<IEventBus>(sp => sp.GetRequiredService<InMemoryTransport>());
builder.Services.AddSingleton<ITriggerState, InMemoryTriggerState>();
builder.Services.AddSingleton<ILeaseService, InMemoryLeaseService>();
Log.Logger.Information("Lite mode: using in-memory transport + trigger state (no Redis)");
}
else
{
// Full mode: Redis Streams.
builder.Services.AddSingleton<RedisStreamsTransport>();
builder.Services.AddSingleton<IJobQueue>(sp => sp.GetRequiredService<RedisStreamsTransport>());
builder.Services.AddSingleton<IEventBus>(sp => sp.GetRequiredService<RedisStreamsTransport>());
builder.Services.AddSingleton<ITriggerState, RedisTriggerState>();
builder.Services.AddSingleton<ILeaseService, LeaseService>();
}
// YAML compile pipeline (step 4).
builder.Services.AddSingleton<LanguageRegistry>();
builder.Services.AddSingleton<WorkflowValidator>();
builder.Services.AddSingleton<WorkflowCompiler>();
// Git sync (step 5): read workflows/*.yaml from the local monorepo checkout.
builder.Services.AddSingleton<IWorkflowSource, LocalRepoWorkflowSource>();
// Workflow source: factory-based — each request creates a tenant-scoped source.
// WorkflowSource:CopiesRoot is REQUIRED; the factory throws if it is empty,
// preventing any request from serving cross-tenant workflows.
builder.Services.AddSingleton<WorkflowSourceFactory>();
builder.Services.AddScoped<IWorkflowSource>(sp =>
{
var http = sp.GetRequiredService<IHttpContextAccessor>();
var factory = sp.GetRequiredService<WorkflowSourceFactory>();
var tenantId = http.HttpContext?.Items["TenantId"] as string
?? throw new InvalidOperationException("TenantId not resolved by auth middleware");
return factory.Create(tenantId);
});
builder.Services.AddScoped<WorkflowSyncService>();
// Mermaid diagrams (step 6).
@ -118,7 +161,6 @@ builder.Services.AddSingleton<RemoteServerExecutor>();
// control-plane entrypoint, never in the worker. They share the run launcher —
// the seam that creates pending runs for the run lifecycle engine (step 9).
builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddSingleton<ITriggerState, RedisTriggerState>();
builder.Services.AddScoped<IRunLauncher, RunLauncher>();
// Run lifecycle engine (step 9): dispatch pending runs + consume task.results +
@ -173,9 +215,7 @@ var app = builder.Build();
app.UseCors();
app.UseMiddleware<AuthMiddleware>();
// Ensure the `workflows` schema exists before serving traffic (mirrors the
// resilient "ensure schema" pattern from w4c-auth). A migration failure must
// not crash the host — health and logs still report the degraded state.
// Ensure the database schema exists before serving traffic.
try
{
using var scope = app.Services.CreateScope();
@ -183,13 +223,15 @@ try
await db.Database.MigrateAsync();
// Additive schema, applied idempotently (raw SQL, no new EF migration) so it is
// safe on every startup against both freshly-created and long-lived databases:
// • "ArchivedAt" — W7 soft-delete so sync can stop wiping historical TaskRuns.
// • "Server" — S8 target managed-server for remote (SSH) task execution.
await db.Database.ExecuteSqlRawAsync(
"ALTER TABLE workflows.\"Tasks\" ADD COLUMN IF NOT EXISTS \"ArchivedAt\" timestamptz NULL;");
await db.Database.ExecuteSqlRawAsync(
"ALTER TABLE workflows.\"Tasks\" ADD COLUMN IF NOT EXISTS \"Server\" text NULL;");
// safe on every startup against both freshly-created and long-lived databases.
// Only applies to PostgreSQL (SQLite handles columns automatically via EF).
if (!liteMode)
{
await db.Database.ExecuteSqlRawAsync(
"ALTER TABLE workflows.\"Tasks\" ADD COLUMN IF NOT EXISTS \"ArchivedAt\" timestamptz NULL;");
await db.Database.ExecuteSqlRawAsync(
"ALTER TABLE workflows.\"Tasks\" ADD COLUMN IF NOT EXISTS \"Server\" text NULL;");
}
}
catch (Exception ex)
{

View file

@ -0,0 +1,38 @@
using System.Collections.Concurrent;
namespace w4c_workflows.Services;
/// <summary>
/// In-memory lease service for lite/self-hosted mode where Redis is not available.
/// Uses a ConcurrentDictionary with TTL tracking. Not distributed — only works
/// for a single-process deployment (which is the lite mode use case).
/// </summary>
public sealed class InMemoryLeaseService : ILeaseService
{
private record Lease(string Owner, DateTime ExpiresAt);
private readonly ConcurrentDictionary<string, Lease> _leases = new();
public Task<bool> AcquireAsync(string tenantId, string runId, string owner, TimeSpan ttl)
{
var key = $"{tenantId}:{runId}";
var lease = new Lease(owner, DateTime.UtcNow + ttl);
return Task.FromResult(_leases.TryAdd(key, lease));
}
public Task<bool> RenewAsync(string tenantId, string runId, string owner, TimeSpan ttl)
{
var key = $"{tenantId}:{runId}";
if (!_leases.TryGetValue(key, out var current) || current.Owner != owner)
return Task.FromResult(false);
_leases[key] = new Lease(owner, DateTime.UtcNow + ttl);
return Task.FromResult(true);
}
public Task<bool> ReleaseAsync(string tenantId, string runId, string owner)
{
var key = $"{tenantId}:{runId}";
// Remove only if the owner still matches (compare-and-delete).
return Task.FromResult(_leases.TryRemove(key, out var lease) && lease.Owner == owner);
}
}

View file

@ -0,0 +1,129 @@
using System.Collections.Concurrent;
using System.Threading.Channels;
namespace w4c_workflows.Services.Messaging;
/// <summary>
/// In-process channel-based transport for <c>IJobQueue</c> and <c>IEventBus</c>.
/// Used in lite/self-hosted mode where Redis is not available. Store-backed
/// (ConcurrentDictionary of channels per stream), survives thread-boundary
/// handoffs but NOT process restarts. For self-hosted single-process deployment
/// this is acceptable (restart = redeliver pending jobs from DB state).
/// </summary>
public sealed class InMemoryTransport : IJobQueue, IEventBus
{
private readonly ConcurrentDictionary<string, Channel<StreamMessage>> _channels = new();
private long _idSeq;
// ---- IJobQueue ----
public Task EnsureGroupAsync(string tenantId, CancellationToken ct)
=> Task.CompletedTask; // No-op: channel consumers are implicit.
public Task<string> EnqueueAsync(string tenantId, IReadOnlyDictionary<string, string> fields, CancellationToken ct)
{
var stream = $"wf:{tenantId}:jobs";
var id = $"0-{Interlocked.Increment(ref _idSeq)}";
var msg = new StreamMessage(id, fields);
GetOrAddChannel(stream).Writer.TryWrite(msg);
return Task.FromResult(id);
}
public Task<IReadOnlyList<StreamMessage>> ReadGroupAsync(string tenantId, string consumer, int count, CancellationToken ct)
=> DrainAsync($"wf:{tenantId}:jobs", count, ct);
public Task<IReadOnlyList<StreamMessage>> ClaimPendingAsync(string tenantId, string consumer, TimeSpan minIdle, int count, CancellationToken ct)
=> Task.FromResult<IReadOnlyList<StreamMessage>>(Array.Empty<StreamMessage>()); // No pending in memory.
public Task<long> AckAsync(string tenantId, string messageId, CancellationToken ct)
=> Task.FromResult(1L); // Ack is a no-op in memory.
public Task DeadLetterAsync(string tenantId, string messageId, IReadOnlyDictionary<string, string> fields, string reason, CancellationToken ct)
{
var dlq = $"wf:{tenantId}:dlq";
var id = $"0-{Interlocked.Increment(ref _idSeq)}";
var dlqFields = new Dictionary<string, string>(fields) { ["deadLetterReason"] = reason };
GetOrAddChannel(dlq).Writer.TryWrite(new StreamMessage(id, dlqFields));
return Task.CompletedTask;
}
public Task PublishDeadLetterAsync(string tenantId, IReadOnlyDictionary<string, string> fields, string reason, CancellationToken ct)
{
var dlq = $"wf:{tenantId}:dlq";
var id = $"0-{Interlocked.Increment(ref _idSeq)}";
var dlqFields = new Dictionary<string, string>(fields) { ["deadLetterReason"] = reason };
GetOrAddChannel(dlq).Writer.TryWrite(new StreamMessage(id, dlqFields));
return Task.CompletedTask;
}
// ---- IEventBus ----
public Task<string> PublishAsync(string tenantId, string stream, IReadOnlyDictionary<string, string> fields, CancellationToken ct)
{
var id = $"0-{Interlocked.Increment(ref _idSeq)}";
var msg = new StreamMessage(id, fields);
// Expand {tenant} placeholder.
var resolved = stream.Replace("{tenant}", tenantId);
GetOrAddChannel(resolved).Writer.TryWrite(msg);
return Task.FromResult(id);
}
Task IEventBus.EnsureGroupAsync(string tenantId, string stream, CancellationToken ct)
=> Task.CompletedTask;
Task<IReadOnlyList<StreamMessage>> IEventBus.ReadGroupAsync(string tenantId, string stream, string consumer, int count, CancellationToken ct)
{
var resolved = stream.Replace("{tenant}", tenantId);
return DrainAsync(resolved, count, ct);
}
Task<IReadOnlyList<StreamMessage>> IEventBus.ClaimPendingAsync(string tenantId, string stream, string consumer, TimeSpan minIdle, int count, CancellationToken ct)
=> Task.FromResult<IReadOnlyList<StreamMessage>>(Array.Empty<StreamMessage>());
Task<long> IEventBus.AckAsync(string tenantId, string stream, string messageId, CancellationToken ct)
=> Task.FromResult(1L);
// ---- Helpers ----
private Channel<StreamMessage> GetOrAddChannel(string key)
=> _channels.GetOrAdd(key, _ => Channel.CreateUnbounded<StreamMessage>(
new UnboundedChannelOptions { SingleReader = false, SingleWriter = false }));
private async Task<IReadOnlyList<StreamMessage>> DrainAsync(string stream, int count, CancellationToken ct)
{
if (!_channels.TryGetValue(stream, out var ch))
return Array.Empty<StreamMessage>();
var result = new List<StreamMessage>(count);
while (result.Count < count)
{
try
{
if (ch.Reader.TryRead(out var msg))
{
result.Add(msg);
}
else
{
// Wait briefly for a message.
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
cts.CancelAfter(TimeSpan.FromMilliseconds(50));
try
{
var read = await ch.Reader.ReadAsync(cts.Token);
result.Add(read);
}
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
{
break; // No more messages available right now.
}
}
}
catch (OperationCanceledException)
{
break;
}
}
return result;
}
}

View file

@ -0,0 +1,248 @@
using System.Diagnostics;
namespace w4c_workflows.Services;
/// <summary>
/// Factory for creating <see cref="IWorkflowSource"/> instances scoped to a
/// specific tenant. The caller provides the tenant ID explicitly (from auth
/// middleware); the factory resolves directories and returns a source bound
/// to that tenant's workflow directory.
///
/// Tenant isolation is <b>mandatory</b>: <c>WorkflowSource:CopiesRoot</c> must
/// be set in configuration. The factory throws at construction time if it is
/// missing, preventing any request from serving cross-tenant workflows.
///
/// Registered as a <b>Singleton</b> (shared config, no per-request state);
/// the created sources are lightweight and backed by filesystem operations.
/// </summary>
public sealed class WorkflowSourceFactory
{
private readonly IConfiguration _config;
private readonly IWebHostEnvironment _env;
private readonly ILoggerFactory _loggers;
private readonly string _copiesRoot;
private readonly string _sharedDir;
private readonly string _repoRoot;
public WorkflowSourceFactory(IConfiguration config, IWebHostEnvironment env, ILoggerFactory loggers)
{
_config = config;
_env = env;
_loggers = loggers;
_copiesRoot = config["WorkflowSource:CopiesRoot"]
?? throw new InvalidOperationException(
"WorkflowSource:CopiesRoot is not configured. " +
"Per-tenant filesystem isolation is required — set it to a writable " +
"directory path (e.g. \"/data/workflow-tenants\" or \".data/workflow-tenants\").");
_sharedDir = config["WorkflowSource:SharedDir"] ?? "workflows";
_repoRoot = ResolveRepoRoot(config, env);
}
/// <summary>Creates a per-tenant workflow source. The tenant directory
/// <c>{CopiesRoot}/{sanitizedTenantId}/</c> is created (and seeded from
/// the shared template dir) on first access.</summary>
public IWorkflowSource Create(string tenantId)
{
var safeId = Sanitize(tenantId);
var tenantDir = Path.GetFullPath(Path.Combine(_copiesRoot, safeId));
var sharedRel = _sharedDir;
string sharedDir;
if (Path.IsPathRooted(sharedRel) && Directory.Exists(sharedRel))
sharedDir = Path.GetFullPath(sharedRel);
else
sharedDir = Path.GetFullPath(Path.Combine(_repoRoot, sharedRel));
var logger = _loggers.CreateLogger<PerTenantWorkflowSource>();
return new PerTenantWorkflowSource(tenantId, tenantDir, sharedDir, logger);
}
private static string Sanitize(string id)
{
if (string.IsNullOrWhiteSpace(id))
return "_";
var sb = new System.Text.StringBuilder(id.Length);
foreach (var ch in id)
sb.Append(char.IsLetterOrDigit(ch) || ch == '-' || ch == '_' ? ch : '_');
var result = sb.ToString();
return result.Length > 120 ? result[..120] : result;
}
private static string ResolveRepoRoot(IConfiguration config, IWebHostEnvironment env)
{
var configured = config["SourceCode:RepoRoot"];
if (!string.IsNullOrWhiteSpace(configured) && Directory.Exists(configured))
return Path.GetFullPath(configured);
var start = AppContext.BaseDirectory;
if (!string.IsNullOrEmpty(env.ContentRootPath) && Directory.Exists(env.ContentRootPath))
start = env.ContentRootPath;
var walk = Path.GetFullPath(start);
while (true)
{
if (Directory.Exists(Path.Combine(walk, ".git")))
return Path.GetFullPath(walk);
var parent = Directory.GetParent(walk)?.FullName;
if (string.IsNullOrEmpty(parent) || parent == walk)
break;
walk = parent;
}
return Path.GetFullPath(Directory.GetCurrentDirectory());
}
}
/// <summary>
/// Per-tenant workflow YAML source with directory isolation. Each tenant gets
/// its own directory at <c>{CopiesRoot}/{tenantId}/</c>, initialized from the
/// shared template directory on first access.
/// </summary>
public sealed class PerTenantWorkflowSource : IWorkflowSource
{
private readonly string _tenantDir;
private readonly string _sharedDir;
private readonly ILogger<PerTenantWorkflowSource> _logger;
public PerTenantWorkflowSource(string tenantId, string tenantDir, string sharedDir, ILogger<PerTenantWorkflowSource> logger)
{
_tenantDir = tenantDir;
_sharedDir = sharedDir;
_logger = logger;
_logger.LogInformation(
"Per-tenant workflow source: tenant={TenantId} dir={Dir} shared={Shared}",
tenantId, _tenantDir, _sharedDir);
}
public Task<IReadOnlyList<WorkflowFile>> ListAsync(CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
EnsureInitialized();
return Task.FromResult(ListYamlFiles(_tenantDir));
}
public async Task<string> ReadAsync(string path, CancellationToken ct)
{
EnsureInitialized();
return await File.ReadAllTextAsync(
Path.Combine(_tenantDir, path.Replace('/', Path.DirectorySeparatorChar)), ct);
}
public WorkflowSourceState GetState()
{
EnsureInitialized();
var head = RunGit(_tenantDir, "rev-parse", "HEAD");
var status = RunGit(_tenantDir, "status", "--porcelain");
return new WorkflowSourceState(
string.IsNullOrWhiteSpace(head) ? null : head,
!string.IsNullOrEmpty(status));
}
/// <summary>
/// Ensures the tenant directory exists and has been seeded from the shared
/// template directory (first-access initialization).
/// </summary>
private void EnsureInitialized()
{
if (Directory.Exists(_tenantDir))
return;
_logger.LogInformation("Initializing tenant workflow directory {Dir} from {Shared}", _tenantDir, _sharedDir);
Directory.CreateDirectory(_tenantDir);
if (!Directory.Exists(_sharedDir))
{
_logger.LogWarning("Shared workflow directory {Shared} does not exist; tenant starts empty", _sharedDir);
return;
}
CopyYamlFiles(_sharedDir, _tenantDir, _logger);
}
private static IReadOnlyList<WorkflowFile> ListYamlFiles(string dir)
{
var files = new List<WorkflowFile>();
if (!Directory.Exists(dir))
return files;
foreach (var full in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories))
{
var ext = Path.GetExtension(full);
if (ext is not (".yaml" or ".yml"))
continue;
// Skip .git
var rel = Path.GetRelativePath(dir, full);
if (rel.StartsWith(".git", StringComparison.OrdinalIgnoreCase))
continue;
files.Add(new WorkflowFile(rel.Replace(Path.DirectorySeparatorChar, '/')));
}
return files.OrderBy(f => f.Path, StringComparer.Ordinal).ToList();
}
/// <summary>
/// Copies YAML files from src to dst, skipping .git and non-yaml files.
/// </summary>
private static void CopyYamlFiles(string src, string dst, ILogger logger)
{
foreach (var file in Directory.EnumerateFiles(src, "*", SearchOption.AllDirectories))
{
var ext = Path.GetExtension(file);
if (ext is not (".yaml" or ".yml"))
continue;
var rel = Path.GetRelativePath(src, file);
if (rel.StartsWith(".git", StringComparison.OrdinalIgnoreCase))
continue;
var destPath = Path.Combine(dst, rel);
var destDir = Path.GetDirectoryName(destPath);
if (destDir != null && !Directory.Exists(destDir))
Directory.CreateDirectory(destDir);
try
{
File.Copy(file, destPath, overwrite: true);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to copy template file {File}", rel);
}
}
}
private string? RunGit(string workDir, params string[] args)
{
try
{
var psi = new ProcessStartInfo("git")
{
WorkingDirectory = workDir,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
};
foreach (var arg in args)
psi.ArgumentList.Add(arg);
using var process = Process.Start(psi);
if (process == null)
return null;
var output = process.StandardOutput.ReadToEnd();
if (!process.WaitForExit(5000))
{
process.Kill(entireProcessTree: true);
return null;
}
return process.ExitCode == 0 ? output.Trim() : null;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "git invocation failed in {Dir}", workDir);
return null;
}
}
}

View file

@ -0,0 +1,26 @@
using System.Collections.Concurrent;
namespace w4c_workflows.Services.Triggers;
/// <summary>
/// In-memory trigger state for lite/self-hosted mode where Redis is not available.
/// Survives only for the process lifetime (restart = re-fire once, which is safe
/// because delivery is at-least-once and the run lifecycle dedups via correlation id).
/// </summary>
public sealed class InMemoryTriggerState : ITriggerState
{
private readonly ConcurrentDictionary<string, DateTimeOffset> _state = new();
public Task<DateTimeOffset?> GetLastFireAsync(string tenantId, Guid workflowId, CancellationToken ct)
{
var key = $"{tenantId}:{workflowId}";
return Task.FromResult(_state.TryGetValue(key, out var ts) ? (DateTimeOffset?)ts : null);
}
public Task SetLastFireAsync(string tenantId, Guid workflowId, DateTimeOffset at, CancellationToken ct)
{
var key = $"{tenantId}:{workflowId}";
_state[key] = at;
return Task.CompletedTask;
}
}

View file

@ -63,5 +63,9 @@
},
"SourceCode": {
"RepoRoot": ""
},
"WorkflowSource": {
"CopiesRoot": ".data/workflow-tenants",
"SharedDir": "workflows"
}
}

View file

@ -26,6 +26,8 @@
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.3" />
<!-- Lite mode (self-hosted): SQLite replaces Postgres, in-process channels replace Redis. -->
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.10" />
<PackageReference Include="Scalar.AspNetCore" Version="2.17.2" />
<PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
<PackageReference Include="Serilog.Sinks.OpenSearch" Version="2.0.0" />