w4c-workflows-api/Program.cs

using Microsoft.EntityFrameworkCore;
using Npgsql;
using Serilog;
using Serilog.Events;
using Serilog.Sinks.OpenSearch;
using StackExchange.Redis;
using Scalar.AspNetCore;
using w4c_workflows.Data;
using w4c_workflows.Middleware;
using w4c_workflows.Services;
using w4c_workflows.Services.Execution;
using w4c_workflows.Services.Messaging;
using w4c_workflows.Services.Runs;
using w4c_workflows.Services.Triggers;

// ---------------------------------------------------------------------------
// w4c-workflows-api — control plane for the custom workflows engine.
//
// Two entrypoints share this image:
//   - default        : HTTP control plane (API, trigger scheduler, Mermaid, keys)
//   - --worker       : per-tenant execution worker (pull/exec/result over Redis)
// ---------------------------------------------------------------------------

var runWorker = args.Any(a => a == "--worker");

var builder = WebApplication.CreateBuilder(args);

var openSearchUrl = builder.Configuration["OpenSearch:Url"] ?? "http://localhost:9200";
var appName = builder.Configuration["APP_NAME"] ?? (runWorker ? "workflows-worker" : "workflows-api");

Log.Logger = new LoggerConfiguration()
    .MinimumLevel.Information()
    .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning)
    // EF Core logs the raw SQL ("Executed DbCommand ...") at Information for
    // EVERY query. The background pollers (run lifecycle, trigger scheduler,
    // handler consumer) fire those several times a second and flood the console
    // / OpenSearch. Keep our own Information logs, but drop the EF command log.
    .MinimumLevel.Override("Microsoft.EntityFrameworkCore.Database.Command", LogEventLevel.Warning)
    .Enrich.FromLogContext()
    .Enrich.WithProperty("App", appName)
    .WriteTo.Console()
    .WriteTo.OpenSearch(new OpenSearchSinkOptions(new Uri(openSearchUrl))
    {
        IndexFormat = "w4c-workflows-{0:yyyy.MM.dd}",
        AutoRegisterTemplate = false,
        BatchAction = OpenOpType.Create,
        EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog,
        BufferBaseFilename = Path.Combine(Path.GetTempPath(), "serilog-opensearch-buffer"),
    })
    .CreateLogger();

builder.Host.UseSerilog();

builder.Services.AddControllers();
builder.Services.AddOpenApi();
builder.Services.AddHttpClient();
builder.Services.AddHttpContextAccessor();

// Lite mode flag: when true, the engine uses SQLite + in-process channels
// instead of PostgreSQL + Redis. Intended for self-hosted single-container
// deployments where external infrastructure is not available.
var liteMode = builder.Configuration.GetValue<bool>("UseLiteMode");

// A transient failure in ONE background service (worker loop, trigger scheduler,
// run lifecycle, handler consumer) must never take the whole control plane down.
// Without this, HostOptions.BackgroundServiceExceptionBehavior defaults to
// StopHost and a single early exception prevents the API from starting.
builder.Services.Configure<HostOptions>(options =>
    options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.Ignore);

if (liteMode)
{
    // Lite mode: SQLite. The connection string should be something like
    // "Data Source=/app/data/workflows.db".
    var sqliteCs = builder.Configuration.GetConnectionString("DefaultConnection")
        ?? "Data Source=workflows.db";
    builder.Services.AddDbContext<WorkflowsDbContext>(options =>
        options.UseSqlite(sqliteCs));
    Log.Logger.Information("Lite mode: using SQLite ({ConnectionString})", sqliteCs);
}
else
{
    builder.Services.AddDbContext<WorkflowsDbContext>(options =>
        options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection")));

    // NpgsqlDataSource for lightweight queries outside EF (e.g. login resolution).
    builder.Services.AddSingleton<NpgsqlDataSource>(_ =>
        NpgsqlDataSource.Create(builder.Configuration.GetConnectionString("DefaultConnection")!));

    builder.Services.AddSingleton<IConnectionMultiplexer>(_ =>
        ConnectionMultiplexer.Connect(
            builder.Configuration["Redis:ConnectionString"] ?? "localhost:6379,abortConnect=false"));
}

builder.Services.AddScoped<ApiKeyService>();

// Transport + lease.
if (liteMode)
{
    // Lite mode: in-process channels replace Redis Streams.
    builder.Services.AddSingleton<InMemoryTransport>();
    builder.Services.AddSingleton<IJobQueue>(sp => sp.GetRequiredService<InMemoryTransport>());
    builder.Services.AddSingleton<IEventBus>(sp => sp.GetRequiredService<InMemoryTransport>());
    builder.Services.AddSingleton<ITriggerState, InMemoryTriggerState>();
    builder.Services.AddSingleton<ILeaseService, InMemoryLeaseService>();
    Log.Logger.Information("Lite mode: using in-memory transport + trigger state (no Redis)");
}
else
{
    // Full mode: Redis Streams.
    builder.Services.AddSingleton<RedisStreamsTransport>();
    builder.Services.AddSingleton<IJobQueue>(sp => sp.GetRequiredService<RedisStreamsTransport>());
    builder.Services.AddSingleton<IEventBus>(sp => sp.GetRequiredService<RedisStreamsTransport>());
    builder.Services.AddSingleton<ITriggerState, RedisTriggerState>();
    builder.Services.AddSingleton<ILeaseService, LeaseService>();
}

// YAML compile pipeline (step 4).
builder.Services.AddSingleton<LanguageRegistry>();
builder.Services.AddSingleton<WorkflowValidator>();
builder.Services.AddSingleton<WorkflowCompiler>();

// Workflow source: factory-based — each request creates a tenant-scoped source.
// WorkflowSource:CopiesRoot is REQUIRED; the factory throws if it is empty,
// preventing any request from serving cross-tenant workflows.
// When Forgejo:AdminToken is configured, the factory creates Forgejo-backed
// sources with per-tenant repos; otherwise it falls back to filesystem-only.
builder.Services.AddSingleton<WorkflowSourceFactory>();
builder.Services.AddScoped<WorkflowRepoStore>();
builder.Services.AddScoped<IWorkflowSource>(sp =>
{
    var http = sp.GetRequiredService<IHttpContextAccessor>();
    var factory = sp.GetRequiredService<WorkflowSourceFactory>();
    // Check if a Forgejo-resolved source was already prepared by the middleware.
    if (http.HttpContext?.Items["WorkflowSource"] is IWorkflowSource forgejoSrc)
        return forgejoSrc;
    var tenantId = http.HttpContext?.Items["TenantId"] as string
        ?? throw new InvalidOperationException("TenantId not resolved by auth middleware");
    return factory.Create(tenantId);
});
builder.Services.AddScoped<WorkflowSyncService>();

// Mermaid diagrams (step 6).
builder.Services.AddSingleton<MermaidGeneratorService>();

// Rich HTML workflow preview (backend-rendered, embeddable in an iframe).
builder.Services.AddSingleton<WorkflowHtmlRenderer>();
builder.Services.AddSingleton<RenderTokenService>();

// Runtime registry + worker executors (step 7).
// The executors map each language to its execution strategy; the registry
// (which also backs GET /api/languages) reports per-runtime availability.
builder.Services.AddSingleton<IScriptExecutor>(new SubprocessScriptExecutor("shell", "sh", builder.Configuration));
builder.Services.AddSingleton<IScriptExecutor>(new SubprocessScriptExecutor("python", "python3", builder.Configuration));
builder.Services.AddSingleton<IScriptExecutor>(new SubprocessScriptExecutor("javascript", "node", builder.Configuration));
builder.Services.AddSingleton<IScriptExecutor>(new TypeScriptExecutor(builder.Configuration));
builder.Services.AddSingleton<IScriptExecutor>(new CSharpScriptExecutor());
// W9: `agent` step type — invokes an LLM agent via chatapi over HTTP (D3), keeping
// workflows-api decoupled from BotSharp.
builder.Services.AddSingleton<IScriptExecutor>(sp =>
    new AgentScriptExecutor(
        sp.GetRequiredService<IHttpClientFactory>(),
        builder.Configuration,
        sp.GetRequiredService<ILogger<AgentScriptExecutor>>()));
builder.Services.AddSingleton<RuntimeRegistry>();

// S8: remote (SSH) task execution via the w4c-webapi server-console exec endpoint.
// Used only when a task carries a `server` reference; local subprocess stays the default.
builder.Services.AddSingleton<RemoteServerExecutor>();

// Trigger engine (step 8): the scheduler + handler consumer run only in the
// control-plane entrypoint, never in the worker. They share the run launcher —
// the seam that creates pending runs for the run lifecycle engine (step 9).
builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddScoped<IRunLauncher, RunLauncher>();

// Run lifecycle engine (step 9): dispatch pending runs + consume task.results +
// advance the nextId chain + durable checkpoint/resume.
builder.Services.AddSingleton<RunLifecycleEngine>();
builder.Services.AddScoped<TaskDispatcher>();
builder.Services.AddScoped<DurableStateStore>();

if (runWorker)
{
    // Dedicated worker process (scaled/separate deployment).
    builder.Services.AddHostedService<WorkerHostService>();
}
else
{
    // Control-plane entrypoint. In single-node / self-hosted deployments one
    // process does both: it accounts, dispatches, AND runs the worker loop so
    // dispatched jobs actually execute. This closes the gap where the API
    // created run + task.run jobs but nothing consumed them, leaving every run
    // 'running' forever. A dedicated --worker can still be used for scale-out.
    builder.Services.AddHostedService<WorkerHostService>();
    builder.Services.AddHostedService<TriggerScheduler>();
    builder.Services.AddHostedService<HandlerStreamConsumer>();
    builder.Services.AddHostedService<RunLifecycleService>();
}

builder.Services.AddCors(options =>
{
    options.AddDefaultPolicy(policy =>
    {
        var origins = builder.Configuration.GetSection("Cors:Origins").Get<string[]>();
        if (origins is { Length: > 0 })
        {
            // Production: restrict to explicitly configured origins.
            policy.WithOrigins(origins)
                  .AllowAnyMethod()
                  .AllowAnyHeader()
                  .AllowCredentials();
        }
        else
        {
            // Dev fallback: no origins configured → allow all.
            policy.AllowAnyOrigin()
                  .AllowAnyMethod()
                  .AllowAnyHeader();
        }
    });
});

var app = builder.Build();

app.UseCors();

// Global exception handler: catches unhandled exceptions from any controller
// or middleware and returns a structured JSON error instead of a raw 500.
app.UseExceptionHandler(handler =>
{
    handler.Run(async context =>
    {
        context.Response.StatusCode = StatusCodes.Status500InternalServerError;
        context.Response.ContentType = "application/json";
        var ex = context.Features.Get<Microsoft.AspNetCore.Diagnostics.IExceptionHandlerFeature>();
        var message = ex?.Error.Message ?? "An unexpected error occurred.";
        var json = System.Text.Json.JsonSerializer.Serialize(new { error = message });
        await context.Response.WriteAsync(json);
    });
});

app.UseMiddleware<AuthMiddleware>();

// Resolve the Forgejo-backed workflow source asynchronously after auth.
// The auth middleware sets TenantId; this middleware resolves the user's Forgejo
// login, clones/pulls the repo, and stores the IWorkflowSource in HttpContext.Items
// so the DI registration can pick it up synchronously.
app.Use(async (context, next) =>
{
    var tenantId = context.Items["TenantId"] as string;
    if (!string.IsNullOrEmpty(tenantId))
    {
        // Scope all workflow queries to the tenant's CURRENT workflow repo, so a
        // workflow compiled from a previously-selected repo is unloaded/disabled.
        var repoName = (context.RequestServices.GetRequiredService<IConfiguration>()["WorkflowSource:WorkflowRepoName"] ?? string.Empty).Trim();
        if (string.IsNullOrWhiteSpace(repoName)) repoName = "workflows";
        try
        {
            var store = context.RequestServices.GetRequiredService<WorkflowRepoStore>();
            repoName = await store.GetNameAsync(tenantId, context.RequestAborted);
        }
        catch (Exception ex)
        {
            var l = context.RequestServices.GetRequiredService<ILoggerFactory>().CreateLogger("WorkflowSourceMiddleware");
            l.LogDebug(ex, "Could not resolve workflow repo for tenant {TenantId}", tenantId);
        }
        var db = context.RequestServices.GetRequiredService<WorkflowsDbContext>();
        db.CurrentRepo = repoName;
        context.Items["WorkflowRepo"] = repoName;

        var factory = context.RequestServices.GetRequiredService<WorkflowSourceFactory>();
        if (factory.IsForgejoBacked)
        {
            try
            {
                var source = await factory.CreateAsync(tenantId, repoName, context.RequestAborted);
                context.Items["WorkflowSource"] = source;
                // Store the resolved Forgejo login for controllers that need
                // to resolve repo paths (WorkflowFilesController, etc.).
                var login = await factory.ResolveForgejoLoginAsync(tenantId, context.RequestAborted);
                if (!string.IsNullOrEmpty(login))
                    context.Items["ForgejoLogin"] = login;
            }
            catch (Exception ex)
            {
                var logger = context.RequestServices.GetRequiredService<ILoggerFactory>()
                    .CreateLogger("WorkflowSourceMiddleware");
                logger.LogWarning(ex, "Forgejo-backed source failed for tenant {TenantId}, falling back to filesystem", tenantId);
            }
        }
    }
    await next();
});

// Ensure the database schema exists before serving traffic.
try
{
    using var scope = app.Services.CreateScope();
    var db = scope.ServiceProvider.GetRequiredService<WorkflowsDbContext>();
    await db.Database.MigrateAsync();

    // Additive schema, applied idempotently (raw SQL, no new EF migration) so it is
    // safe on every startup against both freshly-created and long-lived databases.
    // Only applies to PostgreSQL (SQLite handles columns automatically via EF).
    if (!liteMode)
    {
        await db.Database.ExecuteSqlRawAsync(
            "ALTER TABLE workflows.\"Tasks\" ADD COLUMN IF NOT EXISTS \"ArchivedAt\" timestamptz NULL;");
        await db.Database.ExecuteSqlRawAsync(
            "ALTER TABLE workflows.\"Tasks\" ADD COLUMN IF NOT EXISTS \"Server\" text NULL;");
        // Per-tenant workflow repo setting (which repo holds the workflow files) +
        // which repo a compiled workflow came from. Applied idempotently.
        await db.Database.ExecuteSqlRawAsync(
            "ALTER TABLE workflows.\"Workflows\" ADD COLUMN IF NOT EXISTS \"Repo\" text NOT NULL DEFAULT 'workflows';");
        await db.Database.ExecuteSqlRawAsync(
            "CREATE TABLE IF NOT EXISTS workflows.\"WorkflowRepos\" (" +
            "\"TenantId\" text NOT NULL, \"RepoName\" text NOT NULL, \"UpdatedAt\" timestamptz NULL, " +
            "CONSTRAINT \"PK_WorkflowRepos\" PRIMARY KEY (\"TenantId\"));");
    }
}
catch (Exception ex)
{
    Log.Logger.Error(ex, "Failed to apply EF Core migrations on startup");
}

// Scalar API reference + OpenAPI document are exposed under a dedicated
// per-service prefix (/api/scalar/workflows/...) so Caddy can route them on the
// public origin without colliding with the webapi Scalar reference. The OpenAPI
// document is served at the SAME prefix (not the default /openapi/v1.json) so
// Scalar's relative spec + asset resolution stays consistent behind the proxy.
app.MapOpenApi("/api/scalar/workflows/openapi/{documentName}.json");
app.MapScalarApiReference("/api/scalar/workflows", options =>
{
    options.OpenApiRoutePattern = "/api/scalar/workflows/openapi/{documentName}.json";
});
app.MapControllers();

if (runWorker)
{
    Log.Logger.Information("w4c-workflows-worker entrypoint selected; worker loop running for tenant {TenantId}",
        builder.Configuration["Workflows:TenantId"] ?? Environment.GetEnvironmentVariable("WF_TENANT_ID") ?? "default");
}

app.Run();

Log.CloseAndFlush();