958 lines
35 KiB
C#
958 lines
35 KiB
C#
using System.Globalization;
|
|
using System.Net;
|
|
using System.Net.Http.Headers;
|
|
using System.Text;
|
|
using System.Text.Json.Nodes;
|
|
using w4c_workflows.Models.Credentials;
|
|
using w4c_workflows.Models.Nodes;
|
|
using w4c_workflows.Services.Credentials;
|
|
using w4c_workflows.Services.Nodes.Binary;
|
|
using w4c_workflows.Services.Security;
|
|
|
|
namespace w4c_workflows.Services.Nodes.Executors;
|
|
|
|
/// <summary>
|
|
/// HTTP Request node. Calls an endpoint per input item and maps the response to
|
|
/// items. Non-2xx responses fail the node (routable to the error port) unless
|
|
/// <c>neverError</c> is set; JSON arrays become one item per element.
|
|
///
|
|
/// Every outbound target — the initial URL and each redirect hop — is vetted by
|
|
/// <see cref="EgressGuard"/> before a request is sent, so a workflow cannot reach
|
|
/// private/loopback/link-local addresses. Automatic redirects are disabled at the
|
|
/// transport and followed here instead, which is what makes per-hop vetting
|
|
/// possible; credentials are dropped when a redirect crosses to another host.
|
|
///
|
|
/// <see cref="NodeQuotaPolicy"/> additionally bounds the run: each hop consumes
|
|
/// one request from the per-run budget, and the response body is streamed under
|
|
/// a size cap so an oversized payload fails the node instead of filling memory.
|
|
///
|
|
/// When the node declares a <c>options.pagination</c> plan the same request is
|
|
/// issued repeatedly, driven by <see cref="HttpPaginationPlan"/>: the next URL or
|
|
/// cursor is read from a JSON path in the previous response, and the accumulated
|
|
/// pages are emitted as one item array. Every page is vetted by the egress guard
|
|
/// and consumes quota exactly like a first request, so pagination cannot be used
|
|
/// to reach a blocked target or escape the request budget.
|
|
///
|
|
/// <c>options.responseFormat: file</c> streams the response into the
|
|
/// <see cref="IBinaryStore"/> and emits an item whose <c>binary.data</c> reference
|
|
/// carries the file name, MIME type and size. Request bodies may be
|
|
/// <c>multipart</c> (declared fields plus one input binary attachment) or
|
|
/// <c>binary</c> (the input attachment as the raw body), so a download can be
|
|
/// fed straight back into an upload without inlining bytes in item JSON.
|
|
/// </summary>
|
|
public sealed class HttpRequestNodeExecutor : INodeExecutor
|
|
{
|
|
/// <summary>Credential alias declared on the blueprint.</summary>
|
|
public const string CredentialAlias = "httpAuth";
|
|
|
|
/// <summary>Blueprint type and default HTTP client name.</summary>
|
|
public const string TypeName = "core.httpRequest";
|
|
|
|
/// <summary>Named client used when a node opts out of TLS validation.</summary>
|
|
public const string InsecureClientName = TypeName + ":insecure";
|
|
|
|
/// <summary>Default input/output binary property name.</summary>
|
|
public const string DefaultBinaryProperty = FileSystemBinaryStore.DefaultPropertyName;
|
|
|
|
private const int DefaultTimeoutMs = 30_000;
|
|
private const int DefaultMaxRedirects = 5;
|
|
|
|
private readonly IHttpClientFactory _http;
|
|
private readonly CredentialTypeCatalog _credentialTypes;
|
|
private readonly EgressGuard _egress;
|
|
private readonly NodeQuotaPolicy _quota;
|
|
private readonly IBinaryStore _binary;
|
|
|
|
public HttpRequestNodeExecutor(
|
|
IHttpClientFactory http,
|
|
CredentialTypeCatalog credentialTypes,
|
|
EgressGuard egress,
|
|
NodeQuotaPolicy quota,
|
|
IBinaryStore binary)
|
|
{
|
|
_http = http;
|
|
_credentialTypes = credentialTypes;
|
|
_egress = egress;
|
|
_quota = quota;
|
|
_binary = binary;
|
|
}
|
|
|
|
public string Type => TypeName;
|
|
|
|
public async Task<NodeExecutionOutcome> RunAsync(NodeExecutionContext context, CancellationToken ct)
|
|
{
|
|
var methodName = ReadString(context, "method") ?? "GET";
|
|
var url = ReadString(context, "url");
|
|
if (string.IsNullOrWhiteSpace(url))
|
|
return NodeExecutionOutcome.Failed("url is required", "invalid_parameter");
|
|
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
|
|
return NodeExecutionOutcome.Failed($"'{url}' is not a valid absolute URL", "invalid_url");
|
|
|
|
var options = context.Parameters["options"] as JsonObject ?? new JsonObject();
|
|
var timeoutMs = ReadInt(options, "timeoutMs", DefaultTimeoutMs);
|
|
if (timeoutMs <= 0)
|
|
timeoutMs = DefaultTimeoutMs;
|
|
var neverError = ReadBool(options, "neverError");
|
|
var responseFormat = ReadString(options, "responseFormat") ?? "autodetect";
|
|
var putOutputInField = ReadString(options, "putOutputInField");
|
|
var outputPath = ReadString(options, "outputPath");
|
|
var followRedirects = ReadBool(options, "followRedirects", true);
|
|
// The operator policy is a ceiling: a node may ask for fewer, never more.
|
|
var maxRedirects = Math.Clamp(
|
|
ReadInt(options, "maxRedirects", DefaultMaxRedirects), 0, _egress.MaxRedirects);
|
|
|
|
HttpPaginationPlan? pagination;
|
|
try
|
|
{
|
|
pagination = HttpPaginationPlan.From(options);
|
|
}
|
|
catch (InvalidOperationException ex)
|
|
{
|
|
return NodeExecutionOutcome.Failed(ex.Message, "invalid_pagination");
|
|
}
|
|
|
|
// Pagination only makes sense when a page can be parsed into items.
|
|
if (pagination != null && responseFormat is "text" or "file")
|
|
pagination = null;
|
|
|
|
// Resolve the auth intent once; the credential itself is injected per hop
|
|
// and dropped when a redirect crosses to another host.
|
|
var authentication = ReadString(context, "authentication") ?? "none";
|
|
var hasCredential = context.Credentials.TryGetValue(CredentialAlias, out var credential);
|
|
if (string.Equals(authentication, "credential", StringComparison.Ordinal) && !hasCredential)
|
|
return NodeExecutionOutcome.Failed(
|
|
$"no credential is configured for alias '{CredentialAlias}'", "missing_credential");
|
|
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
|
timeout.CancelAfter(timeoutMs);
|
|
|
|
var client = _http.CreateClient(
|
|
ReadBool(options, "ignoreSslIssues") ? InsecureClientName : Type);
|
|
|
|
var method = new HttpMethod(methodName.ToUpperInvariant());
|
|
var sendBody = ReadBool(context, "sendBody");
|
|
var body = context.Parameters["body"]?.DeepClone();
|
|
var sendCredential = !string.Equals(authentication, "none", StringComparison.Ordinal);
|
|
var queryOverrides = new Dictionary<string, string>(StringComparer.Ordinal);
|
|
var headerOverrides = new Dictionary<string, string>(StringComparer.Ordinal);
|
|
|
|
// pageNumber mode owns its parameter from the first request on, so the
|
|
// starting value is deterministic regardless of what the base URL carries.
|
|
if (pagination?.InitialStep() is { } initial)
|
|
body = ApplyStep(initial, pagination, body, queryOverrides, headerOverrides);
|
|
|
|
var items = new List<FlowItem>();
|
|
var currentUri = uri;
|
|
var page = 0;
|
|
var pageNumber = pagination?.PageStart ?? 1;
|
|
|
|
while (true)
|
|
{
|
|
page++;
|
|
var pageUri = ApplyQuery(currentUri, queryOverrides);
|
|
|
|
var pageResult = await SendPageAsync(
|
|
context, client, timeout, timeoutMs, pageUri, method, sendBody, body,
|
|
headerOverrides, sendCredential, hasCredential, credential,
|
|
followRedirects, maxRedirects, ct);
|
|
|
|
if (pageResult.Failure != null)
|
|
return pageResult.Failure;
|
|
|
|
var status = pageResult.Status;
|
|
if (status >= 400 && !neverError)
|
|
{
|
|
return new NodeExecutionOutcome
|
|
{
|
|
Outputs = Array.Empty<IReadOnlyList<FlowItem>>(),
|
|
Failure = new NodeFailure(
|
|
$"HTTP {status}", "http_error", Truncate(pageResult.Body), status),
|
|
};
|
|
}
|
|
|
|
// A configured stop status ends pagination without contributing the
|
|
// (normally empty) stop page to the output.
|
|
if (pagination != null && pagination.IsStopStatus(status))
|
|
break;
|
|
|
|
var build = await BuildItemsAsync(
|
|
pageResult, responseFormat, putOutputInField, outputPath, timeout.Token);
|
|
if (build.Error != null)
|
|
return NodeExecutionOutcome.Failed(build.Error, "invalid_response");
|
|
|
|
var pageItems = build.Items;
|
|
|
|
var itemLimitReached = AddItems(items, pageItems, pagination?.MaxItems ?? 0);
|
|
|
|
if (pagination == null)
|
|
break;
|
|
if (itemLimitReached)
|
|
break;
|
|
if (pagination.IsStopPage(pageItems.Count))
|
|
break;
|
|
if (pagination.IsPageLimitReached(page))
|
|
break;
|
|
|
|
var step = pagination.TryNext(pageResult.Parsed, ref pageNumber);
|
|
if (step == null)
|
|
break;
|
|
|
|
if (step.NextUrl != null)
|
|
{
|
|
if (!TryResolveNext(step.NextUrl, currentUri, out var nextUri))
|
|
{
|
|
return NodeExecutionOutcome.Failed(
|
|
$"invalid pagination next URL '{step.NextUrl}'", "invalid_redirect");
|
|
}
|
|
currentUri = nextUri;
|
|
// The provider's next URL already carries the query; do not
|
|
// re-apply this run's overrides on top of it.
|
|
queryOverrides.Clear();
|
|
}
|
|
else
|
|
{
|
|
body = ApplyStep(step, pagination, body, queryOverrides, headerOverrides);
|
|
}
|
|
}
|
|
|
|
return NodeExecutionOutcome.Single(items);
|
|
}
|
|
|
|
// ------------------------------------------------------------------ one page
|
|
|
|
private sealed record PageResult(
|
|
int Status,
|
|
string Body,
|
|
byte[] Bytes,
|
|
JsonNode? Parsed,
|
|
string? ContentType,
|
|
string? SuggestedFileName,
|
|
NodeExecutionOutcome? Failure);
|
|
|
|
/// <summary>
|
|
/// Issues one logical request, following redirects by hand so every hop is
|
|
/// egress-vetted. Credentials/body are dropped on a cross-host redirect.
|
|
/// </summary>
|
|
private async Task<PageResult> SendPageAsync(
|
|
NodeExecutionContext context,
|
|
HttpClient client,
|
|
CancellationTokenSource timeout,
|
|
int timeoutMs,
|
|
Uri initialUri,
|
|
HttpMethod method,
|
|
bool sendBody,
|
|
JsonNode? body,
|
|
IReadOnlyDictionary<string, string> headerOverrides,
|
|
bool sendCredential,
|
|
bool hasCredential,
|
|
CredentialData? credential,
|
|
bool followRedirects,
|
|
int maxRedirects,
|
|
CancellationToken ct)
|
|
{
|
|
var currentUri = initialUri;
|
|
var currentMethod = method;
|
|
var currentHost = initialUri.DnsSafeHost;
|
|
var bodyEnabled = sendBody;
|
|
var credEnabled = sendCredential;
|
|
var redirectsFollowed = 0;
|
|
HttpResponseMessage? response = null;
|
|
|
|
while (true)
|
|
{
|
|
var verdict = await _egress.AuthorizeAsync(currentUri, ct);
|
|
if (!verdict.Allowed)
|
|
{
|
|
response?.Dispose();
|
|
return PageFailure(verdict.Reason ?? "blocked by the egress policy", "egress_blocked");
|
|
}
|
|
|
|
if (!_quota.TryReserveRequest(context.State, out var requestsUsed))
|
|
{
|
|
response?.Dispose();
|
|
return PageFailure(
|
|
$"outbound request quota exceeded ({_quota.MaxRequestsPerRun} per run)",
|
|
"quota_exceeded",
|
|
$"this run has already issued {requestsUsed - 1} requests");
|
|
}
|
|
|
|
using var request = new HttpRequestMessage(currentMethod, currentUri);
|
|
ApplyHeaders(context, request, headerOverrides);
|
|
|
|
if (credEnabled && hasCredential)
|
|
{
|
|
var authError = CredentialInjector.Apply(credential!, _credentialTypes, request);
|
|
if (authError != null)
|
|
{
|
|
response?.Dispose();
|
|
return PageFailure(authError, "missing_credential");
|
|
}
|
|
}
|
|
|
|
if (bodyEnabled)
|
|
{
|
|
var bodyError = await ApplyBodyAsync(context, body, request, ct);
|
|
if (bodyError != null)
|
|
{
|
|
response?.Dispose();
|
|
return PageFailure(bodyError, "invalid_parameter");
|
|
}
|
|
}
|
|
|
|
try
|
|
{
|
|
// Headers-only completion so the body is streamed under the size
|
|
// cap below instead of being buffered by the transport first.
|
|
response = await client.SendAsync(
|
|
request, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
|
|
}
|
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
|
{
|
|
throw;
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
return PageFailure($"request timed out after {timeoutMs} ms", "timeout");
|
|
}
|
|
catch (HttpRequestException ex)
|
|
{
|
|
return PageFailure(ex.Message, "request_failed");
|
|
}
|
|
|
|
var location = response.Headers.Location;
|
|
if (!followRedirects || location == null || !IsRedirect(response.StatusCode))
|
|
break;
|
|
|
|
if (redirectsFollowed >= maxRedirects)
|
|
{
|
|
response.Dispose();
|
|
return PageFailure(
|
|
$"exceeded the maximum of {maxRedirects} redirects", "too_many_redirects");
|
|
}
|
|
|
|
if (!TryResolveRedirect(currentUri, location, out var nextUri))
|
|
{
|
|
response.Dispose();
|
|
return PageFailure(
|
|
$"invalid redirect location '{location}'", "invalid_redirect");
|
|
}
|
|
|
|
// 307/308 preserve method and body; the others downgrade to GET.
|
|
if (response.StatusCode != HttpStatusCode.TemporaryRedirect
|
|
&& response.StatusCode != HttpStatusCode.PermanentRedirect
|
|
&& currentMethod != HttpMethod.Head)
|
|
{
|
|
currentMethod = HttpMethod.Get;
|
|
bodyEnabled = false;
|
|
}
|
|
|
|
var nextHost = nextUri.DnsSafeHost;
|
|
if (!string.Equals(nextHost, currentHost, StringComparison.OrdinalIgnoreCase))
|
|
credEnabled = false;
|
|
|
|
response.Dispose();
|
|
response = null;
|
|
currentUri = nextUri;
|
|
currentHost = nextHost;
|
|
redirectsFollowed++;
|
|
}
|
|
|
|
using (response!)
|
|
{
|
|
var status = (int)response.StatusCode;
|
|
var read = await HttpBodyReader.ReadAsync(response, _quota.MaxResponseBytes, timeout.Token);
|
|
if (read.TooLarge)
|
|
return PageFailure(
|
|
$"response exceeded the {_quota.MaxResponseBytes}-byte limit", "response_too_large");
|
|
|
|
var bodyText = read.Body!;
|
|
var contentType = response.Content.Headers.ContentType?.MediaType;
|
|
var fileName = ResolveFileName(response, currentUri);
|
|
return new PageResult(
|
|
status, bodyText, read.Bytes!, TryParse(bodyText), contentType, fileName, null);
|
|
}
|
|
}
|
|
|
|
private static PageResult PageFailure(string message, string? code, string? description = null)
|
|
=> new(
|
|
0, string.Empty, Array.Empty<byte>(), null, null, null,
|
|
NodeExecutionOutcome.Failed(message, code, description));
|
|
|
|
// ------------------------------------------------------------------ pagination
|
|
|
|
private static JsonNode? ApplyStep(
|
|
HttpPaginationStep step,
|
|
HttpPaginationPlan plan,
|
|
JsonNode? body,
|
|
Dictionary<string, string> queryOverrides,
|
|
Dictionary<string, string> headerOverrides)
|
|
{
|
|
if (step.Cursor != null)
|
|
return ApplyParameter(plan.CursorPlace, plan.CursorName, step.Cursor, body, queryOverrides, headerOverrides);
|
|
|
|
if (step.PageNumber is int pageNumber)
|
|
return ApplyParameter(
|
|
plan.PagePlace, plan.PageName,
|
|
pageNumber.ToString(CultureInfo.InvariantCulture),
|
|
body, queryOverrides, headerOverrides);
|
|
|
|
return body;
|
|
}
|
|
|
|
private static JsonNode? ApplyParameter(
|
|
string place,
|
|
string name,
|
|
string value,
|
|
JsonNode? body,
|
|
Dictionary<string, string> queryOverrides,
|
|
Dictionary<string, string> headerOverrides)
|
|
{
|
|
if (string.IsNullOrEmpty(name))
|
|
return body;
|
|
|
|
switch (place)
|
|
{
|
|
case HttpPaginationPlan.PlaceHeader:
|
|
headerOverrides[name] = value;
|
|
return body;
|
|
|
|
case HttpPaginationPlan.PlaceBody:
|
|
if (body is JsonObject obj)
|
|
obj[name] = value;
|
|
else if (body == null)
|
|
body = new JsonObject { [name] = value };
|
|
return body;
|
|
|
|
default:
|
|
queryOverrides[name] = value;
|
|
return body;
|
|
}
|
|
}
|
|
|
|
/// <summary>Merges pagination overrides into the base URL query (same-name keys are replaced).</summary>
|
|
private static Uri ApplyQuery(Uri baseUri, IReadOnlyDictionary<string, string> overrides)
|
|
{
|
|
if (overrides.Count == 0)
|
|
return baseUri;
|
|
|
|
var pairs = new List<KeyValuePair<string, string>>();
|
|
if (!string.IsNullOrEmpty(baseUri.Query))
|
|
{
|
|
foreach (var part in baseUri.Query.TrimStart('?')
|
|
.Split('&', StringSplitOptions.RemoveEmptyEntries))
|
|
{
|
|
var separator = part.IndexOf('=');
|
|
var key = separator < 0 ? part : part[..separator];
|
|
var value = separator < 0 ? string.Empty : part[(separator + 1)..];
|
|
pairs.Add(new KeyValuePair<string, string>(
|
|
Uri.UnescapeDataString(key), Uri.UnescapeDataString(value)));
|
|
}
|
|
}
|
|
|
|
foreach (var (name, value) in overrides)
|
|
{
|
|
pairs.RemoveAll(pair => string.Equals(pair.Key, name, StringComparison.Ordinal));
|
|
pairs.Add(new KeyValuePair<string, string>(name, value));
|
|
}
|
|
|
|
var query = string.Join('&', pairs.Select(pair =>
|
|
$"{Uri.EscapeDataString(pair.Key)}={Uri.EscapeDataString(pair.Value)}"));
|
|
|
|
return new UriBuilder(baseUri) { Query = query }.Uri;
|
|
}
|
|
|
|
/// <summary>Resolves a pagination next URL (absolute or relative) and keeps it HTTP(S).</summary>
|
|
private static bool TryResolveNext(string nextUrl, Uri current, out Uri next)
|
|
{
|
|
next = null!;
|
|
try
|
|
{
|
|
// A leading-slash path parses as an absolute "file" URI on Unix, so
|
|
// only an explicitly http(s) absolute URL bypasses relative resolution.
|
|
if (Uri.TryCreate(nextUrl, UriKind.Absolute, out var absolute)
|
|
&& absolute.Scheme is "http" or "https")
|
|
{
|
|
next = absolute;
|
|
}
|
|
else
|
|
{
|
|
next = new Uri(current, nextUrl);
|
|
}
|
|
|
|
return next.Scheme is "http" or "https";
|
|
}
|
|
catch (UriFormatException)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/// <summary>Accumulates a page under the item cap; returns true when the cap is reached.</summary>
|
|
private static bool AddItems(List<FlowItem> items, IReadOnlyList<FlowItem> pageItems, int maxItems)
|
|
{
|
|
if (maxItems <= 0)
|
|
{
|
|
items.AddRange(pageItems);
|
|
return false;
|
|
}
|
|
|
|
var remaining = maxItems - items.Count;
|
|
if (remaining <= 0)
|
|
return true;
|
|
|
|
if (pageItems.Count > remaining)
|
|
{
|
|
items.AddRange(pageItems.Take(remaining));
|
|
return true;
|
|
}
|
|
|
|
items.AddRange(pageItems);
|
|
return items.Count >= maxItems;
|
|
}
|
|
|
|
// ------------------------------------------------------------------ request
|
|
|
|
private static void ApplyHeaders(
|
|
NodeExecutionContext context,
|
|
HttpRequestMessage request,
|
|
IReadOnlyDictionary<string, string> overrides)
|
|
{
|
|
if (ReadBool(context, "sendHeaders") && context.Parameters["headers"] is JsonArray headers)
|
|
{
|
|
foreach (var entry in headers)
|
|
{
|
|
if (entry is not JsonObject header)
|
|
continue;
|
|
|
|
var name = ReadString(header, "name");
|
|
if (string.IsNullOrWhiteSpace(name))
|
|
continue;
|
|
|
|
request.Headers.TryAddWithoutValidation(name, ReadString(header, "value") ?? string.Empty);
|
|
}
|
|
}
|
|
|
|
// Pagination header overrides win over the static header list.
|
|
foreach (var (name, value) in overrides)
|
|
{
|
|
request.Headers.Remove(name);
|
|
request.Headers.TryAddWithoutValidation(name, value);
|
|
}
|
|
}
|
|
|
|
// ------------------------------------------------------------------ redirects
|
|
|
|
private static bool IsRedirect(HttpStatusCode status)
|
|
=> status is HttpStatusCode.MovedPermanently // 301
|
|
or HttpStatusCode.Found // 302
|
|
or HttpStatusCode.SeeOther // 303
|
|
or HttpStatusCode.TemporaryRedirect // 307
|
|
or HttpStatusCode.PermanentRedirect; // 308
|
|
|
|
/// <summary>Resolves a Location header against the current URL and keeps it HTTP(S).</summary>
|
|
private static bool TryResolveRedirect(Uri current, Uri location, out Uri next)
|
|
{
|
|
next = null!;
|
|
try
|
|
{
|
|
next = location.IsAbsoluteUri ? location : new Uri(current, location);
|
|
return next.Scheme is "http" or "https";
|
|
}
|
|
catch (UriFormatException)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private async Task<string?> ApplyBodyAsync(
|
|
NodeExecutionContext context, JsonNode? body, HttpRequestMessage request, CancellationToken ct)
|
|
{
|
|
var contentType = ReadString(context, "bodyContentType") ?? "json";
|
|
|
|
switch (contentType)
|
|
{
|
|
case "json":
|
|
request.Content = new StringContent(body?.ToJsonString() ?? "null", Encoding.UTF8, "application/json");
|
|
return null;
|
|
|
|
case "form":
|
|
if (body is not JsonObject form)
|
|
return "a form body must be a JSON object";
|
|
request.Content = new FormUrlEncodedContent(
|
|
form.Select(pair => new KeyValuePair<string, string>(pair.Key, ScalarText(pair.Value))));
|
|
return null;
|
|
|
|
case "raw":
|
|
request.Content = new StringContent(ScalarText(body), Encoding.UTF8, "text/plain");
|
|
return null;
|
|
|
|
case "multipart":
|
|
return await ApplyMultipartAsync(context, body, request, ct);
|
|
|
|
case "binary":
|
|
return await ApplyBinaryAsync(context, request, ct);
|
|
|
|
default:
|
|
return $"unknown bodyContentType '{contentType}'";
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Builds a multipart body from the declared fields plus one input binary
|
|
/// attachment. The attachment is streamed from the binary store, so the file
|
|
/// never has to be loaded into item JSON first.
|
|
/// </summary>
|
|
private async Task<string?> ApplyMultipartAsync(
|
|
NodeExecutionContext context, JsonNode? body, HttpRequestMessage request, CancellationToken ct)
|
|
{
|
|
var form = new MultipartFormDataContent();
|
|
if (body is JsonObject fields)
|
|
{
|
|
foreach (var (name, value) in fields)
|
|
form.Add(new StringContent(ScalarText(value), Encoding.UTF8), name);
|
|
}
|
|
|
|
var property = ReadString(context, "binaryField");
|
|
if (string.IsNullOrWhiteSpace(property))
|
|
{
|
|
request.Content = form;
|
|
return null;
|
|
}
|
|
|
|
var attachment = ResolveInputBinary(context, property, out var error);
|
|
if (attachment == null)
|
|
{
|
|
form.Dispose();
|
|
return error;
|
|
}
|
|
|
|
var stream = await _binary.OpenAsync(attachment.AssetId, ct);
|
|
if (stream == null)
|
|
{
|
|
form.Dispose();
|
|
return $"binary asset '{attachment.AssetId}' was not found in the binary store";
|
|
}
|
|
|
|
var file = new StreamContent(stream);
|
|
if (!string.IsNullOrWhiteSpace(attachment.MimeType))
|
|
file.Headers.ContentType = new MediaTypeHeaderValue(attachment.MimeType);
|
|
form.Add(file, property, attachment.FileName ?? property);
|
|
|
|
request.Content = form;
|
|
return null;
|
|
}
|
|
|
|
/// <summary>Sends an input binary attachment as the raw request body.</summary>
|
|
private async Task<string?> ApplyBinaryAsync(
|
|
NodeExecutionContext context, HttpRequestMessage request, CancellationToken ct)
|
|
{
|
|
var property = ReadString(context, "binaryField") ?? DefaultBinaryProperty;
|
|
var attachment = ResolveInputBinary(context, property, out var error);
|
|
if (attachment == null)
|
|
return error;
|
|
|
|
var stream = await _binary.OpenAsync(attachment.AssetId, ct);
|
|
if (stream == null)
|
|
return $"binary asset '{attachment.AssetId}' was not found in the binary store";
|
|
|
|
var content = new StreamContent(stream);
|
|
content.Headers.ContentType = new MediaTypeHeaderValue(
|
|
string.IsNullOrWhiteSpace(attachment.MimeType) ? "application/octet-stream" : attachment.MimeType);
|
|
request.Content = content;
|
|
return null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Looks up a named binary attachment on the first input item. A missing item
|
|
/// or property is a body-configuration error, not an empty body.
|
|
/// </summary>
|
|
private static BinaryAttachment? ResolveInputBinary(
|
|
NodeExecutionContext context, string property, out string? error)
|
|
{
|
|
error = null;
|
|
var item = context.Input(0).FirstOrDefault();
|
|
if (item?.Binary == null || !item.Binary.TryGetValue(property, out var attachment))
|
|
{
|
|
error = $"the input item has no binary field '{property}'";
|
|
return null;
|
|
}
|
|
|
|
return attachment;
|
|
}
|
|
|
|
// ------------------------------------------------------------------ response
|
|
|
|
private sealed record BuildResult(List<FlowItem> Items, string? Error);
|
|
|
|
/// <summary>
|
|
/// Maps one page's body to items. <c>responseFormat: file</c> is the only
|
|
/// asynchronous case: it stores the payload and emits a binary-backed item.
|
|
/// </summary>
|
|
private async Task<BuildResult> BuildItemsAsync(
|
|
PageResult page,
|
|
string responseFormat,
|
|
string? putOutputInField,
|
|
string? outputPath,
|
|
CancellationToken ct)
|
|
{
|
|
if (responseFormat == "file")
|
|
{
|
|
BinaryAttachment attachment;
|
|
try
|
|
{
|
|
await using var content = new MemoryStream(page.Bytes, writable: false);
|
|
attachment = await _binary.SaveAsync(content, page.SuggestedFileName, page.ContentType, ct);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
throw;
|
|
}
|
|
catch (Exception ex) when (ex is InvalidOperationException or IOException)
|
|
{
|
|
return new BuildResult(new List<FlowItem>(), ex.Message);
|
|
}
|
|
|
|
return new BuildResult(new List<FlowItem> { FileItem(attachment, putOutputInField) }, null);
|
|
}
|
|
|
|
if (!TryBuildItems(
|
|
page.Body, page.Parsed, responseFormat, putOutputInField, outputPath,
|
|
out var items, out var error))
|
|
{
|
|
return new BuildResult(new List<FlowItem>(), error);
|
|
}
|
|
|
|
return new BuildResult(items, null);
|
|
}
|
|
|
|
/// <summary>Builds a one-item result whose payload lives in the binary store.</summary>
|
|
private static FlowItem FileItem(BinaryAttachment attachment, string? putOutputInField)
|
|
{
|
|
var meta = new JsonObject
|
|
{
|
|
["fileName"] = attachment.FileName,
|
|
["mimeType"] = attachment.MimeType,
|
|
["sizeBytes"] = attachment.SizeBytes,
|
|
};
|
|
|
|
var extension = Path.GetExtension(attachment.FileName);
|
|
if (!string.IsNullOrEmpty(extension))
|
|
meta["fileExtension"] = extension.TrimStart('.');
|
|
|
|
return new FlowItem
|
|
{
|
|
Json = string.IsNullOrWhiteSpace(putOutputInField)
|
|
? meta
|
|
: new JsonObject { [putOutputInField!] = meta },
|
|
Binary = new Dictionary<string, BinaryAttachment>(StringComparer.Ordinal)
|
|
{
|
|
[DefaultBinaryProperty] = attachment,
|
|
},
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// File name for a downloaded response: the Content-Disposition name, else the
|
|
/// last URL path segment, else null. Directory separators are stripped so a
|
|
/// provider cannot smuggle a path into the stored reference.
|
|
/// </summary>
|
|
private static string? ResolveFileName(HttpResponseMessage response, Uri requestUri)
|
|
{
|
|
try
|
|
{
|
|
var disposition = response.Content.Headers.ContentDisposition;
|
|
var name = TrimName(disposition?.FileNameStar ?? disposition?.FileName);
|
|
if (name != null)
|
|
return name;
|
|
}
|
|
catch (FormatException)
|
|
{
|
|
// Malformed Content-Disposition: fall through to the URL segment.
|
|
}
|
|
|
|
if (requestUri.Segments.Length > 0)
|
|
{
|
|
var name = TrimName(Uri.UnescapeDataString(requestUri.Segments[^1]));
|
|
if (name != null)
|
|
return name;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private static string? TrimName(string? name)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(name))
|
|
return null;
|
|
|
|
var trimmed = name.Trim().Trim('"');
|
|
var separator = trimmed.LastIndexOfAny(new[] { '/', '\\' });
|
|
if (separator >= 0)
|
|
trimmed = trimmed[(separator + 1)..];
|
|
return string.IsNullOrWhiteSpace(trimmed) ? null : trimmed;
|
|
}
|
|
|
|
private static bool TryBuildItems(
|
|
string body,
|
|
JsonNode? parsed,
|
|
string responseFormat,
|
|
string? putOutputInField,
|
|
string? outputPath,
|
|
out List<FlowItem> items,
|
|
out string? error)
|
|
{
|
|
items = new List<FlowItem>();
|
|
error = null;
|
|
|
|
switch (responseFormat)
|
|
{
|
|
case "text":
|
|
items.Add(TextItem(body, putOutputInField));
|
|
return true;
|
|
|
|
case "json":
|
|
if (parsed == null)
|
|
{
|
|
error = "the response is not valid JSON";
|
|
return false;
|
|
}
|
|
break;
|
|
|
|
case "file":
|
|
// File responses are handled by BuildItemsAsync before this point.
|
|
error = "responseFormat 'file' must be built through the binary store";
|
|
return false;
|
|
|
|
default: // autodetect
|
|
if (parsed == null)
|
|
{
|
|
items.Add(TextItem(body, putOutputInField));
|
|
return true;
|
|
}
|
|
break;
|
|
}
|
|
|
|
// Envelope unwrapping: read the page's items from a JSON path so the
|
|
// cursor/next fields that live beside them do not become items.
|
|
if (!string.IsNullOrWhiteSpace(outputPath) && parsed is JsonObject envelope)
|
|
{
|
|
parsed = NodeJsonPath.Read(envelope, outputPath, dotNotation: true);
|
|
if (parsed == null)
|
|
return true; // a page without the data field simply contributes nothing
|
|
}
|
|
|
|
items.AddRange(ToItems(parsed, putOutputInField));
|
|
return true;
|
|
}
|
|
|
|
private static List<FlowItem> ToItems(JsonNode parsed, string? putOutputInField)
|
|
{
|
|
var items = new List<FlowItem>();
|
|
if (!string.IsNullOrWhiteSpace(putOutputInField))
|
|
{
|
|
if (parsed is JsonArray wrapped)
|
|
{
|
|
foreach (var element in wrapped)
|
|
items.Add(Wrap(new JsonObject { [putOutputInField] = element?.DeepClone() }));
|
|
}
|
|
else
|
|
{
|
|
items.Add(Wrap(new JsonObject { [putOutputInField] = parsed.DeepClone() }));
|
|
}
|
|
return items;
|
|
}
|
|
|
|
switch (parsed)
|
|
{
|
|
case JsonArray array:
|
|
foreach (var element in array)
|
|
{
|
|
if (element is JsonObject obj)
|
|
items.Add(Wrap(obj));
|
|
else
|
|
items.Add(Wrap(new JsonObject { ["value"] = element?.DeepClone() }));
|
|
}
|
|
break;
|
|
|
|
case JsonObject obj:
|
|
items.Add(Wrap(obj));
|
|
break;
|
|
|
|
default:
|
|
items.Add(Wrap(new JsonObject { ["value"] = parsed.DeepClone() }));
|
|
break;
|
|
}
|
|
|
|
return items;
|
|
}
|
|
|
|
private static FlowItem TextItem(string body, string? putOutputInField)
|
|
=> string.IsNullOrWhiteSpace(putOutputInField)
|
|
? Wrap(new JsonObject { ["data"] = body })
|
|
: Wrap(new JsonObject { [putOutputInField] = body });
|
|
|
|
private static FlowItem Wrap(JsonNode json)
|
|
=> new() { Json = json is JsonObject obj ? (JsonObject)obj.DeepClone() : new JsonObject { ["value"] = json } };
|
|
|
|
private static JsonNode? TryParse(string body)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(body))
|
|
return null;
|
|
|
|
try
|
|
{
|
|
return JsonNode.Parse(body);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static string Truncate(string text, int max = 512)
|
|
=> text.Length <= max ? text : text[..max] + "…";
|
|
|
|
// ------------------------------------------------------------------ helpers
|
|
|
|
private static string ScalarText(JsonNode? node)
|
|
=> node switch
|
|
{
|
|
null => string.Empty,
|
|
JsonValue value when value.TryGetValue<string>(out var text) => text,
|
|
_ => node.ToJsonString(),
|
|
};
|
|
|
|
private static string? ReadString(NodeExecutionContext context, string name)
|
|
=> context.Parameters[name] is JsonValue value && value.TryGetValue<string>(out var text) ? text : null;
|
|
|
|
private static string? ReadString(JsonObject obj, string name)
|
|
=> obj[name] is JsonValue value && value.TryGetValue<string>(out var text) ? text : null;
|
|
|
|
private static bool ReadBool(NodeExecutionContext context, string name)
|
|
=> context.Parameters[name] is JsonValue value && value.TryGetValue<bool>(out var flag) && flag;
|
|
|
|
private static bool ReadBool(JsonObject obj, string name)
|
|
=> obj[name] is JsonValue value && value.TryGetValue<bool>(out var flag) && flag;
|
|
|
|
private static bool ReadBool(JsonObject obj, string name, bool defaultValue)
|
|
=> obj[name] is JsonValue value && value.TryGetValue<bool>(out var flag) ? flag : defaultValue;
|
|
|
|
private static int ReadInt(JsonObject obj, string name, int defaultValue)
|
|
{
|
|
if (obj[name] is not JsonValue value)
|
|
return defaultValue;
|
|
|
|
if (value.TryGetValue<int>(out var intValue))
|
|
return intValue;
|
|
if (value.TryGetValue<long>(out var longValue))
|
|
return (int)longValue;
|
|
if (value.TryGetValue<decimal>(out var decimalValue))
|
|
return (int)decimalValue;
|
|
if (value.TryGetValue<string>(out var text)
|
|
&& int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
|
|
return parsed;
|
|
|
|
return defaultValue;
|
|
}
|
|
}
|