Implement resumable downloads (#4448)

* Incremental work on persistent file cache

* Rename providers to handlers

* Implement zip file caching for resumable downloads

* Implement partial download for zipped files

* Allow executing workflows using GET verb

* Implement custom header for correlating downloads

* Rename FileName to Filename for consistency

* Update interface XML comment

* Cleanup

* Fix that source Downloadables were executed even when cached zip exists

* Make download correlation ID optional
This commit is contained in:
Sipke Schoorstra 2023-09-17 22:35:13 +02:00 committed by GitHub
parent a21751672d
commit cb87d2ed82
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 558 additions and 147 deletions

View file

@ -21,6 +21,7 @@ ProjectSection(SolutionItems) = preProject
.github\workflows\npm-packages.yml = .github\workflows\npm-packages.yml
update-migrations.sh = update-migrations.sh
.github\workflows\pr-body-generator.yml = .github\workflows\pr-body-generator.yml
packages.props = packages.props
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{0354F050-3992-4DD4-B0EE-5FBA04AC72B6}"

5
packages.props Normal file
View file

@ -0,0 +1,5 @@
<Project>
<ItemGroup>
<PackageReference Include="FluentStorage" Version="5.4.0" />
</ItemGroup>
</Project>

View file

@ -5,9 +5,9 @@ using Elsa.Http.Models;
namespace Elsa.Http.Abstractions;
/// <summary>
/// Provides a base class for <see cref="IDownloadableProvider"/> implementations.
/// Provides a base class for <see cref="IDownloadableContentHandler"/> implementations.
/// </summary>
public abstract class DownloadableProviderBase : IDownloadableProvider
public abstract class DownloadableContentHandlerBase : IDownloadableContentHandler
{
/// <inheritdoc />
public virtual float Priority => 0;
@ -18,19 +18,17 @@ public abstract class DownloadableProviderBase : IDownloadableProvider
/// <summary>
/// Returns a list of downloadables from the specified content.
/// </summary>
protected virtual async ValueTask<IEnumerable<Downloadable>> GetDownloadablesAsync(DownloadableContext context)
protected virtual IEnumerable<Func<ValueTask<Downloadable>>> GetDownloadablesAsync(DownloadableContext context)
{
var downloadable = await GetDownloadableAsync(context);
return new[]{ downloadable };
return new[] { GetDownloadableAsync(context) };
}
/// <summary>
/// Returns a downloadable from the specified content.
/// </summary>
protected virtual ValueTask<Downloadable> GetDownloadableAsync(DownloadableContext context)
protected virtual Func<ValueTask<Downloadable>> GetDownloadableAsync(DownloadableContext context)
{
var downloadable = GetDownloadable(context);
return new (downloadable);
return () => ValueTask.FromResult(GetDownloadable(context));
}
/// <summary>
@ -42,8 +40,8 @@ public abstract class DownloadableProviderBase : IDownloadableProvider
throw new NotImplementedException();
}
async ValueTask<IEnumerable<Downloadable>> IDownloadableProvider.GetDownloadablesAsync(DownloadableContext context)
IEnumerable<Func<ValueTask<Downloadable>>> IDownloadableContentHandler.GetDownloadablesAsync(DownloadableContext context)
{
return await GetDownloadablesAsync(context);
return GetDownloadablesAsync(context);
}
}

View file

@ -1,18 +1,22 @@
using System.IO.Compression;
using System.Net;
using Elsa.Extensions;
using Elsa.Http.Contracts;
using Elsa.Http.Models;
using Elsa.Http.Options;
using Elsa.Http.Services;
using Elsa.Workflows.Core;
using Elsa.Workflows.Core.Attributes;
using Elsa.Workflows.Core.Exceptions;
using Elsa.Workflows.Core.Models;
using FluentStorage.Blobs;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Abstractions;
using Microsoft.AspNetCore.Routing;
using Microsoft.AspNetCore.StaticFiles;
using Microsoft.Extensions.Logging;
using Microsoft.Net.Http.Headers;
using EntityTagHeaderValue = System.Net.Http.Headers.EntityTagHeaderValue;
using RangeHeaderValue = System.Net.Http.Headers.RangeHeaderValue;
namespace Elsa.Http;
@ -32,7 +36,13 @@ public class WriteFileHttpResponse : Activity
/// The name of the file to serve.
/// </summary>
[Input(Description = "The name of the file to serve. Leave empty to let the system determine the file name.")]
public Input<string?> FileName { get; set; } = default!;
public Input<string?> Filename { get; set; } = default!;
/// <summary>
/// The Entity Tag of the file to serve.
/// </summary>
[Input(Description = "The Entity Tag of the file to serve. Leave empty to let the system determine the Entity Tag.")]
public Input<string?> EntityTag { get; set; } = default!;
/// <summary>
/// The file content to serve. Supports byte array, streams, string, Uri and an array of the aforementioned types.
@ -40,6 +50,18 @@ public class WriteFileHttpResponse : Activity
[Input(Description = "The file content to serve. Supports various types, such as byte array, stream, string, Uri, Downloadable and a (mixed) array of the aforementioned types.")]
public Input<object> Content { get; set; } = default!;
/// <summary>
/// Whether to enable resumable downloads. When enabled, the client can resume a download if the connection is lost.
/// </summary>
[Input(Description = "Whether to enable resumable downloads. When enabled, the client can resume a download if the connection is lost.")]
public Input<bool> EnableResumableDownloads { get; set; } = default!;
/// <summary>
/// The correlation ID of the download. Used to resume a download.
/// </summary>
[Input(Description = "The correlation ID of the download used to resume a download. If left empty, the x-elsa-download-id header will be used.")]
public Input<string> DownloadCorrelationId { get; set; } = default!;
/// <inheritdoc />
protected override async ValueTask ExecuteAsync(ActivityExecutionContext context)
{
@ -60,32 +82,25 @@ public class WriteFileHttpResponse : Activity
private async Task WriteResponseAsync(ActivityExecutionContext context, HttpContext httpContext)
{
// Set status code.
var statusCode = HttpStatusCode.OK;
var response = httpContext.Response;
response.StatusCode = (int)statusCode;
// Get content and content type.
var content = context.Get(Content);
if (content == null)
return;
// Write content.
var downloadables = await GetDownloadablesAsync(context, content);
var downloadables = GetDownloadables(context, httpContext, content).ToList();
await SendDownloadablesAsync(context, httpContext, downloadables);
// Complete activity.
await context.CompleteActivityAsync();
}
private async Task SendDownloadablesAsync(ActivityExecutionContext context, HttpContext httpContext, IEnumerable<Downloadable> downloadables)
private async Task SendDownloadablesAsync(ActivityExecutionContext context, HttpContext httpContext, IEnumerable<Func<ValueTask<Downloadable>>> downloadables)
{
var downloadableList = downloadables.ToList();
switch (downloadableList.Count)
{
case 0:
SendNoContent(context, httpContext);
return;
case 1:
{
@ -99,81 +114,140 @@ public class WriteFileHttpResponse : Activity
}
}
private async Task SendSingleFileAsync(ActivityExecutionContext context, HttpContext httpContext, Downloadable downloadable)
private void SendNoContent(ActivityExecutionContext context, HttpContext httpContext)
{
var contentType = ContentType.GetOrDefault(context);
var filename = FileName.GetOrDefault(context);
filename = !string.IsNullOrWhiteSpace(filename) ? filename : !string.IsNullOrWhiteSpace(downloadable.Filename) ? downloadable.Filename : "file.bin";
contentType = !string.IsNullOrWhiteSpace(contentType) ? contentType : !string.IsNullOrWhiteSpace(downloadable.ContentType) ? downloadable.ContentType : GetContentType(context, filename);
await SendFileStream(httpContext, downloadable.Stream, contentType, filename);
httpContext.Response.StatusCode = StatusCodes.Status204NoContent;
}
private async Task SendMultipleFilesAsync(ActivityExecutionContext context, HttpContext httpContext, ICollection<Downloadable> downloadables)
private async Task SendSingleFileAsync(ActivityExecutionContext context, HttpContext httpContext, Func<ValueTask<Downloadable>> downloadableFunc)
{
var logger = context.GetRequiredService<ILogger<WriteFileHttpResponse>>();
// 1. Create a temporary file
var tempFilePath = Path.GetTempFileName();
var currentFileIndex = 0;
// 2. Write the zip archive to the temporary file
await using var tempFileStream = new FileStream(tempFilePath, FileMode.Create);
using var zipArchive = new ZipArchive(tempFileStream, ZipArchiveMode.Create, true);
foreach (var downloadable in downloadables)
{
var entryName = !string.IsNullOrWhiteSpace(downloadable.Filename) ? downloadable.Filename : $"file-{currentFileIndex}.bin";
var entry = zipArchive.CreateEntry(entryName);
var fileStream = downloadable.Stream;
await using var entryStream = entry.Open();
await fileStream.CopyToAsync(entryStream);
await entryStream.FlushAsync();
entryStream.Close();
currentFileIndex++;
}
var contentType = ContentType.GetOrDefault(context);
var filename = FileName.GetOrDefault(context);
var filename = Filename.GetOrDefault(context);
var eTag = EntityTag.GetOrDefault(context);
var downloadable = await downloadableFunc();
filename = !string.IsNullOrWhiteSpace(filename) ? filename : !string.IsNullOrWhiteSpace(downloadable.Filename) ? downloadable.Filename : "file.bin";
contentType = !string.IsNullOrWhiteSpace(contentType) ? contentType : !string.IsNullOrWhiteSpace(downloadable.ContentType) ? downloadable.ContentType : GetContentType(context, filename);
eTag = !string.IsNullOrWhiteSpace(eTag) ? eTag : !string.IsNullOrWhiteSpace(downloadable.ETag) ? downloadable.ETag : default;
var eTagHeaderValue = !string.IsNullOrWhiteSpace(eTag) ? new EntityTagHeaderValue(eTag) : default;
var stream = downloadable.Stream;
await SendFileStream(context, httpContext, stream, contentType, filename, eTagHeaderValue);
}
private async Task SendMultipleFilesAsync(ActivityExecutionContext context, HttpContext httpContext, ICollection<Func<ValueTask<Downloadable>>> downloadables)
{
// If resumable downloads are enabled, check to see if we have a cached file.
var (zipBlob, zipStream, cleanupCallback) = await TryLoadCachedFileAsync(context, httpContext) ?? await GenerateZipFileAsync(context, httpContext, downloadables);
contentType = !string.IsNullOrWhiteSpace(contentType) ? contentType : System.Net.Mime.MediaTypeNames.Application.Zip;
filename = !string.IsNullOrWhiteSpace(filename) ? filename : "download.zip";
// 3. Use FileStreamResult to stream the temporary file back to the client
var resultStream = new FileStream(tempFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 4096, useAsync: true);
await SendFileStream(httpContext, resultStream, contentType, filename);
// 4. Delete the temporary file.
try
{
File.Delete(tempFilePath);
// Send the zip stream the temporary file back to the client.
var contentType = zipBlob.Metadata["ContentType"];
var downloadAsFilename = zipBlob.Metadata["Filename"];
var eTag = $"\"{zipBlob.LastModificationTime?.ToString("O")}\"";
var eTagHeaderValue = new EntityTagHeaderValue(eTag);
await SendFileStream(context, httpContext, zipStream, contentType, downloadAsFilename, eTagHeaderValue);
// TODO: Delete the cached file after the workflow completes.
}
catch (Exception e)
{
logger.LogWarning(e, "Failed to delete temporary file {TempFilePath}", tempFilePath);
var logger = context.GetRequiredService<ILogger<WriteFileHttpResponse>>();
logger.LogWarning(e, "Failed to send zip file to HTTP response");
}
finally
{
// Delete any temporary files.
await cleanupCallback();
}
}
private async Task SendFileStream(HttpContext httpContext, Stream source, string contentType, string filename)
private async Task<(Blob, Stream, Func<ValueTask>)> GenerateZipFileAsync(ActivityExecutionContext context, HttpContext httpContext, ICollection<Func<ValueTask<Downloadable>>> downloadables)
{
var cancellationToken = context.CancellationToken;
var enableResumableDownloads = EnableResumableDownloads.GetOrDefault(context, () => false);
var downloadCorrelationId = GetDownloadCorrelationId(context, httpContext);
var contentType = ContentType.GetOrDefault(context);
var downloadAsFilename = Filename.GetOrDefault(context);
var zipService = context.GetRequiredService<ZipManager>();
var (zipBlob, zipStream, cleanup) = await zipService.CreateAsync(downloadables, enableResumableDownloads, downloadCorrelationId, downloadAsFilename, contentType, cancellationToken);
return (zipBlob, zipStream, Cleanup);
ValueTask Cleanup()
{
cleanup();
return default;
}
}
private async Task<(Blob, Stream, Func<ValueTask>)?> TryLoadCachedFileAsync(ActivityExecutionContext context, HttpContext httpContext)
{
var enableResumableDownloads = EnableResumableDownloads.GetOrDefault(context, () => false);
var downloadCorrelationId = GetDownloadCorrelationId(context, httpContext);
if (!enableResumableDownloads || string.IsNullOrWhiteSpace(downloadCorrelationId))
return null;
var cancellationToken = context.CancellationToken;
var zipService = context.GetRequiredService<ZipManager>();
var tuple = await zipService.LoadAsync(downloadCorrelationId, cancellationToken);
if (tuple == null)
return null;
return (tuple.Value.Item1, tuple.Value.Item2, Noop);
ValueTask Noop() => default;
}
private string GetDownloadCorrelationId(ActivityExecutionContext context, HttpContext httpContext)
{
var downloadCorrelationId = DownloadCorrelationId.GetOrDefault(context);
if (string.IsNullOrWhiteSpace(downloadCorrelationId))
downloadCorrelationId = httpContext.Request.Headers["x-elsa-download-id"];
if (string.IsNullOrWhiteSpace(downloadCorrelationId))
{
var identity = context.WorkflowExecutionContext.Workflow.Identity;
var definitionId = identity.DefinitionId;
var version = identity.Version.ToString();
var correlationId = context.WorkflowExecutionContext.CorrelationId;
var sources = new[] { definitionId, version, correlationId }.Where(x => !string.IsNullOrWhiteSpace(x)).ToArray();
downloadCorrelationId = string.Join("-", sources);
}
return downloadCorrelationId;
}
private async Task SendFileStream(ActivityExecutionContext context, HttpContext httpContext, Stream source, string contentType, string filename, EntityTagHeaderValue? eTag)
{
source.Seek(0, SeekOrigin.Begin);
var result = new FileStreamResult(source, contentType)
{
EnableRangeProcessing = true,
EntityTag = eTag != null ? new Microsoft.Net.Http.Headers.EntityTagHeaderValue(eTag.ToString()) : default,
FileDownloadName = filename
};
var actionContext = new ActionContext(httpContext, httpContext.GetRouteData(), new ActionDescriptor());
await result.ExecuteResultAsync(actionContext);
}
/// <summary>
/// Leverages the <see cref="IDownloadableManager"/> to get a list of <see cref="Downloadable"/> instances from the <paramref name="content"/>.
/// </summary>
private async Task<IEnumerable<Downloadable>> GetDownloadablesAsync(ActivityExecutionContext context, object content)
private IEnumerable<Func<ValueTask<Downloadable>>> GetDownloadables(ActivityExecutionContext context, HttpContext httpContext, object? content)
{
if (content == null)
return Enumerable.Empty<Func<ValueTask<Downloadable>>>();
var manager = context.GetRequiredService<IDownloadableManager>();
return await manager.GetDownloadablesAsync(content, context.CancellationToken);
var headers = httpContext.Request.Headers;
var eTag = headers.TryGetValue(HeaderNames.IfMatch, out var header) ? new EntityTagHeaderValue(header.ToString()) : default;
var range = headers.TryGetValue(HeaderNames.Range, out header) ? RangeHeaderValue.Parse(header.ToString()) : default;
var options = new DownloadableOptions { ETag = eTag, Range = range };
return manager.GetDownloadablesAsync(content, options, context.CancellationToken);
}
private string GetContentType(ActivityExecutionContext context, string filename)
@ -182,26 +256,13 @@ public class WriteFileHttpResponse : Activity
return provider.TryGetContentType(filename, out var contentType) ? contentType : System.Net.Mime.MediaTypeNames.Application.Octet;
}
private static string CreateContentDisposition(string filename)
{
var contentDisposition = new System.Net.Mime.ContentDisposition
{
FileName = filename
};
return contentDisposition.ToString();
}
private async ValueTask OnResumeAsync(ActivityExecutionContext context)
{
var httpContextAccessor = context.GetRequiredService<IHttpContextAccessor>();
var httpContext = httpContextAccessor.HttpContext;
if (httpContext == null)
{
// We're not in an HTTP context, so let's fail.
throw new FaultException("Cannot execute in a non-HTTP context");
}
await WriteResponseAsync(context, httpContext);
}

View file

@ -1,4 +1,5 @@
using Elsa.Http.Contracts;
using Elsa.Http.Options;
namespace Elsa.Http.Contexts;
@ -7,5 +8,10 @@ namespace Elsa.Http.Contexts;
/// </summary>
/// <param name="Manager">The manager.</param>
/// <param name="Content">The content to get downloadables from.</param>
/// <param name="ETag">An optional ETag.</param>
/// <param name="CancellationToken">The cancellation token.</param>
public record DownloadableContext(IDownloadableManager Manager, object Content, CancellationToken CancellationToken);
public record DownloadableContext(
IDownloadableManager Manager,
object Content,
DownloadableOptions Options,
CancellationToken CancellationToken);

View file

@ -1,6 +1,14 @@
namespace Elsa.Http.Contracts;
/// <summary>
/// Provides a way to convert a relative URL to an absolute URL.
/// </summary>
public interface IAbsoluteUrlProvider
{
/// <summary>
/// Converts a relative URL to an absolute URL.
/// </summary>
/// <param name="relativePath">The relative URL.</param>
/// <returns>The absolute URL.</returns>
Uri ToAbsoluteUrl(string relativePath);
}

View file

@ -6,7 +6,7 @@ namespace Elsa.Http.Contracts;
/// <summary>
/// Provides downloadables from the specified content, if supported.
/// </summary>
public interface IDownloadableProvider
public interface IDownloadableContentHandler
{
/// <summary>
/// The priority of this provider. Providers with lower priority are tried first.
@ -22,5 +22,5 @@ public interface IDownloadableProvider
/// <summary>
/// Returns a list of downloadables from the specified content.
/// </summary>
ValueTask<IEnumerable<Downloadable>> GetDownloadablesAsync(DownloadableContext context);
IEnumerable<Func<ValueTask<Downloadable>>> GetDownloadablesAsync(DownloadableContext context);
}

View file

@ -1,4 +1,5 @@
using Elsa.Http.Models;
using Elsa.Http.Options;
namespace Elsa.Http.Contracts;
@ -10,5 +11,5 @@ public interface IDownloadableManager
/// <summary>
/// Returns a list of downloadables from the specified content.
/// </summary>
ValueTask<IEnumerable<Downloadable>> GetDownloadablesAsync(object content, CancellationToken cancellationToken = default);
IEnumerable<Func<ValueTask<Downloadable>>> GetDownloadablesAsync(object content, DownloadableOptions? options = default, CancellationToken cancellationToken = default);
}

View file

@ -0,0 +1,15 @@
using FluentStorage.Blobs;
namespace Elsa.Http.Contracts;
/// <summary>
/// Represents a provider of a file cache storage.
/// </summary>
public interface IFileCacheStorageProvider
{
/// <summary>
/// Gets the storage.
/// </summary>
/// <returns></returns>
IBlobStorage GetStorage();
}

View file

@ -1,3 +1,5 @@
using Elsa.Http.Options;
namespace Elsa.Http.Contracts;
/// <summary>
@ -8,5 +10,5 @@ public interface IFileDownloader
/// <summary>
/// Downloads a file from the specified URL.
/// </summary>
Task<HttpResponseMessage> DownloadAsync(Uri url, CancellationToken cancellationToken = default);
Task<HttpResponseMessage> DownloadAsync(Uri url, FileDownloadOptions? options = default, CancellationToken cancellationToken = default);
}

View file

@ -2,12 +2,12 @@ using Elsa.Http.Abstractions;
using Elsa.Http.Contexts;
using Elsa.Http.Models;
namespace Elsa.Http.DownloadableProviders;
namespace Elsa.Http.DownloadableContentHandlers;
/// <summary>
/// Handles content that represents a downloadable binary file.
/// </summary>
public class BinaryDownloadableProvider : DownloadableProviderBase
public class BinaryDownloadableContentHandler : DownloadableContentHandlerBase
{
/// <inheritdoc />
public override bool GetSupportsContent(object content) => content is byte[];
@ -19,6 +19,6 @@ public class BinaryDownloadableProvider : DownloadableProviderBase
var stream = new MemoryStream(bytes);
var fileName = "file.bin";
var contentType = "application/octet-stream";
return new Downloadable(stream, fileName, contentType);
return new(stream, fileName, contentType);
}
}

View file

@ -2,12 +2,12 @@ using Elsa.Http.Abstractions;
using Elsa.Http.Contexts;
using Elsa.Http.Models;
namespace Elsa.Http.DownloadableProviders;
namespace Elsa.Http.DownloadableContentHandlers;
/// <summary>
/// Handles content that represents a downloadable.
/// </summary>
public class DownloadableDownloadableProvider : DownloadableProviderBase
public class DownloadableDownloadableContentHandler : DownloadableContentHandlerBase
{
/// <inheritdoc />
public override bool GetSupportsContent(object content) => content is Downloadable;

View file

@ -3,27 +3,27 @@ using Elsa.Http.Abstractions;
using Elsa.Http.Contexts;
using Elsa.Http.Models;
namespace Elsa.Http.DownloadableProviders;
namespace Elsa.Http.DownloadableContentHandlers;
/// <summary>
/// Handles content that represents a list of downloadable objects.
/// </summary>
public class MultiDownloadableProvider : DownloadableProviderBase
public class MultiDownloadableContentHandler : DownloadableContentHandlerBase
{
/// <inheritdoc />
public override bool GetSupportsContent(object content) => content is IEnumerable enumerable and not string;
/// <inheritdoc />
protected override async ValueTask<IEnumerable<Downloadable>> GetDownloadablesAsync(DownloadableContext context)
protected override IEnumerable<Func<ValueTask<Downloadable>>> GetDownloadablesAsync(DownloadableContext context)
{
var collectedDownloadables = new List<Downloadable>();
var collectedDownloadables = new List<Func<ValueTask<Downloadable>>>();
var content = context.Content;
var enumerable = (IEnumerable) content;
var manager = context.Manager;
foreach (var item in enumerable)
{
var downloadables = await manager.GetDownloadablesAsync(item, context.CancellationToken);
var downloadables = manager.GetDownloadablesAsync(item, context.Options, context.CancellationToken);
collectedDownloadables.AddRange(downloadables);
}

View file

@ -2,12 +2,12 @@ using Elsa.Http.Abstractions;
using Elsa.Http.Contexts;
using Elsa.Http.Models;
namespace Elsa.Http.DownloadableProviders;
namespace Elsa.Http.DownloadableContentHandlers;
/// <summary>
/// Handles content that represents a downloadable stream.
/// </summary>
public class StreamDownloadableProvider : DownloadableProviderBase
public class StreamDownloadableContentHandler : DownloadableContentHandlerBase
{
/// <inheritdoc />
public override bool GetSupportsContent(object content) => content is Stream;

View file

@ -1,22 +1,22 @@
using System.Text.RegularExpressions;
using Elsa.Http.Abstractions;
using Elsa.Http.Contexts;
using Elsa.Http.Contracts;
using Elsa.Http.Models;
using Elsa.Http.Options;
using Microsoft.AspNetCore.StaticFiles;
namespace Elsa.Http.DownloadableProviders;
namespace Elsa.Http.DownloadableContentHandlers;
/// <summary>
/// Handles content that represents a downloadable URL.
/// </summary>
public class UrlDownloadableProvider : DownloadableProviderBase
public class UrlDownloadableContentHandler : DownloadableContentHandlerBase
{
private readonly IFileDownloader _fileDownloader;
private readonly IContentTypeProvider _contentTypeProvider;
/// <inheritdoc />
public UrlDownloadableProvider(IFileDownloader fileDownloader, IContentTypeProvider contentTypeProvider)
public UrlDownloadableContentHandler(IFileDownloader fileDownloader, IContentTypeProvider contentTypeProvider)
{
_fileDownloader = fileDownloader;
_contentTypeProvider = contentTypeProvider;
@ -26,18 +26,27 @@ public class UrlDownloadableProvider : DownloadableProviderBase
public override bool GetSupportsContent(object content) => (content is string url && url.StartsWith("http", StringComparison.OrdinalIgnoreCase)) || content is Uri;
/// <inheritdoc />
protected override async ValueTask<Downloadable> GetDownloadableAsync(DownloadableContext context)
protected override Func<ValueTask<Downloadable>> GetDownloadableAsync(DownloadableContext context) => async () => await DownloadAsync(context);
private async ValueTask<Downloadable> DownloadAsync(DownloadableContext context)
{
var url = context.Content is string s ? new Uri(s) : (Uri)context.Content;
var cancellationToken = context.CancellationToken;
var response = await _fileDownloader.DownloadAsync(url, cancellationToken);
var options = new FileDownloadOptions
{
// TODO: Uncomment the next two lines if we implement file caching for this handler.
// ETag = context.Options.ETag,
// Range = context.Options.Range
};
var response = await _fileDownloader.DownloadAsync(url, options, cancellationToken);
var eTag = response.Headers.ETag?.Tag;
var filename = GetFilename(response) ?? url.Segments.Last();
var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
var contentType = response.Content.Headers.ContentType?.MediaType ?? GetContentType(filename);
var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
return new Downloadable(stream, filename, contentType);
return new Downloadable(stream, filename, contentType, eTag);
}
private static string? GetFilename(HttpResponseMessage response)
{
if (!response.Content.Headers.TryGetValues("Content-Disposition", out var values))

View file

@ -1,6 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="..\..\..\common.props"/>
<Import Project="..\..\..\packages.props" />
<Import Project="..\..\..\configureawait.props"/>
<PropertyGroup>

View file

@ -5,7 +5,8 @@ using Elsa.Features.Attributes;
using Elsa.Features.Services;
using Elsa.Http.ContentWriters;
using Elsa.Http.Contracts;
using Elsa.Http.DownloadableProviders;
using Elsa.Http.DownloadableContentHandlers;
using Elsa.Http.FileCaches;
using Elsa.Http.Handlers;
using Elsa.Http.HostedServices;
using Elsa.Http.Models;
@ -20,6 +21,7 @@ using Elsa.Liquid.Features;
using Elsa.Workflows.Core.Contracts;
using Elsa.Workflows.Management.Requests;
using Elsa.Workflows.Management.Responses;
using FluentStorage;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using Microsoft.AspNetCore.StaticFiles;
@ -45,6 +47,11 @@ public class HttpFeature : FeatureBase
/// </summary>
public Action<HttpActivityOptions>? ConfigureHttpOptions { get; set; }
/// <summary>
/// A delegate to configure <see cref="HttpFileCacheOptions"/>.
/// </summary>
public Action<HttpFileCacheOptions>? ConfigureHttpFileCacheOptions { get; set; }
/// <summary>
/// A delegate that is invoked when authorizing an inbound HTTP request.
/// </summary>
@ -54,12 +61,21 @@ public class HttpFeature : FeatureBase
/// A delegate that is invoked when an HTTP workflow faults.
/// </summary>
public Func<IServiceProvider, IHttpEndpointWorkflowFaultHandler> HttpEndpointWorkflowFaultHandler { get; set; } = sp => sp.GetRequiredService<DefaultHttpEndpointWorkflowFaultHandler>();
/// <summary>
/// A delegate to configure the <see cref="IContentTypeProvider"/>.
/// </summary>
public Func<IServiceProvider, IContentTypeProvider> ContentTypeProvider { get; set; } = _ => new FileExtensionContentTypeProvider();
/// <summary>
/// A delegate to configure the <see cref="IFileCacheStorageProvider"/>.
/// </summary>
public Func<IServiceProvider, IFileCacheStorageProvider> FileCache { get; set; } = _ =>
{
var blobStorage = StorageFactory.Blobs.DirectoryFiles(Path.GetTempPath());
return new BlobFileCacheStorageProvider(blobStorage);
};
/// <summary>
/// A delegate to configure the <see cref="HttpClient"/> used when by the <see cref="FlowSendHttpRequest"/> activity.
/// </summary>
@ -75,7 +91,7 @@ public class HttpFeature : FeatureBase
/// </summary>
public ICollection<Type> HttpCorrelationIdSelectorTypes { get; } = new List<Type>
{
typeof(HeaderHttpCorrelationIdSelector),
typeof(HeaderHttpCorrelationIdSelector),
typeof(QueryStringHttpCorrelationIdSelector)
};
@ -84,7 +100,7 @@ public class HttpFeature : FeatureBase
/// </summary>
public ICollection<Type> HttpWorkflowInstanceIdSelectorTypes { get; } = new List<Type>
{
typeof(HeaderHttpWorkflowInstanceIdSelector),
typeof(HeaderHttpWorkflowInstanceIdSelector),
typeof(QueryStringHttpWorkflowInstanceIdSelector)
};
@ -123,7 +139,10 @@ public class HttpFeature : FeatureBase
options.BaseUrl = new Uri("http://localhost");
});
var configureFileCacheOptions = ConfigureHttpFileCacheOptions ?? (options => { options.TimeToLive = TimeSpan.FromDays(7); });
Services.Configure(configureOptions);
Services.Configure(configureFileCacheOptions);
var httpClientBuilder = Services.AddHttpClient<SendHttpRequestBase>(HttpClient);
HttpClientBuilder(httpClientBuilder);
@ -161,14 +180,18 @@ public class HttpFeature : FeatureBase
.AddSingleton<DefaultHttpEndpointWorkflowFaultHandler>()
.AddSingleton(HttpEndpointWorkflowFaultHandler)
.AddSingleton(HttpEndpointAuthorizationHandler)
// File related services.
// Downloadable content handlers.
.AddSingleton<IDownloadableManager, DefaultDownloadableManager>()
.AddSingleton<IDownloadableProvider, BinaryDownloadableProvider>()
.AddSingleton<IDownloadableProvider, DownloadableDownloadableProvider>()
.AddSingleton<IDownloadableProvider, MultiDownloadableProvider>()
.AddSingleton<IDownloadableProvider, StreamDownloadableProvider>()
.AddSingleton<IDownloadableProvider, UrlDownloadableProvider>()
.AddSingleton<IDownloadableContentHandler, BinaryDownloadableContentHandler>()
.AddSingleton<IDownloadableContentHandler, DownloadableDownloadableContentHandler>()
.AddSingleton<IDownloadableContentHandler, MultiDownloadableContentHandler>()
.AddSingleton<IDownloadableContentHandler, StreamDownloadableContentHandler>()
.AddSingleton<IDownloadableContentHandler, UrlDownloadableContentHandler>()
// File caches.
.AddSingleton(FileCache)
.AddSingleton<ZipManager>()
// Add mediator handlers.
.AddNotificationHandlersFrom<HttpFeature>()
@ -179,11 +202,11 @@ public class HttpFeature : FeatureBase
// HTTP clients.
Services.AddHttpClient<IFileDownloader, HttpClientFileDownloader>();
// Add selectors.
foreach (var httpCorrelationIdSelectorType in HttpCorrelationIdSelectorTypes)
foreach (var httpCorrelationIdSelectorType in HttpCorrelationIdSelectorTypes)
Services.AddSingleton(typeof(IHttpCorrelationIdSelector), httpCorrelationIdSelectorType);
foreach (var httpWorkflowInstanceIdSelectorType in HttpWorkflowInstanceIdSelectorTypes)
Services.AddSingleton(typeof(IHttpWorkflowInstanceIdSelector), httpWorkflowInstanceIdSelectorType);
}

View file

@ -0,0 +1,26 @@
using Elsa.Http.Contracts;
using FluentStorage.Blobs;
namespace Elsa.Http.FileCaches;
/// <summary>
/// A file cache that stores files in blob storage using FluentStorage.
/// </summary>
public class BlobFileCacheStorageProvider : IFileCacheStorageProvider
{
private readonly IBlobStorage _blobStorage;
/// <summary>
/// Initializes a new instance of the <see cref="BlobFileCacheStorageProvider"/> class.
/// </summary>
public BlobFileCacheStorageProvider(IBlobStorage blobStorage)
{
_blobStorage = blobStorage;
}
/// <inheritdoc />
public IBlobStorage GetStorage()
{
return _blobStorage;
}
}

View file

@ -21,11 +21,13 @@ public class Downloadable
/// <param name="stream">The stream to download.</param>
/// <param name="filename">The filename to use when downloading the stream.</param>
/// <param name="contentType">The content type to use when downloading the stream.</param>
public Downloadable(Stream stream, string? filename = default, string? contentType = default)
/// <param name="eTag">The ETag to use when downloading the stream.</param>
public Downloadable(Stream stream, string? filename = default, string? contentType = default, string? eTag = default)
{
Stream = stream;
Filename = filename;
ContentType = contentType;
ETag = eTag;
}
/// <summary>
@ -42,4 +44,9 @@ public class Downloadable
/// The content type to use when downloading the stream.
/// </summary>
public string? ContentType { get; set; }
/// <summary>
/// The ETag to use when downloading the stream.
/// </summary>
public string? ETag { get; set; }
}

View file

@ -0,0 +1,19 @@
using System.Net.Http.Headers;
namespace Elsa.Http.Options;
/// <summary>
/// Options for downloading a file.
/// </summary>
public class DownloadableOptions
{
/// <summary>
/// Gets or sets the entity tag.
/// </summary>
public EntityTagHeaderValue? ETag { get; set; }
/// <summary>
/// Gets or sets the range.
/// </summary>
public RangeHeaderValue? Range { get; set; }
}

View file

@ -0,0 +1,19 @@
using System.Net.Http.Headers;
namespace Elsa.Http.Options;
/// <summary>
/// Options for downloading a file.
/// </summary>
public class FileDownloadOptions
{
/// <summary>
/// Gets or sets the entity tag.
/// </summary>
public EntityTagHeaderValue? ETag { get; set; }
/// <summary>
/// Gets or sets the range.
/// </summary>
public RangeHeaderValue? Range { get; set; }
}

View file

@ -0,0 +1,12 @@
namespace Elsa.Http.Options;
/// <summary>
/// Provides options for the HTTP file cache.
/// </summary>
public class HttpFileCacheOptions
{
/// <summary>
/// The time to live for cached files.
/// </summary>
public TimeSpan TimeToLive { get; set; } = TimeSpan.FromDays(7);
}

View file

@ -1,33 +1,35 @@
using Elsa.Http.Contexts;
using Elsa.Http.Contracts;
using Elsa.Http.Models;
using Elsa.Http.Options;
namespace Elsa.Http.Services;
/// <inheritdoc />
public class DefaultDownloadableManager : IDownloadableManager
{
private readonly IEnumerable<IDownloadableProvider> _providers;
private readonly IEnumerable<IDownloadableContentHandler> _providers;
/// <summary>
/// Initializes a new instance of the <see cref="DefaultDownloadableManager"/> class.
/// </summary>
public DefaultDownloadableManager(IEnumerable<IDownloadableProvider> providers)
public DefaultDownloadableManager(IEnumerable<IDownloadableContentHandler> providers)
{
_providers = providers.OrderBy(x => x.Priority).ToList();
}
/// <inheritdoc />
public async ValueTask<IEnumerable<Downloadable>> GetDownloadablesAsync(object content, CancellationToken cancellationToken = default)
public IEnumerable<Func<ValueTask<Downloadable>>> GetDownloadablesAsync(object content, DownloadableOptions? options = default, CancellationToken cancellationToken = default)
{
var provider = _providers.FirstOrDefault(x => x.GetSupportsContent(content));
if (provider == null)
return Enumerable.Empty<Downloadable>();
var context = new DownloadableContext(this, content, cancellationToken);
var downloadables = await provider.GetDownloadablesAsync(context);
return Enumerable.Empty<Func<ValueTask<Downloadable>>>();
options ??= new();
var context = new DownloadableContext(this, content, options, cancellationToken);
var downloadables = provider.GetDownloadablesAsync(context);
return downloadables;
}
}

View file

@ -1,4 +1,5 @@
using Elsa.Http.Contracts;
using Elsa.Http.Options;
namespace Elsa.Http.Services;
@ -18,8 +19,16 @@ public class HttpClientFileDownloader : IFileDownloader
}
/// <inheritdoc />
public async Task<HttpResponseMessage> DownloadAsync(Uri url, CancellationToken cancellationToken = default)
public async Task<HttpResponseMessage> DownloadAsync(Uri url, FileDownloadOptions? options = default, CancellationToken cancellationToken = default)
{
return await _httpClient.GetAsync(url, cancellationToken);
var request = new HttpRequestMessage(HttpMethod.Get, url);
if (options?.ETag != null)
request.Headers.IfNoneMatch.Add(options.ETag);
if(options?.Range != null)
request.Headers.Range = options.Range;
return await _httpClient.SendAsync(request, cancellationToken);
}
}

View file

@ -0,0 +1,188 @@
using System.IO.Compression;
using Elsa.Common.Contracts;
using Elsa.Http.Contracts;
using Elsa.Http.Models;
using Elsa.Http.Options;
using FluentStorage.Blobs;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Elsa.Http.Services;
/// <summary>
/// Provides a helper service for zipping downloadable content.
/// </summary>
internal class ZipManager
{
private readonly ISystemClock _clock;
private readonly IFileCacheStorageProvider _fileCacheStorageProvider;
private readonly IOptions<HttpFileCacheOptions> _fileCacheOptions;
private readonly ILogger<ZipManager> _logger;
/// <summary>
/// Initializes a new instance of the <see cref="ZipManager"/> class.
/// </summary>
public ZipManager(ISystemClock clock, IFileCacheStorageProvider fileCacheStorageProvider, IOptions<HttpFileCacheOptions> fileCacheOptions, ILogger<ZipManager> logger)
{
_clock = clock;
_fileCacheStorageProvider = fileCacheStorageProvider;
_fileCacheOptions = fileCacheOptions;
_logger = logger;
}
public async Task<(Blob, Stream, Action)> CreateAsync(
ICollection<Func<ValueTask<Downloadable>>> downloadables,
bool enableResumableDownloads,
string? downloadCorrelationId,
string? downloadAsFilename = default,
string? contentType = default,
CancellationToken cancellationToken = default)
{
// Create a temporary file.
var tempFilePath = Path.GetTempFileName();
// Create a zip archive from the downloadables.
await CreateZipArchiveAsync(tempFilePath, downloadables, cancellationToken);
// Create a blob with metadata for resuming the download.
var zipBlob = CreateBlob(tempFilePath, downloadAsFilename, contentType);
// If resumable downloads are enabled, cache the file.
if (enableResumableDownloads && !string.IsNullOrWhiteSpace(downloadCorrelationId))
await CreateCachedZipBlobAsync(tempFilePath, downloadCorrelationId, downloadAsFilename, contentType, cancellationToken);
var zipStream = File.OpenRead(tempFilePath);
return (zipBlob, zipStream, () => Cleanup(tempFilePath));
}
/// <summary>
/// Loads a cached zip blob for the specified download correlation ID.
/// </summary>
/// <param name="downloadCorrelationId">The download correlation ID.</param>
/// <param name="cancellationToken">An optional cancellation token.</param>
/// <returns>A tuple containing the blob and the stream.</returns>
public async Task<(Blob, Stream)?> LoadAsync(string downloadCorrelationId, CancellationToken cancellationToken = default)
{
var fileCacheStorage = _fileCacheStorageProvider.GetStorage();
var fileCacheFilename = $"{downloadCorrelationId}.tmp";
var blob = await fileCacheStorage.GetBlobAsync(fileCacheFilename, cancellationToken);
if (blob == null)
return null;
// Check if the blob has expired.
var expiresAt = DateTimeOffset.Parse(blob.Metadata["ExpiresAt"]);
if (_clock.UtcNow > expiresAt)
{
// File expired. Try to delete it.
try
{
await fileCacheStorage.DeleteAsync(blob.FullPath, cancellationToken);
}
catch (Exception e)
{
_logger.LogWarning(e, "Failed to delete expired file {FullPath}", blob.FullPath);
}
return null;
}
var stream = await fileCacheStorage.OpenReadAsync(blob.FullPath, cancellationToken);
return (blob, stream);
}
/// <summary>
/// Creates a zip archive from the specified <see cref="Downloadable"/> instances.
/// </summary>
private async Task CreateZipArchiveAsync(string filePath, IEnumerable<Func<ValueTask<Downloadable>>> downloadables, CancellationToken cancellationToken = default)
{
var currentFileIndex = 0;
// Write the zip archive to the temporary file.
await using var tempFileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read, bufferSize: 4096, useAsync: true);
using var zipArchive = new ZipArchive(tempFileStream, ZipArchiveMode.Create, true);
foreach (var downloadableFunc in downloadables)
{
var downloadable = await downloadableFunc();
var entryName = !string.IsNullOrWhiteSpace(downloadable.Filename) ? downloadable.Filename : $"file-{currentFileIndex}.bin";
var entry = zipArchive.CreateEntry(entryName);
var fileStream = downloadable.Stream;
await using var entryStream = entry.Open();
await fileStream.CopyToAsync(entryStream, cancellationToken);
await entryStream.FlushAsync(cancellationToken);
entryStream.Close();
currentFileIndex++;
}
}
/// <summary>
/// Creates a cached zip blob for the specified file.
/// </summary>
/// <param name="localPath">The full path of the file to upload.</param>
/// <param name="downloadCorrelationId">The download correlation ID.</param>
/// <param name="downloadAsFilename">The filename to use when downloading the file.</param>
/// <param name="contentType">The content type of the file.</param>
/// <param name="cancellationToken">An optional cancellation token.</param>
private async Task CreateCachedZipBlobAsync(string localPath, string downloadCorrelationId, string? downloadAsFilename = default, string? contentType = default, CancellationToken cancellationToken = default)
{
var fileCacheStorage = _fileCacheStorageProvider.GetStorage();
var fileCacheFilename = $"{downloadCorrelationId}.tmp";
var expiresAt = _clock.UtcNow.Add(_fileCacheOptions.Value.TimeToLive);
var cachedBlob = CreateBlob(fileCacheFilename, downloadAsFilename, contentType, expiresAt);
await fileCacheStorage.WriteFileAsync(fileCacheFilename, localPath, cancellationToken);
await fileCacheStorage.SetBlobAsync(cachedBlob, cancellationToken: cancellationToken);
}
/// <summary>
/// Creates a blob for the specified file.
/// </summary>
/// <param name="fullPath">The full path of the file.</param>
/// <param name="downloadAsFilename">The filename to use when downloading the file.</param>
/// <param name="contentType">The content type of the file.</param>
/// <param name="expiresAt">The date and time at which the file expires.</param>
/// <returns>The blob.</returns>
private Blob CreateBlob(string fullPath, string? downloadAsFilename, string? contentType, DateTimeOffset? expiresAt = default)
{
(downloadAsFilename, contentType) = GetDownloadableMetadata(downloadAsFilename, contentType);
var now = _clock.UtcNow;
var blob = new Blob(fullPath)
{
Metadata =
{
["ContentType"] = contentType,
["Filename"] = downloadAsFilename
},
CreatedTime = now,
LastModificationTime = now
};
if(expiresAt.HasValue)
blob.Metadata["ExpiresAt"] = expiresAt.Value.ToString("O");
return blob;
}
private (string downloadAsFilename, string contentType) GetDownloadableMetadata(string? contentType, string? downloadAsFilename)
{
contentType = !string.IsNullOrWhiteSpace(contentType) ? contentType : System.Net.Mime.MediaTypeNames.Application.Zip;
downloadAsFilename = !string.IsNullOrWhiteSpace(downloadAsFilename) ? downloadAsFilename : "download.zip";
return (downloadAsFilename, contentType);
}
private void Cleanup(string filePath)
{
try
{
File.Delete(filePath);
}
catch (Exception e)
{
_logger.LogWarning(e, "Failed to delete temporary file {TempFilePath}", filePath);
}
}
}

View file

@ -1,6 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="..\..\..\common.props" />
<Import Project="..\..\..\packages.props" />
<Import Project="..\..\..\configureawait.props" />
<PropertyGroup>
@ -15,8 +16,4 @@
<ProjectReference Include="..\Elsa.Workflows.Runtime\Elsa.Workflows.Runtime.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="FluentStorage" Version="5.0.0" />
</ItemGroup>
</Project>

View file

@ -29,7 +29,7 @@ public static class ModuleExtensions
/// <param name="module">The module.</param>
/// <param name="configure">The configuration delegate.</param>
/// <returns>The module.</returns>
public static IModule UseFluentStorageProvider(this IModule module, Action<FluentStorageFeature>? configure = default)
public static IModule UseFluentStorageProvider(this IModule module, Action<BlobStorageFeature>? configure = default)
{
module.Use(configure);
return module;

View file

@ -7,7 +7,6 @@ using Elsa.WorkflowProviders.BlobStorage.Providers;
using Elsa.Workflows.Management.Features;
using FluentStorage;
using FluentStorage.Blobs;
using JetBrains.Annotations;
using Microsoft.Extensions.DependencyInjection;
namespace Elsa.WorkflowProviders.BlobStorage.Features;
@ -17,11 +16,10 @@ namespace Elsa.WorkflowProviders.BlobStorage.Features;
/// </summary>
[DependsOn(typeof(WorkflowManagementFeature))]
[DependsOn(typeof(DslIntegrationFeature))]
[PublicAPI]
public class FluentStorageFeature : FeatureBase
public class BlobStorageFeature : FeatureBase
{
/// <inheritdoc />
public FluentStorageFeature(IModule module) : base(module)
public BlobStorageFeature(IModule module) : base(module)
{
}

View file

@ -44,7 +44,8 @@ internal class Execute : ElsaEndpoint<Request, Response>
/// <inheritdoc />
public override void Configure()
{
Post("/workflow-definitions/{definitionId}/execute");
Routes("/workflow-definitions/{definitionId}/execute");
Verbs(FastEndpoints.Http.GET, FastEndpoints.Http.POST);
ConfigurePermissions("exec:workflow-definitions");
}
@ -102,7 +103,10 @@ internal class Execute : ElsaEndpoint<Request, Response>
{
// Write a response header to indicate that the response is a workflow state response.
HttpContext.Response.Headers.Add("x-elsa-response", "true");
await SendOkAsync(new Response(workflowState), cancellationToken);
// Only write a response if the status code wasn't changed by the workflow.
if (HttpContext.Response.StatusCode == StatusCodes.Status200OK)
await SendOkAsync(new Response(workflowState), cancellationToken);
}
}
}