110 lines
4.1 KiB
C#
110 lines
4.1 KiB
C#
using System.Text;
|
|
using System.Text.Json.Nodes;
|
|
using w4c_workflows.Models.Nodes;
|
|
|
|
|
|
namespace w4c_workflows.Services.Nodes.Executors;
|
|
|
|
/// <summary>
|
|
/// Remove-duplicates node. Builds a comparison key per item — the whole payload,
|
|
/// a selected set of fields, or the payload minus a set — and keeps either the
|
|
/// first or the last item that produced each distinct key.
|
|
///
|
|
/// Keys are canonicalised (object properties sorted) so two items with the same
|
|
/// fields in a different order count as duplicates. Binary attachments are not
|
|
/// part of the key.
|
|
/// </summary>
|
|
public sealed class RemoveDuplicatesNodeExecutor : INodeExecutor
|
|
{
|
|
public string Type => "core.removeDuplicates";
|
|
|
|
public Task<NodeExecutionOutcome> RunAsync(NodeExecutionContext context, CancellationToken ct)
|
|
{
|
|
var items = context.Input(0);
|
|
var compare = NodeValueAccess.ReadString(context.Parameters, "compare") ?? "allFields";
|
|
var keepLast = string.Equals(
|
|
NodeValueAccess.ReadString(context.Parameters, "keep"), "lastItems", StringComparison.OrdinalIgnoreCase);
|
|
var dotNotation = !NodeValueAccess.ReadBool(context.Parameters, "disableDotNotation");
|
|
var fields = ReadFields(context);
|
|
|
|
if (compare is "selectedFields" or "allFieldsExcept" && fields.Count == 0)
|
|
return Task.FromResult(NodeExecutionOutcome.Failed(
|
|
$"compare '{compare}' needs at least one field", "invalid_parameter"));
|
|
|
|
var output = new List<FlowItem>();
|
|
if (keepLast)
|
|
{
|
|
// The last occurrence's data wins, but the item keeps the position
|
|
// where the key was first seen.
|
|
var indexByKey = new Dictionary<string, int>(StringComparer.Ordinal);
|
|
foreach (var item in items)
|
|
{
|
|
var key = KeyFor(item, compare, fields, dotNotation);
|
|
if (indexByKey.TryGetValue(key, out var index))
|
|
output[index] = item;
|
|
else
|
|
{
|
|
indexByKey[key] = output.Count;
|
|
output.Add(item);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
var seen = new HashSet<string>(StringComparer.Ordinal);
|
|
foreach (var item in items)
|
|
{
|
|
if (seen.Add(KeyFor(item, compare, fields, dotNotation)))
|
|
output.Add(item);
|
|
}
|
|
}
|
|
|
|
return Task.FromResult(NodeExecutionOutcome.Single(output));
|
|
}
|
|
|
|
private static List<string> ReadFields(NodeExecutionContext context)
|
|
{
|
|
var fields = new List<string>();
|
|
foreach (var entry in NodeValueAccess.ReadEntries(context.Parameters, "fields"))
|
|
{
|
|
var name = NodeValueAccess.ReadString(entry, "fieldName");
|
|
if (!string.IsNullOrWhiteSpace(name))
|
|
fields.Add(name);
|
|
}
|
|
return fields;
|
|
}
|
|
|
|
private static string KeyFor(FlowItem item, string compare, IReadOnlyList<string> fields, bool dotNotation)
|
|
=> compare switch
|
|
{
|
|
"selectedFields" => SelectedKey(item, fields, dotNotation),
|
|
"allFieldsExcept" => ExceptKey(item, fields, dotNotation),
|
|
_ => NodeJsonCanonical.For(item.Json),
|
|
};
|
|
|
|
private static string SelectedKey(FlowItem item, IReadOnlyList<string> fields, bool dotNotation)
|
|
{
|
|
var builder = new StringBuilder();
|
|
foreach (var field in fields)
|
|
{
|
|
builder.Append(field).Append('\u001e');
|
|
builder.Append(NodeJsonCanonical.For(NodeJsonPath.Read(item.Json, field, dotNotation)));
|
|
builder.Append('\u001f');
|
|
}
|
|
return builder.ToString();
|
|
}
|
|
|
|
private static string ExceptKey(FlowItem item, IReadOnlyList<string> fields, bool dotNotation)
|
|
{
|
|
var rest = (JsonObject)item.Json.DeepClone();
|
|
foreach (var field in fields)
|
|
{
|
|
// Excluding "user.name" drops the top-level "user" object, matching
|
|
// how the value is read.
|
|
var top = dotNotation ? field.Split('.')[0] : field;
|
|
rest.Remove(top);
|
|
}
|
|
return NodeJsonCanonical.For(rest);
|
|
}
|
|
}
|