using System.Text; using System.Text.Json.Nodes; using w4c_workflows.Models.Nodes; namespace w4c_workflows.Services.Nodes.Executors; /// /// Remove-duplicates node. Builds a comparison key per item — the whole payload, /// a selected set of fields, or the payload minus a set — and keeps either the /// first or the last item that produced each distinct key. /// /// Keys are canonicalised (object properties sorted) so two items with the same /// fields in a different order count as duplicates. Binary attachments are not /// part of the key. /// public sealed class RemoveDuplicatesNodeExecutor : INodeExecutor { public string Type => "core.removeDuplicates"; public Task RunAsync(NodeExecutionContext context, CancellationToken ct) { var items = context.Input(0); var compare = NodeValueAccess.ReadString(context.Parameters, "compare") ?? "allFields"; var keepLast = string.Equals( NodeValueAccess.ReadString(context.Parameters, "keep"), "lastItems", StringComparison.OrdinalIgnoreCase); var dotNotation = !NodeValueAccess.ReadBool(context.Parameters, "disableDotNotation"); var fields = ReadFields(context); if (compare is "selectedFields" or "allFieldsExcept" && fields.Count == 0) return Task.FromResult(NodeExecutionOutcome.Failed( $"compare '{compare}' needs at least one field", "invalid_parameter")); var output = new List(); if (keepLast) { // The last occurrence's data wins, but the item keeps the position // where the key was first seen. var indexByKey = new Dictionary(StringComparer.Ordinal); foreach (var item in items) { var key = KeyFor(item, compare, fields, dotNotation); if (indexByKey.TryGetValue(key, out var index)) output[index] = item; else { indexByKey[key] = output.Count; output.Add(item); } } } else { var seen = new HashSet(StringComparer.Ordinal); foreach (var item in items) { if (seen.Add(KeyFor(item, compare, fields, dotNotation))) output.Add(item); } } return Task.FromResult(NodeExecutionOutcome.Single(output)); } private static List ReadFields(NodeExecutionContext context) { var fields = new List(); foreach (var entry in NodeValueAccess.ReadEntries(context.Parameters, "fields")) { var name = NodeValueAccess.ReadString(entry, "fieldName"); if (!string.IsNullOrWhiteSpace(name)) fields.Add(name); } return fields; } private static string KeyFor(FlowItem item, string compare, IReadOnlyList fields, bool dotNotation) => compare switch { "selectedFields" => SelectedKey(item, fields, dotNotation), "allFieldsExcept" => ExceptKey(item, fields, dotNotation), _ => NodeJsonCanonical.For(item.Json), }; private static string SelectedKey(FlowItem item, IReadOnlyList fields, bool dotNotation) { var builder = new StringBuilder(); foreach (var field in fields) { builder.Append(field).Append('\u001e'); builder.Append(NodeJsonCanonical.For(NodeJsonPath.Read(item.Json, field, dotNotation))); builder.Append('\u001f'); } return builder.ToString(); } private static string ExceptKey(FlowItem item, IReadOnlyList fields, bool dotNotation) { var rest = (JsonObject)item.Json.DeepClone(); foreach (var field in fields) { // Excluding "user.name" drops the top-level "user" object, matching // how the value is read. var top = dotNotation ? field.Split('.')[0] : field; rest.Remove(top); } return NodeJsonCanonical.For(rest); } }