From 3bf9081e4a6292e9efade8298f51638efaa50235 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 22 Feb 2024 10:02:11 -0600 Subject: [PATCH] refine json converter --- .../Conversations/Models/Conversation.cs | 8 +++-- .../RichContentJsonConverter .cs | 18 +++++++++++- .../TemplateMessageJsonConverter.cs | 18 +++++++++++- .../Services/ConversationStorage.cs | 29 +++++++++++++++---- .../FileRepository.Conversation.cs | 12 ++++++-- .../Controllers/ConversationController.cs | 3 +- .../Models/DialogMongoElement.cs | 25 +++++++++------- 7 files changed, 88 insertions(+), 25 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index c8332a3d..45dbc228 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -30,22 +30,24 @@ public class Conversation public class DialogElement { - public DialogMeta MetaData { get; set; } + public DialogMetaData MetaData { get; set; } public string Content { get; set; } + public string? RichContent { get; set; } public DialogElement() { } - public DialogElement(DialogMeta meta, string content) + public DialogElement(DialogMetaData meta, string content, string? richContent = null) { MetaData = meta; Content = content; + RichContent = richContent; } } -public class DialogMeta +public class DialogMetaData { public string Role { get; set; } public string AgentId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs index 7f01132e..33bb30fa 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Messaging.Models.RichContent.Template; using System.Text.Json; namespace BotSharp.Abstraction.Messaging.JsonConverters; @@ -6,7 +7,22 @@ public class RichContentJsonConverter : JsonConverter { public override IRichMessage? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - throw new NotImplementedException(); + using var jsonDoc = JsonDocument.ParseValue(ref reader); + var root = jsonDoc.RootElement; + var jsonText = root.GetRawText(); + JsonElement element; + object? res = null; + + if (root.TryGetProperty("buttons", out element)) + { + res = JsonSerializer.Deserialize(jsonText, options); + } + else if (root.TryGetProperty("options", out element)) + { + res = JsonSerializer.Deserialize(jsonText, options); + } + + return res as IRichMessage; } public override void Write(Utf8JsonWriter writer, IRichMessage value, JsonSerializerOptions options) diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs index 9b990193..db09b7f4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Messaging.Models.RichContent.Template; using System.Text.Json; namespace BotSharp.Abstraction.Messaging.JsonConverters; @@ -6,7 +7,22 @@ public class TemplateMessageJsonConverter : JsonConverter { public override ITemplateMessage? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - throw new NotImplementedException(); + using var jsonDoc = JsonDocument.ParseValue(ref reader); + var root = jsonDoc.RootElement; + var jsonText = root.GetRawText(); + JsonElement element; + object? res = null; + + if (root.TryGetProperty("buttons", out element)) + { + res = JsonSerializer.Deserialize(jsonText, options); + } + else if (root.TryGetProperty("options", out element)) + { + res = JsonSerializer.Deserialize(jsonText, options); + } + + return res as ITemplateMessage; } public override void Write(Utf8JsonWriter writer, ITemplateMessage value, JsonSerializerOptions options) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index cf092b51..f6c2a02c 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -1,3 +1,6 @@ +using BotSharp.Abstraction.Messaging; +using BotSharp.Abstraction.Messaging.JsonConverters; +using BotSharp.Abstraction.Messaging.Models.RichContent; using BotSharp.Abstraction.Repositories; using System; using System.IO; @@ -8,12 +11,25 @@ public class ConversationStorage : IConversationStorage { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; + private readonly JsonSerializerOptions _options; + public ConversationStorage( BotSharpDatabaseSettings dbSettings, IServiceProvider services) { _dbSettings = dbSettings; _services = services; + _options = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + AllowTrailingCommas = true, + Converters = + { + new RichContentJsonConverter(), + new TemplateMessageJsonConverter(), + } + }; } public void Append(string conversationId, RoleDialogModel dialog) @@ -24,7 +40,7 @@ public class ConversationStorage : IConversationStorage if (dialog.Role == AgentRole.Function) { - var meta = new DialogMeta + var meta = new DialogMetaData { Role = dialog.Role, AgentId = agentId, @@ -42,7 +58,7 @@ public class ConversationStorage : IConversationStorage } else { - var meta = new DialogMeta + var meta = new DialogMetaData { Role = dialog.Role, AgentId = agentId, @@ -56,8 +72,8 @@ public class ConversationStorage : IConversationStorage { return; } - - dialogElements.Add(new DialogElement(meta, content)); + var richContent = dialog.RichContent != null ? JsonSerializer.Serialize(dialog.RichContent, _options) : null; + dialogElements.Add(new DialogElement(meta, content, richContent)); } db.AppendConversationDialogs(conversationId, dialogElements); @@ -80,6 +96,8 @@ public class ConversationStorage : IConversationStorage var function = role == AgentRole.Function ? meta.FunctionName : null; var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId; var createdAt = meta.CreateTime; + var richContent = !string.IsNullOrEmpty(dialog.RichContent) ? + JsonSerializer.Deserialize>(dialog.RichContent, _options) : null; var record = new RoleDialogModel(role, content) { @@ -87,7 +105,8 @@ public class ConversationStorage : IConversationStorage MessageId = messageId, CreatedAt = createdAt, SenderId = senderId, - FunctionName = function + FunctionName = function, + RichContent = richContent }; results.Add(record); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 2ae0d1a3..e8609f6f 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -317,7 +317,7 @@ namespace BotSharp.Core.Repository var blocks = rawDialogs[i].Split("|"); var content = rawDialogs[i + 1]; var trimmed = content.Substring(4); - var meta = new DialogMeta + var meta = new DialogMetaData { Role = blocks[1], AgentId = blocks[2], @@ -326,7 +326,13 @@ namespace BotSharp.Core.Repository SenderId = blocks[1] == AgentRole.Function ? null : blocks[4], CreateTime = DateTime.Parse(blocks[0]) }; - dialogs.Add(new DialogElement(meta, trimmed)); + + string? richContent = null; + if (blocks.Count() > 5) + { + richContent = blocks[5]; + } + dialogs.Add(new DialogElement(meta, trimmed, richContent)); } } return dialogs; @@ -342,7 +348,7 @@ namespace BotSharp.Core.Repository var meta = element.MetaData; var createTime = meta.CreateTime.ToString("MM/dd/yyyy hh:mm:ss.fff tt", CultureInfo.InvariantCulture); var source = meta.FunctionName ?? meta.SenderId; - var metaStr = $"{createTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{source}"; + var metaStr = $"{createTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{source}|{element.RichContent}"; dialogTexts.Add(metaStr); var content = $" - {element.Content}"; dialogTexts.Add(content); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 7443b82d..19bcb5a7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -102,7 +102,8 @@ public class ConversationController : ControllerBase { FirstName = agent.Name, Role = message.Role, - } + }, + RichContent = message.RichContent }); } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs index 1940d5d3..a35d3fc1 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -4,8 +4,9 @@ namespace BotSharp.Plugin.MongoStorage.Models; public class DialogMongoElement { - public DialogMetaMongoElement MetaData { get; set; } + public DialogMetaDataMongoElement MetaData { get; set; } public string Content { get; set; } + public string? RichContent { get; set; } public DialogMongoElement() { @@ -16,8 +17,9 @@ public class DialogMongoElement { return new DialogMongoElement { - MetaData = DialogMetaMongoElement.ToMongoElement(dialog.MetaData), - Content = dialog.Content + MetaData = DialogMetaDataMongoElement.ToMongoElement(dialog.MetaData), + Content = dialog.Content, + RichContent = dialog.RichContent }; } @@ -25,13 +27,14 @@ public class DialogMongoElement { return new DialogElement { - MetaData = DialogMetaMongoElement.ToDomainElement(dialog.MetaData), - Content = dialog.Content + MetaData = DialogMetaDataMongoElement.ToDomainElement(dialog.MetaData), + Content = dialog.Content, + RichContent = dialog.RichContent }; } } -public class DialogMetaMongoElement +public class DialogMetaDataMongoElement { public string Role { get; set; } public string AgentId { get; set; } @@ -40,14 +43,14 @@ public class DialogMetaMongoElement public string? SenderId { get; set; } public DateTime CreateTime { get; set; } - public DialogMetaMongoElement() + public DialogMetaDataMongoElement() { } - public static DialogMeta ToDomainElement(DialogMetaMongoElement meta) + public static DialogMetaData ToDomainElement(DialogMetaDataMongoElement meta) { - return new DialogMeta + return new DialogMetaData { Role = meta.Role, AgentId = meta.AgentId, @@ -58,9 +61,9 @@ public class DialogMetaMongoElement }; } - public static DialogMetaMongoElement ToMongoElement(DialogMeta meta) + public static DialogMetaDataMongoElement ToMongoElement(DialogMetaData meta) { - return new DialogMetaMongoElement + return new DialogMetaDataMongoElement { Role = meta.Role, AgentId = meta.AgentId,