diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/LanguageType.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/LanguageType.cs new file mode 100644 index 00000000..64351e45 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/LanguageType.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Infrastructures.Enums; + +public class LanguageType +{ + public const string UNKNOWN = "Unknown"; + public const string ENGLISH = "English"; + public const string SPANISH = "Spanish"; + public const string CHINESE = "Chinese"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/ElementButton.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/ElementButton.cs index 47e7cc9a..b4baf793 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/ElementButton.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/ElementButton.cs @@ -13,6 +13,7 @@ public class ElementButton [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Url { get; set; } + [Translate] public string Title { get; set; } = string.Empty; [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] @@ -28,5 +29,7 @@ public class ElementButton [JsonPropertyName("post_action_disclaimer")] [JsonProperty("post_action_disclaimer")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Translate] public string? PostActionDisclaimer { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/GenericTemplateMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/GenericTemplateMessage.cs index c846e4ea..3d3e5eef 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/GenericTemplateMessage.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/GenericTemplateMessage.cs @@ -11,6 +11,7 @@ public class GenericTemplateMessage : IRichMessage, ITemplateMessage [JsonPropertyName("text")] [JsonProperty("text")] + [Translate] public string Text { get; set; } = string.Empty; [JsonPropertyName("template_type")] @@ -36,6 +37,7 @@ public class GenericTemplateMessage : IRichMessage, ITemplateMessage public class GenericElement { + [Translate] public string Title { get; set; } public string Subtitle { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs index da84ffa3..1577d103 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs @@ -44,5 +44,5 @@ public interface IRoutingService Task GetConversationContent(List dialogs, int maxDialogCount = 50); - bool HasMissingRequiredField(RoleDialogModel message, out string agentId); + (bool, string) HasMissingRequiredField(RoleDialogModel message, out string agentId); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs index 31311b57..105c0c57 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs @@ -50,6 +50,12 @@ public class RoutingArgs [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string UserGoal { get; set; } + [JsonPropertyName("language")] + public string Language { get; set; } = LanguageType.ENGLISH; + + [JsonPropertyName("lastest_message_translated_to_english")] + public string UserMessageInEnglish { get; set; } + public override string ToString() { var route = string.IsNullOrEmpty(AgentName) ? "" : $""; diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/Attributes/TranslateAttribute.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/Attributes/TranslateAttribute.cs new file mode 100644 index 00000000..1c39c415 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/Attributes/TranslateAttribute.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Translation.Attributes; + +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, Inherited = false)] +public class TranslateAttribute : Attribute +{ + public TranslateAttribute() + { + + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/ITranslationService.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/ITranslationService.cs new file mode 100644 index 00000000..e69e35b1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/ITranslationService.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Translation; + +public interface ITranslationService +{ + Task Translate(Agent router, string messageId, T data, string language = "Spanish", bool clone = true) where T : class; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index f9b708dc..91f672b8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -14,3 +14,4 @@ global using BotSharp.Abstraction.Models; global using BotSharp.Abstraction.Routing.Models; global using BotSharp.Abstraction.Routing.Planning; global using BotSharp.Abstraction.Templating; +global using BotSharp.Abstraction.Translation.Attributes; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index b4dfa0b5..1650b719 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -62,6 +62,7 @@ + @@ -118,6 +119,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index 5c215884..8fb6d9a5 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -8,6 +8,7 @@ using BotSharp.Core.Instructs; using BotSharp.Core.Messaging; using BotSharp.Core.Routing.Planning; using BotSharp.Core.Templating; +using BotSharp.Core.Translation; using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Conversations; @@ -35,6 +36,7 @@ public class ConversationPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); // Rich content messaging services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index c4dd9302..0306ccaf 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -71,15 +71,6 @@ public partial class ConversationService } } - // Persist to storage - if (!message.StopCompletion) - { - _storage.Append(_conversationId, message); - - // Add to thread - dialogs.Add(RoleDialogModel.From(message)); - } - if (!stopCompletion) { // Routing with reasoning @@ -139,7 +130,7 @@ public partial class ConversationService response.RichContent is RichContent template && string.IsNullOrEmpty(template.Message.Text)) { - template.Message.Text = response.Content; + template.Message.Text = response.SecondaryContent ?? response.Content; } // Only read content from RichContent for UI rendering. When richContent is null, create a basic text message for richContent. @@ -147,7 +138,7 @@ public partial class ConversationService response.RichContent = response.RichContent ?? new RichContent { Recipient = new Recipient { Id = state.GetConversationId() }, - Message = new TextMessage(response.Content) + Message = new TextMessage(response.SecondaryContent ?? response.Content) }; // Patch return function name diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs index 2450a6fc..639c907b 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/RouteToAgentFn.cs @@ -83,11 +83,11 @@ public partial class RouteToAgentFn : IFunctionCallback } var routing = _services.GetRequiredService(); - var missingfield = routing.HasMissingRequiredField(message, out var agentId); + var (missingfield, reason) = routing.HasMissingRequiredField(message, out var agentId); if (missingfield && message.CurrentAgentId != agentId) { // Stack redirection agent - _context.Push(agentId, reason: $"REDIRECTION {message.Content}"); + _context.Push(agentId, reason: $"REDIRECTION {reason}"); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs index 2b010183..92a3b137 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs @@ -10,10 +10,21 @@ public class ResponseToUserRoutingHandler : RoutingHandlerBase, IRoutingHandler public List Parameters => new List { - new ParameterPropertyDef("reason", "why response to user directly without go to other agents"), - new ParameterPropertyDef("response", "response content to user in courteous words. If the user wants to end the conversation, you must set conversation_end to true and response politely."), - new ParameterPropertyDef("conversation_end", "whether to end this conversation", type: "boolean"), - new ParameterPropertyDef("task_completed ", "whether the user's task request has been completed.", type: "boolean") + new ParameterPropertyDef("reason", + "why response to user directly without go to other agents."), + new ParameterPropertyDef("response", + "response content to user in courteous words with language English. If the user wants to end the conversation, you must set conversation_end to true and response politely."), + new ParameterPropertyDef("conversation_end", + "whether to end this conversation.", + type: "boolean"), + new ParameterPropertyDef("task_completed ", + "whether the user's task request has been completed.", + type: "boolean"), + new ParameterPropertyDef("language", + "User preferred language, considering the whole conversation. Language could be English, Spanish or Chinese.", + required: true), + new ParameterPropertyDef("lastest_message_translated_to_english", + "Translate user lastest message in [CONVERSATION] to English"), }; public ResponseToUserRoutingHandler(IServiceProvider services, ILogger logger, RoutingSettings settings) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index b5c6a59e..5c08d679 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -12,23 +12,28 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler public List Parameters => new List { new ParameterPropertyDef("next_action_reason", - "the reason why route to this virtual agent", + "the reason why route to this virtual agent.", required: true), new ParameterPropertyDef("next_action_agent", - "agent for next action based on user latest response, if user is replying last agent's question, you must route to this agent", + "agent for next action based on user latest response, if user is replying last agent's question, you must route to this agent.", required: true), + new ParameterPropertyDef("args", + "useful parameters of next action agent, format: { }", + type: "object"), new ParameterPropertyDef("user_goal_description", "user goal based on user initial task.", required: true), new ParameterPropertyDef("user_goal_agent", "agent who can acheive user initial task, must align with user_goal_description.", required: true), - new ParameterPropertyDef("args", - "useful parameters of next action agent, format: { }", - type: "object"), new ParameterPropertyDef("is_new_task", "whether the user is requesting a new task that is different from the previous topic.", - type: "boolean") + type: "boolean"), + new ParameterPropertyDef("language", + "User preferred language, considering the whole conversation. Language could be English, Spanish or Chinese.", + required: true), + new ParameterPropertyDef("lastest_message_translated_to_english", + "Translate lastest user message in [CONVERSATION] to English"), }; public RouteToAgentRoutingHandler(IServiceProvider services, ILogger logger, RoutingSettings settings) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs index 4d8eba43..6b126a37 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs @@ -128,7 +128,7 @@ public class RoutingContext : IRoutingContext }; var routing = _services.GetRequiredService(); - var missingfield = routing.HasMissingRequiredField(message, out agentId); + var (missingfield, _) = routing.HasMissingRequiredField(message, out agentId); if (missingfield) { if (currentAgentId != agentId) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.HasMissingRequiredField.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.HasMissingRequiredField.cs index 2a6b676c..60f3f331 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.HasMissingRequiredField.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.HasMissingRequiredField.cs @@ -10,8 +10,9 @@ public partial class RoutingService /// If the target agent needs some required fields but the /// /// - public bool HasMissingRequiredField(RoleDialogModel message, out string agentId) + public (bool, string) HasMissingRequiredField(RoleDialogModel message, out string agentId) { + var reason = string.Empty; var args = JsonSerializer.Deserialize(message.FunctionArgs); var routing = _services.GetRequiredService(); @@ -20,7 +21,7 @@ public partial class RoutingService if (routingRules == null || !routingRules.Any()) { agentId = message.CurrentAgentId; - return false; + return (false, reason); } agentId = routingRules.First().AgentId; @@ -68,9 +69,13 @@ public partial class RoutingService if (missingFields.Any()) { + var logger = _services.GetRequiredService>(); + // Add field to args message.FunctionArgs = AppendPropertyToArgs(message.FunctionArgs, "missing_fields", missingFields); - message.Content = $"missing some information: {string.Join(", ", missingFields)}"; + reason = $"missing some information: {string.Join(", ", missingFields)}"; + // message.Content = reason; + logger.LogWarning(reason); // Handle redirect var routingRule = routingRules.FirstOrDefault(x => missingFields.Contains(x.Field)); @@ -82,7 +87,6 @@ public partial class RoutingService // Add redirected agent message.FunctionArgs = AppendPropertyToArgs(message.FunctionArgs, "redirect_to", record.Name); agentId = routingRule.RedirectTo; - var logger = _services.GetRequiredService>(); #if DEBUG Console.WriteLine($"*** Routing redirect to {record.Name.ToUpper()} ***", Color.Yellow); #else @@ -96,7 +100,7 @@ public partial class RoutingService } } - return missingFields.Any(); + return (missingFields.Any(), reason); } private string AppendPropertyToArgs(string args, string key, string value) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 86839b67..42465533 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -4,9 +4,6 @@ namespace BotSharp.Core.Routing; public partial class RoutingService { - private List _functionCallStack = new List(); - public List FunctionCallStack => _functionCallStack; - public async Task InvokeFunction(string name, RoleDialogModel message) { var function = _services.GetServices().FirstOrDefault(x => x.Name == name); @@ -38,14 +35,6 @@ public partial class RoutingService { result = await function.Execute(clonedMessage); - _functionCallStack.Add(new FunctionCallingResponse - { - Role = AgentRole.Function, - FunctionName = clonedMessage.FunctionName, - Args = JsonDocument.Parse(clonedMessage.FunctionArgs ?? "{}"), - Content = clonedMessage.Content - }); - // After functions have been executed foreach (var hook in hooks) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 4a17f67a..36fb21be 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -1,7 +1,19 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Abstraction.Routing.Settings; +using BotSharp.Abstraction.Templating; +using BotSharp.Core.Routing.Planning; +using Fluid.Ast; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Diagnostics.Metrics; using System.Drawing; +using System.Runtime.InteropServices; +using System.Security.Cryptography.X509Certificates; +using ThirdParty.Json.LitJson; +using static System.Net.Mime.MediaTypeNames; namespace BotSharp.Core.Routing; @@ -39,15 +51,10 @@ public partial class RoutingService : IRoutingService var handler = handlers.FirstOrDefault(x => x.Name == "route_to_agent"); var conv = _services.GetRequiredService(); - var dialogs = new List(); - if (conv.States.GetState("hide_context", "false") == "true") - { - dialogs.Add(message); - } - else - { - dialogs = conv.GetDialogHistory(); - } + var storage = _services.GetRequiredService(); + storage.Append(conv.ConversationId, message); + + var dialogs = conv.GetDialogHistory(); handler.SetDialogs(dialogs); var inst = new FunctionCallFromLlm @@ -71,11 +78,14 @@ public partial class RoutingService : IRoutingService public async Task InstructLoop(RoleDialogModel message, List dialogs) { - var agentService = _services.GetRequiredService(); - _router = await agentService.LoadAgent(message.CurrentAgentId); - RoleDialogModel response = default; + var agentService = _services.GetRequiredService(); + var convService = _services.GetRequiredService(); + var storage = _services.GetRequiredService(); + + _router = await agentService.LoadAgent(message.CurrentAgentId); + var states = _services.GetRequiredService(); var executor = _services.GetRequiredService(); @@ -83,17 +93,23 @@ public partial class RoutingService : IRoutingService _context.Push(_router.Id); - int loopCount = 0; - while (loopCount < planner.MaxLoopCount && !_context.IsEmpty) + dialogs.Add(message); + + // Get first instruction + _router.TemplateDict["conversation"] = await GetConversationContent(dialogs); + var inst = await planner.GetNextInstruction(_router, message.MessageId, dialogs); + + // Handle multi-language for input + + if (inst.Language != LanguageType.UNKNOWN && inst.Language != LanguageType.ENGLISH) { - loopCount++; - - var conversation = await GetConversationContent(dialogs); - _router.TemplateDict["conversation"] = conversation; - - // Get instruction from Planner - var inst = await planner.GetNextInstruction(_router, message.MessageId, dialogs); + message.Content = inst.UserMessageInEnglish; + } + storage.Append(convService.ConversationId, message); + int loopCount = 1; + while (true) + { await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(inst, message) ); @@ -121,6 +137,36 @@ public partial class RoutingService : IRoutingService } await planner.AgentExecuted(_router, inst, response, dialogs); + + if (loopCount >= planner.MaxLoopCount || _context.IsEmpty) + { + break; + } + + // Get next instruction from Planner + _router.TemplateDict["conversation"] = await GetConversationContent(dialogs); + inst = await planner.GetNextInstruction(_router, message.MessageId, dialogs); + loopCount++; + } + + // Handle multi-language for output + if (inst.Language != LanguageType.UNKNOWN && inst.Language != LanguageType.ENGLISH) + { + var translator = _services.GetRequiredService(); + if (response.RichContent != null) + { + response.RichContent.Message = await translator.Translate(_router, + message.MessageId, + response.RichContent.Message, + language: inst.Language); + } + else + { + response.SecondaryContent = await translator.Translate(_router, + message.MessageId, + response.Content, + language: inst.Language); + } } return response; diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs new file mode 100644 index 00000000..815b2dc9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -0,0 +1,308 @@ +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Templating; +using BotSharp.Abstraction.Translation.Attributes; +using Newtonsoft.Json; +using System.Collections; +using System.Reflection; + +namespace BotSharp.Core.Translation; + +public class TranslationService : ITranslationService +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private Agent _router; + private string _messageId; + + public TranslationService(IServiceProvider services, + ILogger logger, + BotSharpOptions options) + { + _services = services; + _logger = logger; + _options = options; + } + + public async Task Translate(Agent router, string messageId, T data, string language = "Spanish", bool clone = true) where T : class + { + _router = router; + _messageId = messageId; + + var unique = new HashSet(); + Collect(data, ref unique); + if (unique.Count == 0) + { + return data; + } + + var cloned = data; + if (clone) + { + cloned = Clone(data); + } + + var map = await InnerTranslate(unique, language); + cloned = Assign(cloned, map); + + return cloned; + } + + private T Clone(T data) where T : class + { + if (data == null) return data; + + var str = System.Text.Json.JsonSerializer.Serialize(data, _options.JsonSerializerOptions); + var cloned = System.Text.Json.JsonSerializer.Deserialize(str, _options.JsonSerializerOptions); + return cloned; + } + + /// + /// Collect unique strings in data + /// + /// + /// + /// + private void Collect(T data, ref HashSet res) where T : class + { + if (data == null) return; + + var dataType = data.GetType(); + if (dataType == typeof(string)) + { + res.Add(data.ToString()); + return; + } + + var interfaces = dataType.GetTypeInfo().ImplementedInterfaces; + if (interfaces.Any(x => x.Name == typeof(IDictionary<,>).Name)) + { + return; + } + + var isList = interfaces.Any(x => x.Name == typeof(IEnumerable).Name); + if (dataType.IsArray || isList) + { + var elementType = dataType.IsArray ? dataType.GetElementType() : dataType.GetGenericArguments().FirstOrDefault(); + if (elementType == typeof(string)) + { + foreach (var item in (data as IEnumerable)) + { + if (item == null) continue; + res.Add(item); + } + } + else if (elementType != null && (elementType.IsClass || elementType.IsInterface)) + { + foreach (var item in (data as IEnumerable)) + { + if (item == null) continue; + Collect(item, ref res); + } + } + return; + } + + + var props = dataType.GetProperties(); + foreach (var prop in props) + { + var value = prop.GetValue(data, null); + var propType = prop.PropertyType; + var translate = prop.GetCustomAttributes(true).FirstOrDefault(x => x.GetType() == typeof(TranslateAttribute)); + + if (value == null) continue; + + if (propType == typeof(string)) + { + if (translate != null) + { + Collect(value, ref res); + } + } + else if (propType.IsClass || propType.IsInterface) + { + interfaces = propType.GetTypeInfo().ImplementedInterfaces; + isList = interfaces.Any(x => x.Name == typeof(IEnumerable).Name); + if (interfaces.Any(x => x.Name == typeof(IDictionary<,>).Name)) + { + Collect(value, ref res); + } + else if (propType.IsArray || isList) + { + var elementType = propType.IsArray ? propType.GetElementType() : propType.GetGenericArguments().FirstOrDefault(); + if (elementType == typeof(string)) + { + if (translate != null) + { + Collect(value, ref res); + } + } + else if (elementType != null && (elementType.IsClass || elementType.IsInterface)) + { + Collect(value, ref res); + } + } + else + { + Collect(value, ref res); + } + } + } + } + + /// + /// Assign translated values to corresponding attributes + /// + /// + /// + /// + /// + private T Assign(T data, Dictionary map) where T : class + { + if (data == null) return data; + + var dataType = data.GetType(); + if (dataType == typeof(string) && map.TryGetValue(data.ToString(), out var target)) + { + return target as T; + } + + var interfaces = dataType.GetTypeInfo().ImplementedInterfaces; + if (interfaces.Any(x => x.Name == typeof(IDictionary<,>).Name)) + { + return data; + } + + var isList = interfaces.Any(x => x.Name == typeof(IEnumerable).Name); + if (dataType.IsArray || isList) + { + var elementType = dataType.IsArray ? dataType.GetElementType() : dataType.GetGenericArguments().FirstOrDefault(); + if (elementType == typeof(string)) + { + var list = new List(); + foreach (var item in (data as IEnumerable)) + { + if (map.TryGetValue(item, out target)) + { + list.Add(target); + } + else + { + list.Add(item?.ToString()); + } + } + + data = dataType.IsArray ? list.ToArray() as T : list as T; + } + else if (elementType != null && (elementType.IsClass || elementType.IsInterface)) + { + foreach (var item in (data as IEnumerable)) + { + if (item == null) continue; + Assign(item, map); + } + } + return data; + } + + + var props = dataType.GetProperties(); + foreach (var prop in props) + { + var value = prop.GetValue(data, null); + var propType = prop.PropertyType; + var translate = prop.GetCustomAttributes(true).FirstOrDefault(x => x.GetType() == typeof(TranslateAttribute)); + + if (value == null) continue; + + if (propType == typeof(string)) + { + if (translate != null) + { + prop.SetValue(data, Assign(value, map)); + } + } + else if (propType.IsClass || propType.IsInterface) + { + interfaces = propType.GetTypeInfo().ImplementedInterfaces; + isList = interfaces.Any(x => x.Name == typeof(IEnumerable).Name); + if (interfaces.Any(x => x.Name == typeof(IDictionary<,>).Name)) + { + Assign(value, map); + } + else if (propType.IsArray || isList) + { + var elementType = propType.IsArray ? propType.GetElementType() : propType.GetGenericArguments().FirstOrDefault(); + if (elementType == typeof(string)) + { + if (translate != null) + { + var targetValue = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(Assign(value, map)), propType); + prop.SetValue(data, targetValue); + } + } + else if (elementType != null && (elementType.IsClass || elementType.IsInterface)) + { + prop.SetValue(data, Assign(value, map)); + } + } + else + { + Assign(value, map); + } + } + } + + return data; + } + + /// + /// Translate + /// + /// + /// + /// + private async Task> InnerTranslate(HashSet list, string language) + { + // chat completion + var completion = CompletionProvider.GetChatCompletion(_services, + provider: _router?.LlmConfig?.Provider, + model: _router?.LlmConfig?.Model); + + var texts = list.ToArray(); + var translator = new Agent + { + Id = Guid.Empty.ToString(), + Name = "Translator", + TemplateDict = new Dictionary + { + { "text_list", JsonConvert.SerializeObject(texts) }, + { "language", language } + } + }; + + var template = _router.Templates.First(x => x.Name == "translation_prompt").Content; + var render = _services.GetRequiredService(); + var prompt = render.Render(template, translator.TemplateDict); + + var translationDialogs = new List + { + new RoleDialogModel(AgentRole.User, prompt) + { + FunctionName = "translation_prompt", + MessageId = _messageId + } + }; + var translationResponse = await completion.GetChatCompletions(translator, translationDialogs); + var translatedTexts = translationResponse.Content.JsonArrayContent(); + var map = new Dictionary(); + + for (var i = 0; i < list.Count; i++) + { + map.Add(texts[i], translatedTexts[i]); + } + + return map; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index fbb1c8df..769f318f 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -22,6 +22,7 @@ global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Repositories.Filters; +global using BotSharp.Abstraction.Translation; global using BotSharp.Core.Repository; global using BotSharp.Core.Routing; global using BotSharp.Core.Agents.Services; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/agent.json index d32e4680..24508fcc 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/agent.json @@ -1,11 +1,11 @@ { "id": "01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b", "name": "Human Support", - "description": "Reach out to human customer service representative.", + "description": "Reach out to human customer service.", "type": "task", "createdDateTime": "2024-04-22T10:00:00Z", "updatedDateTime": "2024-04-22T10:00:00Z", "disabled": false, "isPublic": true, "profiles": [ "human" ] -} \ No newline at end of file +} diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instruction.liquid index 3b1aab42..58795499 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instruction.liquid @@ -1,2 +1,2 @@ You are a human customer service connection program. -When other AI customer service cannot solve user problems, you know how to call the API to transfer users to human customer service for answers. \ No newline at end of file +When other AI customer service agents cannot solve user problems, you know how to call the API to transfer users to human customer service for answers. diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid index 93960782..85cd1ff9 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid @@ -1,4 +1,6 @@ -You're {{router.name}} ({{router.description}}). Follow these steps to handle user request: +You're {{router.name}} ({{router.description}}). +You can understand messages sent by users in different languages. +Follow these steps to handle user request: 1. Read the [CONVERSATION] content. 2. Select a appropriate function from [FUNCTIONS]. 3. Determine which agent is suitable to handle this conversation. diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.naive.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.naive.liquid index f7388be6..5177a002 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.naive.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.naive.liquid @@ -10,3 +10,4 @@ Expected user goal agent is {{ expected_user_goal_agent }}. {%- else -%} User goal agent is inferred based on user initial request. {%- endif %} +Detect language based on the overall content in [CONVERSATION] and only include user message. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid new file mode 100644 index 00000000..729f0f72 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid @@ -0,0 +1,4 @@ +{{ text_list }} + +===== +Translate the sentences in the list into {{ language }}, only output the translated text in string list [""]. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid index d0bed0db..bc80b14c 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid @@ -1,7 +1,7 @@ -This is a model evaluation program, which interactive with model to complete a certain task based on the background information given to you. +This is a model evaluation program, which interacts with model to complete a certain task based on the background information given to you. {{ task_prompt }} user: Hi! assistant: Hello, How can I help you? -user: \ No newline at end of file +user: diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 04e61773..2f00911d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -83,8 +83,7 @@ public class ConversationController : ControllerBase ConversationId = conversationId, MessageId = message.MessageId, CreatedAt = message.CreatedAt, - Text = message.Content, - SecondaryText = message.SecondaryContent, + Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content, Data = message.Data, Sender = UserViewModel.FromUser(user) }); @@ -97,8 +96,7 @@ public class ConversationController : ControllerBase ConversationId = conversationId, MessageId = message.MessageId, CreatedAt = message.CreatedAt, - Text = message.Content, - SecondaryText = message.SecondaryContent, + Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content, Function = message.FunctionName, Data = message.Data, Sender = new UserViewModel @@ -106,8 +104,7 @@ public class ConversationController : ControllerBase FirstName = agent.Name, Role = message.Role, }, - RichContent = message.RichContent, - SecondaryRichContent = message.SecondaryRichContent + RichContent = message.SecondaryRichContent ?? message.RichContent }); } } @@ -180,9 +177,9 @@ public class ConversationController : ControllerBase replyMessage: input.Postback, async msg => { - response.Text = msg.Content; + response.Text = !string.IsNullOrEmpty(msg.SecondaryContent) ? msg.SecondaryContent : msg.Content; response.Function = msg.FunctionName; - response.RichContent = msg.RichContent; + response.RichContent = msg.SecondaryRichContent ?? msg.RichContent; response.Instruction = msg.Instruction; response.Data = msg.Data; }, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs index 86903800..cf71b8f9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ChatResponseModel.cs @@ -21,9 +21,6 @@ public class ChatResponseModel : InstructResult [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public FunctionCallFromLlm? Instruction { get; set; } - [JsonPropertyName("secondary_text")] - public string? SecondaryText { get; set; } - /// /// Rich message for UI rendering /// @@ -31,10 +28,6 @@ public class ChatResponseModel : InstructResult [JsonPropertyName("rich_content")] public RichContent? RichContent { get; set; } - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [JsonPropertyName("secondary_rich_content")] - public RichContent? SecondaryRichContent { get; set; } - [JsonPropertyName("created_at")] public DateTime CreatedAt { get; set; } = DateTime.UtcNow; } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 0e5cc0ab..820dcb84 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -300,6 +300,7 @@ public class ChatCompletionProvider : IChatCompletion })); prompt += $"{verbose}\r\n"; + prompt += "\r\n[CONVERSATION]\r\n"; verbose = string.Join("\r\n", chatCompletionsOptions.Messages .Where(x => x.Role != AgentRole.System).Select(x => { @@ -311,7 +312,7 @@ public class ChatCompletionProvider : IChatCompletion else if (x.Role == ChatRole.User) { var m = x as ChatRequestUserMessage; - return !string.IsNullOrEmpty(m.Name) ? + return !string.IsNullOrEmpty(m.Name) && m.Name != "route_to_agent" ? $"{m.Name}: {m.Content}" : $"{m.Role}: {m.Content}"; } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 0ebff14c..3b14494b 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -45,7 +45,7 @@ public class ChatHubConversationHook : ConversationHookBase { ConversationId = conv.ConversationId, MessageId = message.MessageId, - Text = message.Content, + Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content, Sender = UserViewModel.FromUser(sender) }); @@ -71,9 +71,9 @@ public class ChatHubConversationHook : ConversationHookBase { ConversationId = conv.ConversationId, MessageId = message.MessageId, - Text = message.Content, + Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content, Function = message.FunctionName, - RichContent = message.RichContent, + RichContent = message.SecondaryRichContent ?? message.RichContent, Data = message.Data, Sender = new UserViewModel() { diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index e1b5fd0f..26e894c7 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -7,6 +7,8 @@ using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Routing; using Microsoft.AspNetCore.SignalR; +using System.Text.Encodings.Web; +using System.Text.Unicode; namespace BotSharp.Plugin.ChatHub.Hooks; @@ -45,7 +47,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnMessageReceived(RoleDialogModel message) { var conversationId = _state.GetConversationId(); - var log = $"{message.Content}"; + var log = $"{GetMessageContent(message)}"; var input = new ContentLogInputModel(conversationId, message) { @@ -59,7 +61,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnPostbackMessageReceived(RoleDialogModel message, PostbackMessageModel replyMsg) { var conversationId = _state.GetConversationId(); - var log = $"{message.Content}"; + var log = $"{GetMessageContent(message)}"; var replyContent = JsonSerializer.Serialize(replyMsg, _options.JsonSerializerOptions); log += $"\r\n```json\r\n{replyContent}\r\n```"; @@ -183,10 +185,18 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR if (message.Role == AgentRole.Assistant) { var agent = await _agentService.LoadAgent(message.CurrentAgentId); - var log = $"{message.Content}"; - if (message.RichContent != null) + var log = $"{GetMessageContent(message)}"; + if (message.RichContent != null || message.SecondaryRichContent != null) { - var richContent = JsonSerializer.Serialize(message.RichContent, _options.JsonSerializerOptions); + var jsonOptions = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + AllowTrailingCommas = true, + WriteIndented = true, + Encoder = JavaScriptEncoder.Create(UnicodeRanges.All) + }; + var richContent = JsonSerializer.Serialize(message.SecondaryRichContent ?? message.RichContent, jsonOptions); log += $"\r\n```json\r\n{richContent}\r\n```"; } @@ -204,7 +214,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR public override async Task OnTaskCompleted(RoleDialogModel message) { var conversationId = _state.GetConversationId(); - var log = $"{message.Content}"; + var log = $"{GetMessageContent(message)}"; var agent = await _agentService.LoadAgent(message.CurrentAgentId); var input = new ContentLogInputModel(conversationId, message) @@ -479,4 +489,9 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR return JsonSerializer.Serialize(model, _options.JsonSerializerOptions); } + + private string GetMessageContent(RoleDialogModel message) + { + return !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content; + } }