From 237645033ccb0f47e914589b2a7e95365c12cdbc Mon Sep 17 00:00:00 2001 From: "LAPTOP-3CFGGVOS\\rabbit" Date: Tue, 11 Jun 2024 21:05:29 +0800 Subject: [PATCH 01/63] optimize BotSharpMessageParser --- .../BotSharp.Abstraction.csproj | 1 + .../Messaging/BotSharpMessageParser.cs | 101 +++++++++++------- .../Messaging/Enums/RichTypeEnum.cs | 1 + .../Template/ProductTemplateMessage.cs | 2 +- .../Models/RichContent/TextMessage.cs | 2 + 5 files changed, 70 insertions(+), 37 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 65424adb..b97758f1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -19,6 +19,7 @@ + diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs index 1f495a44..483900a4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs @@ -2,57 +2,53 @@ using BotSharp.Abstraction.Messaging.Models.RichContent.Template; using BotSharp.Abstraction.Messaging.Models.RichContent; using System.Text.Json; using System.Reflection; +using System.Linq; namespace BotSharp.Abstraction.Messaging; public static class BotSharpMessageParser { + private static Dictionary GenericTemplateTypeMap = new(); + private static Dictionary ElementTypeMap = new(); + private static Dictionary NonGenericTemplateTypeMap = new(); + + static BotSharpMessageParser() + { + var types = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(assembly => assembly.GetTypes()) + .ToList(); + ElementTypeMap = types + .Where(type => type.Name.EndsWith("Element")) + .ToDictionary(k => k.Name, v => v); + + var richMessageTypes = types + .Where(type => typeof(IRichMessage).IsAssignableFrom(type) && !type.IsInterface && !type.IsAbstract) + .ToDictionary(k => GetRichTypeValue(k), v => v); + GenericTemplateTypeMap = richMessageTypes.Where(p => p.Value.IsGenericType).ToDictionary(k => k.Key, v => v.Value); + NonGenericTemplateTypeMap = richMessageTypes.Where(p => !p.Value.IsGenericType).ToDictionary(k => k.Key, v => v.Value); + } public static IRichMessage? ParseRichMessage(JsonElement root, JsonSerializerOptions options) { IRichMessage? res = null; Type? targetType = null; - JsonElement element; var jsonText = root.GetRawText(); - if (root.TryGetProperty("rich_type", out element)) - { - var richType = element.GetString(); - if (richType == RichTypeEnum.ButtonTemplate) - { - targetType = typeof(ButtonTemplateMessage); - } - else if (richType == RichTypeEnum.MultiSelectTemplate) - { - targetType = typeof(MultiSelectTemplateMessage); - } - else if (richType == RichTypeEnum.QuickReply) - { - targetType = typeof(QuickReplyMessage); - } - else if (richType == RichTypeEnum.CouponTemplate) - { - targetType = typeof(CouponTemplateMessage); - } - else if (richType == RichTypeEnum.Text) - { - targetType = typeof(TextMessage); - } - else if (richType == RichTypeEnum.GenericTemplate) - { - if (root.TryGetProperty("element_type", out element)) - { - var elementType = element.GetString(); - var wrapperType = typeof(GenericTemplateMessage<>); - var genericType = Assembly.GetExecutingAssembly().GetTypes().FirstOrDefault(x => x.Name == elementType); + if (!root.TryGetProperty("rich_type", out var richTypeElement)) return res; - if (wrapperType != null && genericType != null) - { - targetType = wrapperType.MakeGenericType(genericType); - } - } + string? richType = richTypeElement.GetString(); + if (GenericTemplateTypeMap.TryGetValue(richType, out var wrapperType)) + { + if (root.TryGetProperty("element_type", out var elementTypeElement)) + { + string? elementType = elementTypeElement.GetString(); + targetType = CreateGenericElementType(wrapperType, elementType); } } + else if (NonGenericTemplateTypeMap.TryGetValue(richType, out targetType)) + { + // targetType is already set by the dictionary lookup + } if (targetType != null) { @@ -62,6 +58,39 @@ public static class BotSharpMessageParser return res; } + private static Type? CreateGenericElementType(Type wrapperType, string elementTypeName) + { + if (wrapperType != null && ElementTypeMap.TryGetValue(elementTypeName, out var elementType)) + { + return wrapperType.MakeGenericType(elementType); + } + + return null; + } + + private static string GetRichTypeValue(Type type) + { + var richTypeProperty = type.GetProperty("RichType", BindingFlags.Public | BindingFlags.Instance); + if (richTypeProperty != null && richTypeProperty.PropertyType == typeof(string)) + { + return CreateRichMessage(type)?.RichType; + } + return null; + } + + private static dynamic CreateRichMessage(Type type) + { + if (!type.IsGenericType) + { + return Activator.CreateInstance(type); + } + else + { + var genericType = type.MakeGenericType(typeof(object)); + return Activator.CreateInstance(genericType); + } + } + public static ITemplateMessage? ParseTemplateMessage(JsonElement root, JsonSerializerOptions options) { ITemplateMessage? res = null; diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs index 7603f790..968aa30c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs @@ -9,4 +9,5 @@ public static class RichTypeEnum public const string QuickReply = "quick_reply"; public const string Text = "text"; public const string Attachment = "attachment"; + public const string ProductTemplate = "product_template"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs index 05af097a..201131be 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Messaging.Models.RichContent.Template; public class ProductTemplateMessage : IRichMessage, ITemplateMessage { [JsonPropertyName("rich_type")] - public string RichType => RichTypeEnum.GenericTemplate; + public string RichType => RichTypeEnum.ProductTemplate; [JsonPropertyName("text")] [Translate] diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs index 3a37ad54..966701bd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs @@ -8,6 +8,8 @@ public class TextMessage : IRichMessage [Translate] public string Text { get; set; } = string.Empty; + public TextMessage() { } + public TextMessage(string text) { Text = text; From 2ecd6c3649393671ad7c7c6c21a4ed1e837eceda Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 2 Aug 2024 16:24:44 -0500 Subject: [PATCH 02/63] fix audio range request --- .../BotSharp.Abstraction/BotSharp.Abstraction.csproj | 2 +- .../Files/Constants/FileConstants.cs | 9 +++++++++ .../Controllers/ConversationController.cs | 5 ++++- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 446b8869..570f5c92 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs new file mode 100644 index 00000000..683d037c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Files.Constants; + +public class FileConstants +{ + public static readonly IEnumerable AudioTypes = new List + { + ".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma" + }; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 22649d72..1999cfd1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Constants; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; @@ -413,7 +414,9 @@ public class ConversationController : ControllerBase using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read); var bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); - return File(bytes, "application/octet-stream", Path.GetFileName(file)); + var fileExtensions = Path.GetExtension(file).ToLower(); + var enableRangeProcessing = FileConstants.AudioTypes.Contains(fileExtensions); + return File(bytes, "application/octet-stream", Path.GetFileName(file), enableRangeProcessing: enableRangeProcessing); } private async Task OnChunkReceived(HttpResponse response, ChatResponseModel message) From b84aef83215e5ba5c3b6af72910d73d6467d3a6d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 2 Aug 2024 16:26:11 -0500 Subject: [PATCH 03/63] rename --- .../BotSharp.Abstraction/Files/Constants/FileConstants.cs | 2 +- .../BotSharp.OpenAPI/Controllers/ConversationController.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs index 683d037c..dab5bc81 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Constants; public class FileConstants { - public static readonly IEnumerable AudioTypes = new List + public static readonly IEnumerable AudioExtensions = new List { ".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma" }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 1999cfd1..261eb04b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -414,8 +414,8 @@ public class ConversationController : ControllerBase using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read); var bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); - var fileExtensions = Path.GetExtension(file).ToLower(); - var enableRangeProcessing = FileConstants.AudioTypes.Contains(fileExtensions); + var fileExtension = Path.GetExtension(file).ToLower(); + var enableRangeProcessing = FileConstants.AudioExtensions.Contains(fileExtension); return File(bytes, "application/octet-stream", Path.GetFileName(file), enableRangeProcessing: enableRangeProcessing); } From fb9ad1b7d0177530b29aa8171e33b5bd776b6901 Mon Sep 17 00:00:00 2001 From: "LAPTOP-3CFGGVOS\\rabbit" Date: Sun, 4 Aug 2024 23:01:08 +0800 Subject: [PATCH 04/63] optimize paramter type check --- .../Agents/Services/AgentService.LoadAgent.cs | 45 +++++++++++++++++++ .../Services/ConversationStateService.cs | 21 ++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 89c45510..757d9937 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -1,7 +1,12 @@ +using BotSharp.Abstraction.Routing.Models; +using System.Collections.Concurrent; + namespace BotSharp.Core.Agents.Services; public partial class AgentService { + public static ConcurrentDictionary> AgentParameterTypes = new(); + [MemoryCache(10 * 60, perInstanceCache: true)] public async Task LoadAgent(string id) { @@ -49,6 +54,7 @@ public partial class AgentService agent.Instruction = inheritedAgent.Instruction; } } + AddOrUpdateParameters(agent); agent.TemplateDict = new Dictionary(); @@ -96,4 +102,43 @@ public partial class AgentService dict[t.Key] = t.Value; } } + + private void AddOrUpdateParameters(Agent agent) + { + var agentId = agent.Id ?? agent.Name; + if (AgentParameterTypes.ContainsKey(agentId)) return; + + AddOrUpdateRoutesParameters(agentId, agent.RoutingRules); + AddOrUpdateFunctionsParameters(agentId, agent.Functions); + } + + private void AddOrUpdateRoutesParameters(string agentId, List routingRules) + { + if(!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) parameterTypes = new(); + foreach (var rule in routingRules.Where(x => x.Required)) + { + if (string.IsNullOrEmpty(rule.FieldType)) continue; + parameterTypes.TryAdd(rule.Field, rule.FieldType); + } + AgentParameterTypes.TryAdd(agentId, parameterTypes); + } + + private void AddOrUpdateFunctionsParameters(string agentId, List functions) + { + if (!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) parameterTypes = new(); + var parameters = functions.Select(p => p.Parameters); + foreach (var param in parameters) + { + foreach (JsonProperty prop in param.Properties.RootElement.EnumerateObject()) + { + var name = prop.Name; + var node = prop.Value; + if (node.TryGetProperty("type", out var type)) + { + parameterTypes.TryAdd(name, type.GetString()); + } + } + } + AgentParameterTypes.TryAdd(agentId, parameterTypes); + } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 57f09c71..aa518ee7 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -360,9 +360,28 @@ public class ConversationStateService : IConversationStateService, IDisposable stateValue = stateValue?.ToLower(); } - SetState(property.Name, stateValue, source: StateSource.Application); + if (CheckArgType(property.Name, stateValue)) + { + SetState(property.Name, stateValue, source: StateSource.Application); + } } } } } + + private bool CheckArgType(string name, string value) + { + var agentTypes = AgentService.AgentParameterTypes.SelectMany(p => p.Value).ToList(); + var filed = agentTypes.FirstOrDefault(t => t.Key == name); + if (filed.Key != null) + { + return filed.Value switch + { + "boolean" => bool.TryParse(value, out _), + "number" => long.TryParse(value, out _), + _ => true, + }; + } + return true; + } } From e5f1180d32954d5d85785b200c315255e1569a13 Mon Sep 17 00:00:00 2001 From: "LAPTOP-3CFGGVOS\\rabbit" Date: Sun, 4 Aug 2024 23:05:42 +0800 Subject: [PATCH 05/63] Revert "optimize BotSharpMessageParser" This reverts commit 237645033ccb0f47e914589b2a7e95365c12cdbc. --- .../BotSharp.Abstraction.csproj | 1 - .../Messaging/BotSharpMessageParser.cs | 99 +++++++------------ .../Messaging/Enums/RichTypeEnum.cs | 1 - .../Template/ProductTemplateMessage.cs | 2 +- .../Models/RichContent/TextMessage.cs | 2 - 5 files changed, 36 insertions(+), 69 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 94ec0f12..570f5c92 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -25,7 +25,6 @@ - diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs index 483900a4..1f495a44 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs @@ -2,52 +2,56 @@ using BotSharp.Abstraction.Messaging.Models.RichContent.Template; using BotSharp.Abstraction.Messaging.Models.RichContent; using System.Text.Json; using System.Reflection; -using System.Linq; namespace BotSharp.Abstraction.Messaging; public static class BotSharpMessageParser { - private static Dictionary GenericTemplateTypeMap = new(); - private static Dictionary ElementTypeMap = new(); - private static Dictionary NonGenericTemplateTypeMap = new(); - - static BotSharpMessageParser() - { - var types = AppDomain.CurrentDomain.GetAssemblies() - .SelectMany(assembly => assembly.GetTypes()) - .ToList(); - ElementTypeMap = types - .Where(type => type.Name.EndsWith("Element")) - .ToDictionary(k => k.Name, v => v); - - var richMessageTypes = types - .Where(type => typeof(IRichMessage).IsAssignableFrom(type) && !type.IsInterface && !type.IsAbstract) - .ToDictionary(k => GetRichTypeValue(k), v => v); - GenericTemplateTypeMap = richMessageTypes.Where(p => p.Value.IsGenericType).ToDictionary(k => k.Key, v => v.Value); - NonGenericTemplateTypeMap = richMessageTypes.Where(p => !p.Value.IsGenericType).ToDictionary(k => k.Key, v => v.Value); - } public static IRichMessage? ParseRichMessage(JsonElement root, JsonSerializerOptions options) { IRichMessage? res = null; Type? targetType = null; + JsonElement element; var jsonText = root.GetRawText(); - if (!root.TryGetProperty("rich_type", out var richTypeElement)) return res; - - string? richType = richTypeElement.GetString(); - if (GenericTemplateTypeMap.TryGetValue(richType, out var wrapperType)) + if (root.TryGetProperty("rich_type", out element)) { - if (root.TryGetProperty("element_type", out var elementTypeElement)) + var richType = element.GetString(); + if (richType == RichTypeEnum.ButtonTemplate) { - string? elementType = elementTypeElement.GetString(); - targetType = CreateGenericElementType(wrapperType, elementType); + targetType = typeof(ButtonTemplateMessage); + } + else if (richType == RichTypeEnum.MultiSelectTemplate) + { + targetType = typeof(MultiSelectTemplateMessage); + } + else if (richType == RichTypeEnum.QuickReply) + { + targetType = typeof(QuickReplyMessage); + } + else if (richType == RichTypeEnum.CouponTemplate) + { + targetType = typeof(CouponTemplateMessage); + } + else if (richType == RichTypeEnum.Text) + { + targetType = typeof(TextMessage); + } + else if (richType == RichTypeEnum.GenericTemplate) + { + if (root.TryGetProperty("element_type", out element)) + { + var elementType = element.GetString(); + var wrapperType = typeof(GenericTemplateMessage<>); + var genericType = Assembly.GetExecutingAssembly().GetTypes().FirstOrDefault(x => x.Name == elementType); + + if (wrapperType != null && genericType != null) + { + targetType = wrapperType.MakeGenericType(genericType); + } + } } - } - else if (NonGenericTemplateTypeMap.TryGetValue(richType, out targetType)) - { - // targetType is already set by the dictionary lookup } if (targetType != null) @@ -58,39 +62,6 @@ public static class BotSharpMessageParser return res; } - private static Type? CreateGenericElementType(Type wrapperType, string elementTypeName) - { - if (wrapperType != null && ElementTypeMap.TryGetValue(elementTypeName, out var elementType)) - { - return wrapperType.MakeGenericType(elementType); - } - - return null; - } - - private static string GetRichTypeValue(Type type) - { - var richTypeProperty = type.GetProperty("RichType", BindingFlags.Public | BindingFlags.Instance); - if (richTypeProperty != null && richTypeProperty.PropertyType == typeof(string)) - { - return CreateRichMessage(type)?.RichType; - } - return null; - } - - private static dynamic CreateRichMessage(Type type) - { - if (!type.IsGenericType) - { - return Activator.CreateInstance(type); - } - else - { - var genericType = type.MakeGenericType(typeof(object)); - return Activator.CreateInstance(genericType); - } - } - public static ITemplateMessage? ParseTemplateMessage(JsonElement root, JsonSerializerOptions options) { ITemplateMessage? res = null; diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs index 968aa30c..7603f790 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs @@ -9,5 +9,4 @@ public static class RichTypeEnum public const string QuickReply = "quick_reply"; public const string Text = "text"; public const string Attachment = "attachment"; - public const string ProductTemplate = "product_template"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs index 201131be..05af097a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProductTemplateMessage.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Messaging.Models.RichContent.Template; public class ProductTemplateMessage : IRichMessage, ITemplateMessage { [JsonPropertyName("rich_type")] - public string RichType => RichTypeEnum.ProductTemplate; + public string RichType => RichTypeEnum.GenericTemplate; [JsonPropertyName("text")] [Translate] diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs index 966701bd..3a37ad54 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs @@ -8,8 +8,6 @@ public class TextMessage : IRichMessage [Translate] public string Text { get; set; } = string.Empty; - public TextMessage() { } - public TextMessage(string text) { Text = text; From a16602a4fdaff2af7a73aaee1f19aa68837e7c32 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 4 Aug 2024 22:46:41 -0500 Subject: [PATCH 06/63] Add ExcludeResponseUrls to PageActionArgs --- .../BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs | 4 ++++ .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 5 +++-- .../Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index 661b0da4..54707cec 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -15,6 +15,10 @@ public class PageActionArgs /// This value has to be set to true if you want to get the page XHR/ Fetch responses /// public bool OpenNewTab { get; set; } = false; + /// + /// Exclude urls for XHR/ Fetch responses + /// + public string[]? ExcludeResponseUrls { get; set; } public bool UseExistingPage { get; set; } = false; public bool WaitForNetworkIdle { get; set; } = true; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 79eec038..0a1eea3b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -113,7 +113,7 @@ public class PlaywrightInstance : IDisposable return _contexts[ctxId]; } - public async Task NewPage(MessageInfo message) + public async Task NewPage(MessageInfo message, string[]? excludeResponseUrls = null) { var context = await GetContext(message.ContextId); var page = await context.NewPageAsync(); @@ -128,7 +128,8 @@ public class PlaywrightInstance : IDisposable if (e.Status != 204 && e.Headers.ContainsKey("content-type") && e.Headers["content-type"].Contains("application/json") && - (e.Request.ResourceType == "fetch" || e.Request.ResourceType == "xhr")) + (e.Request.ResourceType == "fetch" || e.Request.ResourceType == "xhr") && + (excludeResponseUrls == null || !excludeResponseUrls.Any(url => e.Url.ToLower().Contains(url)))) { Serilog.Log.Information($"{e.Request.Method}: {e.Url}"); JsonElement? json = null; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 76e63171..c3121855 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -10,7 +10,7 @@ public partial class PlaywrightWebDriver { var page = args.UseExistingPage ? _instance.GetPage(message.ContextId, pattern: args.Url) : - await _instance.NewPage(message); + await _instance.NewPage(message, excludeResponseUrls: args.ExcludeResponseUrls); if (args.UseExistingPage && page != null && page.Url != "about:blank") { @@ -23,7 +23,7 @@ public partial class PlaywrightWebDriver if (args.UseExistingPage && args.OpenNewTab && page != null && page.Url == "about:blank") { - page = await _instance.NewPage(message); + page = await _instance.NewPage(message, excludeResponseUrls: args.ExcludeResponseUrls); } var response = await page.GotoAsync(args.Url, new PageGotoOptions From b1ea7fdbbc292c12ef2b7edb35f2bfbe5ce44c80 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 5 Aug 2024 14:25:37 -0500 Subject: [PATCH 07/63] use 4o-mini --- .../01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 536ca340..9fc0c389 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -7,5 +7,11 @@ "updatedDateTime": "2024-01-15T14:39:32Z", "iconUrl": "/images/logo.png", "disabled": false, - "isPublic": true + "isPublic": true, + "llmConfig": { + "is_inherit": false, + "provider": "openai", + "model": "gpt-4o-mini", + "max_recursion_depth": 3 + } } \ No newline at end of file From ea441c75e8ffde6bb87cb39aa371d68fea9c61e7 Mon Sep 17 00:00:00 2001 From: Haiping Date: Mon, 5 Aug 2024 16:32:22 -0500 Subject: [PATCH 08/63] Fix conversation_end --- .../Conversations/Services/ConversationService.SendMessage.cs | 1 + .../Routing/Handlers/RouteToAgentRoutingHandler.cs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 26c62b62..36b588ae 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -150,6 +150,7 @@ public partial class ConversationService await HookEmitter.Emit(_services, async hook => await hook.OnConversationEnding(response) ); + response.FunctionName = "conversation_end"; } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index 4a686c6b..e8431cce 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -26,6 +26,10 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler new ParameterPropertyDef("user_goal_agent", "agent who can acheive user initial task, must align with user_goal_description.", required: true), + new ParameterPropertyDef("conversation_end", + "user is ending the conversation.", + type: "boolean", + required: true), new ParameterPropertyDef("is_new_task", "whether the user is requesting a new task that is different from the previous topic.", type: "boolean") From 7e51ea6da27a337078153d70cb2be6b4a4519c7f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 6 Aug 2024 14:01:26 -0500 Subject: [PATCH 09/63] add enums --- .../Knowledges/Enums/KnowledgeCollectionName.cs | 6 ++++++ .../Knowledges/Enums/KnowledgePayloadName.cs | 10 ++++++++++ .../Functions/ConfirmKnowledgePersistenceFn.cs | 6 ------ .../Functions/KnowledgeRetrievalFn.cs | 13 ++----------- .../Functions/MemorizeKnowledgeFn.cs | 5 ++--- .../Hooks/KnowledgeBaseAgentHook.cs | 6 ------ .../Hooks/KnowledgeBaseUtilityHook.cs | 2 -- .../Services/KnowledgeService.cs | 6 +++--- .../Services/TextChopperService.cs | 2 +- src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs | 13 ++++++++++++- src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 10 +--------- src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs | 1 - src/Plugins/BotSharp.Plugin.Qdrant/Using.cs | 6 ++++++ 13 files changed, 43 insertions(+), 43 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs create mode 100644 src/Plugins/BotSharp.Plugin.Qdrant/Using.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs new file mode 100644 index 00000000..7d92e504 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Enums; + +public static class KnowledgeCollectionName +{ + public static string BotSharp = nameof(BotSharp); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs new file mode 100644 index 00000000..9d95967f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Knowledges.Enums; + +public static class KnowledgePayloadName +{ + public static string Text = "text"; + public static string Question = "question"; + public static string Answer = "answer"; + public static string Request = "request"; + public static string Response = "response"; +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs index 521dc221..bac82d2e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs @@ -1,9 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Messaging.Enums; -using BotSharp.Abstraction.Messaging.Models.RichContent.Template; -using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.Messaging; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class ConfirmKnowledgePersistenceFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 3eaa72f5..f8d83e0c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback @@ -23,15 +20,9 @@ public class KnowledgeRetrievalFn : IFunctionCallback var embedding = _services.GetServices() .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); - var vector = await embedding.GetVectorsAsync(new List - { - args.Question - }); - + var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetRequiredService(); - - var id = Utilities.HashTextMd5(args.Question); - var knowledges = await vectorDb.Search("lessen", vector[0], "answer"); + var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer); if (knowledges.Count > 0) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 23b49b94..1944ad99 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Functions; using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.KnowledgeBase.Functions; @@ -30,10 +29,10 @@ public class MemorizeKnowledgeFn : IFunctionCallback var vectorDb = _services.GetRequiredService(); - await vectorDb.CreateCollection("lessen", vector[0].Length); + await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length); var id = Utilities.HashTextMd5(args.Question); - var result = await vectorDb.Upsert("lessen", id, vector[0], + var result = await vectorDb.Upsert(KnowledgeCollectionName.BotSharp, id, vector[0], args.Question, new Dictionary { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs index 2eebf75c..f8296520 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs @@ -1,9 +1,3 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; -using BotSharp.Plugin.KnowledgeBase.Enum; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseAgentHook : AgentHookBase, IAgentHook diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs index fd163cbf..cd428136 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.KnowledgeBase.Enum; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseUtilityHook : IAgentUtilityHook diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 7ac5cf4c..a5d42442 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -28,11 +28,11 @@ public partial class KnowledgeService : IKnowledgeService var db = GetVectorDb(); var textEmbedding = GetTextEmbedding(); - await db.CreateCollection("shared", textEmbedding.Dimension); + await db.CreateCollection(KnowledgeCollectionName.BotSharp, textEmbedding.Dimension); foreach (var line in lines) { var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert("shared", idStart.ToString(), vec, line); + await db.Upsert(KnowledgeCollectionName.BotSharp, idStart.ToString(), vec, line); idStart++; Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); } @@ -68,7 +68,7 @@ public partial class KnowledgeService : IKnowledgeService // Vector search var db = GetVectorDb(); - var result = await db.Search("shared", vector, "answer", limit: 10); + var result = await db.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer, limit: 10); // Restore return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}")); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs index 96dc021d..88c77641 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs @@ -16,7 +16,7 @@ public class TextChopperService : ITextChopper var chunks = new List(); var words = content.Split(' ') - .Where(x => !string.IsNullOrEmpty(x)) + .Where(x => !string.IsNullOrWhiteSpace(x)) .ToList(); var chunk = ""; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 60db13a6..b92f919f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -17,7 +17,18 @@ global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Agents.Settings; global using BotSharp.Abstraction.Conversations.Settings; global using BotSharp.Abstraction.Knowledges.Settings; +global using BotSharp.Abstraction.Knowledges.Enums; global using BotSharp.Abstraction.VectorStorage; global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.Abstraction.MLTasks; -global using BotSharp.Plugin.KnowledgeBase.Services; \ No newline at end of file +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Abstraction.Messaging.Models.RichContent.Template; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Plugin.KnowledgeBase.Services; +global using BotSharp.Plugin.KnowledgeBase.Enum; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index e698d5e3..0c93613b 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,13 +1,5 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.VectorStorage; -using Microsoft.Extensions.DependencyInjection; using Qdrant.Client; using Qdrant.Client.Grpc; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; namespace BotSharp.Plugin.Qdrant; @@ -80,7 +72,7 @@ public class QdrantDb : IVectorDb Payload = { - { "text", text } + { KnowledgePayloadName.Text, text } } }; diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs index 8cfd137b..a3bc116a 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; -using BotSharp.Abstraction.VectorStorage; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs new file mode 100644 index 00000000..b144270c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs @@ -0,0 +1,6 @@ +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Threading.Tasks; +global using BotSharp.Abstraction.VectorStorage; +global using BotSharp.Abstraction.Knowledges.Enums; \ No newline at end of file From a015b4bfd0b5d3db0662f724f0aafff5c237bfd8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 6 Aug 2024 14:05:40 -0500 Subject: [PATCH 10/63] use enum --- .../Functions/MemorizeKnowledgeFn.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 1944ad99..0ef0f950 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -36,7 +36,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback args.Question, new Dictionary { - { "answer", args.Answer } + { KnowledgePayloadName.Answer, args.Answer } }); message.Content = result ? "Saved to my brain" : "I forgot it"; From 93646a01aedc5163cc15130786fdf67bf4f51807 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 6 Aug 2024 16:02:02 -0500 Subject: [PATCH 11/63] add knowledge collection info --- .../Knowledges/IKnowledgeService.cs | 4 ++++ .../Models/KnowledgeCollectionInfo.cs | 7 ++++++ .../VectorStorage/IVectorDb.cs | 3 +++ .../Controllers/KnowledgeBaseController.cs | 9 ++++++++ .../KnowledgeCollectionInfoViewModel.cs | 22 +++++++++++++++++++ .../MemVecDb/MemVectorDatabase.cs | 14 ++++++++++++ .../Services/KnowledgeService.List.cs | 18 +++++++++++++++ .../Services/KnowledgeService.cs | 5 ++++- .../Providers/FaissDb.cs | 6 +++++ .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 11 ++++++++++ 10 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 5b2b795b..60c86549 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -11,4 +11,8 @@ public interface IKnowledgeService Task EmbedKnowledge(KnowledgeCreationModel knowledge); Task GetKnowledges(KnowledgeRetrievalModel retrievalModel); Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); + + #region List + Task GetKnowledgeCollectionInfo(string collectionName); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs new file mode 100644 index 00000000..714c2b3f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionInfo +{ + public ulong DataCount { get; set; } + public ulong VectorCount { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 68aaaa78..2e9271a2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -1,8 +1,11 @@ +using BotSharp.Abstraction.Knowledges.Models; + namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { Task> GetCollections(); + Task GetCollectionInfo(string collectionName); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 6ad463b3..92482b79 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.OpenAPI.ViewModels.Knowledges; using Microsoft.AspNetCore.Http; namespace BotSharp.OpenAPI.Controllers; @@ -90,4 +91,12 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } + + [HttpGet("/knowledge/info")] + public async Task GetKnowledgeCollectionInfo([FromQuery] string collectionName) + { + var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName); + return KnowledgeCollectionInfoViewModel.ToViewModel(info); + } + } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs new file mode 100644 index 00000000..65f5d979 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs @@ -0,0 +1,22 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionInfoViewModel +{ + [JsonPropertyName("data_count")] + public ulong DataCount { get; set; } + + [JsonPropertyName("vector_count")] + public ulong VectorCount { get; set; } + + public static KnowledgeCollectionInfoViewModel ToViewModel(KnowledgeCollectionInfo info) + { + return new KnowledgeCollectionInfoViewModel + { + DataCount = info.DataCount, + VectorCount = info.VectorCount + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index c46ccdf5..d0d6080d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -18,6 +18,20 @@ public class MemVectorDatabase : IVectorDb return _collections.Select(x => x.Key).ToList(); } + public async Task GetCollectionInfo(string collectionName) + { + if (_vectors.TryGetValue(collectionName, out var info)) + { + info = new List(); + } + + return new KnowledgeCollectionInfo + { + DataCount = (ulong)(info?.Count ?? 0), + VectorCount = (ulong)(info?.Count(x => x.Vector != null && x.Vector.Length > 0) ?? 0) + }; + } + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { if (!_vectors.ContainsKey(collectionName)) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs new file mode 100644 index 00000000..ee8562d2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task GetKnowledgeCollectionInfo(string collectionName) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionInfo(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collectio info. {ex.Message}\r\n{ex.InnerException}"); + return new KnowledgeCollectionInfo(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index a5d42442..cdfff9db 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -5,14 +5,17 @@ public partial class KnowledgeService : IKnowledgeService private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; private readonly ITextChopper _textChopper; + private readonly ILogger _logger; public KnowledgeService(IServiceProvider services, KnowledgeBaseSettings settings, - ITextChopper textChopper) + ITextChopper textChopper, + ILogger logger) { _services = services; _settings = settings; _textChopper = textChopper; + _logger = logger; } public async Task EmbedKnowledge(KnowledgeCreationModel knowledge) diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 48e2be17..805861fa 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage; using System; using System.Collections.Generic; @@ -12,6 +13,11 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } + public Task GetCollectionInfo(string collectionName) + { + throw new NotImplementedException(); + } + public Task> GetCollections() { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 0c93613b..72e70e92 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Models; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -38,6 +39,16 @@ public class QdrantDb : IVectorDb return collections.ToList(); } + public async Task GetCollectionInfo(string collectionName) + { + var info = await GetClient().GetCollectionInfoAsync(collectionName); + return new KnowledgeCollectionInfo + { + DataCount = info.PointsCount, + VectorCount = info.VectorsCount + }; + } + public async Task CreateCollection(string collectionName, int dim) { var collections = await GetCollections(); From 6ab7c0254572c097787321e4b1f81497f775b721 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 6 Aug 2024 17:43:11 -0500 Subject: [PATCH 12/63] add knowledge data --- .../Knowledges/IKnowledgeService.cs | 1 + .../Models/KnowledgeCollectionData.cs | 9 +++++ .../Knowledges/Models/KnowledgeFilter.cs | 10 +++++ .../Utilities/UuidPagination.cs | 15 +++++++ .../VectorStorage/IVectorDb.cs | 1 + .../Controllers/KnowledgeBaseController.cs | 16 +++++++- .../KnowledgeCollectionDataViewModel.cs | 31 ++++++++++++++ .../MemVecDb/MemVectorDatabase.cs | 5 +++ .../Services/KnowledgeService.List.cs | 14 +++++++ .../Providers/FaissDb.cs | 6 +++ .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 40 +++++++++++++++++-- src/Plugins/BotSharp.Plugin.Qdrant/Using.cs | 3 +- 12 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 60c86549..cd0035c7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,5 +14,6 @@ public interface IKnowledgeService #region List Task GetKnowledgeCollectionInfo(string collectionName); + Task> GetKnowledgeCollectionData(KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs new file mode 100644 index 00000000..de3e521b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeCollectionData +{ + public string Id { get; set; } + public string Text { get; set; } + public string Answer { get; set; } + public float[]? Vector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs new file mode 100644 index 00000000..86957b48 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeFilter : UuidPagination +{ + [JsonPropertyName("collection_name")] + public string CollectionName { get; set; } + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs new file mode 100644 index 00000000..45f46422 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Abstraction.Utilities; + +public class UuidPagination : Pagination +{ + [JsonPropertyName("start_id")] + public string? StartId { get; set; } +} + +public class UuidPagedItems : PagedItems +{ + public new ulong Count { get; set; } + + [JsonPropertyName("next_id")] + public string? NextId { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 2e9271a2..1e1122a5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,6 +6,7 @@ public interface IVectorDb { Task> GetCollections(); Task GetCollectionInfo(string collectionName); + Task> GetCollectionData(KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 92482b79..21df9840 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -92,11 +92,25 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } - [HttpGet("/knowledge/info")] + [HttpGet("/knowledge/collection/info")] public async Task GetKnowledgeCollectionInfo([FromQuery] string collectionName) { var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName); return KnowledgeCollectionInfoViewModel.ToViewModel(info); } + [HttpPost("/knowledge/collection/data")] + public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) + { + var data = await _knowledgeService.GetKnowledgeCollectionData(filter); + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? + .ToList() ?? new List(); + + return new UuidPagedItems + { + Count = data.Count, + NextId = data.NextId, + Items = items + }; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs new file mode 100644 index 00000000..7f8aa087 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -0,0 +1,31 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeCollectionDataViewModel +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("text")] + public string Text { get; set; } + + [JsonPropertyName("answer")] + public string Answer { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + public static KnowledgeCollectionDataViewModel ToViewModel(KnowledgeCollectionData data) + { + return new KnowledgeCollectionDataViewModel + { + Id = data.Id, + Text = data.Text, + Answer = data.Answer, + Vector = data.Vector + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index d0d6080d..de74e669 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -32,6 +32,11 @@ public class MemVectorDatabase : IVectorDb }; } + public Task> GetCollectionData(KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { if (!_vectors.ContainsKey(collectionName)) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index ee8562d2..950a5246 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -15,4 +15,18 @@ public partial class KnowledgeService return new KnowledgeCollectionInfo(); } } + + public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionData(filter); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}"); + return new UuidPagedItems(); + } + } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 805861fa..b6831e05 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using System; using System.Collections.Generic; @@ -18,6 +19,11 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } + public Task> GetCollectionData(KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + public Task> GetCollections() { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 72e70e92..9fb9e57d 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -41,7 +41,12 @@ public class QdrantDb : IVectorDb public async Task GetCollectionInfo(string collectionName) { - var info = await GetClient().GetCollectionInfoAsync(collectionName); + var client = GetClient(); + + var exists = await client.CollectionExistsAsync(collectionName); + if (!exists) return new KnowledgeCollectionInfo(); + + var info = await client.GetCollectionInfoAsync(collectionName); return new KnowledgeCollectionInfo { DataCount = info.PointsCount, @@ -49,6 +54,35 @@ public class QdrantDb : IVectorDb }; } + public async Task> GetCollectionData(KnowledgeFilter filter) + { + var client = GetClient(); + var exists = await client.CollectionExistsAsync(filter.CollectionName); + if (!exists) + { + return new UuidPagedItems(); + } + + var totalPointCount = await client.CountAsync(filter.CollectionName); + var response = await client.ScrollAsync(filter.CollectionName, limit: (uint)filter.Size, + offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, + vectorsSelector: filter.WithVector); + var points = response?.Result?.Select(x => new KnowledgeCollectionData + { + Id = x.Id?.Uuid ?? string.Empty, + Text = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty, + Answer = x.Payload.ContainsKey(KnowledgePayloadName.Answer) ? x.Payload[KnowledgePayloadName.Answer].StringValue : string.Empty, + Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null + })?.ToList() ?? new List(); + + return new UuidPagedItems + { + Count = totalPointCount, + NextId = response?.NextPageOffset?.Uuid, + Items = points + }; + } + public async Task CreateCollection(string collectionName, int dim) { var collections = await GetCollections(); @@ -81,7 +115,7 @@ public class QdrantDb : IVectorDb }, Vectors = vector, - Payload = + Payload = { { KnowledgePayloadName.Text, text } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs index b144270c..dc7e9747 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs @@ -3,4 +3,5 @@ global using System.Collections.Generic; global using System.Linq; global using System.Threading.Tasks; global using BotSharp.Abstraction.VectorStorage; -global using BotSharp.Abstraction.Knowledges.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Knowledges.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file From 98a02caf714e50a428c9a45cfad50ddacfe6c6b6 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 6 Aug 2024 19:18:55 -0500 Subject: [PATCH 13/63] _activePage --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 9 +++++++-- .../PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 0a1eea3b..65b45013 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -9,6 +9,7 @@ public class PlaywrightInstance : IDisposable public IServiceProvider Services => _services; Dictionary _contexts = new Dictionary(); Dictionary> _pages = new Dictionary>(); + IPage? _activePage = null; /// /// ContextId and BrowserContext @@ -25,17 +26,19 @@ public class PlaywrightInstance : IDisposable _services = services; } - public IPage GetPage(string contextId, string? pattern = null) + public IPage? GetPage(string contextId, string? pattern = null) { if (string.IsNullOrEmpty(pattern)) { - return _contexts[contextId].Pages.LastOrDefault(); + return _activePage ?? _contexts[contextId].Pages.LastOrDefault(); } foreach (var page in _contexts[contextId].Pages) { if (page.Url.ToLower() == pattern.ToLower()) { + _activePage = page; + page.BringToFrontAsync().Wait(); return page; } } @@ -84,6 +87,7 @@ public class PlaywrightInstance : IDisposable _contexts[ctxId].Page += async (sender, page) => { + _activePage = page; _pages[ctxId].Add(page); page.Close += async (sender, e) => { @@ -200,6 +204,7 @@ public class PlaywrightInstance : IDisposable if (page != null) { await page.CloseAsync(); + _activePage = _pages[ctxId].LastOrDefault(); } } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index c3121855..5ddbfcbd 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -12,7 +12,7 @@ public partial class PlaywrightWebDriver _instance.GetPage(message.ContextId, pattern: args.Url) : await _instance.NewPage(message, excludeResponseUrls: args.ExcludeResponseUrls); - if (args.UseExistingPage && page != null && page.Url != "about:blank") + if (args.UseExistingPage && page != null && page.Url == args.Url) { Serilog.Log.Information($"goto existing page: {args.Url}"); result.IsSuccess = true; From ba9180120db22ac8fd25c2839c6fc4e71a926827 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 09:57:07 -0500 Subject: [PATCH 14/63] rename --- .../BotSharp.Abstraction/Knowledges/IKnowledgeService.cs | 2 +- .../Knowledges/Models/KnowledgeCollectionData.cs | 2 +- .../Knowledges/Models/KnowledgeFilter.cs | 2 +- .../{UuidPagination.cs => StringIdPagination.cs} | 4 ++-- .../BotSharp.Abstraction/VectorStorage/IVectorDb.cs | 2 +- .../Controllers/KnowledgeBaseController.cs | 4 ++-- .../Knowledges/KnowledgeCollectionDataViewModel.cs | 6 +++--- .../MemVecDb/MemVectorDatabase.cs | 2 +- .../Services/KnowledgeService.List.cs | 4 ++-- src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs | 2 +- src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 8 ++++---- 11 files changed, 19 insertions(+), 19 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Utilities/{UuidPagination.cs => StringIdPagination.cs} (71%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index cd0035c7..f7e1464e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,6 +14,6 @@ public interface IKnowledgeService #region List Task GetKnowledgeCollectionInfo(string collectionName); - Task> GetKnowledgeCollectionData(KnowledgeFilter filter); + Task> GetKnowledgeCollectionData(KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs index de3e521b..d013529f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeCollectionData { public string Id { get; set; } - public string Text { get; set; } + public string Question { get; set; } public string Answer { get; set; } public float[]? Vector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs index 86957b48..553b0b2c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Knowledges.Models; -public class KnowledgeFilter : UuidPagination +public class KnowledgeFilter : StringIdPagination { [JsonPropertyName("collection_name")] public string CollectionName { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs similarity index 71% rename from src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs rename to src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs index 45f46422..d8e4d355 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/UuidPagination.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs @@ -1,12 +1,12 @@ namespace BotSharp.Abstraction.Utilities; -public class UuidPagination : Pagination +public class StringIdPagination : Pagination { [JsonPropertyName("start_id")] public string? StartId { get; set; } } -public class UuidPagedItems : PagedItems +public class StringIdPagedItems : PagedItems { public new ulong Count { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 1e1122a5..ce61b134 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,7 +6,7 @@ public interface IVectorDb { Task> GetCollections(); Task GetCollectionInfo(string collectionName); - Task> GetCollectionData(KnowledgeFilter filter); + Task> GetCollectionData(KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 21df9840..df17bfe2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -100,13 +100,13 @@ public class KnowledgeBaseController : ControllerBase } [HttpPost("/knowledge/collection/data")] - public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) + public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) { var data = await _knowledgeService.GetKnowledgeCollectionData(filter); var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? .ToList() ?? new List(); - return new UuidPagedItems + return new StringIdPagedItems { Count = data.Count, NextId = data.NextId, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs index 7f8aa087..16ebadda 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -8,8 +8,8 @@ public class KnowledgeCollectionDataViewModel [JsonPropertyName("id")] public string Id { get; set; } - [JsonPropertyName("text")] - public string Text { get; set; } + [JsonPropertyName("question")] + public string Question { get; set; } [JsonPropertyName("answer")] public string Answer { get; set; } @@ -23,7 +23,7 @@ public class KnowledgeCollectionDataViewModel return new KnowledgeCollectionDataViewModel { Id = data.Id, - Text = data.Text, + Question = data.Question, Answer = data.Answer, Vector = data.Vector }; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index de74e669..a7c7a7f4 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -32,7 +32,7 @@ public class MemVectorDatabase : IVectorDb }; } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index 950a5246..3f9ae419 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -16,7 +16,7 @@ public partial class KnowledgeService } } - public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) + public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) { try { @@ -26,7 +26,7 @@ public partial class KnowledgeService catch (Exception ex) { _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}"); - return new UuidPagedItems(); + return new StringIdPagedItems(); } } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index b6831e05..a06df43c 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -19,7 +19,7 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 9fb9e57d..2a75a414 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -54,13 +54,13 @@ public class QdrantDb : IVectorDb }; } - public async Task> GetCollectionData(KnowledgeFilter filter) + public async Task> GetCollectionData(KnowledgeFilter filter) { var client = GetClient(); var exists = await client.CollectionExistsAsync(filter.CollectionName); if (!exists) { - return new UuidPagedItems(); + return new StringIdPagedItems(); } var totalPointCount = await client.CountAsync(filter.CollectionName); @@ -70,12 +70,12 @@ public class QdrantDb : IVectorDb var points = response?.Result?.Select(x => new KnowledgeCollectionData { Id = x.Id?.Uuid ?? string.Empty, - Text = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty, + Question = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty, Answer = x.Payload.ContainsKey(KnowledgePayloadName.Answer) ? x.Payload[KnowledgePayloadName.Answer].StringValue : string.Empty, Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null })?.ToList() ?? new List(); - return new UuidPagedItems + return new StringIdPagedItems { Count = totalPointCount, NextId = response?.NextPageOffset?.Uuid, From ac4e856f7b286e405c9d4e5d77eb02ac2292921d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 10:17:28 -0500 Subject: [PATCH 15/63] fix missing implementation --- .../SemanticKernelMemoryStoreProvider.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 20fdeecc..8a06d266 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -1,8 +1,8 @@ +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using Microsoft.SemanticKernel.Memory; -using System; using System.Collections.Generic; -using System.Text; using System.Threading.Tasks; namespace BotSharp.Plugin.SemanticKernel @@ -24,6 +24,16 @@ namespace BotSharp.Plugin.SemanticKernel await _memoryStore.CreateCollectionAsync(collectionName); } + public Task> GetCollectionData(KnowledgeFilter filter) + { + throw new System.NotImplementedException(); + } + + public Task GetCollectionInfo(string collectionName) + { + throw new System.NotImplementedException(); + } + public async Task> GetCollections() { var result = new List(); From 1ba98db5c97457790f6bf852da85cd772c38ea1a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 11:50:23 -0500 Subject: [PATCH 16/63] split collection name --- .../Knowledges/IKnowledgeService.cs | 2 +- .../Knowledges/Models/KnowledgeFilter.cs | 3 --- .../VectorStorage/IVectorDb.cs | 2 +- .../Controllers/KnowledgeBaseController.cs | 14 +++++++------- .../MemVecDb/MemVectorDatabase.cs | 2 +- .../Services/KnowledgeService.List.cs | 4 ++-- .../BotSharp.Plugin.MetaAI/Providers/FaissDb.cs | 2 +- src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 8 ++++---- .../SemanticKernelMemoryStoreProvider.cs | 2 +- 9 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index f7e1464e..b93e905b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,6 +14,6 @@ public interface IKnowledgeService #region List Task GetKnowledgeCollectionInfo(string collectionName); - Task> GetKnowledgeCollectionData(KnowledgeFilter filter); + Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs index 553b0b2c..d2d9c490 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs @@ -2,9 +2,6 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeFilter : StringIdPagination { - [JsonPropertyName("collection_name")] - public string CollectionName { get; set; } - [JsonPropertyName("with_vector")] public bool WithVector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index ce61b134..266dd968 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -6,7 +6,7 @@ public interface IVectorDb { Task> GetCollections(); Task GetCollectionInfo(string collectionName); - Task> GetCollectionData(KnowledgeFilter filter); + Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index df17bfe2..ae67ca37 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -92,17 +92,17 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } - [HttpGet("/knowledge/collection/info")] - public async Task GetKnowledgeCollectionInfo([FromQuery] string collectionName) + [HttpGet("/knowledge/{collection}/info")] + public async Task GetKnowledgeCollectionInfo([FromRoute] string collection) { - var info = await _knowledgeService.GetKnowledgeCollectionInfo(collectionName); + var info = await _knowledgeService.GetKnowledgeCollectionInfo(collection); return KnowledgeCollectionInfoViewModel.ToViewModel(info); } - [HttpPost("/knowledge/collection/data")] - public async Task> GetKnowledgeCollectionData([FromBody] KnowledgeFilter filter) - { - var data = await _knowledgeService.GetKnowledgeCollectionData(filter); + [HttpPost("/knowledge/{collection}/data")] + public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) + {; + var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? .ToList() ?? new List(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index a7c7a7f4..b481d127 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -32,7 +32,7 @@ public class MemVectorDatabase : IVectorDb }; } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index 3f9ae419..dff57ec6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -16,12 +16,12 @@ public partial class KnowledgeService } } - public async Task> GetKnowledgeCollectionData(KnowledgeFilter filter) + public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) { try { var db = GetVectorDb(); - return await db.GetCollectionData(filter); + return await db.GetCollectionData(collectionName, filter); } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index a06df43c..ad408824 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -19,7 +19,7 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 2a75a414..997215ca 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -54,17 +54,17 @@ public class QdrantDb : IVectorDb }; } - public async Task> GetCollectionData(KnowledgeFilter filter) + public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { var client = GetClient(); - var exists = await client.CollectionExistsAsync(filter.CollectionName); + var exists = await client.CollectionExistsAsync(collectionName); if (!exists) { return new StringIdPagedItems(); } - var totalPointCount = await client.CountAsync(filter.CollectionName); - var response = await client.ScrollAsync(filter.CollectionName, limit: (uint)filter.Size, + var totalPointCount = await client.CountAsync(collectionName); + var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, vectorsSelector: filter.WithVector); var points = response?.Result?.Select(x => new KnowledgeCollectionData diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 8a06d266..876bd149 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -24,7 +24,7 @@ namespace BotSharp.Plugin.SemanticKernel await _memoryStore.CreateCollectionAsync(collectionName); } - public Task> GetCollectionData(KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new System.NotImplementedException(); } From cdf02c10c5e791035b0b99c99797c524ab8ad6a3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 13:18:41 -0500 Subject: [PATCH 17/63] remove knowledge info --- .../Knowledges/IKnowledgeService.cs | 1 - .../VectorStorage/IVectorDb.cs | 1 - .../Controllers/KnowledgeBaseController.cs | 6 ----- .../KnowledgeCollectionInfoViewModel.cs | 22 ------------------- .../MemVecDb/MemVectorDatabase.cs | 14 ------------ .../Services/KnowledgeService.List.cs | 14 ------------ .../Providers/FaissDb.cs | 5 ----- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 16 +------------- .../SemanticKernelMemoryStoreProvider.cs | 5 ----- 9 files changed, 1 insertion(+), 83 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index b93e905b..f2eb82fe 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -13,7 +13,6 @@ public interface IKnowledgeService Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); #region List - Task GetKnowledgeCollectionInfo(string collectionName); Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 266dd968..50788756 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -5,7 +5,6 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { Task> GetCollections(); - Task GetCollectionInfo(string collectionName); Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index ae67ca37..cd01ec05 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -92,12 +92,6 @@ public class KnowledgeBaseController : ControllerBase return Ok(new { count = files.Count, size }); } - [HttpGet("/knowledge/{collection}/info")] - public async Task GetKnowledgeCollectionInfo([FromRoute] string collection) - { - var info = await _knowledgeService.GetKnowledgeCollectionInfo(collection); - return KnowledgeCollectionInfoViewModel.ToViewModel(info); - } [HttpPost("/knowledge/{collection}/data")] public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs deleted file mode 100644 index 65f5d979..00000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionInfoViewModel.cs +++ /dev/null @@ -1,22 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Models; -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class KnowledgeCollectionInfoViewModel -{ - [JsonPropertyName("data_count")] - public ulong DataCount { get; set; } - - [JsonPropertyName("vector_count")] - public ulong VectorCount { get; set; } - - public static KnowledgeCollectionInfoViewModel ToViewModel(KnowledgeCollectionInfo info) - { - return new KnowledgeCollectionInfoViewModel - { - DataCount = info.DataCount, - VectorCount = info.VectorCount - }; - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index b481d127..61598a71 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -18,20 +18,6 @@ public class MemVectorDatabase : IVectorDb return _collections.Select(x => x.Key).ToList(); } - public async Task GetCollectionInfo(string collectionName) - { - if (_vectors.TryGetValue(collectionName, out var info)) - { - info = new List(); - } - - return new KnowledgeCollectionInfo - { - DataCount = (ulong)(info?.Count ?? 0), - VectorCount = (ulong)(info?.Count(x => x.Vector != null && x.Vector.Length > 0) ?? 0) - }; - } - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index dff57ec6..2f8547f4 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -2,20 +2,6 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task GetKnowledgeCollectionInfo(string collectionName) - { - try - { - var db = GetVectorDb(); - return await db.GetCollectionInfo(collectionName); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting knowledge collectio info. {ex.Message}\r\n{ex.InnerException}"); - return new KnowledgeCollectionInfo(); - } - } - public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) { try diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index ad408824..5983d49a 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -14,11 +14,6 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task GetCollectionInfo(string collectionName) - { - throw new NotImplementedException(); - } - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 997215ca..d331e28e 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -39,24 +39,10 @@ public class QdrantDb : IVectorDb return collections.ToList(); } - public async Task GetCollectionInfo(string collectionName) - { - var client = GetClient(); - - var exists = await client.CollectionExistsAsync(collectionName); - if (!exists) return new KnowledgeCollectionInfo(); - - var info = await client.GetCollectionInfoAsync(collectionName); - return new KnowledgeCollectionInfo - { - DataCount = info.PointsCount, - VectorCount = info.VectorsCount - }; - } - public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { var client = GetClient(); + var exists = await client.CollectionExistsAsync(collectionName); if (!exists) { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 876bd149..c831d15b 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -29,11 +29,6 @@ namespace BotSharp.Plugin.SemanticKernel throw new System.NotImplementedException(); } - public Task GetCollectionInfo(string collectionName) - { - throw new System.NotImplementedException(); - } - public async Task> GetCollections() { var result = new List(); From ec5535c72d07e25f1597181ee97db82df42a87ec Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 16:56:57 -0500 Subject: [PATCH 18/63] split file service --- .../BotSharp.Abstraction.csproj | 3 +- ...arpFileService.cs => IFileBasicService.cs} | 32 +---- .../Files/IFileInstructService.cs | 28 ++++ .../Files/Models/FileSelectContext.cs | 8 ++ .../Files/Utilities/FileUtility.cs | 40 ++++++ .../BotSharp.Core/BotSharp.Core.csproj | 19 ++- .../ConversationService.TruncateMessage.cs | 2 +- .../Services/ConversationService.cs | 2 +- .../BotSharp.Core/Files/FilePlugin.cs | 3 +- .../Services/Basic/FileBasicService.Common.cs | 54 ++++++++ .../FileBasicService.Conversation.cs} | 32 ++--- .../FileBasicService.User.cs} | 6 +- .../FileBasicService.cs} | 19 +-- .../Services/BotSharpFileService.Common.cs | 63 --------- .../FileInstructService.Image.cs} | 26 +++- .../FileInstructService.Pdf.cs} | 38 ++--- .../FileInstructService.SelectFile.cs | 106 ++++++++++++++ .../Services/Instruct/FileInstructService.cs | 32 +++++ src/Infrastructure/BotSharp.Core/Using.cs | 3 +- .../templates/select_file_prompt.liquid | 15 ++ .../Controllers/ConversationController.cs | 12 +- .../Controllers/InstructModeController.cs | 36 ++--- .../Controllers/UserController.cs | 6 +- .../Providers/Chat/ChatCompletionProvider.cs | 6 +- .../Functions/HandleEmailSenderFn.cs | 10 +- .../Functions/EditImageFn.cs | 65 +-------- .../Functions/GenerateImageFn.cs | 2 +- .../Functions/ReadImageFn.cs | 2 +- .../Functions/ReadPdfFn.cs | 2 +- .../Providers/Chat/ChatCompletionProvider.cs | 6 +- .../Services/TencentCosService.Common.cs | 52 +++---- .../TencentCosService.Conversation.cs | 23 ++-- .../Services/TencentCosService.Image.cs | 107 -------------- .../Services/TencentCosService.Pdf.cs | 130 ------------------ .../Services/TencentCosService.User.cs | 6 +- .../Services/TencentCosService.cs | 14 +- .../TencentCosPlugin.cs | 2 +- 37 files changed, 442 insertions(+), 570 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Files/{IBotSharpFileService.cs => IFileBasicService.cs} (67%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs rename src/Infrastructure/BotSharp.Core/Files/Services/{BotSharpFileService.Conversation.cs => Basic/FileBasicService.Conversation.cs} (93%) rename src/Infrastructure/BotSharp.Core/Files/Services/{BotSharpFileService.User.cs => Basic/FileBasicService.User.cs} (91%) rename src/Infrastructure/BotSharp.Core/Files/Services/{BotSharpFileService.cs => Basic/FileBasicService.cs} (70%) delete mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs rename src/Infrastructure/BotSharp.Core/Files/Services/{BotSharpFileService.Image.cs => Instruct/FileInstructService.Image.cs} (84%) rename src/Infrastructure/BotSharp.Core/Files/Services/{BotSharpFileService.Pdf.cs => Instruct/FileInstructService.Pdf.cs} (80%) create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs delete mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 570f5c92..51e18819 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -25,6 +25,7 @@ + diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs similarity index 67% rename from src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs index dd91a2bb..4a985950 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Abstraction.Files; -public interface IBotSharpFileService +public interface IFileBasicService { #region Conversation /// @@ -28,7 +28,7 @@ public interface IBotSharpFileService /// /// /// - IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, bool imageOnly = false); + IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); bool SaveMessageFiles(string conversationId, string messageId, string source, List files); @@ -45,38 +45,18 @@ public interface IBotSharpFileService bool DeleteConversationFiles(IEnumerable conversationIds); #endregion - #region Image - Task GenerateImage(string? provider, string? model, string text); - Task VaryImage(string? provider, string? model, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); - #endregion - - #region Pdf - /// - /// Take screenshots of pdf pages and get response from llm - /// - /// - /// Pdf files - /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); - #endregion - #region User string GetUserAvatar(); bool SaveUserAvatar(BotSharpFile file); #endregion #region Common - /// - /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" - /// - /// - /// - (string, byte[]) GetFileInfoFromData(string data); string GetDirectory(string conversationId); - string GetFileContentType(string filePath); byte[] GetFileBytes(string fileStorageUrl); bool SavefileToPath(string filePath, Stream stream); + bool ExistDirectory(string? dir); + void CreateDirectory(string dir); + void DeleteDirectory(string dir); + string BuildDirectory(params string[] segments); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs new file mode 100644 index 00000000..7d717fd0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -0,0 +1,28 @@ +namespace BotSharp.Abstraction.Files; + +public interface IFileInstructService +{ + #region Image + Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task GenerateImage(string? provider, string? model, string text); + Task VaryImage(string? provider, string? model, BotSharpFile image); + Task EditImage(string? provider, string? model, string text, BotSharpFile image); + Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); + #endregion + + #region Pdf + /// + /// Take screenshots of pdf pages and get response from llm + /// + /// + /// Pdf files + /// + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + #endregion + + #region Select file + Task> SelectMessageFiles(string conversationId, + string? agentId = null, string? template = null, bool includeBotFile = false, bool fromBreakpoint = false, + int? offset = null, IEnumerable? contentTypes = null); + #endregion +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs new file mode 100644 index 00000000..d13b4f1e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileSelectContext +{ + [JsonPropertyName("selected_ids")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? Selecteds { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs new file mode 100644 index 00000000..df33906d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -0,0 +1,40 @@ +using Microsoft.AspNetCore.StaticFiles; + +namespace BotSharp.Abstraction.Files.Utilities; + +public static class FileUtility +{ + /// + /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" + /// + /// + /// + public static (string, byte[]) GetFileInfoFromData(string data) + { + if (string.IsNullOrEmpty(data)) + { + return (string.Empty, new byte[0]); + } + + var typeStartIdx = data.IndexOf(':'); + var typeEndIdx = data.IndexOf(';'); + var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); + + var base64startIdx = data.IndexOf(','); + var base64Str = data.Substring(base64startIdx + 1); + + return (contentType, Convert.FromBase64String(base64Str)); + } + + public static string GetFileContentType(string filePath) + { + string contentType; + var provider = new FileExtensionContentTypeProvider(); + if (!provider.TryGetContentType(filePath, out contentType)) + { + contentType = string.Empty; + } + + return contentType; + } +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 667e7e71..dec4d9f6 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -45,6 +45,15 @@ 1701;1702 + + + + + + + + + @@ -69,6 +78,7 @@ + @@ -155,6 +165,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest @@ -172,7 +185,6 @@ - @@ -181,9 +193,4 @@ - - - - - diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs index 451cdeed..3d6cc79b 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs @@ -5,7 +5,7 @@ public partial class ConversationService : IConversationService public async Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true); fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 87beba41..74b49d3d 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -37,7 +37,7 @@ public partial class ConversationService : IConversationService public async Task DeleteConversations(IEnumerable ids) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var isDeleted = db.DeleteConversations(ids); fileService.DeleteConversationFiles(ids); return await Task.FromResult(isDeleted); diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index 90397b93..d429ca28 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -20,7 +20,8 @@ public class FilePlugin : IBotSharpPlugin if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) { - services.AddScoped(); + services.AddScoped(); } + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs new file mode 100644 index 00000000..df0417fb --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs @@ -0,0 +1,54 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class FileBasicService +{ + public string GetDirectory(string conversationId) + { + var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + return dir; + } + + public byte[] GetFileBytes(string fileStorageUrl) + { + using var stream = File.OpenRead(fileStorageUrl); + var bytes = new byte[stream.Length]; + stream.Read(bytes, 0, (int)stream.Length); + return bytes; + } + + public bool SavefileToPath(string filePath, Stream stream) + { + using (var fileStream = new FileStream(filePath, FileMode.Create)) + { + stream.CopyTo(fileStream); + } + return true; + } + + public string BuildDirectory(params string[] segments) + { + var relativePath = Path.Combine(segments); + return Path.Combine(_baseDir, relativePath); + } + + public void CreateDirectory(string dir) + { + Directory.CreateDirectory(dir); + } + + public bool ExistDirectory(string? dir) + { + return !string.IsNullOrEmpty(dir) && Directory.Exists(dir); + } + + public void DeleteDirectory(string dir) + { + Directory.Delete(dir, true); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs similarity index 93% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs index 1b98f0d9..f2624fda 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs @@ -4,7 +4,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileBasicService { public async Task> GetChatFiles(string conversationId, string source, IEnumerable conversations, IEnumerable contentTypes, @@ -29,7 +29,7 @@ public partial class BotSharpFileService var file = Directory.GetFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = GetFileContentType(file); + var contentType = FileUtility.GetFileContentType(file); if (contentTypes?.Contains(contentType) != true) continue; var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); @@ -43,7 +43,7 @@ public partial class BotSharpFileService } public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, bool imageOnly = false) + string source, IEnumerable? contentTypes = null) { var files = new List(); if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; @@ -62,8 +62,8 @@ public partial class BotSharpFileService foreach (var file in Directory.GetFiles(subDir)) { - var contentType = GetFileContentType(file); - if (imageOnly && !_imageTypes.Contains(contentType)) + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType)) { continue; } @@ -141,7 +141,7 @@ public partial class BotSharpFileService try { - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = Path.Combine(dir, source, $"{i + 1}"); if (!ExistDirectory(subDir)) { @@ -180,7 +180,7 @@ public partial class BotSharpFileService { if (ExistDirectory(newDir)) { - Directory.Delete(newDir, true); + DeleteDirectory(newDir); } Directory.Move(prevDir, newDir); @@ -189,7 +189,7 @@ public partial class BotSharpFileService var botDir = Path.Combine(newDir, BOT_FILE_FOLDER); if (ExistDirectory(botDir)) { - Directory.Delete(botDir, true); + DeleteDirectory(botDir); } } } @@ -200,7 +200,7 @@ public partial class BotSharpFileService if (!ExistDirectory(dir)) continue; Thread.Sleep(100); - Directory.Delete(dir, true); + DeleteDirectory(dir); } return true; @@ -215,7 +215,7 @@ public partial class BotSharpFileService var convDir = GetConversationDirectory(conversationId); if (!ExistDirectory(convDir)) continue; - Directory.Delete(convDir, true); + DeleteDirectory(convDir); } return true; } @@ -248,13 +248,9 @@ public partial class BotSharpFileService { if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 0) + if (offset.HasValue && offset < 1) { - offset = MIN_OFFSET; - } - else if (offset > MAX_OFFSET) - { - offset = MAX_OFFSET; + offset = 1; } var messageIds = new List(); @@ -285,7 +281,7 @@ public partial class BotSharpFileService { foreach (var screenShot in Directory.GetFiles(screenShotDir)) { - contentType = GetFileContentType(screenShot); + contentType = FileUtility.GetFileContentType(screenShot); if (!_imageTypes.Contains(contentType)) continue; var fileName = Path.GetFileNameWithoutExtension(screenShot); @@ -307,7 +303,7 @@ public partial class BotSharpFileService var images = await ConvertPdfToImages(file, screenShotDir); foreach (var image in images) { - contentType = GetFileContentType(image); + contentType = FileUtility.GetFileContentType(image); var fileName = Path.GetFileNameWithoutExtension(image); var fileType = Path.GetExtension(image).Substring(1); var model = new MessageFileModel() diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs similarity index 91% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs index fa99b13f..f26763c9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileBasicService { public string GetUserAvatar() { @@ -30,11 +30,11 @@ public partial class BotSharpFileService if (Directory.Exists(dir)) { - Directory.Delete(dir, true); + DeleteDirectory(dir); } dir = GetUserAvatarDir(user?.Id, createNewDir: true); - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); File.WriteAllBytes(Path.Combine(dir, file.FileName), bytes); return true; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs similarity index 70% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs index b7c9a946..1d2079b9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs @@ -1,14 +1,13 @@ -using Microsoft.AspNetCore.StaticFiles; using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService : IBotSharpFileService +public partial class FileBasicService : IFileBasicService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly string _baseDir; private readonly IEnumerable _imageTypes = new List { @@ -25,13 +24,10 @@ public partial class BotSharpFileService : IBotSharpFileService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - private const int MIN_OFFSET = 1; - private const int MAX_OFFSET = 5; - - public BotSharpFileService( + public FileBasicService( BotSharpDatabaseSettings dbSettings, IUserIdentity user, - ILogger logger, + ILogger logger, IServiceProvider services) { _dbSettings = dbSettings; @@ -40,11 +36,4 @@ public partial class BotSharpFileService : IBotSharpFileService _services = services; _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository); } - - #region Private methods - private bool ExistDirectory(string? dir) - { - return !string.IsNullOrEmpty(dir) && Directory.Exists(dir); - } - #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs deleted file mode 100644 index be5f3180..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Microsoft.AspNetCore.StaticFiles; -using System.IO; - -namespace BotSharp.Core.Files.Services; - -public partial class BotSharpFileService -{ - public string GetDirectory(string conversationId) - { - var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - return dir; - } - - public (string, byte[]) GetFileInfoFromData(string data) - { - if (string.IsNullOrEmpty(data)) - { - return (string.Empty, new byte[0]); - } - - var typeStartIdx = data.IndexOf(':'); - var typeEndIdx = data.IndexOf(';'); - var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); - - var base64startIdx = data.IndexOf(','); - var base64Str = data.Substring(base64startIdx + 1); - - return (contentType, Convert.FromBase64String(base64Str)); - } - - public string GetFileContentType(string filePath) - { - string contentType; - var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) - { - contentType = string.Empty; - } - - return contentType; - } - - public byte[] GetFileBytes(string fileStorageUrl) - { - using var stream = File.OpenRead(fileStorageUrl); - var bytes = new byte[stream.Length]; - stream.Read(bytes, 0, (int)stream.Length); - return bytes; - } - - public bool SavefileToPath(string filePath, Stream stream) - { - using (var fileStream = new FileStream(filePath, FileMode.Create)) - { - stream.CopyTo(fileStream); - } - return true; - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs similarity index 84% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 619360d2..c9d35cb7 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -2,8 +2,24 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileInstructService { + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + { + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); + var message = await completion.GetChatCompletions(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, text) + { + Files = images?.ToList() ?? new List() + } + }); + return message; + } + public async Task GenerateImage(string? provider, string? model, string text) { var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); @@ -31,7 +47,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); - + stream.Close(); return message; } @@ -53,7 +69,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); - + stream.Close(); return message; } @@ -82,7 +98,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); - + imageStream.Close(); maskStream.Close(); return message; @@ -100,7 +116,7 @@ public partial class BotSharpFileService } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = GetFileInfoFromData(file.FileData); + (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); } return bytes; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs similarity index 80% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index daca7711..4fbe01aa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,8 +1,9 @@ +using BotSharp.Abstraction.Files.Converters; using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileInstructService { public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) { @@ -14,11 +15,9 @@ public partial class BotSharpFileService } var guid = Guid.NewGuid().ToString(); - var sessionDir = GetSessionDirectory(guid); - if (!ExistDirectory(sessionDir)) - { - Directory.CreateDirectory(sessionDir); - } + + var sessionDir = _fileBasic.BuildDirectory(SESSION_FOLDER, guid); + DeleteIfExistDirectory(sessionDir); try { @@ -38,9 +37,7 @@ public partial class BotSharpFileService Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() } }); - - content = message.Content; - return content; + return message.Content; } catch (Exception ex) { @@ -49,17 +46,11 @@ public partial class BotSharpFileService } finally { - Directory.Delete(sessionDir, true); + _fileBasic.DeleteDirectory(sessionDir); } } #region Private methods - private string GetSessionDirectory(string id) - { - var dir = Path.Combine(_baseDir, SESSION_FOLDER, id); - return dir; - } - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) @@ -81,19 +72,16 @@ public partial class BotSharpFileService } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = GetFileInfoFromData(file.FileData); + (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); } if (!bytes.IsNullOrEmpty()) { var guid = Guid.NewGuid().ToString(); - var fileDir = Path.Combine(dir, guid); - if (!ExistDirectory(fileDir)) - { - Directory.CreateDirectory(fileDir); - } + var fileDir = _fileBasic.BuildDirectory(dir, guid); + DeleteIfExistDirectory(fileDir); - var pdfDir = Path.Combine(fileDir, $"{guid}.{extension}"); + var pdfDir = _fileBasic.BuildDirectory(fileDir, $"{guid}.{extension}"); using (var fs = new FileStream(pdfDir, FileMode.Create)) { fs.Write(bytes, 0, bytes.Length); @@ -115,7 +103,7 @@ public partial class BotSharpFileService private async Task> ConvertPdfToImages(IEnumerable files) { var images = new List(); - var converter = GetPdf2ImageConverter(); + var converter = _services.GetServices().FirstOrDefault(); if (converter == null || files.IsNullOrEmpty()) { return images; @@ -127,7 +115,7 @@ public partial class BotSharpFileService { var segs = file.Split(Path.DirectorySeparatorChar); var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); - var folder = Path.Combine(dir, "screenshots"); + var folder = _fileBasic.BuildDirectory(dir, "screenshots"); var urls = await converter.ConvertPdfToImages(file, folder); images.AddRange(urls); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs new file mode 100644 index 00000000..e4602b9f --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -0,0 +1,106 @@ +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Templating; + +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService +{ + public async Task> SelectMessageFiles(string conversationId, + string? agentId = null, string? template = null, bool includeBotFile = false, bool fromBreakpoint = false, + int? offset = null, IEnumerable? contentTypes = null) + { + if (string.IsNullOrEmpty(conversationId)) + { + return Enumerable.Empty(); + } + + var convService = _services.GetRequiredService(); + var dialogs = convService.GetDialogHistory(fromBreakpoint: fromBreakpoint); + var messageIds = GetMessageIds(dialogs, offset); + + var files = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes); + if (includeBotFile) + { + var botFiles = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes); + files = files.Concat(botFiles); + } + + if (files.IsNullOrEmpty()) + { + return Enumerable.Empty(); + } + + return await SelectFiles(agentId, template, files, dialogs); + } + + private async Task> SelectFiles(string? agentId, string? template, IEnumerable files, List dialogs) + { + if (files.IsNullOrEmpty()) return new List(); + + var llmProviderService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + var db = _services.GetRequiredService(); + + try + { + var promptFiles = files.Select((x, idx) => + { + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; + }).ToList(); + + agentId = !string.IsNullOrWhiteSpace(agentId) ? agentId : BuiltInAgentId.UtilityAssistant; + template = !string.IsNullOrWhiteSpace(template) ? template : "select_file_prompt"; + + var foundAgent = db.GetAgent(agentId); + var prompt = db.GetAgentTemplate(agentId, template); + prompt = render.Render(prompt, new Dictionary + { + { "file_list", promptFiles } + }); + + var agent = new Agent + { + Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = foundAgent?.Name ?? "Utility Assistant", + Instruction = prompt + }; + + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); + var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); + var latest = dialogs.Last(); + var response = await completion.GetChatCompletions(agent, new List { latest }); + var content = response?.Content ?? string.Empty; + var selecteds = JsonSerializer.Deserialize(content); + var fids = selecteds?.Selecteds ?? new List(); + return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when selecting files. {ex.Message}\r\n{ex.InnerException}"); + return new List(); + } + } + + private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + { + if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + + if (offset.HasValue && offset < 1) + { + offset = 1; + } + + var messageIds = new List(); + if (offset.HasValue) + { + messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + } + else + { + messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + } + + return messageIds; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs new file mode 100644 index 00000000..f5d7ede1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -0,0 +1,32 @@ +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService : IFileInstructService +{ + private readonly IFileBasicService _fileBasic; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + private const string SESSION_FOLDER = "sessions"; + + public FileInstructService( + IFileBasicService fileBasic, + ILogger logger, + IServiceProvider services) + { + _fileBasic = fileBasic; + _logger = logger; + _services = services; + } + + private void DeleteIfExistDirectory(string? dir) + { + if (_fileBasic.ExistDirectory(dir)) + { + _fileBasic.DeleteDirectory(dir); + } + else + { + _fileBasic.CreateDirectory(dir); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 88293d97..fa3e94d2 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -29,6 +29,7 @@ global using BotSharp.Abstraction.Translation; global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Utilities; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; global using BotSharp.Core.Repository; @@ -37,4 +38,4 @@ global using BotSharp.Core.Agents.Services; global using BotSharp.Core.Conversations.Services; global using BotSharp.Core.Infrastructures; global using BotSharp.Core.Users.Services; -global using Aspects.Cache; +global using Aspects.Cache; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid new file mode 100644 index 00000000..f1267212 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid @@ -0,0 +1,15 @@ +Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. + +** Ensure the output is only in JSON format without any additional text. +** If no files are selected, you must output an empty list []. +** You may need to look at the file_name as a reference to find the correct file id or ids. + +Here is the JSON format to use: +{ + "selected_ids": a list of id selected from the [FILES] section +} + +[FILES] +{% for file in file_list -%} +{{ file }}{{ "\r\n" }} +{%- endfor %} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 261eb04b..a0d65f8a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -81,7 +81,7 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var messageIds = history.Select(x => x.MessageId).Distinct().ToList(); var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds); @@ -349,7 +349,7 @@ public class ConversationController : ControllerBase { if (files != null && files.Length > 0) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var dir = fileService.GetDirectory(conversationId); foreach (var file in files) { @@ -372,7 +372,7 @@ public class ConversationController : ControllerBase var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); var conv = await convService.GetConversationRecordOrCreateNew(agentId); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var messageId = Guid.NewGuid().ToString(); var isSaved = fileService.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); return isSaved ? messageId : string.Empty; @@ -381,15 +381,15 @@ public class ConversationController : ControllerBase [HttpGet("/conversation/{conversationId}/files/{messageId}/{source}")] public IEnumerable GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source) { - var fileService = _services.GetRequiredService(); - var files = fileService.GetMessageFiles(conversationId, new List { messageId }, source, imageOnly: false); + var fileService = _services.GetRequiredService(); + var files = fileService.GetMessageFiles(conversationId, new List { messageId }, source); return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List(); } [HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")] public IActionResult GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source, [FromRoute] string index, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var file = fileService.GetMessageFile(conversationId, messageId, source, index, fileName); if (string.IsNullOrEmpty(file)) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 2189af2f..48fdafe1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; -using NetTopologySuite.IO; namespace BotSharp.OpenAPI.Controllers; @@ -87,18 +86,8 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", - model: input.Model ?? "gpt-4o", multiModal: true); - var message = await completion.GetChatCompletions(new Agent() - { - Id = Guid.Empty.ToString(), - }, new List - { - new RoleDialogModel(AgentRole.User, input.Text) - { - Files = input.Files - } - }); + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); return message.Content; } catch (Exception ex) @@ -114,14 +103,14 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-generation")] public async Task ImageGeneration([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - var message = await fileService.GenerateImage(input.Provider, input.Model, input.Text); + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.GenerateImage(input.Provider, input.Model, input.Text); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -140,7 +129,6 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-variation")] public async Task ImageVariation([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -152,7 +140,9 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } - var message = await fileService.VaryImage(input.Provider, input.Model, image); + + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.VaryImage(input.Provider, input.Model, image); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -169,7 +159,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-edit")] public async Task ImageEdit([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); + var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -181,7 +171,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } - var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -198,7 +188,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-mask-edit")] public async Task ImageMaskEdit([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); + var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -211,7 +201,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" }; } - var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image, mask); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -236,8 +226,8 @@ public class InstructModeController : ControllerBase try { - var fileService = _services.GetRequiredService(); - var content = await fileService.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); viewModel.Content = content; return viewModel; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 966f51c9..74db3bd0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -137,14 +137,14 @@ public class UserController : ControllerBase [HttpPost("/user/avatar")] public bool UploadUserAvatar([FromBody] BotSharpFile file) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); return fileService.SaveUserAvatar(file); } [HttpGet("/user/avatar")] public IActionResult GetUserAvatar() { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var file = fileService.GetUserAvatar(); if (string.IsNullOrEmpty(file)) { @@ -158,7 +158,7 @@ public class UserController : ControllerBase #region Private methods private FileContentResult BuildFileResult(string file) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var bytes = fileService.GetFileBytes(file); return File(bytes, "application/octet-stream", Path.GetFileName(file)); } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 16e2841a..72034317 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; namespace BotSharp.Plugin.AzureOpenAI.Providers.Chat; @@ -196,7 +197,6 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); @@ -270,13 +270,13 @@ public class ChatCompletionProvider : IChatCompletion } else if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); using var stream = File.OpenRead(file.FileStorageUrl); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 45dc65be..072c22d8 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -74,13 +74,11 @@ public class HandleEmailSenderFn : IFunctionCallback private async Task> GetConversationFiles() { var convService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var conversationId = convService.ConversationId; - var dialogs = convService.GetDialogHistory(fromBreakpoint: false); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var userFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User); - var botFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot); - return await SelectFiles(userFiles.Concat(botFiles), dialogs); + + var fileInstruct = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(conversationId, includeBotFile: true); + return selecteds; } private async Task> SelectFiles(IEnumerable files, List dialogs) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 63938c5e..52cb1249 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -28,7 +28,7 @@ public class EditImageFn : IFunctionCallback Init(message); SetImageOptions(); - var image = await SelectConversationImage(descrpition); + var image = await SelectImage(descrpition); var response = await GetImageEditGeneration(message, descrpition, image); message.Content = response; return true; @@ -48,64 +48,11 @@ public class EditImageFn : IFunctionCallback state.SetState("image_count", "1"); } - private async Task SelectConversationImage(string? description) + private async Task SelectImage(string? description) { - var convService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var userImages = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true); - return await SelectImage(userImages, dialogs.LastOrDefault(), description); - } - - private async Task SelectImage(IEnumerable images, RoleDialogModel message, string? description) - { - if (images.IsNullOrEmpty()) return null; - - var llmProviderService = _services.GetRequiredService(); - var render = _services.GetRequiredService(); - var db = _services.GetRequiredService(); - - try - { - var promptImages = images.Where(x => x.ContentType == MediaTypeNames.Image.Png).Select((x, idx) => - { - return $"id: {idx + 1}, image_name: {x.FileName}.{x.FileType}"; - }).ToList(); - - if (promptImages.IsNullOrEmpty()) return null; - - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_edit_image_prompt"); - prompt = render.Render(prompt, new Dictionary - { - { "image_list", promptImages } - }); - - var agent = new Agent - { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant", - Instruction = prompt - }; - - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - - var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; - var dialog = RoleDialogModel.From(message, AgentRole.User, text); - - var response = await completion.GetChatCompletions(agent, new List { dialog }); - var content = response?.Content ?? string.Empty; - var selected = JsonSerializer.Deserialize(content); - var fid = selected?.Selected ?? -1; - return fid > 0 ? images.Where((x, idx) => idx == fid - 1).FirstOrDefault() : null; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting the image edit response. {ex.Message}\r\n{ex.InnerException}"); - return null; - } + var fileInstruct = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, contentTypes: new List { MediaTypeNames.Image.Png }); + return selecteds?.FirstOrDefault(); } private async Task GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image) @@ -154,7 +101,7 @@ public class EditImageFn : IFunctionCallback } }; - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4d53f880..3869a419 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -83,7 +83,7 @@ public class GenerateImageFn : IFunctionCallback FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" }).ToList(); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 66f69353..cdff6cf4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -51,7 +51,7 @@ public class ReadImageFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var images = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); foreach (var dialog in dialogs) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index 85c2afbc..d3c21737 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -50,7 +50,7 @@ public class ReadPdfFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var files = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); foreach (var dialog in dialogs) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 23084ead..f12e8b34 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -197,7 +198,6 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); @@ -271,13 +271,13 @@ public class ChatCompletionProvider : IChatCompletion } else if (!string.IsNullOrEmpty(file.FileData)) { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); using var stream = File.OpenRead(file.FileStorageUrl); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index e9631786..61b58c6a 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -1,5 +1,3 @@ -using Microsoft.AspNetCore.StaticFiles; - namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -9,41 +7,11 @@ public partial class TencentCosService return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } - public (string, byte[]) GetFileInfoFromData(string data) - { - if (string.IsNullOrEmpty(data)) - { - return (string.Empty, new byte[0]); - } - - var typeStartIdx = data.IndexOf(':'); - var typeEndIdx = data.IndexOf(';'); - var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); - - var base64startIdx = data.IndexOf(','); - var base64Str = data.Substring(base64startIdx + 1); - - return (contentType, Convert.FromBase64String(base64Str)); - } - - public string GetFileContentType(string filePath) - { - string contentType; - var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) - { - contentType = string.Empty; - } - - return contentType; - } - public byte[] GetFileBytes(string fileStorageUrl) { try { var fileData = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); - return fileData; } catch (Exception ex) @@ -67,4 +35,24 @@ public partial class TencentCosService return false; } } + + public string BuildDirectory(params string[] segments) + { + return string.Join("/", segments); + } + + public void CreateDirectory(string dir) + { + + } + + public bool ExistDirectory(string? dir) + { + return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir); + } + + public void DeleteDirectory(string dir) + { + _cosClient.BucketClient.DeleteDir(dir); + } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index df39e0a9..207fb464 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; +using BotSharp.Abstraction.Files.Utilities; using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; @@ -28,7 +29,7 @@ public partial class TencentCosService var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = GetFileContentType(file); + var contentType = FileUtility.GetFileContentType(file); if (contentTypes?.Contains(contentType) != true) continue; var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); @@ -42,7 +43,7 @@ public partial class TencentCosService } public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, bool imageOnly = false) + string source, IEnumerable? contentTypes = null) { var files = new List(); if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; @@ -59,8 +60,8 @@ public partial class TencentCosService { foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir)) { - var contentType = GetFileContentType(file); - if (imageOnly && !_imageTypes.Contains(contentType)) + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType)) { continue; } @@ -135,7 +136,7 @@ public partial class TencentCosService try { - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = $"{dir}/{source}/{i + 1}"; @@ -225,13 +226,9 @@ public partial class TencentCosService { if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 0) + if (offset <= 1) { - offset = MIN_OFFSET; - } - else if (offset > MAX_OFFSET) - { - offset = MAX_OFFSET; + offset = 1; } var messageIds = new List(); @@ -264,7 +261,7 @@ public partial class TencentCosService { foreach (var screenShot in fileList) { - contentType = GetFileContentType(screenShot); + contentType = FileUtility.GetFileContentType(screenShot); if (!_imageTypes.Contains(contentType)) continue; var fileName = Path.GetFileNameWithoutExtension(screenShot); @@ -286,7 +283,7 @@ public partial class TencentCosService var images = await ConvertPdfToImages(file, screenShotDir); foreach (var image in images) { - contentType = GetFileContentType(image); + contentType = FileUtility.GetFileContentType(image); var fileName = Path.GetFileNameWithoutExtension(image); var fileType = Path.GetExtension(image).Substring(1); var model = new MessageFileModel() diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs deleted file mode 100644 index e8628ce3..00000000 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace BotSharp.Plugin.TencentCos.Services; - -public partial class TencentCosService -{ - public async Task GenerateImage(string? provider, string? model, string text) - { - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); - var message = await completion.GetImageGeneration(new Agent() - { - Id = Guid.Empty.ToString(), - }, new RoleDialogModel(AgentRole.User, text)); - return message; - } - - public async Task VaryImage(string? provider, string? model, BotSharpFile image) - { - if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) - { - throw new ArgumentException($"Cannot find image url or data!"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); - stream.Position = 0; - - var message = await completion.GetImageVariation(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); - - stream.Close(); - return message; - } - - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image) - { - if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) - { - throw new ArgumentException($"Cannot find image url or data!"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); - stream.Position = 0; - - var message = await completion.GetImageEdits(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); - - stream.Close(); - return message; - } - - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask) - { - if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || - (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) - { - throw new ArgumentException($"Cannot find image/mask url or data"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var imageBytes = await DownloadFile(image); - var maskBytes = await DownloadFile(mask); - - using var imageStream = new MemoryStream(); - imageStream.Write(imageBytes, 0, imageBytes.Length); - imageStream.Position = 0; - - using var maskStream = new MemoryStream(); - maskStream.Write(maskBytes, 0, maskBytes.Length); - maskStream.Position = 0; - - var message = await completion.GetImageEdits(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); - - imageStream.Close(); - maskStream.Close(); - return message; - } - - #region Private methods - private async Task DownloadFile(BotSharpFile file) - { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, bytes) = GetFileInfoFromData(file.FileData); - } - - return bytes; - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs deleted file mode 100644 index 1efbad6d..00000000 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs +++ /dev/null @@ -1,130 +0,0 @@ -namespace BotSharp.Plugin.TencentCos.Services; - -public partial class TencentCosService -{ - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) - { - var content = string.Empty; - - if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty()) - { - return content; - } - - var guid = Guid.NewGuid().ToString(); - var sessionDir = GetSessionDirectory(guid); - - try - { - var pdfFiles = await DownloadFiles(sessionDir, files); - var images = await ConvertPdfToImages(pdfFiles); - if (images.IsNullOrEmpty()) return content; - - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", - model: model, modelId: modelId ?? "gpt-4", multiModal: true); - var message = await completion.GetChatCompletions(new Agent() - { - Id = Guid.Empty.ToString(), - }, new List - { - new RoleDialogModel(AgentRole.User, prompt) - { - Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() - } - }); - - content = message.Content; - return content; - } - catch (Exception ex) - { - _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}\r\n{ex.InnerException}"); - return content; - } - finally - { - Directory.Delete(sessionDir, true); - } - } - - #region Private methods - private string GetSessionDirectory(string id) - { - var dir = $"{SESSION_FOLDER}/{id}"; - return dir; - } - - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") - { - if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) - { - return Enumerable.Empty(); - } - - var locs = new List(); - foreach (var file in files) - { - try - { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, bytes) = GetFileInfoFromData(file.FileData); - } - - if (!bytes.IsNullOrEmpty()) - { - var guid = Guid.NewGuid().ToString(); - var fileDir = $"{dir}/{guid}"; - - var pdfDir = $"{fileDir}/{guid}.{extension}"; - - - _cosClient.BucketClient.UploadBytes(pdfDir, bytes); - locs.Add(pdfDir); - } - } - catch (Exception ex) - { - _logger.LogWarning($"Error when saving pdf file: {ex.Message}\r\n{ex.InnerException}"); - continue; - } - } - return locs; - } - - private async Task> ConvertPdfToImages(IEnumerable files) - { - var images = new List(); - var converter = GetPdf2ImageConverter(); - if (converter == null || files.IsNullOrEmpty()) - { - return images; - } - - foreach (var file in files) - { - try - { - var segs = file.Split(Path.DirectorySeparatorChar); - var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); - var folder = Path.Combine(dir, "screenshots"); - var urls = await converter.ConvertPdfToImages(file, folder); - images.AddRange(urls); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}\r\n{ex.InnerException}"); - continue; - } - } - return images; - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index 23ce68c0..55e26d81 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Files.Utilities; + namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -26,10 +28,8 @@ public partial class TencentCosService if (string.IsNullOrEmpty(dir)) return false; - var (_, bytes) = GetFileInfoFromData(file.FileData); - + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var extension = Path.GetExtension(file.FileName); - var fileName = user?.Id == null ? file.FileName : $"{user?.Id}{extension}"; return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", bytes); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index 80b8fd78..78c6bcc9 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -5,8 +5,9 @@ using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; -public partial class TencentCosService : IBotSharpFileService +public partial class TencentCosService : IFileBasicService { + private readonly TencentCosClient _cosClient; private readonly TencentCosSettings _settings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; @@ -27,10 +28,6 @@ public partial class TencentCosService : IBotSharpFileService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - private const int MIN_OFFSET = 1; - private const int MAX_OFFSET = 5; - - private readonly TencentCosClient _cosClient; public TencentCosService( TencentCosSettings settings, @@ -46,11 +43,4 @@ public partial class TencentCosService : IBotSharpFileService _fullBuketName = $"{_settings.BucketName}-{_settings.AppId}"; _cosClient = cosClient; } - - #region Private methods - private bool ExistDirectory(string? dir) - { - return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir); - } - #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 93a99c14..25cdb277 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -31,7 +31,7 @@ public class TencentCosPlugin : IBotSharpPlugin services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } } From 0d8532c3c71703af0b37cebd17b2748c1cfa408a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 17:14:56 -0500 Subject: [PATCH 19/63] minor change --- .../Files/IFileBasicService.cs | 4 ++- .../Services/Basic/FileBasicService.Common.cs | 21 ++++++++++++++- .../Instruct/FileInstructService.Pdf.cs | 12 +++------ .../Controllers/ConversationController.cs | 2 +- .../Services/TencentCosService.Common.cs | 27 +++++++++++++++++-- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs index 4a985950..50d9413a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs @@ -53,7 +53,9 @@ public interface IFileBasicService #region Common string GetDirectory(string conversationId); byte[] GetFileBytes(string fileStorageUrl); - bool SavefileToPath(string filePath, Stream stream); + bool SaveFileStreamToPath(string filePath, Stream stream); + bool SaveFileBytesToPath(string filePath, byte[] bytes); + string GetParentDir(string dir, int level = 1); bool ExistDirectory(string? dir); void CreateDirectory(string dir); void DeleteDirectory(string dir); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs index df0417fb..a1208a31 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs @@ -22,8 +22,10 @@ public partial class FileBasicService return bytes; } - public bool SavefileToPath(string filePath, Stream stream) + public bool SaveFileStreamToPath(string filePath, Stream stream) { + if (string.IsNullOrEmpty(filePath)) return false; + using (var fileStream = new FileStream(filePath, FileMode.Create)) { stream.CopyTo(fileStream); @@ -31,6 +33,23 @@ public partial class FileBasicService return true; } + public bool SaveFileBytesToPath(string filePath, byte[] bytes) + { + using (var fs = new FileStream(filePath, FileMode.Create)) + { + fs.Write(bytes, 0, bytes.Length); + fs.Flush(); + fs.Close(); + } + return true; + } + + public string GetParentDir(string dir, int level = 1) + { + var segs = dir.Split(Path.DirectorySeparatorChar); + return string.Join(Path.DirectorySeparatorChar, segs.SkipLast(level)); + } + public string BuildDirectory(params string[] segments) { var relativePath = Path.Combine(segments); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 4fbe01aa..d4413983 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -82,13 +82,8 @@ public partial class FileInstructService DeleteIfExistDirectory(fileDir); var pdfDir = _fileBasic.BuildDirectory(fileDir, $"{guid}.{extension}"); - using (var fs = new FileStream(pdfDir, FileMode.Create)) - { - fs.Write(bytes, 0, bytes.Length); - fs.Close(); - locs.Add(pdfDir); - Thread.Sleep(100); - } + _fileBasic.SaveFileBytesToPath(pdfDir, bytes); + locs.Add(pdfDir); } } catch (Exception ex) @@ -113,8 +108,7 @@ public partial class FileInstructService { try { - var segs = file.Split(Path.DirectorySeparatorChar); - var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); + var dir = _fileBasic.GetParentDir(file); var folder = _fileBasic.BuildDirectory(dir, "screenshots"); var urls = await converter.ConvertPdfToImages(file, folder); images.AddRange(urls); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index a0d65f8a..d81f069a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -357,7 +357,7 @@ public class ConversationController : ControllerBase var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"'); var filePath = Path.Combine(dir, fileName); - fileService.SavefileToPath(filePath, file.OpenReadStream()); + fileService.SaveFileStreamToPath(filePath, file.OpenReadStream()); } return Ok(new { message = "File uploaded successfully." }); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index 61b58c6a..9de15321 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -1,3 +1,5 @@ +using System.IO; + namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -21,7 +23,7 @@ public partial class TencentCosService return Array.Empty(); } - public bool SavefileToPath(string filePath, Stream stream) + public bool SaveFileStreamToPath(string filePath, Stream stream) { if (string.IsNullOrEmpty(filePath)) return false; @@ -31,11 +33,32 @@ public partial class TencentCosService } catch (Exception ex) { - _logger.LogWarning($"Error when saving file to path: {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when saving file stream to path: {ex.Message}\r\n{ex.InnerException}"); return false; } } + public bool SaveFileBytesToPath(string filePath, byte[] bytes) + { + if (string.IsNullOrEmpty(filePath)) return false; + + try + { + return _cosClient.BucketClient.UploadBytes(filePath, bytes); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving file bytes to path: {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + + public string GetParentDir(string dir, int level = 1) + { + var segs = dir.Split("/"); + return string.Join("/", segs.SkipLast(level)); + } + public string BuildDirectory(params string[] segments) { return string.Join("/", segments); From 2a10294d70ed9cb45372b002a06b5da984f33d98 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 17:40:39 -0500 Subject: [PATCH 20/63] clean file select prompt --- .../Files/IFileBasicService.cs | 2 +- .../Files/Models/MessageFileModel.cs | 2 +- .../Basic/FileBasicService.Conversation.cs | 9 ++-- .../templates/select_file_prompt.liquid | 29 ++++++++++++ .../BotSharp.Plugin.EmailHandler.csproj | 3 -- .../Functions/HandleEmailSenderFn.cs | 44 ------------------- .../templates/select_attachment_prompt.liquid | 44 ------------------- .../BotSharp.Plugin.FileHandler.csproj | 3 -- .../templates/select_edit_image_prompt.liquid | 41 ----------------- .../TencentCosService.Conversation.cs | 9 ++-- 10 files changed, 43 insertions(+), 143 deletions(-) delete mode 100644 src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs index 50d9413a..d8baf6e7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs @@ -17,7 +17,7 @@ public interface IFileBasicService /// /// Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, + IEnumerable conversations, IEnumerable? contentTypes, bool includeScreenShot = false, int? offset = null); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 7cd93269..05568e66 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -39,6 +39,6 @@ public class MessageFileModel public override string ToString() { - return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}"; + return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}"; } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs index f2624fda..fa5df123 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs @@ -7,7 +7,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileBasicService { public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, + IEnumerable conversations, IEnumerable? contentTypes = null, bool includeScreenShot = false, int? offset = null) { var files = new List(); @@ -30,7 +30,10 @@ public partial class FileBasicService if (file == null) continue; var contentType = FileUtility.GetFileContentType(file); - if (contentTypes?.Contains(contentType) != true) continue; + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + { + continue; + } var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); if (foundFiles.IsNullOrEmpty()) continue; @@ -63,7 +66,7 @@ public partial class FileBasicService foreach (var file in Directory.GetFiles(subDir)) { var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType)) + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) { continue; } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid index f1267212..57f9895c 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid @@ -9,6 +9,35 @@ Here is the JSON format to use: "selected_ids": a list of id selected from the [FILES] section } +Suppose there are four files: + +id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user +id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user +id: 3, file_name: example_file.png, content_type: image/png, author: bot +id: 4, file_name: example_file.png, content_type: image/png, author: bot + +===== +Example 1: +USER: I want to send the first file and the third file. +OUTPUT: { "selected_ids": [1, 3] } + +Example 2: +USER: Send all the images. +OUTPUT: { "selected_ids": [1, 2, 4] } + +Example 3: +USER: Send all the images I uploaded. +OUTPUT: { "selected_ids": [1] } + +Example 4: +USER: Send the image and the pdf file. +OUTPUT: { "selected_ids": [1, 2] } + +Example 5: +USER: Send the images generated by bot +OUTPUT: { "selected_ids": [3, 4] } +===== + [FILES] {% for file in file_list -%} {{ file }}{{ "\r\n" }} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index f5926a53..3aa65e97 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -28,9 +28,6 @@ PreserveNewest - - PreserveNewest - PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 072c22d8..8169844e 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -81,50 +81,6 @@ public class HandleEmailSenderFn : IFunctionCallback return selecteds; } - private async Task> SelectFiles(IEnumerable files, List dialogs) - { - if (files.IsNullOrEmpty()) return new List(); - - var llmProviderService = _services.GetRequiredService(); - var render = _services.GetRequiredService(); - var db = _services.GetRequiredService(); - - try - { - var promptFiles = files.Select((x, idx) => - { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; - }).ToList(); - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt"); - prompt = render.Render(prompt, new Dictionary - { - { "file_list", promptFiles } - }); - - var agent = new Agent - { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant", - Instruction = prompt - }; - - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var latest = dialogs.LastOrDefault(); - var response = await completion.GetChatCompletions(agent, new List { latest }); - var content = response?.Content ?? string.Empty; - var selecteds = JsonSerializer.Deserialize(content); - var fids = selecteds?.Selecteds ?? new List(); - return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}"); - return new List(); - } - } - private void BuildEmailAttachments(BodyBuilder builder, IEnumerable files) { if (files.IsNullOrEmpty()) return; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid deleted file mode 100644 index f4295baa..00000000 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid +++ /dev/null @@ -1,44 +0,0 @@ -Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. - -** Ensure the output is only in JSON format without any additional text. -** If no files are selected, you must output an empty list []. -** You may need to look at the file_name as a reference to find the correct file id. - -Here is the JSON format to use: -{ - "selected_ids": a list of id selected from the [FILES] section -} - -Suppose there are four files: - -id: 1, file_name: example_file.jpg, content_type: image/jpeg, author: user -id: 2, file_name: example_file.pdf, content_type: application/pdf, author: user -id: 3, file_name: example_file.png, content_type: image/png, author: bot -id: 4, file_name: example_file.png, content_type: image/png, author: bot - -===== -Example 1: -USER: I want to send the first file and the third file. -OUTPUT: { "selected_ids": [1, 3] } - -Example 2: -USER: Send all the images. -OUTPUT: { "selected_ids": [1, 2, 4] } - -Example 3: -USER: Send all the images I uploaded. -OUTPUT: { "selected_ids": [1] } - -Example 4: -USER: Send the image and the pdf file. -OUTPUT: { "selected_ids": [1, 2] } - -Example 5: -USER: Send the images generated by bot -OUTPUT: { "selected_ids": [3, 4] } -===== - -[FILES] -{% for file in file_list -%} -{{ file }}{{ "\r\n" }} -{%- endfor %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 78d77ac4..9b097fed 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -47,9 +47,6 @@ PreserveNewest - - PreserveNewest - diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid deleted file mode 100644 index 9e67faad..00000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid +++ /dev/null @@ -1,41 +0,0 @@ -Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user. - -** Ensure the output is only in JSON format without any additional text. -** You may need to look at the image_name as a reference to find the correct image id. - -Here is the JSON format to use: -{ - "selected_id": the id selected from the [IMAGES] section -} - - -Suppose there are four images: - -id: 1, image_name: example_image_a.png -id: 2, image_name: example_image_b.png -id: 3, image_name: example_image_c.png -id: 4, image_name: example_image_d.png - -===== -Example 1: -USER: I want to add a dog in the first file. -OUTPUT: { "selected_id": 1 } - -Example 2: -USER: Add a coffee cup in the second image I uploaded. -OUTPUT: { "selected_id": 2 } - -Example 3: -USER: Please remove the left tree in the third and the first images. -OUTPUT: { "selected_id": 3 } - -Example 4: -USER: Circle the head of the dog in example_image_b.png. -OUTPUT: { "selected_id": 4 } -===== - - -[IMAGES] -{% for image in image_list -%} -{{ image }}{{ "\r\n" }} -{%- endfor %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 207fb464..ac661509 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -8,7 +8,7 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, + IEnumerable conversations, IEnumerable? contentTypes = null, bool includeScreenShot = false, int? offset = null) { var files = new List(); @@ -30,7 +30,10 @@ public partial class TencentCosService if (file == null) continue; var contentType = FileUtility.GetFileContentType(file); - if (contentTypes?.Contains(contentType) != true) continue; + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + { + continue; + } var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); if (foundFiles.IsNullOrEmpty()) continue; @@ -61,7 +64,7 @@ public partial class TencentCosService foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir)) { var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && contentTypes.Contains(contentType)) + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) { continue; } From c9a1761228fdb3d9382c2f7125b68c59ccf0d178 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 17:53:58 -0500 Subject: [PATCH 21/63] change param name --- .../Files/IFileBasicService.cs | 4 ++-- .../Files/IFileInstructService.cs | 3 ++- .../Basic/FileBasicService.Conversation.cs | 14 ++++++------- .../FileInstructService.SelectFile.cs | 20 +++++++++++++------ .../Functions/EditImageFn.cs | 2 +- .../TencentCosService.Conversation.cs | 6 +++--- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs index d8baf6e7..b0e5d261 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs @@ -11,13 +11,13 @@ public interface IFileBasicService /// /// /// - /// + /// /// /// /// /// Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable? contentTypes, + IEnumerable dialogs, IEnumerable? contentTypes, bool includeScreenShot = false, int? offset = null); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 7d717fd0..433a582f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -22,7 +22,8 @@ public interface IFileInstructService #region Select file Task> SelectMessageFiles(string conversationId, - string? agentId = null, string? template = null, bool includeBotFile = false, bool fromBreakpoint = false, + string? agentId = null, string? template = null, string? description = null, + bool includeBotFile = false, bool fromBreakpoint = false, int? offset = null, IEnumerable? contentTypes = null); #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs index fa5df123..403e2da5 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs @@ -7,16 +7,16 @@ namespace BotSharp.Core.Files.Services; public partial class FileBasicService { public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable? contentTypes = null, + IEnumerable dialogs, IEnumerable? contentTypes = null, bool includeScreenShot = false, int? offset = null) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(conversations, offset); + var messageIds = GetMessageIds(dialogs, offset); var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); foreach (var messageId in messageIds) @@ -247,9 +247,9 @@ public partial class FileBasicService return dir; } - private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) { - if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + if (dialogs.IsNullOrEmpty()) return Enumerable.Empty(); if (offset.HasValue && offset < 1) { @@ -259,11 +259,11 @@ public partial class FileBasicService var messageIds = new List(); if (offset.HasValue) { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); } else { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); } return messageIds; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index e4602b9f..dfed9f77 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -6,7 +6,8 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { public async Task> SelectMessageFiles(string conversationId, - string? agentId = null, string? template = null, bool includeBotFile = false, bool fromBreakpoint = false, + string? agentId = null, string? template = null, string? description = null, + bool includeBotFile = false, bool fromBreakpoint = false, int? offset = null, IEnumerable? contentTypes = null) { if (string.IsNullOrEmpty(conversationId)) @@ -30,10 +31,11 @@ public partial class FileInstructService return Enumerable.Empty(); } - return await SelectFiles(agentId, template, files, dialogs); + return await SelectFiles(agentId, template, description, files, dialogs); } - private async Task> SelectFiles(string? agentId, string? template, IEnumerable files, List dialogs) + private async Task> SelectFiles(string? agentId, string? template, string? description, + IEnumerable files, List dialogs) { if (files.IsNullOrEmpty()) return new List(); @@ -52,7 +54,7 @@ public partial class FileInstructService template = !string.IsNullOrWhiteSpace(template) ? template : "select_file_prompt"; var foundAgent = db.GetAgent(agentId); - var prompt = db.GetAgentTemplate(agentId, template); + var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template); prompt = render.Render(prompt, new Dictionary { { "file_list", promptFiles } @@ -68,8 +70,14 @@ public partial class FileInstructService var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var latest = dialogs.Last(); - var response = await completion.GetChatCompletions(agent, new List { latest }); + + var message = dialogs.Last(); + if (!string.IsNullOrWhiteSpace(description)) + { + message = RoleDialogModel.From(message, AgentRole.User, description); + } + + var response = await completion.GetChatCompletions(agent, new List { message }); var content = response?.Content ?? string.Empty; var selecteds = JsonSerializer.Deserialize(content); var fids = selecteds?.Selecteds ?? new List(); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 52cb1249..e9d1851b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -51,7 +51,7 @@ public class EditImageFn : IFunctionCallback private async Task SelectImage(string? description) { var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, contentTypes: new List { MediaTypeNames.Image.Png }); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, description: description, contentTypes: new List { MediaTypeNames.Image.Png }); return selecteds?.FirstOrDefault(); } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index ac661509..d67d69f0 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -8,16 +8,16 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable? contentTypes = null, + IEnumerable dialogs, IEnumerable? contentTypes = null, bool includeScreenShot = false, int? offset = null) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(conversations, offset); + var messageIds = GetMessageIds(dialogs, offset); var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; foreach (var messageId in messageIds) From f14c634b3e03ef70014e16481fd316f1fe3666ce Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 7 Aug 2024 18:01:00 -0500 Subject: [PATCH 22/63] clean using --- src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index e9d1851b..8855d352 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Templating; using System.IO; namespace BotSharp.Plugin.FileHandler.Functions; From ef0468e21333a966a4281cd9fe847e1aa0784e96 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 7 Aug 2024 19:26:22 -0500 Subject: [PATCH 23/63] use select file option --- .../Files/IFileInstructService.cs | 5 +- .../Files/Models/SelectFileOptions.cs | 14 ++++++ .../FileInstructService.SelectFile.cs | 47 ++++++++++--------- .../Functions/HandleEmailSenderFn.cs | 2 +- .../Functions/EditImageFn.cs | 6 ++- 5 files changed, 46 insertions(+), 28 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 433a582f..78a400e1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -21,9 +21,6 @@ public interface IFileInstructService #endregion #region Select file - Task> SelectMessageFiles(string conversationId, - string? agentId = null, string? template = null, string? description = null, - bool includeBotFile = false, bool fromBreakpoint = false, - int? offset = null, IEnumerable? contentTypes = null); + Task> SelectMessageFiles(string conversationId, SelectFileOptions options); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs new file mode 100644 index 00000000..6ba6cdd2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class SelectFileOptions +{ + public string? Provider { get; set; } + public string? ModelId { get; set; } + public string? AgentId { get; set; } + public string? Template { get; set; } + public string? Description { get; set; } + public bool IncludeBotFile { get; set; } + public bool FromBreakpoint { get; set; } + public int? Offset { get; set; } + public IEnumerable? ContentTypes { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index dfed9f77..c6a9e06c 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -5,10 +5,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task> SelectMessageFiles(string conversationId, - string? agentId = null, string? template = null, string? description = null, - bool includeBotFile = false, bool fromBreakpoint = false, - int? offset = null, IEnumerable? contentTypes = null) + public async Task> SelectMessageFiles(string conversationId, SelectFileOptions options) { if (string.IsNullOrEmpty(conversationId)) { @@ -16,13 +13,13 @@ public partial class FileInstructService } var convService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(fromBreakpoint: fromBreakpoint); - var messageIds = GetMessageIds(dialogs, offset); + var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); + var messageIds = GetMessageIds(dialogs, options.Offset); - var files = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes); - if (includeBotFile) + var files = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes); + if (options.IncludeBotFile) { - var botFiles = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes); + var botFiles = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); files = files.Concat(botFiles); } @@ -31,11 +28,10 @@ public partial class FileInstructService return Enumerable.Empty(); } - return await SelectFiles(agentId, template, description, files, dialogs); + return await SelectFiles(files, dialogs, options); } - private async Task> SelectFiles(string? agentId, string? template, string? description, - IEnumerable files, List dialogs) + private async Task> SelectFiles(IEnumerable files, IEnumerable dialogs, SelectFileOptions options) { if (files.IsNullOrEmpty()) return new List(); @@ -50,8 +46,8 @@ public partial class FileInstructService return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; }).ToList(); - agentId = !string.IsNullOrWhiteSpace(agentId) ? agentId : BuiltInAgentId.UtilityAssistant; - template = !string.IsNullOrWhiteSpace(template) ? template : "select_file_prompt"; + var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant; + var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "select_file_prompt"; var foundAgent = db.GetAgent(agentId); var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template); @@ -67,16 +63,23 @@ public partial class FileInstructService Instruction = prompt }; - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); + var message = dialogs.LastOrDefault(); + var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : message?.Content; + if (message == null) + { + message = new RoleDialogModel(AgentRole.User, text); + } + else + { + message = RoleDialogModel.From(message, AgentRole.User, text); + } + + var providerName = options.Provider ?? "openai"; + var modelId = options?.ModelId ?? "gpt-4"; + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); + var model = llmProviderService.GetProviderModel(provider: provider, id: modelId); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var message = dialogs.Last(); - if (!string.IsNullOrWhiteSpace(description)) - { - message = RoleDialogModel.From(message, AgentRole.User, description); - } - var response = await completion.GetChatCompletions(agent, new List { message }); var content = response?.Content ?? string.Empty; var selecteds = JsonSerializer.Deserialize(content); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 8169844e..634be215 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -77,7 +77,7 @@ public class HandleEmailSenderFn : IFunctionCallback var conversationId = convService.ConversationId; var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(conversationId, includeBotFile: true); + var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions { IncludeBotFile = true }); return selecteds; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index e9d1851b..1f99c287 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -51,7 +51,11 @@ public class EditImageFn : IFunctionCallback private async Task SelectImage(string? description) { var fileInstruct = _services.GetRequiredService(); - var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, description: description, contentTypes: new List { MediaTypeNames.Image.Png }); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions + { + Description = description, + ContentTypes = new List { MediaTypeNames.Image.Png } + }); return selecteds?.FirstOrDefault(); } From 1ecec6b21a3f0fa5655517927483d833ca11399c Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 7 Aug 2024 20:56:29 -0500 Subject: [PATCH 24/63] rename to file storage service --- .../BotSharp.Abstraction.csproj | 4 +- ...BasicService.cs => IFileStorageService.cs} | 2 +- .../Files/Models/BotSharpFile.cs | 20 +------- .../Files/Models/FileBase.cs | 46 +++++++++++++++++++ .../Files/Models/MessageFileModel.cs | 26 +---------- .../Files/Utilities/FileUtility.cs | 27 +++++++++++ .../ConversationService.TruncateMessage.cs | 5 +- .../Services/ConversationService.cs | 4 +- .../BotSharp.Core/Files/FilePlugin.cs | 2 +- .../Services/Basic/FileBasicService.Common.cs | 2 +- .../Basic/FileBasicService.Conversation.cs | 2 +- .../Services/Basic/FileBasicService.User.cs | 2 +- ...cService.cs => LocalFileStorageService.cs} | 8 ++-- .../Services/Instruct/FileInstructService.cs | 4 +- .../Controllers/ConversationController.cs | 22 ++++----- .../Controllers/UserController.cs | 12 ++--- .../Functions/HandleEmailSenderFn.cs | 13 +++--- .../Functions/EditImageFn.cs | 10 ++-- .../Functions/GenerateImageFn.cs | 4 +- .../Functions/ReadImageFn.cs | 5 +- .../Functions/ReadPdfFn.cs | 5 +- .../TencentCosService.Conversation.cs | 13 ++++-- .../Services/TencentCosService.cs | 2 +- .../TencentCosPlugin.cs | 2 +- 24 files changed, 143 insertions(+), 99 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Files/{IFileBasicService.cs => IFileStorageService.cs} (98%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs rename src/Infrastructure/BotSharp.Core/Files/Services/Basic/{FileBasicService.cs => LocalFileStorageService.cs} (84%) diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 51e18819..ea456184 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -28,9 +28,11 @@ + + diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs similarity index 98% rename from src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index b0e5d261..2db2dc43 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Abstraction.Files; -public interface IFileBasicService +public interface IFileStorageService { #region Conversation /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 7e556e67..11a11e46 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,23 +1,7 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile +public class BotSharpFile : FileBase { - [JsonPropertyName("file_name")] - public string FileName { get; set; } = string.Empty; - - /// - /// File data, e.g., "data:image/png;base64,aaaaaaaa" - /// - [JsonPropertyName("file_data")] - public string FileData { get; set; } = string.Empty; - - [JsonPropertyName("file_url")] - public string FileUrl { get; set; } = string.Empty; - - [JsonPropertyName("content_type")] - public string ContentType { get; set; } = string.Empty; - - [JsonPropertyName("file_storage_url")] - public string FileStorageUrl { get; set; } = string.Empty; + } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs new file mode 100644 index 00000000..c54c31ff --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -0,0 +1,46 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileBase +{ + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; + + /// + /// Internal file storage url + /// + [JsonPropertyName("file_storage_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileStorageUrl { get; set; } = string.Empty; + + /// + /// File name without extension + /// + [JsonPropertyName("file_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileName { get; set; } = string.Empty; + + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileData { get; set; } = string.Empty; + + /// + /// File content type + /// + [JsonPropertyName("content_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ContentType { get; set; } = string.Empty; + + /// + /// File extension without dot + /// + [JsonPropertyName("file_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileType { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 05568e66..e06e1a0f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,34 +1,10 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel +public class MessageFileModel : FileBase { [JsonPropertyName("message_id")] public string MessageId { get; set; } - /// - /// External file url - /// - [JsonPropertyName("file_url")] - public string FileUrl { get; set; } - - /// - /// Internal file storage url - /// - [JsonPropertyName("file_storage_url")] - public string FileStorageUrl { get; set; } - - /// - /// File name without extension - /// - [JsonPropertyName("file_name")] - public string FileName { get; set; } - - [JsonPropertyName("file_type")] - public string FileType { get; set; } - - [JsonPropertyName("content_type")] - public string ContentType { get; set; } - [JsonPropertyName("file_source")] public string FileSource { get; set; } = FileSourceType.User; diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index df33906d..5c575df5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -1,4 +1,10 @@ +using BotSharp.Abstraction.Repositories.Enums; using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.IO; +using System.Net.Http; +using System.Net.Mime; namespace BotSharp.Abstraction.Files.Utilities; @@ -37,4 +43,25 @@ public static class FileUtility return contentType; } + + public static async Task GetFileBytes(IServiceProvider services, FileBase file) + { + var bytes = new byte[0]; + var settings = services.GetRequiredService(); + + if (settings.Default == FileStorageEnum.LocalFileStorage) + { + using var fs = File.OpenRead(file.FileStorageUrl); + var binary = BinaryData.FromStream(fs); + bytes = binary.ToArray(); + fs.Close(); + } + else + { + var http = services.GetRequiredService(); + using var client = http.CreateClient(); + bytes = await client.GetByteArrayAsync(file.FileUrl); + } + return bytes; + } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs index 3d6cc79b..ccc074e8 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs @@ -5,10 +5,9 @@ public partial class ConversationService : IConversationService public async Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true); - - fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId); + fileStorage.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId); var hooks = _services.GetServices().ToList(); foreach (var hook in hooks) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 74b49d3d..f2a93b3f 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -37,9 +37,9 @@ public partial class ConversationService : IConversationService public async Task DeleteConversations(IEnumerable ids) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var isDeleted = db.DeleteConversations(ids); - fileService.DeleteConversationFiles(ids); + fileStorage.DeleteConversationFiles(ids); return await Task.FromResult(isDeleted); } diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index d429ca28..f02c9b41 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -20,7 +20,7 @@ public class FilePlugin : IBotSharpPlugin if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) { - services.AddScoped(); + services.AddScoped(); } services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs index a1208a31..153b47d0 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class FileBasicService +public partial class LocalFileStorageService { public string GetDirectory(string conversationId) { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs index 403e2da5..597b8922 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs @@ -4,7 +4,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class FileBasicService +public partial class LocalFileStorageService { public async Task> GetChatFiles(string conversationId, string source, IEnumerable dialogs, IEnumerable? contentTypes = null, diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs index f26763c9..43ff9eed 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class FileBasicService +public partial class LocalFileStorageService { public string GetUserAvatar() { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs similarity index 84% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs index 1d2079b9..45d449bd 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs @@ -2,12 +2,12 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class FileBasicService : IFileBasicService +public partial class LocalFileStorageService : IFileStorageService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly string _baseDir; private readonly IEnumerable _imageTypes = new List { @@ -24,10 +24,10 @@ public partial class FileBasicService : IFileBasicService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - public FileBasicService( + public LocalFileStorageService( BotSharpDatabaseSettings dbSettings, IUserIdentity user, - ILogger logger, + ILogger logger, IServiceProvider services) { _dbSettings = dbSettings; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index f5d7ede1..4109145d 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -2,14 +2,14 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService : IFileInstructService { - private readonly IFileBasicService _fileBasic; + private readonly IFileStorageService _fileBasic; private readonly IServiceProvider _services; private readonly ILogger _logger; private const string SESSION_FOLDER = "sessions"; public FileInstructService( - IFileBasicService fileBasic, + IFileStorageService fileBasic, ILogger logger, IServiceProvider services) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index d81f069a..89cdc6e9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -81,10 +81,10 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var messageIds = history.Select(x => x.MessageId).Distinct().ToList(); - var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds); + var fileMessages = fileStorage.GetMessagesWithFile(conversationId, messageIds); var dialogs = new List(); foreach (var message in history) @@ -349,15 +349,15 @@ public class ConversationController : ControllerBase { if (files != null && files.Length > 0) { - var fileService = _services.GetRequiredService(); - var dir = fileService.GetDirectory(conversationId); + var fileStorage = _services.GetRequiredService(); + var dir = fileStorage.GetDirectory(conversationId); foreach (var file in files) { // Save the file, process it, etc. var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"'); var filePath = Path.Combine(dir, fileName); - fileService.SaveFileStreamToPath(filePath, file.OpenReadStream()); + fileStorage.SaveFileStreamToPath(filePath, file.OpenReadStream()); } return Ok(new { message = "File uploaded successfully." }); @@ -372,25 +372,25 @@ public class ConversationController : ControllerBase var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); var conv = await convService.GetConversationRecordOrCreateNew(agentId); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var messageId = Guid.NewGuid().ToString(); - var isSaved = fileService.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); + var isSaved = fileStorage.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); return isSaved ? messageId : string.Empty; } [HttpGet("/conversation/{conversationId}/files/{messageId}/{source}")] public IEnumerable GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source) { - var fileService = _services.GetRequiredService(); - var files = fileService.GetMessageFiles(conversationId, new List { messageId }, source); + var fileStorage = _services.GetRequiredService(); + var files = fileStorage.GetMessageFiles(conversationId, new List { messageId }, source); return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List(); } [HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")] public IActionResult GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source, [FromRoute] string index, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); - var file = fileService.GetMessageFile(conversationId, messageId, source, index, fileName); + var fileStorage = _services.GetRequiredService(); + var file = fileStorage.GetMessageFile(conversationId, messageId, source, index, fileName); if (string.IsNullOrEmpty(file)) { return NotFound(); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 74db3bd0..a7d4c449 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -137,15 +137,15 @@ public class UserController : ControllerBase [HttpPost("/user/avatar")] public bool UploadUserAvatar([FromBody] BotSharpFile file) { - var fileService = _services.GetRequiredService(); - return fileService.SaveUserAvatar(file); + var fileStorage = _services.GetRequiredService(); + return fileStorage.SaveUserAvatar(file); } [HttpGet("/user/avatar")] public IActionResult GetUserAvatar() { - var fileService = _services.GetRequiredService(); - var file = fileService.GetUserAvatar(); + var fileStorage = _services.GetRequiredService(); + var file = fileStorage.GetUserAvatar(); if (string.IsNullOrEmpty(file)) { return NotFound(); @@ -158,8 +158,8 @@ public class UserController : ControllerBase #region Private methods private FileContentResult BuildFileResult(string file) { - var fileService = _services.GetRequiredService(); - var bytes = fileService.GetFileBytes(file); + var fileStorage = _services.GetRequiredService(); + var bytes = fileStorage.GetFileBytes(file); return File(bytes, "application/octet-stream", Path.GetFileName(file)); } #endregion diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 634be215..8bed9169 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using MailKit.Net.Smtp; using MailKit.Security; using MimeKit; @@ -52,7 +53,7 @@ public class HandleEmailSenderFn : IFunctionCallback if (isNeedAttachments) { var files = await GetConversationFiles(); - BuildEmailAttachments(bodyBuilder, files); + await BuildEmailAttachments(bodyBuilder, files); } mailMessage.Body = bodyBuilder.ToMessageBody(); @@ -65,7 +66,7 @@ public class HandleEmailSenderFn : IFunctionCallback catch (Exception ex) { var msg = $"Failed to send the email. {ex.Message}"; - _logger.LogError($"{msg}\n(Error: {ex.Message})"); + _logger.LogError($"{msg}\n(Error: {ex.Message}\r\n{ex.InnerException})"); message.Content = msg; return false; } @@ -81,7 +82,7 @@ public class HandleEmailSenderFn : IFunctionCallback return selecteds; } - private void BuildEmailAttachments(BodyBuilder builder, IEnumerable files) + private async Task BuildEmailAttachments(BodyBuilder builder, IEnumerable files) { if (files.IsNullOrEmpty()) return; @@ -89,10 +90,8 @@ public class HandleEmailSenderFn : IFunctionCallback { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - using var fs = File.OpenRead(file.FileStorageUrl); - var binary = BinaryData.FromStream(fs); - builder.Attachments.Add($"{file.FileName}.{file.FileType}", binary.ToArray(), ContentType.Parse(file.ContentType)); - fs.Close(); + var fileBytes = await FileUtility.GetFileBytes(_services, file); + builder.Attachments.Add($"{file.FileName}.{file.FileType}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 1f99c287..28e74a5e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Templating; using System.IO; @@ -77,7 +78,10 @@ public class EditImageFn : IFunctionCallback Name = "Utility Assistant" }; - using var stream = File.OpenRead(image.FileStorageUrl); + var fileBytes = await FileUtility.GetFileBytes(_services, image); + using var stream = new MemoryStream(); + stream.Write(fileBytes); + stream.Position = 0; var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); @@ -105,7 +109,7 @@ public class EditImageFn : IFunctionCallback } }; - var fileService = _services.GetRequiredService(); - fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + var fileStorage = _services.GetRequiredService(); + fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 3869a419..4102ff7b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -83,7 +83,7 @@ public class GenerateImageFn : IFunctionCallback FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" }).ToList(); - var fileService = _services.GetRequiredService(); - fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + var fileStorage = _services.GetRequiredService(); + fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index cdff6cf4..183b775b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -51,8 +51,8 @@ public class ReadImageFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); - var images = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); + var fileStorage = _services.GetRequiredService(); + var images = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); foreach (var dialog in dialogs) { @@ -62,6 +62,7 @@ public class ReadImageFn : IFunctionCallback dialog.Files = found.Select(x => new BotSharpFile { ContentType = x.ContentType, + FileUrl = x.FileUrl, FileStorageUrl = x.FileStorageUrl }).ToList(); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index d3c21737..ef34f96d 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -50,8 +50,8 @@ public class ReadPdfFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); - var files = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); + var fileStorage = _services.GetRequiredService(); + var files = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); foreach (var dialog in dialogs) { @@ -61,6 +61,7 @@ public class ReadPdfFn : IFunctionCallback dialog.Files = found.Select(x => new BotSharpFile { ContentType = x.ContentType, + FileUrl = x.FileUrl, FileStorageUrl = x.FileStorageUrl }).ToList(); } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index d67d69f0..9db083c8 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -74,7 +74,7 @@ public partial class TencentCosService var model = new MessageFileModel() { MessageId = messageId, - FileUrl = $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}", + FileUrl = BuilFileUrl(file), FileStorageUrl = file, FileName = fileName, FileType = fileType, @@ -140,9 +140,7 @@ public partial class TencentCosService try { var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); - var subDir = $"{dir}/{source}/{i + 1}"; - _cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", bytes); } catch (Exception ex) @@ -257,7 +255,6 @@ public partial class TencentCosService if (!_imageTypes.Contains(contentType) && includeScreenShot) { var screenShotDir = $"{fileDir}/{SCREENSHOT_FILE_FOLDER}/"; - var fileList = _cosClient.BucketClient.GetDirFiles(screenShotDir); if (!fileList.IsNullOrEmpty()) @@ -274,6 +271,7 @@ public partial class TencentCosService MessageId = messageId, FileName = fileName, FileType = fileType, + FileUrl = BuilFileUrl(screenShot), FileStorageUrl = screenShot, ContentType = contentType, FileSource = source @@ -294,6 +292,7 @@ public partial class TencentCosService MessageId = messageId, FileName = fileName, FileType = fileType, + FileUrl = BuilFileUrl(image), FileStorageUrl = image, ContentType = contentType, FileSource = source @@ -311,6 +310,7 @@ public partial class TencentCosService MessageId = messageId, FileName = fileName, FileType = fileType, + FileUrl = BuilFileUrl(file), FileStorageUrl = file, ContentType = contentType, FileSource = source @@ -346,5 +346,10 @@ public partial class TencentCosService var converters = _services.GetServices(); return converters.FirstOrDefault(); } + + private string BuilFileUrl(string file) + { + return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}"; + } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index 78c6bcc9..12bd9e17 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -5,7 +5,7 @@ using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; -public partial class TencentCosService : IFileBasicService +public partial class TencentCosService : IFileStorageService { private readonly TencentCosClient _cosClient; private readonly TencentCosSettings _settings; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 25cdb277..757b6c7f 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -31,7 +31,7 @@ public class TencentCosPlugin : IBotSharpPlugin services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } } From be260034bdabe4e162c4d6069ef7bcd429783f60 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 7 Aug 2024 21:10:45 -0500 Subject: [PATCH 25/63] rename --- .../Instruct/FileInstructService.Pdf.cs | 18 +++++++++--------- .../Instruct/FileInstructService.SelectFile.cs | 4 ++-- .../Services/Instruct/FileInstructService.cs | 16 ++++++++-------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index d4413983..67dd8cb9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -16,8 +16,8 @@ public partial class FileInstructService var guid = Guid.NewGuid().ToString(); - var sessionDir = _fileBasic.BuildDirectory(SESSION_FOLDER, guid); - DeleteIfExistDirectory(sessionDir); + var sessionDir = _fileStorage.BuildDirectory(SESSION_FOLDER, guid); + DeleteIfExistDirectory(sessionDir, true); try { @@ -46,7 +46,7 @@ public partial class FileInstructService } finally { - _fileBasic.DeleteDirectory(sessionDir); + _fileStorage.DeleteDirectory(sessionDir); } } @@ -78,11 +78,11 @@ public partial class FileInstructService if (!bytes.IsNullOrEmpty()) { var guid = Guid.NewGuid().ToString(); - var fileDir = _fileBasic.BuildDirectory(dir, guid); - DeleteIfExistDirectory(fileDir); + var fileDir = _fileStorage.BuildDirectory(dir, guid); + DeleteIfExistDirectory(fileDir, true); - var pdfDir = _fileBasic.BuildDirectory(fileDir, $"{guid}.{extension}"); - _fileBasic.SaveFileBytesToPath(pdfDir, bytes); + var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); + _fileStorage.SaveFileBytesToPath(pdfDir, bytes); locs.Add(pdfDir); } } @@ -108,8 +108,8 @@ public partial class FileInstructService { try { - var dir = _fileBasic.GetParentDir(file); - var folder = _fileBasic.BuildDirectory(dir, "screenshots"); + var dir = _fileStorage.GetParentDir(file); + var folder = _fileStorage.BuildDirectory(dir, "screenshots"); var urls = await converter.ConvertPdfToImages(file, folder); images.AddRange(urls); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index c6a9e06c..70b3b80f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -16,10 +16,10 @@ public partial class FileInstructService var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); var messageIds = GetMessageIds(dialogs, options.Offset); - var files = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes); + var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes); if (options.IncludeBotFile) { - var botFiles = _fileBasic.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); + var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); files = files.Concat(botFiles); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index 4109145d..acd0ddaa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -2,31 +2,31 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService : IFileInstructService { - private readonly IFileStorageService _fileBasic; + private readonly IFileStorageService _fileStorage; private readonly IServiceProvider _services; private readonly ILogger _logger; private const string SESSION_FOLDER = "sessions"; public FileInstructService( - IFileStorageService fileBasic, + IFileStorageService fileStorate, ILogger logger, IServiceProvider services) { - _fileBasic = fileBasic; + _fileStorage = fileStorate; _logger = logger; _services = services; } - private void DeleteIfExistDirectory(string? dir) + private void DeleteIfExistDirectory(string? dir, bool createNew = false) { - if (_fileBasic.ExistDirectory(dir)) + if (_fileStorage.ExistDirectory(dir)) { - _fileBasic.DeleteDirectory(dir); + _fileStorage.DeleteDirectory(dir); } - else + else if (createNew) { - _fileBasic.CreateDirectory(dir); + _fileStorage.CreateDirectory(dir); } } } From aa324bc3bf7e9016ef29e8c5ffadb0713919944c Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 7 Aug 2024 22:00:18 -0500 Subject: [PATCH 26/63] fix setting --- .../BotSharp.Core/Files/FilePlugin.cs | 3 ++- .../FileInstructService.SelectFile.cs | 5 +++- .../Functions/HandleEmailReaderFn.cs | 25 ++++++------------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs index f02c9b41..46eded0f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs @@ -10,13 +10,14 @@ public class FilePlugin : IBotSharpPlugin public string Name => "File"; - public string Description => "Provides file analysis."; + public string Description => "Provides file storage and analysis."; public void RegisterDI(IServiceCollection services, IConfiguration config) { var myFileStorageSettings = new FileStorageSettings(); config.Bind("FileStorage", myFileStorageSettings); + services.AddSingleton(myFileStorageSettings); if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 70b3b80f..30031362 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -82,7 +82,10 @@ public partial class FileInstructService var response = await completion.GetChatCompletions(agent, new List { message }); var content = response?.Content ?? string.Empty; - var selecteds = JsonSerializer.Deserialize(content); + var selecteds = JsonSerializer.Deserialize(content, new JsonSerializerOptions + { + AllowTrailingCommas = true + }); var fids = selecteds?.Selecteds ?? new List(); return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs index 121638e7..d0f11466 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs @@ -1,17 +1,7 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Messaging.Models.RichContent.Template; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.EmailHandler.Models; using BotSharp.Plugin.EmailHandler.Providers; using MailKit; -using MailKit.Net.Imap; -using MailKit.Search; -using MailKit.Security; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using MimeKit; namespace BotSharp.Plugin.EmailReader.Functions; @@ -31,13 +21,14 @@ public class HandleEmailReaderFn : IFunctionCallback private readonly IConversationStateService _state; private readonly IEmailReader _emailProvider; - public HandleEmailReaderFn(IServiceProvider services, - ILogger logger, - IHttpContextAccessor context, - BotSharpOptions options, - EmailReaderSettings emailPluginSettings, - IConversationStateService state, - IEmailReader emailProvider) + public HandleEmailReaderFn( + IServiceProvider services, + ILogger logger, + IHttpContextAccessor context, + BotSharpOptions options, + EmailReaderSettings emailPluginSettings, + IConversationStateService state, + IEmailReader emailProvider) { _services = services; _logger = logger; From 7a3dcc77a5df6ad7550850c9fd932e8bf23bffa2 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 8 Aug 2024 00:51:15 -0500 Subject: [PATCH 27/63] get file bytes --- .../Files/IFileStorageService.cs | 2 +- .../Files/Utilities/FileUtility.cs | 21 ------------------- ...n.cs => LocalFileStorageService.Common.cs} | 6 +++--- ...> LocalFileStorageService.Conversation.cs} | 3 +-- ...ser.cs => LocalFileStorageService.User.cs} | 0 .../Instruct/FileInstructService.Pdf.cs | 1 - .../Functions/HandleEmailSenderFn.cs | 7 ++++--- .../Functions/EditImageFn.cs | 3 ++- .../Services/TencentCosService.Common.cs | 6 ++---- .../TencentCosService.Conversation.cs | 10 ++++----- .../Services/TencentCosService.User.cs | 1 - .../TencentCosClient.cs | 1 - .../TencentCosPlugin.cs | 1 - 13 files changed, 18 insertions(+), 44 deletions(-) rename src/Infrastructure/BotSharp.Core/Files/Services/Basic/{FileBasicService.Common.cs => LocalFileStorageService.Common.cs} (88%) rename src/Infrastructure/BotSharp.Core/Files/Services/Basic/{FileBasicService.Conversation.cs => LocalFileStorageService.Conversation.cs} (99%) rename src/Infrastructure/BotSharp.Core/Files/Services/Basic/{FileBasicService.User.cs => LocalFileStorageService.User.cs} (100%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 2db2dc43..5ce60519 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -52,7 +52,7 @@ public interface IFileStorageService #region Common string GetDirectory(string conversationId); - byte[] GetFileBytes(string fileStorageUrl); + byte[] GetFileBytes(string filePath); bool SaveFileStreamToPath(string filePath, Stream stream); bool SaveFileBytesToPath(string filePath, byte[] bytes); string GetParentDir(string dir, int level = 1); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index 5c575df5..e10ba740 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -43,25 +43,4 @@ public static class FileUtility return contentType; } - - public static async Task GetFileBytes(IServiceProvider services, FileBase file) - { - var bytes = new byte[0]; - var settings = services.GetRequiredService(); - - if (settings.Default == FileStorageEnum.LocalFileStorage) - { - using var fs = File.OpenRead(file.FileStorageUrl); - var binary = BinaryData.FromStream(fs); - bytes = binary.ToArray(); - fs.Close(); - } - else - { - var http = services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - return bytes; - } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs similarity index 88% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs index 153b47d0..c7a3cec9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs @@ -6,7 +6,7 @@ public partial class LocalFileStorageService { public string GetDirectory(string conversationId) { - var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); + var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, "attachments"); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); @@ -14,9 +14,9 @@ public partial class LocalFileStorageService return dir; } - public byte[] GetFileBytes(string fileStorageUrl) + public byte[] GetFileBytes(string filePath) { - using var stream = File.OpenRead(fileStorageUrl); + using var stream = File.OpenRead(filePath); var bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); return bytes; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs similarity index 99% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs index 597b8922..bae18cae 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs @@ -87,7 +87,6 @@ public partial class LocalFileStorageService } } } - return files; } @@ -202,8 +201,8 @@ public partial class LocalFileStorageService var dir = GetConversationFileDirectory(conversationId, messageId); if (!ExistDirectory(dir)) continue; - Thread.Sleep(100); DeleteDirectory(dir); + Thread.Sleep(100); } return true; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.User.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.User.cs diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 67dd8cb9..6e6d4168 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Files.Converters; -using System.IO; namespace BotSharp.Core.Files.Services; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 8bed9169..7b59048e 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -53,7 +53,7 @@ public class HandleEmailSenderFn : IFunctionCallback if (isNeedAttachments) { var files = await GetConversationFiles(); - await BuildEmailAttachments(bodyBuilder, files); + BuildEmailAttachments(bodyBuilder, files); } mailMessage.Body = bodyBuilder.ToMessageBody(); @@ -82,7 +82,7 @@ public class HandleEmailSenderFn : IFunctionCallback return selecteds; } - private async Task BuildEmailAttachments(BodyBuilder builder, IEnumerable files) + private void BuildEmailAttachments(BodyBuilder builder, IEnumerable files) { if (files.IsNullOrEmpty()) return; @@ -90,7 +90,8 @@ public class HandleEmailSenderFn : IFunctionCallback { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - var fileBytes = await FileUtility.GetFileBytes(_services, file); + var fileStorage = _services.GetRequiredService(); + var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); builder.Attachments.Add($"{file.FileName}.{file.FileType}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 28e74a5e..9d5b6ad0 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -78,7 +78,8 @@ public class EditImageFn : IFunctionCallback Name = "Utility Assistant" }; - var fileBytes = await FileUtility.GetFileBytes(_services, image); + var fileStorage = _services.GetRequiredService(); + var fileBytes = fileStorage.GetFileBytes(image.FileStorageUrl); using var stream = new MemoryStream(); stream.Write(fileBytes); stream.Position = 0; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index 9de15321..0424d287 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -1,5 +1,3 @@ -using System.IO; - namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -9,11 +7,11 @@ public partial class TencentCosService return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } - public byte[] GetFileBytes(string fileStorageUrl) + public byte[] GetFileBytes(string filePath) { try { - var fileData = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); + var fileData = _cosClient.BucketClient.DownloadFileBytes(filePath); return fileData; } catch (Exception ex) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 9db083c8..bea7d137 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -223,11 +223,11 @@ public partial class TencentCosService return dir; } - private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) { - if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + if (dialogs.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 1) + if (offset.HasValue && offset < 1) { offset = 1; } @@ -235,11 +235,11 @@ public partial class TencentCosService var messageIds = new List(); if (offset.HasValue) { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); } else { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); } return messageIds; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index 55e26d81..de222c43 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -51,7 +51,6 @@ public partial class TencentCosService } var dir = $"{USERS_FOLDER}/{userId}/{USER_AVATAR_FOLDER}/"; - return dir; } #endregion diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs index b7e600f3..527be651 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs @@ -19,7 +19,6 @@ namespace BotSharp.Plugin.TencentCos settings.SecretId, settings.SecretKey, settings.KeyDurationSecond); var cosXml = new CosXmlServer(cosXmlConfig, cosCredentialProvider); - BucketClient = new BucketClient(cosXml, $"{settings.BucketName}-{settings.AppId}", settings.AppId, settings.Region); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 757b6c7f..1dcb5edb 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -30,7 +30,6 @@ public class TencentCosPlugin : IBotSharpPlugin }); services.AddScoped(); - services.AddScoped(); } } From 32e1da62c02414f3a85885c56275f2554ae65fb7 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 8 Aug 2024 13:34:50 -0500 Subject: [PATCH 28/63] add get message file screenshots --- .../Files/IFileStorageService.cs | 13 +- .../Files/Models/FileBase.cs | 4 +- .../Files/Models/MessageFileModel.cs | 2 +- .../Files/Models/SelectFileOptions.cs | 35 +++++ .../BotSharp.Core/BotSharp.Core.csproj | 2 +- .../FileInstructService.SelectFile.cs | 2 +- .../LocalFileStorageService.Common.cs | 0 .../LocalFileStorageService.Conversation.cs | 140 ++++++----------- .../LocalFileStorageService.User.cs | 0 .../LocalFileStorageService.cs | 5 - .../ViewModels/Files/MessageFileViewModel.cs | 6 +- .../Providers/Chat/ChatCompletionProvider.cs | 19 +-- .../Functions/HandleEmailSenderFn.cs | 4 +- .../Functions/EditImageFn.cs | 2 +- .../Functions/ReadImageFn.cs | 15 +- .../Functions/ReadPdfFn.cs | 9 +- .../Providers/Chat/ChatCompletionProvider.cs | 19 +-- .../TencentCosService.Conversation.cs | 146 ++++++------------ .../Services/TencentCosService.cs | 2 +- 19 files changed, 176 insertions(+), 249 deletions(-) rename src/Infrastructure/BotSharp.Core/Files/Services/{Basic => Storage}/LocalFileStorageService.Common.cs (100%) rename src/Infrastructure/BotSharp.Core/Files/Services/{Basic => Storage}/LocalFileStorageService.Conversation.cs (77%) rename src/Infrastructure/BotSharp.Core/Files/Services/{Basic => Storage}/LocalFileStorageService.User.cs (100%) rename src/Infrastructure/BotSharp.Core/Files/Services/{Basic => Storage}/LocalFileStorageService.cs (88%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 5ce60519..99840dff 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -6,19 +6,12 @@ public interface IFileStorageService { #region Conversation /// - /// Get the files that have been uploaded in the chat. - /// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file. + /// Get the message file screenshots for specific content types, e.g., pdf /// /// - /// - /// - /// - /// - /// + /// /// - Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes, - bool includeScreenShot = false, int? offset = null); + Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs index c54c31ff..3483921a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -40,7 +40,7 @@ public class FileBase /// /// File extension without dot /// - [JsonPropertyName("file_type")] + [JsonPropertyName("file_extension")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileType { get; set; } = string.Empty; + public string? FileExtension { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index e06e1a0f..2a0128b6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -15,6 +15,6 @@ public class MessageFileModel : FileBase public override string ToString() { - return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}"; + return $"File name: {FileName}, File extension: {FileExtension}, Content type: {ContentType}, Source: {FileSource}"; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs index 6ba6cdd2..d61c1b7c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -2,13 +2,48 @@ namespace BotSharp.Abstraction.Files.Models; public class SelectFileOptions { + /// + /// Llm provider + /// public string? Provider { get; set; } + + /// + /// Llm model id + /// public string? ModelId { get; set; } + + /// + /// Agent id + /// public string? AgentId { get; set; } + + /// + /// Template (prompt) name + /// public string? Template { get; set; } + + /// + /// Description that user provides to select files + /// public string? Description { get; set; } + + /// + /// Whether include bot generated files + /// public bool IncludeBotFile { get; set; } + + /// + /// Conversation breakpoint + /// public bool FromBreakpoint { get; set; } + + /// + /// Message offset from last + /// public int? Offset { get; set; } + + /// + /// File content types. If null, all types of files will be retrived + /// public IEnumerable? ContentTypes { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index dec4d9f6..dfa39aad 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 30031362..5f185a6f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -43,7 +43,7 @@ public partial class FileInstructService { var promptFiles = files.Select((x, idx) => { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}"; }).ToList(); var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs similarity index 77% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index bae18cae..5307d198 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,22 +6,20 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes = null, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(dialogs, offset); + var source = FileSourceType.User; var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); foreach (var messageId in messageIds) { - var dir = Path.Combine(pathPrefix, messageId, source); + var dir = Path.Combine(pathPrefix, messageId, FileSourceType.User); if (!ExistDirectory(dir)) continue; foreach (var subDir in Directory.GetDirectories(dir)) @@ -30,18 +28,49 @@ public partial class LocalFileStorageService if (file == null) continue; var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + var screenshotDir = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER); + + if (ExistDirectory(screenshotDir) && !Directory.GetFiles(screenshotDir).IsNullOrEmpty()) { - continue; + foreach (var screenshot in Directory.GetFiles(screenshotDir)) + { + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = screenshot, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = image, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } } - - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; - - files.AddRange(foundFiles); } } - return files; } @@ -72,14 +101,14 @@ public partial class LocalFileStorageService } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}", FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -268,85 +297,6 @@ public partial class LocalFileStorageService return messageIds; } - - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) - { - var files = new List(); - - try - { - if (!_imageTypes.Contains(contentType) && includeScreenShot) - { - var screenShotDir = Path.Combine(fileDir, SCREENSHOT_FILE_FOLDER); - if (ExistDirectory(screenShotDir) && !Directory.GetFiles(screenShotDir).IsNullOrEmpty()) - { - foreach (var screenShot in Directory.GetFiles(screenShotDir)) - { - contentType = FileUtility.GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) - { - contentType = FileUtility.GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - } - else - { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - - return files; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); - return files; - } - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs similarity index 88% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 45d449bd..d83cae03 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -9,11 +9,6 @@ public partial class LocalFileStorageService : IFileStorageService private readonly IUserIdentity _user; private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _imageTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg - }; private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs index 131a9baf..787ab147 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs @@ -10,8 +10,8 @@ public class MessageFileViewModel [JsonPropertyName("file_name")] public string FileName { get; set; } - [JsonPropertyName("file_type")] - public string FileType { get; set; } + [JsonPropertyName("file_extension")] + public string FileExtension { get; set; } [JsonPropertyName("content_type")] public string ContentType { get; set; } @@ -30,7 +30,7 @@ public class MessageFileViewModel { FileUrl = model.FileUrl, FileName = model.FileName, - FileType = model.FileType, + FileExtension = model.FileExtension, ContentType = model.ContentType, FileSource = model.FileSource }; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 72034317..af82b037 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -198,6 +198,7 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -262,13 +263,7 @@ public class ChatCompletionProvider : IChatCompletion { foreach (var file in message.Files) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileData)) + if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); @@ -277,8 +272,14 @@ public class ChatCompletionProvider : IChatCompletion else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 7b59048e..b4f9a6e4 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -1,8 +1,6 @@ -using BotSharp.Abstraction.Files.Utilities; using MailKit.Net.Smtp; using MailKit.Security; using MimeKit; -using System.IO; namespace BotSharp.Plugin.EmailHandler.Functions; @@ -92,7 +90,7 @@ public class HandleEmailSenderFn : IFunctionCallback var fileStorage = _services.GetRequiredService(); var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); - builder.Attachments.Add($"{file.FileName}.{file.FileType}", fileBytes, ContentType.Parse(file.ContentType)); + builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 9d5b6ad0..a12bdfd4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -87,7 +87,7 @@ public class EditImageFn : IFunctionCallback stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); - return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted."; + return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted."; } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 183b775b..1d9a8112 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -8,12 +8,6 @@ public class ReadImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly IEnumerable _imageContentTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg, - }; - public ReadImageFn( IServiceProvider services, ILogger logger) @@ -52,7 +46,14 @@ public class ReadImageFn : IFunctionCallback } var fileStorage = _services.GetRequiredService(); - var images = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); + var fileInstruct = _services.GetRequiredService(); + + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List + { + MediaTypeNames.Image.Png, + MediaTypeNames.Image.Jpeg + }); foreach (var dialog in dialogs) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index ef34f96d..2a5d7197 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -51,11 +51,16 @@ public class ReadPdfFn : IFunctionCallback } var fileStorage = _services.GetRequiredService(); - var files = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); + var fileInstruct = _services.GetRequiredService(); + + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + + if (screenshots.IsNullOrEmpty()) return dialogs; foreach (var dialog in dialogs) { - var found = files.Where(x => x.MessageId == dialog.MessageId).ToList(); + var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index f12e8b34..00f47149 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -199,6 +199,7 @@ public class ChatCompletionProvider : IChatCompletion { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -263,13 +264,7 @@ public class ChatCompletionProvider : IChatCompletion { foreach (var file in message.Files) { - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileData)) + if (!string.IsNullOrEmpty(file.FileData)) { var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); @@ -278,8 +273,14 @@ public class ChatCompletionProvider : IChatCompletion else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index bea7d137..8008a5e2 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,3 +1,4 @@ +using AspectInjector.Broker; using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; @@ -7,38 +8,68 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes = null, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(dialogs, offset); + var source = FileSourceType.User; var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; - foreach (var messageId in messageIds) { var dir = $"{pathPrefix}/{messageId}/{source}"; - foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) { var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); if (file == null) continue; var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) + var screenshotDir = $"{subDir}/{SCREENSHOT_FILE_FOLDER}/"; + var screenshots = _cosClient.BucketClient.GetDirFiles(screenshotDir); + if (!screenshots.IsNullOrEmpty()) { - continue; + foreach (var screenshot in screenshots) + { + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(screenshot), + FileStorageUrl = screenshot, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(image), + FileStorageUrl = image, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } } - - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; - - files.AddRange(foundFiles); } } @@ -70,14 +101,14 @@ public partial class TencentCosService } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, FileUrl = BuilFileUrl(file), FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -94,7 +125,6 @@ public partial class TencentCosService var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/"; var fileList = _cosClient.BucketClient.GetDirFiles(dir); - var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName)); return found; } @@ -246,88 +276,6 @@ public partial class TencentCosService } - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) - { - var files = new List(); - try - { - if (!_imageTypes.Contains(contentType) && includeScreenShot) - { - var screenShotDir = $"{fileDir}/{SCREENSHOT_FILE_FOLDER}/"; - var fileList = _cosClient.BucketClient.GetDirFiles(screenShotDir); - - if (!fileList.IsNullOrEmpty()) - { - foreach (var screenShot in fileList) - { - contentType = FileUtility.GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileUrl = BuilFileUrl(screenShot), - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) - { - contentType = FileUtility.GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileUrl = BuilFileUrl(image), - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - } - else - { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileUrl = BuilFileUrl(file), - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - - return files; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); - return files; - } - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index 12bd9e17..ddb5e031 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -40,7 +40,7 @@ public partial class TencentCosService : IFileStorageService _user = user; _logger = logger; _services = services; - _fullBuketName = $"{_settings.BucketName}-{_settings.AppId}"; + _fullBuketName = $"{settings.BucketName}-{settings.AppId}"; _cosClient = cosClient; } } From 1abf3f4aae68d5164e3a883bead2fa9de3e597d8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 8 Aug 2024 13:37:07 -0500 Subject: [PATCH 29/63] change param name --- .../Files/IFileStorageService.cs | 26 ++++++++++--------- .../Storage/LocalFileStorageService.Common.cs | 4 +-- .../Services/TencentCosService.Common.cs | 5 ++-- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 99840dff..065cdc55 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -4,6 +4,19 @@ namespace BotSharp.Abstraction.Files; public interface IFileStorageService { + #region Common + string GetDirectory(string conversationId); + byte[] GetFileBytes(string fileStorageUrl); + bool SaveFileStreamToPath(string filePath, Stream stream); + bool SaveFileBytesToPath(string filePath, byte[] bytes); + string GetParentDir(string dir, int level = 1); + bool ExistDirectory(string? dir); + void CreateDirectory(string dir); + void DeleteDirectory(string dir); + string BuildDirectory(params string[] segments); + #endregion + + #region Conversation /// /// Get the message file screenshots for specific content types, e.g., pdf @@ -38,20 +51,9 @@ public interface IFileStorageService bool DeleteConversationFiles(IEnumerable conversationIds); #endregion + #region User string GetUserAvatar(); bool SaveUserAvatar(BotSharpFile file); #endregion - - #region Common - string GetDirectory(string conversationId); - byte[] GetFileBytes(string filePath); - bool SaveFileStreamToPath(string filePath, Stream stream); - bool SaveFileBytesToPath(string filePath, byte[] bytes); - string GetParentDir(string dir, int level = 1); - bool ExistDirectory(string? dir); - void CreateDirectory(string dir); - void DeleteDirectory(string dir); - string BuildDirectory(params string[] segments); - #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index c7a3cec9..c49edbce 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -14,9 +14,9 @@ public partial class LocalFileStorageService return dir; } - public byte[] GetFileBytes(string filePath) + public byte[] GetFileBytes(string fileStorageUrl) { - using var stream = File.OpenRead(filePath); + using var stream = File.OpenRead(fileStorageUrl); var bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); return bytes; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index 0424d287..e94ab6ce 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -7,12 +7,11 @@ public partial class TencentCosService return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } - public byte[] GetFileBytes(string filePath) + public byte[] GetFileBytes(string fileStorageUrl) { try { - var fileData = _cosClient.BucketClient.DownloadFileBytes(filePath); - return fileData; + return _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); } catch (Exception ex) { From 3b606974f004713ffcf9811ff5633f4c6cc77d9d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 8 Aug 2024 13:50:45 -0500 Subject: [PATCH 30/63] refine screenshots --- .../LocalFileStorageService.Conversation.cs | 102 +++++++++------- .../TencentCosService.Conversation.cs | 109 +++++++++++------- 2 files changed, 126 insertions(+), 85 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 5307d198..9e14677d 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -27,53 +27,16 @@ public partial class LocalFileStorageService var file = Directory.GetFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = FileUtility.GetFileContentType(file); - var screenshotDir = Path.Combine(subDir, SCREENSHOT_FILE_FOLDER); + var screenshots = await GetScreenshots(file, subDir, messageId, source); + if (screenshots.IsNullOrEmpty()) continue; - if (ExistDirectory(screenshotDir) && !Directory.GetFiles(screenshotDir).IsNullOrEmpty()) - { - foreach (var screenshot in Directory.GetFiles(screenshotDir)) - { - var fileName = Path.GetFileNameWithoutExtension(screenshot); - var fileExtension = Path.GetExtension(screenshot).Substring(1); - var screenshotContentType = FileUtility.GetFileContentType(screenshot); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileExtension = fileExtension, - FileStorageUrl = screenshot, - ContentType = screenshotContentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenshotDir); - foreach (var image in images) - { - var fileName = Path.GetFileNameWithoutExtension(image); - var fileExtension = Path.GetExtension(image).Substring(1); - var screenshotContentType = FileUtility.GetFileContentType(image); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileExtension = fileExtension, - FileStorageUrl = image, - ContentType = screenshotContentType, - FileSource = source - }; - files.Add(model); - } - } + files.AddRange(screenshots); } } return files; } + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null) { @@ -315,5 +278,62 @@ public partial class LocalFileStorageService var converters = _services.GetServices(); return converters.FirstOrDefault(); } + + private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + { + var files = new List(); + + try + { + var contentType = FileUtility.GetFileContentType(file); + var screenshotDir = Path.Combine(parentDir, SCREENSHOT_FILE_FOLDER); + + if (ExistDirectory(screenshotDir) && !Directory.GetFiles(screenshotDir).IsNullOrEmpty()) + { + foreach (var screenshot in Directory.GetFiles(screenshotDir)) + { + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = screenshot, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = image, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } + } + return files; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + return files; + } + } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 8008a5e2..6037b1e7 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -26,50 +26,10 @@ public partial class TencentCosService var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = FileUtility.GetFileContentType(file); - var screenshotDir = $"{subDir}/{SCREENSHOT_FILE_FOLDER}/"; - var screenshots = _cosClient.BucketClient.GetDirFiles(screenshotDir); - if (!screenshots.IsNullOrEmpty()) - { - foreach (var screenshot in screenshots) - { - var screenshotContentType = FileUtility.GetFileContentType(screenshot); - var fileName = Path.GetFileNameWithoutExtension(screenshot); - var fileExtension = Path.GetExtension(screenshot).Substring(1); - var model = new MessageFileModel - { - MessageId = messageId, - FileName = fileName, - FileExtension = fileExtension, - FileUrl = BuilFileUrl(screenshot), - FileStorageUrl = screenshot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenshotDir); - foreach (var image in images) - { - var fileName = Path.GetFileNameWithoutExtension(image); - var fileExtension = Path.GetExtension(image).Substring(1); - var screenshotContentType = FileUtility.GetFileContentType(image); - var model = new MessageFileModel - { - MessageId = messageId, - FileName = fileName, - FileExtension = fileExtension, - FileUrl = BuilFileUrl(image), - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } + var screenshots = await GetScreenshots(file, subDir, messageId, source); + if (screenshots.IsNullOrEmpty()) continue; + + files.AddRange(screenshots); } } @@ -120,6 +80,8 @@ public partial class TencentCosService return files; } + + public string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName) { var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/"; @@ -299,5 +261,64 @@ public partial class TencentCosService { return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}"; } + + private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + { + var files = new List(); + + try + { + var contentType = FileUtility.GetFileContentType(file); + var screenshotDir = $"{parentDir}/{SCREENSHOT_FILE_FOLDER}/"; + var screenshots = _cosClient.BucketClient.GetDirFiles(screenshotDir); + if (!screenshots.IsNullOrEmpty()) + { + foreach (var screenshot in screenshots) + { + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(screenshot), + FileStorageUrl = screenshot, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(image), + FileStorageUrl = image, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + return files; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + return files; + } + } #endregion } From 5eb241d26b43d067aa9d5b2f0227b0077cbe50f1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 8 Aug 2024 14:54:12 -0500 Subject: [PATCH 31/63] merge message files --- .../FileInstructService.SelectFile.cs | 20 ++++++++++++++++++- .../Functions/ReadImageFn.cs | 6 ++---- .../Functions/ReadPdfFn.cs | 2 -- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 5f185a6f..41e0becc 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -20,7 +20,7 @@ public partial class FileInstructService if (options.IncludeBotFile) { var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); - files = files.Concat(botFiles); + files = MergeMessageFiles(messageIds, files, botFiles); } if (files.IsNullOrEmpty()) @@ -31,6 +31,24 @@ public partial class FileInstructService return await SelectFiles(files, dialogs, options); } + private IEnumerable MergeMessageFiles(IEnumerable messageIds, IEnumerable userFiles, IEnumerable botFiles) + { + var files = new List(); + + if (messageIds.IsNullOrEmpty()) return files; + + foreach (var messageId in messageIds) + { + var users = userFiles.Where(x => x.MessageId == messageId).ToList(); + var bots = botFiles.Where(x => x.MessageId == messageId).ToList(); + + if (!users.IsNullOrEmpty()) files.AddRange(users); + if (!bots.IsNullOrEmpty()) files.AddRange(bots); + } + + return files; + } + private async Task> SelectFiles(IEnumerable files, IEnumerable dialogs, SelectFileOptions options) { if (files.IsNullOrEmpty()) return new List(); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 1d9a8112..a415207e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -23,7 +23,7 @@ public class ReadImageFn : IFunctionCallback var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); var fileAgent = new Agent { @@ -38,7 +38,7 @@ public class ReadImageFn : IFunctionCallback return true; } - private async Task> AssembleFiles(string conversationId, List dialogs) + private List AssembleFiles(string conversationId, List dialogs) { if (dialogs.IsNullOrEmpty()) { @@ -46,8 +46,6 @@ public class ReadImageFn : IFunctionCallback } var fileStorage = _services.GetRequiredService(); - var fileInstruct = _services.GetRequiredService(); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index 2a5d7197..e2b465c8 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -51,8 +51,6 @@ public class ReadPdfFn : IFunctionCallback } var fileStorage = _services.GetRequiredService(); - var fileInstruct = _services.GetRequiredService(); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); From 6f62ca37c02c53882282e8031c7d52e5e6aef1c6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 8 Aug 2024 15:57:15 -0500 Subject: [PATCH 32/63] clean using --- .../BotSharp.Abstraction/Files/Utilities/FileUtility.cs | 6 ------ .../Services/TencentCosService.Conversation.cs | 1 - 2 files changed, 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index e10ba740..df33906d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -1,10 +1,4 @@ -using BotSharp.Abstraction.Repositories.Enums; using Microsoft.AspNetCore.StaticFiles; -using Microsoft.Extensions.DependencyInjection; -using System; -using System.IO; -using System.Net.Http; -using System.Net.Mime; namespace BotSharp.Abstraction.Files.Utilities; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 6037b1e7..f04d0be6 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,4 +1,3 @@ -using AspectInjector.Broker; using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; From c041d5fcb9ca98291a877baf45af2b35553a2905 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 8 Aug 2024 16:06:30 -0500 Subject: [PATCH 33/63] clean using --- .../BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index a12bdfd4..759f94b6 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Files.Utilities; -using BotSharp.Abstraction.Templating; using System.IO; namespace BotSharp.Plugin.FileHandler.Functions; From 608dc1b4cf92844d71e5e0e8e620ec8f65efa2fc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 8 Aug 2024 17:15:51 -0500 Subject: [PATCH 34/63] temp save --- .../Knowledges/IKnowledgeService.cs | 2 ++ .../Models/KnowledgeRetrievalResult.cs | 9 +++++++++ .../Knowledges/Models/RetrievedResult.cs | 2 -- .../VectorStorage/IVectorDb.cs | 5 +++-- .../Controllers/KnowledgeBaseController.cs | 19 +++++++++++++++++-- .../KnowledgeCollectionDataViewModel.cs | 2 ++ .../MemVecDb/MemVectorDatabase.cs | 16 ++++++++++++---- .../Services/KnowledgeService.List.cs | 16 +++++++++++++++- .../Providers/FaissDb.cs | 9 +++++++-- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 17 +++++++++++------ .../SemanticKernelMemoryStoreProvider.cs | 10 ++++++++-- 11 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index f2eb82fe..f145aa6f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -14,5 +14,7 @@ public interface IKnowledgeService #region List Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); + Task> GetSimilarKnowledgeData(string collectionName, KnowledgeFilter filter); + Task DeleteKnowledgeCollectionData(string collectionName, string id); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs new file mode 100644 index 00000000..131bacc0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeRetrievalResult +{ + public string Id { get; set; } + public string Text { get; set; } + public float Score { get; set; } + public float[]? Vector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs index 296e1ed6..18e55634 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs @@ -1,5 +1,3 @@ -using System.Text.Json.Serialization; - namespace BotSharp.Abstraction.Knowledges.Models; public class RetrievedResult diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 50788756..413e998a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -4,9 +4,10 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { - Task> GetCollections(); + Task> GetCollections(); Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index cd01ec05..debec995 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.OpenAPI.ViewModels.Knowledges; -using Microsoft.AspNetCore.Http; namespace BotSharp.OpenAPI.Controllers; @@ -95,7 +94,7 @@ public class KnowledgeBaseController : ControllerBase [HttpPost("/knowledge/{collection}/data")] public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) - {; + { var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? .ToList() ?? new List(); @@ -107,4 +106,20 @@ public class KnowledgeBaseController : ControllerBase Items = items }; } + + [HttpPost("/knowledge/{collection}/similar")] + public async Task> GetSimilarKnowledgeData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) + { + var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? + .ToList() ?? new List(); + + return + } + + [HttpDelete("/knowledge/{collection}/data/{id}")] + public async Task DeleteKnowledgeCollectionData([FromRoute] string collection, [FromRoute] string id) + { + return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id); + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs index 16ebadda..5ce43369 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -9,9 +9,11 @@ public class KnowledgeCollectionDataViewModel public string Id { get; set; } [JsonPropertyName("question")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Question { get; set; } [JsonPropertyName("answer")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string Answer { get; set; } [JsonPropertyName("vector")] diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs index 61598a71..f7c9276d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs @@ -7,13 +7,14 @@ public class MemVectorDatabase : IVectorDb { private readonly Dictionary _collections = new Dictionary(); private readonly Dictionary> _vectors = new Dictionary>(); + public async Task CreateCollection(string collectionName, int dim) { _collections[collectionName] = dim; _vectors[collectionName] = new List(); } - public async Task> GetCollections() + public async Task> GetCollections() { return _collections.Select(x => x.Key).ToList(); } @@ -23,7 +24,7 @@ public class MemVectorDatabase : IVectorDb throw new NotImplementedException(); } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { if (!_vectors.ContainsKey(collectionName)) { @@ -54,6 +55,12 @@ public class MemVectorDatabase : IVectorDb return true; } + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } + + #region Private methods private float[] CalEuclideanDistance(float[] vec, List records) { var a = np.zeros((records.Count, vec.Length), np.float32); @@ -69,7 +76,7 @@ public class MemVectorDatabase : IVectorDb return c.ToArray(); } - public NDArray CalCosineSimilarity(float[] vec, List records) + private NDArray CalCosineSimilarity(float[] vec, List records) { var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); @@ -113,7 +120,7 @@ public class MemVectorDatabase : IVectorDb return resIndex.ToArray(); } - public (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) + private (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) { var squaredX = np.sum(np.multiply(x, x), axis: 1); var normX = np.sqrt(squaredX); @@ -128,4 +135,5 @@ public class MemVectorDatabase : IVectorDb return (x / normX, normX); } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs index 2f8547f4..d3b5ed84 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs @@ -11,8 +11,22 @@ public partial class KnowledgeService } catch (Exception ex) { - _logger.LogWarning($"Error when getting knowledge collectio data. {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); return new StringIdPagedItems(); } } + + public async Task DeleteKnowledgeCollectionData(string collectionName, string id) + { + try + { + var db = GetVectorDb(); + return await db.DeleteCollectionData(collectionName, id); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting knowledge collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 5983d49a..7e884632 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -19,12 +19,12 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollections() + public Task> GetCollections() { throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) + public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) { throw new NotImplementedException(); } @@ -33,4 +33,9 @@ public class FaissDb : IVectorDb { throw new NotImplementedException(); } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index d331e28e..f7bb25c5 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -32,7 +32,7 @@ public class QdrantDb : IVectorDb return _client; } - public async Task> GetCollections() + public async Task> GetCollections() { // List all the collections var collections = await GetClient().ListCollectionsAsync(); @@ -100,7 +100,6 @@ public class QdrantDb : IVectorDb Uuid = id }, Vectors = vector, - Payload = { { KnowledgePayloadName.Text, text } @@ -125,13 +124,19 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { var client = GetClient(); - var points = await client.SearchAsync(collectionName, vector, - limit: (ulong)limit, - scoreThreshold: confidence); + var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); return points.Select(x => x.Payload[returnFieldName].StringValue).ToList(); } + + public async Task DeleteCollectionData(string collectionName, string id) + { + var client = GetClient(); + var guid = Guid.Parse(id); + var result = await client.DeleteAsync(collectionName, guid); + return result.Status == UpdateStatus.Completed; + } } diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index c831d15b..b52ab71e 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using Microsoft.SemanticKernel.Memory; +using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -29,7 +30,7 @@ namespace BotSharp.Plugin.SemanticKernel throw new System.NotImplementedException(); } - public async Task> GetCollections() + public async Task> GetCollections() { var result = new List(); await foreach (var collection in _memoryStore.GetCollectionsAsync()) @@ -39,7 +40,7 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); @@ -60,5 +61,10 @@ namespace BotSharp.Plugin.SemanticKernel #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. return true; } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } } } From e2c73cbbc5ae8120f2d6e7fad38f08a3d95d0d2f Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 8 Aug 2024 22:44:46 -0500 Subject: [PATCH 35/63] Add JWT ExpireInMinutes --- src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj | 2 +- src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 279d735c..ad46fb56 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index bf833c8a..2fa38215 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -205,11 +205,12 @@ public class UserService : IUserService var config = _services.GetRequiredService(); var issuer = config["Jwt:Issuer"]; var audience = config["Jwt:Audience"]; + var expireInMinutes = int.Parse(config["Jwt:ExpireInMinutes"] ?? "120"); var key = Encoding.ASCII.GetBytes(config["Jwt:Key"]); var tokenDescriptor = new SecurityTokenDescriptor { Subject = new ClaimsIdentity(claims), - Expires = DateTime.UtcNow.AddHours(2), + Expires = DateTime.UtcNow.AddMinutes(expireInMinutes), Issuer = issuer, Audience = audience, SigningCredentials = new SigningCredentials(new SymmetricSecurityKey(key), From 280b1bcd2a2918dd8030fedf6a3af58c5f5b212f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 9 Aug 2024 16:03:29 -0500 Subject: [PATCH 36/63] refine file settings and knowledge service --- .../Files/Converters/IPdf2ImageConverter.cs | 2 + .../Files/FileCoreSettings.cs | 10 ++ .../Files/FileStorageSettings.cs | 8 - .../Knowledges/IKnowledgeHook.cs | 2 - .../Knowledges/IKnowledgeService.cs | 13 +- .../Knowledges/IPdf2TextConverter.cs | 6 +- .../Models/KnowledgeCollectionInfo.cs | 7 - .../Models/KnowledgeCreationModel.cs | 3 + .../Knowledges/Models/KnowledgeFeedModel.cs | 7 - .../Models/KnowledgeRetrievalModel.cs | 10 +- .../Models/KnowledgeRetrievalResult.cs | 9 -- .../Models/KnowledgeSearchResult.cs | 12 ++ .../Knowledges/Models/RetrievedResult.cs | 12 -- .../Settings/KnowledgeBaseSettings.cs | 9 +- .../BotSharp.Abstraction/Using.cs | 3 +- .../VectorStorage/IVectorDb.cs | 6 +- .../{FilePlugin.cs => FileCorePlugin.cs} | 12 +- .../Instruct/FileInstructService.Pdf.cs | 3 +- .../LocalFileStorageService.Conversation.cs | 5 +- .../Controllers/KnowledgeBaseController.cs | 121 +++++---------- .../KnowledgeCollectionDataViewModel.cs | 2 +- .../Knowledges/KnowledgeRetrivalViewModel.cs | 27 ++++ .../Knowledges/SearchKnowledgeModel.cs | 25 ++++ .../Functions/KnowledgeRetrievalFn.cs | 8 +- .../Functions/MemorizeKnowledgeFn.cs | 9 +- .../MemVecDb/MemVecDbPlugin.cs | 2 +- .../MemVecDb/MemVectorDatabase.cs | 139 ------------------ .../MemVecDb/MemoryVectorDb.cs | 71 +++++++++ .../Services/KnowledgeService.Create.cs | 28 ++++ ...ice.List.cs => KnowledgeService.Delete.cs} | 14 -- .../Services/KnowledgeService.Get.cs | 38 +++++ .../Services/KnowledgeService.cs | 104 ++----------- .../Services/KnowledgeService.i.cs | 14 -- .../Services/PigPdf2TextConverter.cs | 2 + .../Utilities/VectorUtility.cs | 83 +++++++++++ .../Providers/FaissDb.cs | 5 +- .../Providers/Pdf2TextConverter.cs | 5 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 62 ++++++-- .../Providers/IntentClassifier.cs | 8 +- .../SemanticKernelMemoryStoreProvider.cs | 30 +++- .../TencentCosService.Conversation.cs | 6 +- .../TencentCosPlugin.cs | 6 +- src/WebStarter/appsettings.json | 17 ++- 43 files changed, 495 insertions(+), 470 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs rename src/Infrastructure/BotSharp.Core/Files/{FilePlugin.cs => FileCorePlugin.cs} (63%) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs rename src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/{KnowledgeService.List.cs => KnowledgeService.Delete.cs} (50%) create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs delete mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs create mode 100644 src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs index 54ad3a6d..87df6137 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs @@ -2,6 +2,8 @@ namespace BotSharp.Abstraction.Files.Converters; public interface IPdf2ImageConverter { + public string Name { get; } + /// /// Convert pdf pages to images, and return a list of image file paths /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs new file mode 100644 index 00000000..10ccd1a0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Repositories.Enums; + +namespace BotSharp.Abstraction.Files; + +public class FileCoreSettings +{ + public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; + public string Pdf2TextConverter { get; set; } + public string Pdf2ImageConverter { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs deleted file mode 100644 index 23ba12c6..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs +++ /dev/null @@ -1,8 +0,0 @@ -using BotSharp.Abstraction.Repositories.Enums; - -namespace BotSharp.Abstraction.Files; - -public class FileStorageSettings -{ - public string Default { get; set; } = FileStorageEnum.LocalFileStorage; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs index aefcdec6..3a5ab788 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeHook diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index f145aa6f..827b2ba1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -4,17 +4,8 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { - Task> CollectChunkedKnowledge(); - Task EmbedKnowledge(List chunks); - - Task Feed(KnowledgeFeedModel knowledge); - Task EmbedKnowledge(KnowledgeCreationModel knowledge); - Task GetKnowledges(KnowledgeRetrievalModel retrievalModel); - Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); - - #region List + Task> SearchKnowledge(KnowledgeRetrievalModel model); + Task FeedKnowledge(KnowledgeCreationModel model); Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); - Task> GetSimilarKnowledgeData(string collectionName, KnowledgeFilter filter); Task DeleteKnowledgeCollectionData(string collectionName, string id); - #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index d2ca2940..b8f2d47b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -1,12 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Microsoft.AspNetCore.Http; - namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { + public string Name { get; } Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs deleted file mode 100644 index 714c2b3f..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionInfo.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeCollectionInfo -{ - public ulong DataCount { get; set; } - public ulong VectorCount { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs index 33d09bd6..b43bce67 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs @@ -1,6 +1,9 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeCreationModel { + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; public string Content { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs deleted file mode 100644 index 7e3a315e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeFeedModel -{ - public string AgentId { get; set; } = string.Empty; - public string Content { get; set; } = string.Empty; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs index 03f66eb5..76e5e77d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs @@ -1,7 +1,13 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeRetrievalModel { - public string AgentId { get; set; } = string.Empty; - public string Question { get; set; } = string.Empty; + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; + public string Text { get; set; } = string.Empty; + public IEnumerable? Fields { get; set; } = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + public int? Limit { get; set; } = 5; + public float? Confidence { get; set; } = 0.5f; + public bool WithVector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs deleted file mode 100644 index 131bacc0..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalResult.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeRetrievalResult -{ - public string Id { get; set; } - public string Text { get; set; } - public float Score { get; set; } - public float[]? Vector { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs new file mode 100644 index 00000000..b0deaa0f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeSearchResult +{ + public IDictionary Data { get; set; } = new Dictionary(); + public double Score { get; set; } + public float[]? Vector { get; set; } +} + +public class KnowledgeRetrievalResult : KnowledgeSearchResult +{ +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs deleted file mode 100644 index 18e55634..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class RetrievedResult -{ - public int Paragraph { get; set; } - - [JsonPropertyName("cite_source")] - public string CiteSource { get; set; } = "related text"; - - [JsonPropertyName("reasoning")] - public string Reasoning { get; set; } = ""; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 97f7f55c..0c3f8d45 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -3,7 +3,12 @@ namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { public string VectorDb { get; set; } - public string TextEmbedding { get; set; } - public string TextCompletion { get; set; } + public KnowledgeModelSetting TextEmbedding { get; set; } public string Pdf2TextConverter { get; set; } } + +public class KnowledgeModelSetting +{ + public string Provider { get; set; } + public string Model { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index 89c9f7db..825f7d8f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -17,4 +17,5 @@ global using BotSharp.Abstraction.Templating; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; global using BotSharp.Abstraction.Files.Models; -global using BotSharp.Abstraction.Files.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 413e998a..a079aaef 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -1,13 +1,13 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { + string Name { get; } + Task> GetCollections(); Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task> Search(string collectionName, float[] vector, IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs similarity index 63% rename from src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs rename to src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs index 46eded0f..0b09d6aa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs @@ -4,22 +4,22 @@ using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Files; -public class FilePlugin : IBotSharpPlugin +public class FileCorePlugin : IBotSharpPlugin { public string Id => "6a8473c0-04eb-4346-be32-24755ce5973d"; - public string Name => "File"; + public string Name => "File Core"; public string Description => "Provides file storage and analysis."; public void RegisterDI(IServiceCollection services, IConfiguration config) { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); - services.AddSingleton(myFileStorageSettings); + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); + services.AddSingleton(fileCoreSettings); - if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) + if (fileCoreSettings.Storage == FileStorageEnum.LocalFileStorage) { services.AddScoped(); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 6e6d4168..2aec257b 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -97,7 +97,8 @@ public partial class FileInstructService private async Task> ConvertPdfToImages(IEnumerable files) { var images = new List(); - var converter = _services.GetServices().FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 9e14677d..8c2ed831 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -275,8 +275,9 @@ public partial class LocalFileStorageService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; } private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index debec995..7f6a06de 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.OpenAPI.ViewModels.Knowledges; @@ -17,86 +18,28 @@ public class KnowledgeBaseController : ControllerBase _services = services; } - [HttpGet("/knowledge/{agentId}")] - public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) + [HttpPost("/knowledge/search")] + public async Task> SearchKnowledge([FromBody] SearchKnowledgeModel model) { - return await _knowledgeService.GetAnswer(new KnowledgeRetrievalModel + var searchModel = new KnowledgeRetrievalModel { - AgentId = agentId, - Question = question - }); + Collection = model.Collection, + Text = model.Text, + Fields = model.Fields, + Limit = model.Limit ?? 5, + Confidence = model.Confidence ?? 0.5f, + WithVector = model.WithVector + }; + + var results = await _knowledgeService.SearchKnowledge(searchModel); + return results.Select(x => KnowledgeRetrivalViewModel.From(x)).ToList(); } - [HttpPost("/knowledge-base/upload")] - public async Task UploadKnowledge(IFormFile file, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) - { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); - - var filePath = Path.GetTempFileName(); - using (var stream = System.IO.File.Create(filePath)) - { - await file.CopyToAsync(stream); - } - - var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.EmbedKnowledge(new KnowledgeCreationModel - { - Content = content - }); - - return Ok(new { count = 1, file.Length }); - } - - [HttpPost("/knowledge/{agentId}")] - public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) - { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); - long size = files.Sum(f => f.Length); - - foreach (var formFile in files) - { - var filePath = Path.GetTempFileName(); - - - using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) - { - await formFile.CopyToAsync(stream); - await stream.FlushAsync(); // Ensure all data is written to the file - } - - var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.Feed(new KnowledgeFeedModel - { - AgentId = agentId, - Content = content - }); - - // Delete the temp file after processing to clean up - System.IO.File.Delete(filePath); - } - - return Ok(new { count = files.Count, size }); - } - - [HttpPost("/knowledge/{collection}/data")] public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) { var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); - var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? + var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.From(x))? .ToList() ?? new List(); return new StringIdPagedItems @@ -107,19 +50,33 @@ public class KnowledgeBaseController : ControllerBase }; } - [HttpPost("/knowledge/{collection}/similar")] - public async Task> GetSimilarKnowledgeData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) - { - var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); - var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.ToViewModel(x))? - .ToList() ?? new List(); - - return - } - [HttpDelete("/knowledge/{collection}/data/{id}")] public async Task DeleteKnowledgeCollectionData([FromRoute] string collection, [FromRoute] string id) { return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id); } + + [HttpPost("/knowledge/upload")] + public async Task UploadKnowledge(IFormFile file, [FromQuery] string? collection, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + { + var setttings = _services.GetRequiredService(); + var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); + + var filePath = Path.GetTempFileName(); + using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) + { + await file.CopyToAsync(stream); + await stream.FlushAsync(); + } + + var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); + await _knowledgeService.FeedKnowledge(new KnowledgeCreationModel + { + Collection = collection ?? KnowledgeCollectionName.BotSharp, + Content = content + }); + + System.IO.File.Delete(filePath); + return Ok(new { count = 1, file.Length }); + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs index 5ce43369..f77b8777 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs @@ -20,7 +20,7 @@ public class KnowledgeCollectionDataViewModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public float[]? Vector { get; set; } - public static KnowledgeCollectionDataViewModel ToViewModel(KnowledgeCollectionData data) + public static KnowledgeCollectionDataViewModel From(KnowledgeCollectionData data) { return new KnowledgeCollectionDataViewModel { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs new file mode 100644 index 00000000..2e2b9e08 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeRetrivalViewModel +{ + [JsonPropertyName("data")] + public IDictionary Data { get; set; } + + [JsonPropertyName("score")] + public double Score { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + public static KnowledgeRetrivalViewModel From(KnowledgeRetrievalResult model) + { + return new KnowledgeRetrivalViewModel + { + Data = model.Data, + Score = model.Score, + Vector = model.Vector + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs new file mode 100644 index 00000000..afb311ed --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Knowledges.Enums; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class SearchKnowledgeModel +{ + [JsonPropertyName("collection")] + public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; + + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("fields")] + public IEnumerable? Fields { get; set; } + + [JsonPropertyName("limit")] + public int? Limit { get; set; } = 5; + + [JsonPropertyName("confidence")] + public float? Confidence { get; set; } = 0.5f; + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index f8d83e0c..7d2af5b9 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -17,14 +17,14 @@ public class KnowledgeRetrievalFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetRequiredService(); - var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer); + var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, new List { KnowledgePayloadName.Answer }); - if (knowledges.Count > 0) + if (!knowledges.IsNullOrEmpty()) { message.Content = string.Join("\r\n\r\n=====\r\n", knowledges); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 0ef0f950..619ea685 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class MemorizeKnowledgeFn : IFunctionCallback @@ -19,8 +17,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); var vector = await embedding.GetVectorsAsync(new List { @@ -28,10 +26,9 @@ public class MemorizeKnowledgeFn : IFunctionCallback }); var vectorDb = _services.GetRequiredService(); - await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length); - var id = Utilities.HashTextMd5(args.Question); + var id = Guid.NewGuid().ToString(); var result = await vectorDb.Upsert(KnowledgeCollectionName.BotSharp, id, vector[0], args.Question, new Dictionary diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs index 4710faea..df97bc34 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs @@ -9,6 +9,6 @@ public class MemVecDbPlugin : IBotSharpPlugin public string Description => "Store text embedding, search similar text from memory."; public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddSingleton(); + services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs deleted file mode 100644 index f7c9276d..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ /dev/null @@ -1,139 +0,0 @@ -using Tensorflow.NumPy; -using static Tensorflow.Binding; - -namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; - -public class MemVectorDatabase : IVectorDb -{ - private readonly Dictionary _collections = new Dictionary(); - private readonly Dictionary> _vectors = new Dictionary>(); - - public async Task CreateCollection(string collectionName, int dim) - { - _collections[collectionName] = dim; - _vectors[collectionName] = new List(); - } - - public async Task> GetCollections() - { - return _collections.Select(x => x.Key).ToList(); - } - - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) - { - throw new NotImplementedException(); - } - - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) - { - if (!_vectors.ContainsKey(collectionName)) - { - return new List(); - } - - var similarities = CalCosineSimilarity(vector, _vectors[collectionName]); - // var similarities2 = CalEuclideanDistance(vector, _vectors[collectionName]); - - var texts = np.argsort(similarities).ToArray() - .Reverse() - .Take(limit) - .Select(i => _vectors[collectionName][i].Text) - .ToList(); - - return texts; - } - - public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) - { - _vectors[collectionName].Add(new VecRecord - { - Id = id, - Vector = vector, - Text = text - }); - - return true; - } - - public Task DeleteCollectionData(string collectionName, string id) - { - throw new NotImplementedException(); - } - - #region Private methods - private float[] CalEuclideanDistance(float[] vec, List records) - { - var a = np.zeros((records.Count, vec.Length), np.float32); - var b = np.zeros((records.Count, vec.Length), np.float32); - for (var i = 0; i < records.Count; i++) - { - a[i] = vec; - b[i] = records[i].Vector; - } - - var c = np.sqrt(np.sum(np.square(a - b), axis: 1)); - // var c = -np.prod(np.linalg.norm(a, axis: 1) * np.linalg.norm(b, axis: 1), axis: 1); - return c.ToArray(); - } - - private NDArray CalCosineSimilarity(float[] vec, List records) - { - var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); - - for (int i = 0; i < records.Count; i++) - { - recordsArray[i] = records[i].Vector; - } - - var vecArray = np.expand_dims(np.array(vec, dtype: np.float32), axis: 0); // [1. 300] - - (var normVecArray, var _) = SafeNormalize(vecArray); - (var normRecordsArray, var _) = SafeNormalize(recordsArray); - - var simiMatix = tf.matmul(tf.cast(normVecArray, tf.float32), tf.transpose(tf.cast(normRecordsArray, tf.float32))).numpy(); // [1, num_records] - - simiMatix = np.squeeze(simiMatix, axis: 0); - - return simiMatix; - } - - public (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) - { - var simiMatix = CalCosineSimilarity(vec, records); - - topK = Math.Min(topK, records.Count); - var topIndex = np.argsort(simiMatix)["::-1"][$":{topK}"]; - - var resIndex = new List<(int, float)>(); - - for (int i = 0; i < topK; i++) - { - var index = topIndex[i]; - var value = simiMatix[index]; - - if (value > filterProb) - { - resIndex.Add((topIndex[i], value)); - } - } - - return resIndex.ToArray(); - } - - private (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) - { - var squaredX = np.sum(np.multiply(x, x), axis: 1); - var normX = np.sqrt(squaredX); - - var epsTensor = tf.cast(tf.convert_to_tensor(eps), dtype: tf.float32); - var normXTensor = tf.cast(normX, tf.float32); - var contantMask = (normXTensor < epsTensor); - var divideTensor = tf.ones_like(normXTensor, dtype: tf.float32); - - normX = tf.where(contantMask, divideTensor, normXTensor).numpy(); - normX = np.expand_dims(normX, axis: 1); - - return (x / normX, normX); - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs new file mode 100644 index 00000000..ba5e6383 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -0,0 +1,71 @@ +using BotSharp.Plugin.KnowledgeBase.Utilities; +using Tensorflow.NumPy; + +namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; + +public class MemoryVectorDb : IVectorDb +{ + private readonly Dictionary _collections = new Dictionary(); + private readonly Dictionary> _vectors = new Dictionary>(); + + + public string Name => "MemoryVector"; + + public async Task CreateCollection(string collectionName, int dim) + { + _collections[collectionName] = dim; + _vectors[collectionName] = new List(); + } + + public async Task> GetCollections() + { + return _collections.Select(x => x.Key).ToList(); + } + + public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + { + throw new NotImplementedException(); + } + + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + { + if (!_vectors.ContainsKey(collectionName)) + { + return new List(); + } + + var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); + // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); + + var results = np.argsort(similarities).ToArray() + .Reverse() + .Take(limit) + .Select(i => new KnowledgeSearchResult + { + Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, + Score = similarities[i], + Vector = withVector ? _vectors[collectionName][i].Vector : null, + }) + .ToList(); + + return await Task.FromResult(results); + } + + public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) + { + _vectors[collectionName].Add(new VecRecord + { + Id = id, + Vector = vector, + Text = text + }); + + return true; + } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs new file mode 100644 index 00000000..81d923a3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -0,0 +1,28 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task FeedKnowledge(KnowledgeCreationModel knowledge) + { + var index = 0; + var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + { + Size = 1024, + Conjunction = 32, + SplitByWord = true, + }); + + var db = GetVectorDb(); + var textEmbedding = GetTextEmbedding(); + + await db.CreateCollection(knowledge.Collection, textEmbedding.Dimension); + foreach (var line in lines) + { + var vec = await textEmbedding.GetVectorAsync(line); + var id = Guid.NewGuid().ToString(); + await db.Upsert(knowledge.Collection, id, vec, line); + index++; + Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs similarity index 50% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index d3b5ed84..5020c529 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.List.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -2,20 +2,6 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) - { - try - { - var db = GetVectorDb(); - return await db.GetCollectionData(collectionName, filter); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new StringIdPagedItems(); - } - } - public async Task DeleteKnowledgeCollectionData(string collectionName, string id) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs new file mode 100644 index 00000000..648ba120 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -0,0 +1,38 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) + { + try + { + var db = GetVectorDb(); + return await db.GetCollectionData(collectionName, filter); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return new StringIdPagedItems(); + } + } + + public async Task> SearchKnowledge(KnowledgeRetrievalModel model) + { + var textEmbedding = GetTextEmbedding(); + var vector = await textEmbedding.GetVectorAsync(model.Text); + + // Vector search + var db = GetVectorDb(); + var collection = !string.IsNullOrWhiteSpace(model.Collection) ? model.Collection : KnowledgeCollectionName.BotSharp; + var fields = !model.Fields.IsNullOrEmpty() ? model.Fields : new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + var found = await db.Search(collection, vector, fields, limit: model.Limit ?? 5, confidence: model.Confidence ?? 0.5f, withVector: model.WithVector); + + var results = found.Select(x => new KnowledgeRetrievalResult + { + Data = x.Data, + Score = x.Score, + Vector = x.Vector + }).ToList(); + return results; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index cdfff9db..dff80134 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -7,7 +7,8 @@ public partial class KnowledgeService : IKnowledgeService private readonly ITextChopper _textChopper; private readonly ILogger _logger; - public KnowledgeService(IServiceProvider services, + public KnowledgeService( + IServiceProvider services, KnowledgeBaseSettings settings, ITextChopper textChopper, ILogger logger) @@ -18,104 +19,19 @@ public partial class KnowledgeService : IKnowledgeService _logger = logger; } - public async Task EmbedKnowledge(KnowledgeCreationModel knowledge) + private IVectorDb GetVectorDb() { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection(KnowledgeCollectionName.BotSharp, textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert(KnowledgeCollectionName.BotSharp, idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task Feed(KnowledgeFeedModel knowledge) - { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection(knowledge.AgentId, textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert(knowledge.AgentId, idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task GetKnowledges(KnowledgeRetrievalModel retrievalModel) - { - var textEmbedding = GetTextEmbedding(); - var vector = await textEmbedding.GetVectorAsync(retrievalModel.Question); - - // Vector search - var db = GetVectorDb(); - var result = await db.Search(KnowledgeCollectionName.BotSharp, vector, KnowledgePayloadName.Answer, limit: 10); - - // Restore - return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}")); - } - - public async Task> GetAnswer(KnowledgeRetrievalModel retrievalModel) - { - // Restore - var prompt = await GetKnowledges(retrievalModel); - - var sb = new StringBuilder(prompt); - sb.AppendLine(); - sb.AppendLine(); - sb.AppendLine("------"); - sb.AppendLine("Answer question based on the given information above. Keep your answers concise. Please response with paragraph number, cite sources and reasoning in JSON format, if multiple paragraphs are found, put them in a JSON array. make sure the paragraph number is real. If you don't know the answer just output empty."); - sb.AppendLine("[" + JsonSerializer.Serialize(new RetrievedResult()) + "]"); - sb.AppendLine("------"); - sb.AppendLine($"QUESTION: \"{retrievalModel.Question}\""); - sb.AppendLine("Which paragraphs are relevant in order to answer the above question?"); - sb.AppendLine("ANSWER: "); - prompt = sb.ToString().Trim(); - - var completion = await GetTextCompletion().GetCompletion(prompt, Guid.Empty.ToString(), Guid.Empty.ToString()); - return JsonSerializer.Deserialize>(completion); - } - - public IVectorDb GetVectorDb() - { - var db = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.VectorDb)); + var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); return db; } - public ITextEmbedding GetTextEmbedding() + private ITextEmbedding GetTextEmbedding() { - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + if (embedding != null) + { + embedding.SetModelName(_settings.TextEmbedding.Model); + } return embedding; } - - public ITextCompletion GetTextCompletion() - { - var textCompletion = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextCompletion)); - return textCompletion; - } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs deleted file mode 100644 index eccfed78..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> CollectChunkedKnowledge() - { - throw new NotImplementedException(); - } - - public async Task EmbedKnowledge(List chunks) - { - throw new NotImplementedException(); - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs index 17706c13..09401f20 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs @@ -5,6 +5,8 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public class PigPdf2TextConverter : IPdf2TextConverter { + public string Name => "Pig"; + public Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { // since PdfDocument.Open is not async, we dont need to make this method async diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs new file mode 100644 index 00000000..f0538af0 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs @@ -0,0 +1,83 @@ +using BotSharp.Plugin.KnowledgeBase.MemVecDb; +using Tensorflow.NumPy; +using static Tensorflow.Binding; + +namespace BotSharp.Plugin.KnowledgeBase.Utilities; + +public static class VectorUtility +{ + public static float[] CalEuclideanDistance(float[] vec, List records) + { + var a = np.zeros((records.Count, vec.Length), np.float32); + var b = np.zeros((records.Count, vec.Length), np.float32); + for (var i = 0; i < records.Count; i++) + { + a[i] = vec; + b[i] = records[i].Vector; + } + + var c = np.sqrt(np.sum(np.square(a - b), axis: 1)); + // var c = -np.prod(np.linalg.norm(a, axis: 1) * np.linalg.norm(b, axis: 1), axis: 1); + return c.ToArray(); + } + + public static NDArray CalCosineSimilarity(float[] vec, List records) + { + var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); + + for (int i = 0; i < records.Count; i++) + { + recordsArray[i] = records[i].Vector; + } + + var vecArray = np.expand_dims(np.array(vec, dtype: np.float32), axis: 0); // [1. 300] + + (var normVecArray, var _) = SafeNormalize(vecArray); + (var normRecordsArray, var _) = SafeNormalize(recordsArray); + + var simiMatix = tf.matmul(tf.cast(normVecArray, tf.float32), tf.transpose(tf.cast(normRecordsArray, tf.float32))).numpy(); // [1, num_records] + + simiMatix = np.squeeze(simiMatix, axis: 0); + + return simiMatix; + } + + public static (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) + { + var simiMatix = CalCosineSimilarity(vec, records); + + topK = Math.Min(topK, records.Count); + var topIndex = np.argsort(simiMatix)["::-1"][$":{topK}"]; + + var resIndex = new List<(int, float)>(); + + for (int i = 0; i < topK; i++) + { + var index = topIndex[i]; + var value = simiMatix[index]; + + if (value > filterProb) + { + resIndex.Add((topIndex[i], value)); + } + } + + return resIndex.ToArray(); + } + + private static (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) + { + var squaredX = np.sum(np.multiply(x, x), axis: 1); + var normX = np.sqrt(squaredX); + + var epsTensor = tf.cast(tf.convert_to_tensor(eps), dtype: tf.float32); + var normXTensor = tf.cast(normX, tf.float32); + var contantMask = (normXTensor < epsTensor); + var divideTensor = tf.ones_like(normXTensor, dtype: tf.float32); + + normX = tf.where(contantMask, divideTensor, normXTensor).numpy(); + normX = np.expand_dims(normX, axis: 1); + + return (x / normX, normX); + } +} diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 7e884632..785acd4d 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -9,6 +9,8 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { + public string Name => "Faiss"; + public Task CreateCollection(string collectionName, int dim) { throw new NotImplementedException(); @@ -24,7 +26,8 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) + public Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 10, float confidence = 0.5f, bool withVector = false) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index a66cb8f9..81c53f83 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -19,15 +19,18 @@ using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter -{ +{ private Dictionary _mappings = new Dictionary(); private FullOcrModel _model; private PaddleSharpSettings _paddleSharpSettings; + public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) { _paddleSharpSettings = paddleSharpSettings; } + public string Name => "Paddle"; + public async Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { await ConvertPdfToLocalImagesAsync(filePath, startPageNum, endPageNum); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index f7bb25c5..f03b4ab6 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -10,14 +10,16 @@ public class QdrantDb : IVectorDb private readonly QdrantSetting _setting; private readonly IServiceProvider _services; - public QdrantDb(QdrantSetting setting, + public QdrantDb( + QdrantSetting setting, IServiceProvider services) { _setting = setting; _services = services; - } + public string Name => "Qdrant"; + private QdrantClient GetClient() { if (_client == null) @@ -42,9 +44,8 @@ public class QdrantDb : IVectorDb public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) { var client = GetClient(); - - var exists = await client.CollectionExistsAsync(collectionName); - if (!exists) + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) { return new StringIdPagedItems(); } @@ -71,11 +72,12 @@ public class QdrantDb : IVectorDb public async Task CreateCollection(string collectionName, int dim) { - var collections = await GetCollections(); - if (!collections.Contains(collectionName)) + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) { // Create a new collection - await GetClient().CreateCollectionAsync(collectionName, new VectorParams() + await client.CreateCollectionAsync(collectionName, new VectorParams() { Size = (ulong)dim, Distance = Distance.Cosine @@ -83,7 +85,7 @@ public class QdrantDb : IVectorDb } // Get collection info - var collectionInfo = await _client.GetCollectionInfoAsync(collectionName); + var collectionInfo = await client.GetCollectionInfoAsync(collectionName); if (collectionInfo == null) { throw new Exception($"Create {collectionName} failed."); @@ -115,7 +117,6 @@ public class QdrantDb : IVectorDb } var client = GetClient(); - var result = await client.UpsertAsync(collectionName, points: new List { point @@ -124,19 +125,54 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var client = GetClient(); var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); - return points.Select(x => x.Payload[returnFieldName].StringValue).ToList(); + var results = new List(); + foreach (var point in points) + { + var data = new Dictionary(); + foreach (var field in fields) + { + if (point.Payload.ContainsKey(field)) + { + data[field] = point.Payload[field].StringValue; + } + else + { + data[field] = ""; + } + } + + results.Add(new KnowledgeSearchResult + { + Data = data, + Score = point.Score, + Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null + }); + } + + return results; } public async Task DeleteCollectionData(string collectionName, string id) { + if (!Guid.TryParse(id, out var guid)) + { + return false; + } + var client = GetClient(); - var guid = Guid.Parse(id); var result = await client.DeleteAsync(collectionName, guid); return result.Status == UpdateStatus.Completed; } + + + private async Task DoesCollectionExist(QdrantClient client, string collectionName) + { + return await client.CollectionExistsAsync(collectionName); + } } diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs index 089b5168..c52baf59 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs @@ -57,8 +57,8 @@ public class IntentClassifier return; } - var vector = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding)); + var vector = _services.GetServices().FirstOrDefault(x => x.Provider == _knowledgeBaseSettings.TextEmbedding.Provider); + vector.SetModelName(_knowledgeBaseSettings.TextEmbedding.Model); var layers = new List { @@ -136,8 +136,8 @@ public class IntentClassifier public NDArray GetTextEmbedding(string text) { var knowledgeSettings = _services.GetRequiredService(); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(knowledgeSettings.TextEmbedding)); + var embedding = _services.GetServices() .FirstOrDefault(x => x.Provider == knowledgeSettings.TextEmbedding.Provider); + embedding.SetModelName(knowledgeSettings.TextEmbedding.Model); var x = np.zeros((1, embedding.Dimension), dtype: np.float32); x[0] = embedding.GetVectorAsync(text).GetAwaiter().GetResult(); diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index b52ab71e..a5db6de8 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -20,6 +20,10 @@ namespace BotSharp.Plugin.SemanticKernel { this._memoryStore = memoryStore; } + + + public string Name => "SemanticKernel"; + public async Task CreateCollection(string collectionName, int dim) { await _memoryStore.CreateCollectionAsync(collectionName); @@ -40,18 +44,23 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); - var resultTexts = new List(); - await foreach (var (record, _) in results) + var resultTexts = new List(); + await foreach (var (record, score) in results) { - resultTexts.Add(record.Metadata.Text); + resultTexts.Add(new KnowledgeSearchResult + { + Data = new Dictionary { { "text", record.Metadata.Text } }, + Score = score, + Vector = withVector ? record.Embedding.ToArray() : null + }); } return resultTexts; - } public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload) @@ -62,9 +71,16 @@ namespace BotSharp.Plugin.SemanticKernel return true; } - public Task DeleteCollectionData(string collectionName, string id) + public async Task DeleteCollectionData(string collectionName, string id) { - throw new NotImplementedException(); + var exist = await _memoryStore.DoesCollectionExistAsync(collectionName); + + if (exist) + { + await _memoryStore.RemoveAsync(collectionName, id); + return true; + } + return false; } } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index f04d0be6..7e2462de 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; @@ -252,8 +253,9 @@ public partial class TencentCosService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; } private string BuilFileUrl(string file) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 1dcb5edb..791e7f98 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -18,10 +18,10 @@ public class TencentCosPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); - if (myFileStorageSettings.Default == FileStorageEnum.TencentCosStorage) + if (fileCoreSettings.Storage == FileStorageEnum.TencentCosStorage) { services.AddScoped(provider => { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index b10e2160..155dae1b 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -230,9 +230,13 @@ "FileRepository": "data", "Assemblies": [ "BotSharp.Core" ] }, - "FileStorage": { - "Default": "LocalFileStorage" + + "FileCore": { + "Storage": "LocalFileStorage", + "Pdf2TextConverter": "", + "Pdf2ImageConverter": "" }, + "TencentCos": { "AppId": "", "SecretId": "", @@ -254,10 +258,11 @@ }, "KnowledgeBase": { - "VectorDb": "MemVectorDatabase", - "TextEmbedding": "fastTextEmbeddingProvider", - "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", - "Pdf2TextConverter": "PigPdf2TextConverter" + "VectorDb": "Qdrant", + "TextEmbedding": { + "Provider": "openai", + "Model": "text-embedding-3-small" + } }, "SparkDesk": { From 3264892ce562db07aa81e3b7685fdc0b7e71b2f4 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 9 Aug 2024 16:06:48 -0500 Subject: [PATCH 37/63] fix setting --- .../Knowledges/Settings/KnowledgeBaseSettings.cs | 1 - .../BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 0c3f8d45..ac3f0500 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -4,7 +4,6 @@ public class KnowledgeBaseSettings { public string VectorDb { get; set; } public KnowledgeModelSetting TextEmbedding { get; set; } - public string Pdf2TextConverter { get; set; } } public class KnowledgeModelSetting diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 7f6a06de..5f73adb9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -59,7 +59,7 @@ public class KnowledgeBaseController : ControllerBase [HttpPost("/knowledge/upload")] public async Task UploadKnowledge(IFormFile file, [FromQuery] string? collection, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) { - var setttings = _services.GetRequiredService(); + var setttings = _services.GetRequiredService(); var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); var filePath = Path.GetTempFileName(); From 23f547ab85a8a38955158c6b54239874c7310c9b Mon Sep 17 00:00:00 2001 From: Bo Yin Date: Thu, 8 Aug 2024 14:54:32 -0500 Subject: [PATCH 38/63] initial --- .../Files/IFileStorageService.cs | 4 + .../MLTasks/ITextToSpeech.cs | 23 ++++ .../Services/BotSharpFileService.Speech.cs | 26 +++++ .../Storage/LocalFileStorageService.cs | 1 + .../Infrastructures/CompletionProvider.cs | 16 +++ .../BotSharp.Plugin.OpenAI/OpenAiPlugin.cs | 2 + .../Providers/Audio/TextToSpeechProvider.cs | 30 ++++++ .../BotSharp.Plugin.Twilio.csproj | 2 + .../Controllers/TwilioVoiceController.cs | 102 +++++++++++++++++- .../Models/CallerMessage.cs | 15 +++ .../Services/ITwilioSessionManager.cs | 12 +++ .../Services/TwilioMessageQueue.cs | 31 ++++++ .../Services/TwilioMessageQueueService.cs | 91 ++++++++++++++++ .../Services/TwilioService.cs | 40 +++++++ .../Services/TwilioSessionManager.cs | 46 ++++++++ .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 9 +- 16 files changed, 447 insertions(+), 3 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 065cdc55..a3e5a32d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -56,4 +56,8 @@ public interface IFileStorageService string GetUserAvatar(); bool SaveUserAvatar(BotSharpFile file); #endregion + #region Speech + Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data); + Task RetrieveSpeechFileAsync(string conversationId, string fileName); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs new file mode 100644 index 00000000..344fad0e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs @@ -0,0 +1,23 @@ +namespace BotSharp.Abstraction.MLTasks +{ + public interface ITextToSpeech + { + /// + /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI + /// + string Provider { get; } + + /// + /// Set model name, one provider can consume different model or version(s) + /// + /// deployment name + void SetModelName(string model); + + Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null); + } + + public interface ITextToSpeechOptions + { + + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs new file mode 100644 index 00000000..b81d37f3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs @@ -0,0 +1,26 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services +{ + public partial class BotSharpFileService + { + public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + { + var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, TEXT_TO_SPEECH_FOLDER, conversationId); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + using var file = File.Create(Path.Combine(dir, fileName)); + using var input = data.ToStream(); + await input.CopyToAsync(file); + } + + public async Task RetrieveSpeechFileAsync(string conversationId, string fileName) + { + var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, TEXT_TO_SPEECH_FOLDER, conversationId, fileName); + using var file = new FileStream(path, FileMode.Open, FileAccess.Read); + return await BinaryData.FromStreamAsync(file); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index d83cae03..750803c4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -18,6 +18,7 @@ public partial class LocalFileStorageService : IFileStorageService private const string USERS_FOLDER = "users"; private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; + private const string TEXT_TO_SPEECH_FOLDER = "speeches"; public LocalFileStorageService( BotSharpDatabaseSettings dbSettings, diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index dd6b99af..f6a12188 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -115,6 +115,22 @@ public class CompletionProvider return completer; } + public static ITextToSpeech GetTextToSpeech( + IServiceProvider services, + string provider, + string model) + { + var completions = services.GetServices(); + var completer = completions.FirstOrDefault(x => x.Provider == provider); + if (completer == null) + { + var logger = services.GetRequiredService>(); + logger.LogError($"Can't resolve text2speech provider by {provider}"); + } + completer.SetModelName(model); + return completer; + } + private static (string, string) GetProviderAndModel(IServiceProvider services, string? provider = null, string? model = null, diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index 1bc69aaf..1fdb5bfb 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -5,6 +5,7 @@ using BotSharp.Plugin.OpenAI.Providers.Image; using BotSharp.Plugin.OpenAI.Providers.Text; using BotSharp.Plugin.OpenAI.Providers.Chat; using Microsoft.Extensions.Configuration; +using BotSharp.Plugin.OpenAI.Providers.Audio; namespace BotSharp.Plugin.OpenAI; @@ -30,5 +31,6 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs new file mode 100644 index 00000000..e559e109 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs @@ -0,0 +1,30 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio +{ + public partial class TextToSpeechProvider : ITextToSpeech + { + public string Provider => "openai"; + private readonly IServiceProvider _services; + private string? _model; + + public TextToSpeechProvider( + IServiceProvider services) + { + _services = services; + } + + public void SetModelName(string model) + { + _model = model; + } + + public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) + { + var client = ProviderHelper + .GetClient(Provider, _model, _services) + .GetAudioClient(_model); + return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index ed99a926..0ad04dcd 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -9,6 +9,7 @@ + @@ -16,6 +17,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 30facac9..43c4dc51 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -1,12 +1,16 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using System.IdentityModel.Tokens.Jwt; -using BotSharp.Plugin.Twilio.Services; -using BotSharp.Abstraction.Routing; namespace BotSharp.Plugin.Twilio.Controllers; [AllowAnonymous] +[Route("[controller]")] public class TwilioVoiceController : TwilioController { private readonly TwilioSetting _settings; @@ -80,4 +84,98 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } + + + [HttpPost("anonymous/start")] + public TwiMLResult InitiateConversation(VoiceRequest request) + { + if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + string sessionId = $"TwilioVoice_{request.CallSid}"; + var twilio = _services.GetRequiredService(); + var url = $"twiliovoice/anonymous/{sessionId}/send/0"; + var response = twilio.DummyInstructions("Hello, how may I help you?", url, false); + return TwiML(response); + } + + [HttpPost("anonymous/{sessionId}/send/{seqNum}")] + public async Task SendCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request) + { + var twilio = _services.GetRequiredService(); + var messageQueue = _services.GetRequiredService(); + var sessionManager = _services.GetRequiredService(); + var url = $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}"; + var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(sessionId, seqNum); + if (!string.IsNullOrWhiteSpace(request.SpeechResult)) + { + messages.Add(request.SpeechResult); + } + var messageContent = string.Join("\r\n", messages); + VoiceResponse response; + if (!string.IsNullOrWhiteSpace(messageContent)) + { + var callerMessage = new CallerMessage() + { + SessionId = sessionId, + SeqNumber = seqNum, + Content = messageContent, + From = request.From + }; + await messageQueue.EnqueueAsync(callerMessage); + response = twilio.DummyInstructions("Please hold on and wait a moment.", url, true); + } + else + { + response = twilio.HangUp("Thanks for calling. Good bye."); + } + return TwiML(response); + } + + [HttpPost("anonymous/{sessionId}/reply/{seqNum}")] + public async Task ReplyCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request) + { + var nextSeqNum = seqNum + 1; + var sessionManager = _services.GetRequiredService(); + var twilio = _services.GetRequiredService(); + if (request.SpeechResult != null) + { + await sessionManager.StageCallerMessageAsync(sessionId, nextSeqNum, request.SpeechResult); + } + var reply = await sessionManager.GetAssistantReplyAsync(sessionId, seqNum); + VoiceResponse response; + if (string.IsNullOrEmpty(reply)) + { + response = twilio.ReturnInstructions(null, $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}", true); + } + else + { + + var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); + var fileService = _services.GetRequiredService(); + var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply); + var fileName = $"{seqNum}.mp3"; + await fileService.SaveSpeechFileAsync(sessionId, fileName, data); + response = twilio.ReturnInstructions($"twiliovoice/anonymous/speeches/{sessionId}/{fileName}", $"twiliovoice/anonymous/{sessionId}/send/{nextSeqNum}", true); + } + return TwiML(response); + } + + [HttpGet("anonymous/speeches/{conversationId}/{fileName}")] + public async Task RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) + { + var fileService = _services.GetRequiredService(); + var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName); + var result = new FileContentResult(data.ToArray(), "application/octet-stream"); + result.FileDownloadName = fileName; + return result; + } + + [HttpGet("anonymous/text-to-speech")] + public async Task TextToSpeech([FromQuery] string text) + { + var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); + var data = await textToSpeechService.GenerateSpeechFromTextAsync(text); + var fileService = _services.GetRequiredService(); + await fileService.SaveSpeechFileAsync("123", "sample.mp3", data); + return Ok(); + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs new file mode 100644 index 00000000..be270a41 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Plugin.Twilio.Models +{ + public class CallerMessage + { + public string SessionId { get; set; } + public int SeqNumber { get; set; } + public string Content { get; set; } + public string From { get; set; } + + public override string ToString() + { + return $"({SessionId}-{SeqNumber}) {Content}"; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs new file mode 100644 index 00000000..3ad027f7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs @@ -0,0 +1,12 @@ +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services +{ + public interface ITwilioSessionManager + { + Task SetAssistantReplyAsync(string sessionId, int seqNum, string message); + Task GetAssistantReplyAsync(string sessionId, int seqNum); + Task StageCallerMessageAsync(string sessionId, int seqNum, string message); + Task> RetrieveStagedCallerMessagesAsync(string sessionId, int seqNum); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs new file mode 100644 index 00000000..456e43c1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs @@ -0,0 +1,31 @@ +using BotSharp.Plugin.Twilio.Models; +using System.Threading.Channels; + +namespace BotSharp.Plugin.Twilio.Services +{ + public class TwilioMessageQueue + { + private readonly Channel _queue; + internal ChannelReader Reader => _queue.Reader; + public TwilioMessageQueue() + { + BoundedChannelOptions options = new(100) + { + FullMode = BoundedChannelFullMode.Wait + }; + _queue = Channel.CreateBounded(options); + } + + public async ValueTask EnqueueAsync(CallerMessage request) + { + if (request == null) throw new ArgumentNullException(nameof(request)); + Console.WriteLine($"[{DateTime.UtcNow}] Enqueue {request}"); + await _queue.Writer.WriteAsync(request); + } + + internal void Stop() + { + _queue.Writer.TryComplete(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs new file mode 100644 index 00000000..5e01f2a9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -0,0 +1,91 @@ +using BotSharp.Abstraction.Routing; +using BotSharp.Plugin.Twilio.Models; +using Microsoft.Extensions.Hosting; +using System.Security.Cryptography; +using System.Threading; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services +{ + public class TwilioMessageQueueService : BackgroundService + { + private readonly TwilioMessageQueue _queue; + private readonly IServiceProvider _serviceProvider; + private readonly SemaphoreSlim _throttler; + + public TwilioMessageQueueService( + TwilioMessageQueue queue, + IServiceProvider serviceProvider) + { + _queue = queue; + _serviceProvider = serviceProvider; + _throttler = new SemaphoreSlim(4, 4); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await foreach (var message in _queue.Reader.ReadAllAsync(stoppingToken)) + { + await _throttler.WaitAsync(stoppingToken); + _ = Task.Run(async () => + { + try + { + Console.WriteLine("Processing {message}.", message); + await ProcessUserMessageAsync(message); + } + catch (Exception ex) + { + Console.WriteLine("Processing {message} failed due to {ex}.", message, ex.Message); + } + finally + { + _throttler.Release(); + } + }); + } + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + _queue.Stop(); + await base.StopAsync(cancellationToken); + } + + private async Task ProcessUserMessageAsync(CallerMessage message) + { + using var scope = _serviceProvider.CreateScope(); + var sp = scope.ServiceProvider; + string reply = null; + //await Task.Delay(2000); + //reply = $"response for sequence number {message.SeqNumber}"; + var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); + var conv = sp.GetRequiredService(); + var routing = sp.GetRequiredService(); + routing.Context.SetMessageId(message.SessionId, inputMsg.MessageId); + conv.SetConversationId(message.SessionId, new List + { + new MessageState("channel", ConversationChannel.Phone), + new MessageState("calling_phone", message.From) + }); + var result = await conv.SendMessage("2cd4b805-7078-4405-87e9-2ec9aadf8a11", + inputMsg, + replyMessage: null, + async msg => + { + reply = msg.Content; + }, + async functionExecuting => + { }, + async functionExecuted => + { } + ); + if (string.IsNullOrWhiteSpace(reply)) + { + reply = "Bye."; + } + var sessionManager = sp.GetRequiredService(); + await sessionManager.SetAssistantReplyAsync(message.SessionId, message.SeqNumber, reply); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index f01a7e29..2428b90b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -64,6 +64,46 @@ public class TwilioService return response; } + public VoiceResponse ReturnInstructions(string speechPath, string callbackPath, bool actionOnEmptyResult) + { + var response = new VoiceResponse(); + var gather = new Gather() + { + Input = new List() + { + Gather.InputEnum.Speech + }, + Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), + ActionOnEmptyResult = actionOnEmptyResult + }; + if (!string.IsNullOrEmpty(speechPath)) + { + gather.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } + response.Append(gather); + return response; + } + + public VoiceResponse DummyInstructions(string message, string callbackPath, bool actionOnEmptyResult) + { + var response = new VoiceResponse(); + var gather = new Gather() + { + Input = new List() + { + Gather.InputEnum.Speech + }, + Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), + ActionOnEmptyResult = actionOnEmptyResult + }; + if (!string.IsNullOrEmpty(message)) + { + gather.Say(message); + } + response.Append(gather); + return response; + } + public VoiceResponse HangUp(string message) { var response = new VoiceResponse(); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs new file mode 100644 index 00000000..daefbdcc --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -0,0 +1,46 @@ +using StackExchange.Redis; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services +{ + public class TwilioSessionManager : ITwilioSessionManager + { + private readonly ConnectionMultiplexer _redis; + + public TwilioSessionManager(ConnectionMultiplexer redis) + { + _redis = redis; + } + + public async Task GetAssistantReplyAsync(string sessionId, int seqNum) + { + var db = _redis.GetDatabase(); + var key = $"{sessionId}:Assisist:{seqNum}"; + return await db.StringGetAsync(key); + } + + public async Task> RetrieveStagedCallerMessagesAsync(string sessionId, int seqNum) + { + var db = _redis.GetDatabase(); + var key = $"{sessionId}:Caller:{seqNum}"; + return (await db.ListRangeAsync(key)) + .Select(x => (string)x) + .ToList(); + } + + public async Task SetAssistantReplyAsync(string sessionId, int seqNum, string message) + { + var db = _redis.GetDatabase(); + var key = $"{sessionId}:Assisist:{seqNum}"; + await db.StringSetAsync(key, message, TimeSpan.FromMinutes(5)); + } + + public async Task StageCallerMessageAsync(string sessionId, int seqNum, string message) + { + var db = _redis.GetDatabase(); + var key = $"{sessionId}:Caller:{seqNum}"; + await db.ListRightPushAsync(key, message); + await db.KeyExpireAsync(key, DateTime.UtcNow.AddMinutes(10)); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index 1a2065ca..0a93b6a9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Settings; using BotSharp.Plugin.Twilio.Services; +using StackExchange.Redis; namespace BotSharp.Plugin.Twilio; @@ -11,12 +12,18 @@ public class TwilioPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddSingleton(provider => + services.AddScoped(provider => { var settingService = provider.GetRequiredService(); return settingService.Bind("Twilio"); }); services.AddScoped(); + var conn = ConnectionMultiplexer.Connect("10.2.3.227"); + var sessionManager = new TwilioSessionManager(conn); + services.AddSingleton(sessionManager); + services.AddSingleton(); + services.AddHostedService(); + } } From de44da5425aec9bc4c0639d0c48931bfb1d2a352 Mon Sep 17 00:00:00 2001 From: Bo Yin Date: Thu, 8 Aug 2024 22:28:39 -0500 Subject: [PATCH 39/63] draft --- .../Services/BotSharpFileService.Speech.cs | 2 +- .../Controllers/TwilioVoiceController.cs | 38 +++++++------------ .../Models/CallerMessage.cs | 2 +- .../Services/TwilioMessageQueue.cs | 1 + .../Services/TwilioMessageQueueService.cs | 12 +++--- .../Services/TwilioService.cs | 28 ++------------ .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 2 +- 7 files changed, 27 insertions(+), 58 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs index b81d37f3..9788631e 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Core.Files.Services { - public partial class BotSharpFileService + public partial class LocalFileStorageService { public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 43c4dc51..6e163ce8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -86,24 +86,24 @@ public class TwilioVoiceController : TwilioController } - [HttpPost("anonymous/start")] + [HttpPost("start")] public TwiMLResult InitiateConversation(VoiceRequest request) { if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); string sessionId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); - var url = $"twiliovoice/anonymous/{sessionId}/send/0"; - var response = twilio.DummyInstructions("Hello, how may I help you?", url, false); + var url = $"twiliovoice/{sessionId}/send/0"; + var response = twilio.ReturnInstructions("twilio/welcome.mp3", url, false); return TwiML(response); } - [HttpPost("anonymous/{sessionId}/send/{seqNum}")] + [HttpPost("{sessionId}/send/{seqNum}")] public async Task SendCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request) { var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); - var url = $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}"; + var url = $"twiliovoice/{sessionId}/reply/{seqNum}"; var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(sessionId, seqNum); if (!string.IsNullOrWhiteSpace(request.SpeechResult)) { @@ -121,16 +121,16 @@ public class TwilioVoiceController : TwilioController From = request.From }; await messageQueue.EnqueueAsync(callerMessage); - response = twilio.DummyInstructions("Please hold on and wait a moment.", url, true); + response = twilio.ReturnInstructions("twilio/holdon.mp3", url, true); } else { - response = twilio.HangUp("Thanks for calling. Good bye."); + response = twilio.HangUp("twilio/holdon.mp3"); } return TwiML(response); } - [HttpPost("anonymous/{sessionId}/reply/{seqNum}")] + [HttpPost("{sessionId}/reply/{seqNum}")] public async Task ReplyCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request) { var nextSeqNum = seqNum + 1; @@ -144,38 +144,28 @@ public class TwilioVoiceController : TwilioController VoiceResponse response; if (string.IsNullOrEmpty(reply)) { - response = twilio.ReturnInstructions(null, $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}", true); + response = twilio.ReturnInstructions(null, $"twiliovoice/{sessionId}/reply/{seqNum}", true); } else { var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply); var fileName = $"{seqNum}.mp3"; await fileService.SaveSpeechFileAsync(sessionId, fileName, data); - response = twilio.ReturnInstructions($"twiliovoice/anonymous/speeches/{sessionId}/{fileName}", $"twiliovoice/anonymous/{sessionId}/send/{nextSeqNum}", true); + response = twilio.ReturnInstructions($"twiliovoice/speeches/{sessionId}/{fileName}", $"twiliovoice/{sessionId}/send/{nextSeqNum}", true); } return TwiML(response); } - [HttpGet("anonymous/speeches/{conversationId}/{fileName}")] + [HttpGet("speeches/{conversationId}/{fileName}")] public async Task RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); + var fileService = _services.GetRequiredService(); var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName); - var result = new FileContentResult(data.ToArray(), "application/octet-stream"); + var result = new FileContentResult(data.ToArray(), "audio/mpeg"); result.FileDownloadName = fileName; return result; } - - [HttpGet("anonymous/text-to-speech")] - public async Task TextToSpeech([FromQuery] string text) - { - var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(text); - var fileService = _services.GetRequiredService(); - await fileService.SaveSpeechFileAsync("123", "sample.mp3", data); - return Ok(); - } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs index be270a41..e0f4463a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs @@ -9,7 +9,7 @@ namespace BotSharp.Plugin.Twilio.Models public override string ToString() { - return $"({SessionId}-{SeqNumber}) {Content}"; + return $"{SessionId}-{SeqNumber}"; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs index 456e43c1..455e314c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs @@ -25,6 +25,7 @@ namespace BotSharp.Plugin.Twilio.Services internal void Stop() { + Console.WriteLine($"[{DateTime.UtcNow}] Complete queue"); _queue.Writer.TryComplete(); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 5e01f2a9..dab06c4b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Routing; using BotSharp.Plugin.Twilio.Models; using Microsoft.Extensions.Hosting; -using System.Security.Cryptography; using System.Threading; using Task = System.Threading.Tasks.Task; @@ -31,12 +30,12 @@ namespace BotSharp.Plugin.Twilio.Services { try { - Console.WriteLine("Processing {message}.", message); + Console.WriteLine($"Start processing {message}."); await ProcessUserMessageAsync(message); } catch (Exception ex) { - Console.WriteLine("Processing {message} failed due to {ex}.", message, ex.Message); + Console.WriteLine($"Processing {message} failed due to {ex.Message}."); } finally { @@ -57,18 +56,17 @@ namespace BotSharp.Plugin.Twilio.Services using var scope = _serviceProvider.CreateScope(); var sp = scope.ServiceProvider; string reply = null; - //await Task.Delay(2000); - //reply = $"response for sequence number {message.SeqNumber}"; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService(); var routing = sp.GetRequiredService(); + var config = sp.GetRequiredService(); routing.Context.SetMessageId(message.SessionId, inputMsg.MessageId); conv.SetConversationId(message.SessionId, new List { new MessageState("channel", ConversationChannel.Phone), new MessageState("calling_phone", message.From) }); - var result = await conv.SendMessage("2cd4b805-7078-4405-87e9-2ec9aadf8a11", + var result = await conv.SendMessage(config.AgentId, inputMsg, replyMessage: null, async msg => @@ -82,7 +80,7 @@ namespace BotSharp.Plugin.Twilio.Services ); if (string.IsNullOrWhiteSpace(reply)) { - reply = "Bye."; + reply = "Sorry, something was wrong."; } var sessionManager = sp.GetRequiredService(); await sessionManager.SetAssistantReplyAsync(message.SessionId, message.SeqNumber, reply); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 2428b90b..09405817 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,4 +1,3 @@ -using BotSharp.Plugin.Twilio.Settings; using Twilio.Jwt.AccessToken; using Token = Twilio.Jwt.AccessToken.Token; @@ -74,6 +73,7 @@ public class TwilioService Gather.InputEnum.Speech }, Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), + SpeechTimeout = "3", ActionOnEmptyResult = actionOnEmptyResult }; if (!string.IsNullOrEmpty(speechPath)) @@ -84,32 +84,12 @@ public class TwilioService return response; } - public VoiceResponse DummyInstructions(string message, string callbackPath, bool actionOnEmptyResult) + public VoiceResponse HangUp(string speechPath) { var response = new VoiceResponse(); - var gather = new Gather() + if (!string.IsNullOrEmpty(speechPath)) { - Input = new List() - { - Gather.InputEnum.Speech - }, - Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), - ActionOnEmptyResult = actionOnEmptyResult - }; - if (!string.IsNullOrEmpty(message)) - { - gather.Say(message); - } - response.Append(gather); - return response; - } - - public VoiceResponse HangUp(string message) - { - var response = new VoiceResponse(); - if (!string.IsNullOrEmpty(message)) - { - response.Say(message); + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); } response.Hangup(); return response; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index 0a93b6a9..38d278a6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -19,7 +19,7 @@ public class TwilioPlugin : IBotSharpPlugin }); services.AddScoped(); - var conn = ConnectionMultiplexer.Connect("10.2.3.227"); + var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]); var sessionManager = new TwilioSessionManager(conn); services.AddSingleton(sessionManager); services.AddSingleton(); From 31214852f097398cfbcd544afae557722ac00af6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 9 Aug 2024 16:43:16 -0500 Subject: [PATCH 40/63] fix string join --- .../Functions/KnowledgeRetrievalFn.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 7d2af5b9..ca81a4ba 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -26,7 +26,8 @@ public class KnowledgeRetrievalFn : IFunctionCallback if (!knowledges.IsNullOrEmpty()) { - message.Content = string.Join("\r\n\r\n=====\r\n", knowledges); + var answers = knowledges.Select(x => x.Data[KnowledgePayloadName.Answer]).ToList(); + message.Content = string.Join("\r\n\r\n=====\r\n", answers); } else { From bb6fd363c32e4524492f38bdd504d09a6b581bdc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 9 Aug 2024 17:39:54 -0500 Subject: [PATCH 41/63] resolve conflict --- .../Knowledges/IKnowledgeService.cs | 6 ++---- .../Models/KnowledgeCreationModel.cs | 3 --- ...alModel.cs => KnowledgeRetrievalOptions.cs} | 3 +-- .../LocalFileStorageService.Audio.cs} | 0 .../Controllers/KnowledgeBaseController.cs | 18 ++++++++---------- .../Functions/MemorizeKnowledgeFn.cs | 1 - .../Services/KnowledgeService.Create.cs | 6 +++--- .../Services/KnowledgeService.Get.cs | 9 ++++----- .../Services/TencentCosService.Audio.cs | 14 ++++++++++++++ 9 files changed, 32 insertions(+), 28 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/{KnowledgeRetrievalModel.cs => KnowledgeRetrievalOptions.cs} (78%) rename src/Infrastructure/BotSharp.Core/Files/Services/{BotSharpFileService.Speech.cs => Storage/LocalFileStorageService.Audio.cs} (100%) create mode 100644 src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 827b2ba1..657ac42a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -1,11 +1,9 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { - Task> SearchKnowledge(KnowledgeRetrievalModel model); - Task FeedKnowledge(KnowledgeCreationModel model); + Task> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options); + Task FeedKnowledge(string collectionName, KnowledgeCreationModel model); Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); Task DeleteKnowledgeCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs index b43bce67..33d09bd6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCreationModel.cs @@ -1,9 +1,6 @@ -using BotSharp.Abstraction.Knowledges.Enums; - namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeCreationModel { - public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; public string Content { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalOptions.cs similarity index 78% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalOptions.cs index 76e5e77d..8a608ddc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalOptions.cs @@ -2,9 +2,8 @@ using BotSharp.Abstraction.Knowledges.Enums; namespace BotSharp.Abstraction.Knowledges.Models; -public class KnowledgeRetrievalModel +public class KnowledgeRetrievalOptions { - public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; public string Text { get; set; } = string.Empty; public IEnumerable? Fields { get; set; } = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; public int? Limit { get; set; } = 5; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 5f73adb9..d86b5d21 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,7 +1,7 @@ using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.OpenAPI.ViewModels.Knowledges; +using Microsoft.Extensions.Options; namespace BotSharp.OpenAPI.Controllers; @@ -18,12 +18,11 @@ public class KnowledgeBaseController : ControllerBase _services = services; } - [HttpPost("/knowledge/search")] - public async Task> SearchKnowledge([FromBody] SearchKnowledgeModel model) + [HttpPost("/knowledge/{collection}/search")] + public async Task> SearchKnowledge([FromQuery] string collection, [FromBody] SearchKnowledgeModel model) { - var searchModel = new KnowledgeRetrievalModel + var options = new KnowledgeRetrievalOptions { - Collection = model.Collection, Text = model.Text, Fields = model.Fields, Limit = model.Limit ?? 5, @@ -31,7 +30,7 @@ public class KnowledgeBaseController : ControllerBase WithVector = model.WithVector }; - var results = await _knowledgeService.SearchKnowledge(searchModel); + var results = await _knowledgeService.SearchKnowledge(collection, options); return results.Select(x => KnowledgeRetrivalViewModel.From(x)).ToList(); } @@ -56,8 +55,8 @@ public class KnowledgeBaseController : ControllerBase return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id); } - [HttpPost("/knowledge/upload")] - public async Task UploadKnowledge(IFormFile file, [FromQuery] string? collection, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + [HttpPost("/knowledge/{collection}/upload")] + public async Task UploadKnowledge([FromRoute] string collection, [FromForm] IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) { var setttings = _services.GetRequiredService(); var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); @@ -70,9 +69,8 @@ public class KnowledgeBaseController : ControllerBase } var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - await _knowledgeService.FeedKnowledge(new KnowledgeCreationModel + await _knowledgeService.FeedKnowledge(collection, new KnowledgeCreationModel { - Collection = collection ?? KnowledgeCollectionName.BotSharp, Content = content }); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 619ea685..218b33d4 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -37,7 +37,6 @@ public class MemorizeKnowledgeFn : IFunctionCallback }); message.Content = result ? "Saved to my brain" : "I forgot it"; - return true; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs index 81d923a3..f68d550b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task FeedKnowledge(KnowledgeCreationModel knowledge) + public async Task FeedKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; var lines = _textChopper.Chop(knowledge.Content, new ChunkOption @@ -15,12 +15,12 @@ public partial class KnowledgeService var db = GetVectorDb(); var textEmbedding = GetTextEmbedding(); - await db.CreateCollection(knowledge.Collection, textEmbedding.Dimension); + await db.CreateCollection(collectionName, textEmbedding.Dimension); foreach (var line in lines) { var vec = await textEmbedding.GetVectorAsync(line); var id = Guid.NewGuid().ToString(); - await db.Upsert(knowledge.Collection, id, vec, line); + await db.Upsert(collectionName, id, vec, line); index++; Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs index 648ba120..96a5c139 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -16,16 +16,15 @@ public partial class KnowledgeService } } - public async Task> SearchKnowledge(KnowledgeRetrievalModel model) + public async Task> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options) { var textEmbedding = GetTextEmbedding(); - var vector = await textEmbedding.GetVectorAsync(model.Text); + var vector = await textEmbedding.GetVectorAsync(options.Text); // Vector search var db = GetVectorDb(); - var collection = !string.IsNullOrWhiteSpace(model.Collection) ? model.Collection : KnowledgeCollectionName.BotSharp; - var fields = !model.Fields.IsNullOrEmpty() ? model.Fields : new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; - var found = await db.Search(collection, vector, fields, limit: model.Limit ?? 5, confidence: model.Confidence ?? 0.5f, withVector: model.WithVector); + var fields = !options.Fields.IsNullOrEmpty() ? options.Fields : new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + var found = await db.Search(collectionName, vector, fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); var results = found.Select(x => new KnowledgeRetrievalResult { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs new file mode 100644 index 00000000..4d803628 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.TencentCos.Services; + +public partial class TencentCosService +{ + public Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + { + throw new NotImplementedException(); + } + + public Task RetrieveSpeechFileAsync(string conversationId, string fileName) + { + throw new NotImplementedException(); + } +} From fee39d0e3883709950cee258de3895d39e51f2d2 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 9 Aug 2024 18:12:34 -0500 Subject: [PATCH 42/63] fix typo --- .../BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs | 4 +--- .../ViewModels/Knowledges/SearchKnowledgeModel.cs | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index d86b5d21..2df4d9f5 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,7 +1,5 @@ -using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.Knowledges.Models; using BotSharp.OpenAPI.ViewModels.Knowledges; -using Microsoft.Extensions.Options; namespace BotSharp.OpenAPI.Controllers; @@ -19,7 +17,7 @@ public class KnowledgeBaseController : ControllerBase } [HttpPost("/knowledge/{collection}/search")] - public async Task> SearchKnowledge([FromQuery] string collection, [FromBody] SearchKnowledgeModel model) + public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeModel model) { var options = new KnowledgeRetrievalOptions { diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs index afb311ed..9a91e004 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs @@ -5,9 +5,6 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges; public class SearchKnowledgeModel { - [JsonPropertyName("collection")] - public string Collection { get; set; } = KnowledgeCollectionName.BotSharp; - [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; From e0b442efc0bf007dab3b1f936e5aa9413af3a38f Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sun, 11 Aug 2024 14:44:57 -0500 Subject: [PATCH 43/63] remove from form --- .../BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 2df4d9f5..e20632f5 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -54,7 +54,7 @@ public class KnowledgeBaseController : ControllerBase } [HttpPost("/knowledge/{collection}/upload")] - public async Task UploadKnowledge([FromRoute] string collection, [FromForm] IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) + public async Task UploadKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) { var setttings = _services.GetRequiredService(); var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); From 5b041b19e5689e2ab54056bc149f9d555c7c23c7 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 12 Aug 2024 09:43:39 -0500 Subject: [PATCH 44/63] Add EnableResponseCallback --- .../Browsing/Models/PageActionArgs.cs | 9 +++++++++ .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 12 +++++++++--- .../PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs | 8 ++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index 54707cec..098b9538 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -15,10 +15,19 @@ public class PageActionArgs /// This value has to be set to true if you want to get the page XHR/ Fetch responses /// public bool OpenNewTab { get; set; } = false; + + public bool EnableResponseCallback { get; set; } = false; + /// /// Exclude urls for XHR/ Fetch responses /// public string[]? ExcludeResponseUrls { get; set; } + + /// + /// Only include urls for XHR/ Fetch responses + /// + public string[]? IncludeResponseUrls { get; set; } + public bool UseExistingPage { get; set; } = false; public bool WaitForNetworkIdle { get; set; } = true; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 65b45013..cd6daa34 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -117,7 +117,7 @@ public class PlaywrightInstance : IDisposable return _contexts[ctxId]; } - public async Task NewPage(MessageInfo message, string[]? excludeResponseUrls = null) + public async Task NewPage(MessageInfo message, bool enableResponseCallback = false, string[]? excludeResponseUrls = null, string[]? includeResponseUrls = null) { var context = await GetContext(message.ContextId); var page = await context.NewPageAsync(); @@ -127,13 +127,19 @@ public class PlaywrightInstance : IDisposable var js = @"Object.defineProperties(navigator, {webdriver:{get:()=>false}});"; await page.AddInitScriptAsync(js); + if (!enableResponseCallback) + { + return page; + } + page.Response += async (sender, e) => { if (e.Status != 204 && e.Headers.ContainsKey("content-type") && e.Headers["content-type"].Contains("application/json") && (e.Request.ResourceType == "fetch" || e.Request.ResourceType == "xhr") && - (excludeResponseUrls == null || !excludeResponseUrls.Any(url => e.Url.ToLower().Contains(url)))) + (excludeResponseUrls == null || !excludeResponseUrls.Any(url => e.Url.ToLower().Contains(url))) && + (includeResponseUrls == null || includeResponseUrls.Any(url => e.Url.ToLower().Contains(url)))) { Serilog.Log.Information($"{e.Request.Method}: {e.Url}"); JsonElement? json = null; @@ -160,7 +166,7 @@ public class PlaywrightInstance : IDisposable } catch (Exception ex) { - Serilog.Log.Error(ex.ToString()); + Serilog.Log.Error($"{e.Url}\r\n" + ex.ToString()); } } }; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 5ddbfcbd..40f617d2 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -10,7 +10,9 @@ public partial class PlaywrightWebDriver { var page = args.UseExistingPage ? _instance.GetPage(message.ContextId, pattern: args.Url) : - await _instance.NewPage(message, excludeResponseUrls: args.ExcludeResponseUrls); + await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCallback, + excludeResponseUrls: args.ExcludeResponseUrls, + includeResponseUrls: args.IncludeResponseUrls); if (args.UseExistingPage && page != null && page.Url == args.Url) { @@ -23,7 +25,9 @@ public partial class PlaywrightWebDriver if (args.UseExistingPage && args.OpenNewTab && page != null && page.Url == "about:blank") { - page = await _instance.NewPage(message, excludeResponseUrls: args.ExcludeResponseUrls); + page = await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCallback, + excludeResponseUrls: args.ExcludeResponseUrls, + includeResponseUrls: args.IncludeResponseUrls); } var response = await page.GotoAsync(args.Url, new PageGotoOptions From 30f3de9bc4aa928fd218ea2eca6e4de041e824e2 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 12 Aug 2024 14:22:47 -0500 Subject: [PATCH 45/63] python interpreter --- BotSharp.sln | 14 +++++ .../Models/InterpretationRequest.cs | 10 ++++ .../BotSharp.Plugin.PythonInterpreter.csproj | 35 +++++++++++++ .../Enums/UtilityName.cs | 6 +++ .../Functions/InterpretationFn.cs | 46 +++++++++++++++++ .../Hooks/InterpreterAgentHook.cs | 51 +++++++++++++++++++ .../Hooks/InterpreterUtilityHook.cs | 9 ++++ .../InterpreterPlugin.cs | 17 +++++++ .../Using.cs | 18 +++++++ .../functions/python_interpreter.json | 19 +++++++ .../templates/python_interpreter.fn.liquid | 1 + src/WebStarter/Program.cs | 8 +++ src/WebStarter/WebStarter.csproj | 1 + src/WebStarter/appsettings.json | 3 +- 14 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Interpreters/Models/InterpretationRequest.cs create mode 100644 src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj create mode 100644 src/Plugins/BotSharp.Plugin.PythonInterpreter/Enums/UtilityName.cs create mode 100644 src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/InterpretationFn.cs create mode 100644 src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterAgentHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterUtilityHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.PythonInterpreter/InterpreterPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.PythonInterpreter/Using.cs create mode 100644 src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/python_interpreter.json create mode 100644 src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/python_interpreter.fn.liquid diff --git a/BotSharp.sln b/BotSharp.sln index b7545cdc..8c1030f9 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -105,6 +105,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FileStorages", "FileStorage EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.TencentCos", "src\Plugins\BotSharp.Plugin.TencentCos\BotSharp.Plugin.TencentCos.csproj", "{BF029B0A-768B-43A1-8D91-E70B95505716}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Interpreters", "Interpreters", "{C4C59872-3C8A-450D-83D5-2BE402D610D5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.PythonInterpreter", "src\Plugins\BotSharp.Plugin.PythonInterpreter\BotSharp.Plugin.PythonInterpreter.csproj", "{05E6E405-5021-406E-8A5E-0A7CEC881F6D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -425,6 +429,14 @@ Global {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|Any CPU.Build.0 = Release|Any CPU {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|x64.ActiveCfg = Release|Any CPU {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|x64.Build.0 = Release|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Debug|x64.ActiveCfg = Debug|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Debug|x64.Build.0 = Debug|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|Any CPU.Build.0 = Release|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|x64.ActiveCfg = Release|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -475,6 +487,8 @@ Global {54E83C6F-54EE-4ADC-8D72-93C009CC4FB4} = {51AFE054-AE99-497D-A593-69BAEFB5106F} {38B37C0D-1930-4D47-BCBF-E358EC1096B1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} {BF029B0A-768B-43A1-8D91-E70B95505716} = {38B37C0D-1930-4D47-BCBF-E358EC1096B1} + {C4C59872-3C8A-450D-83D5-2BE402D610D5} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} + {05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/Interpreters/Models/InterpretationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Interpreters/Models/InterpretationRequest.cs new file mode 100644 index 00000000..5a8f7552 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Interpreters/Models/InterpretationRequest.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Interpreters.Models; + +public class InterpretationRequest +{ + [JsonPropertyName("script")] + public string Script { get; set; } = null!; + + [JsonPropertyName("language")] + public string Language { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj new file mode 100644 index 00000000..b3293a0f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj @@ -0,0 +1,35 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Enums/UtilityName.cs new file mode 100644 index 00000000..95a5b8a2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Enums/UtilityName.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.PythonInterpreter.Enums; + +public class UtilityName +{ + public const string PythonInterpreter = "python-interpreter"; +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/InterpretationFn.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/InterpretationFn.cs new file mode 100644 index 00000000..b12cb739 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/InterpretationFn.cs @@ -0,0 +1,46 @@ +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Interpreters.Models; +using Microsoft.Extensions.Logging; +using Python.Runtime; +using System.Text.Json; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.PythonInterpreter.Functions; + +public class InterpretationFn : IFunctionCallback +{ + public string Name => "python_interpreter"; + public string Indication => "Interpreting python code"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + using (Py.GIL()) + { + // Import necessary Python modules + dynamic sys = Py.Import("sys"); + dynamic io = Py.Import("io"); + + // Redirect standard output to capture it + dynamic stringIO = io.StringIO(); + sys.stdout = stringIO; + + // Execute a simple Python script + using var locals = new PyDict(); + PythonEngine.Exec(args.Script, null, locals); + + // Console.WriteLine($"Result from Python: {result}"); + message.Content = stringIO.getvalue(); + + // Restore the original stdout + sys.stdout = sys.__stdout__; + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterAgentHook.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterAgentHook.cs new file mode 100644 index 00000000..64e0ec27 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterAgentHook.cs @@ -0,0 +1,51 @@ +namespace BotSharp.Plugin.PythonInterpreter.Hooks; + +public class InterpreterAgentHook : AgentHookBase +{ + private static string FUNCTION_NAME = "python_interpreter"; + + public override string SelfId => string.Empty; + + public InterpreterAgentHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) + { + } + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.PythonInterpreter); + + if (isConvMode && isEnabled) + { + var (prompt, fn) = GetPromptAndFunction(); + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); + } + } + } + + base.OnAgentLoaded(agent); + } + + private (string, FunctionDef?) GetPromptAndFunction() + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); + return (prompt, loadAttachmentFn); + } +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterUtilityHook.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterUtilityHook.cs new file mode 100644 index 00000000..be37bfa1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterUtilityHook.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.PythonInterpreter.Hooks; + +public class InterpreterUtilityHook : IAgentUtilityHook +{ + public void AddUtilities(List utilities) + { + utilities.Add(UtilityName.PythonInterpreter); + } +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/InterpreterPlugin.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/InterpreterPlugin.cs new file mode 100644 index 00000000..075038e1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/InterpreterPlugin.cs @@ -0,0 +1,17 @@ +using BotSharp.Plugin.PythonInterpreter.Hooks; + +namespace BotSharp.Plugin.PythonInterpreter; + +public class InterpreterPlugin : IBotSharpPlugin +{ + public string Id => "23174e08-e866-4173-824a-cf1d97afa8d0"; + public string Name => "Python Interpreter"; + public string Description => "Python Interpreter enables AI to write and execute Python code within a secure, sandboxed environment."; + public string? IconUrl => "https://static.vecteezy.com/system/resources/previews/012/697/295/non_2x/3d-python-programming-language-logo-free-png.png"; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + services.AddScoped(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Using.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Using.cs new file mode 100644 index 00000000..8cc31aa1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Using.cs @@ -0,0 +1,18 @@ +global using System; +global using System.Linq; +global using System.Collections.Generic; + +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; + +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Agents.Settings; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; + +global using BotSharp.Plugin.PythonInterpreter.Enums; diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/python_interpreter.json b/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/python_interpreter.json new file mode 100644 index 00000000..ac2b2a91 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/python_interpreter.json @@ -0,0 +1,19 @@ +{ + "name": "python_interpreter", + "description": "write and execute python code, print the result in Console", + "parameters": { + "type": "object", + "properties": { + "script": { + "type": "string", + "description": "python code" + }, + "language": { + "type": "string", + "enum": [ "python" ], + "description": "python code" + } + }, + "required": [ "language", "script" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/python_interpreter.fn.liquid b/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/python_interpreter.fn.liquid new file mode 100644 index 00000000..8dd2425f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/python_interpreter.fn.liquid @@ -0,0 +1 @@ +Write and execute Python script in python_interpreter function, and use python function print(a) to output the result in stand output. \ No newline at end of file diff --git a/src/WebStarter/Program.cs b/src/WebStarter/Program.cs index ed0f8baf..a13d9d0f 100644 --- a/src/WebStarter/Program.cs +++ b/src/WebStarter/Program.cs @@ -4,6 +4,7 @@ using BotSharp.Logger; using BotSharp.Plugin.ChatHub; using Serilog; using BotSharp.Abstraction.Messaging.JsonConverters; +using Python.Runtime; var builder = WebApplication.CreateBuilder(args); @@ -41,4 +42,11 @@ app.UseBotSharp() .UseBotSharpOpenAPI(app.Environment) .UseBotSharpUI(); +Runtime.PythonDLL = @"C:\Users\xxx\AppData\Local\Programs\Python\Python311\python311.dll"; +PythonEngine.Initialize(); +PythonEngine.BeginAllowThreads(); + app.Run(); + +// Shut down the Python engine +PythonEngine.Shutdown(); \ No newline at end of file diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 5a5b1ced..c7443dc1 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -30,6 +30,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 155dae1b..e2aed92b 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -319,7 +319,8 @@ "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", "BotSharp.Plugin.EmailHandler", - "BotSharp.Plugin.TencentCos" + "BotSharp.Plugin.TencentCos", + "BotSharp.Plugin.PythonInterpreter" ] } } From 64d088c87937aea72f4f4521a726b4ac17533d32 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 12 Aug 2024 21:58:06 -0500 Subject: [PATCH 46/63] Fix GoToPage --- .../Drivers/PlaywrightDriver/PlaywrightInstance.cs | 5 +++++ .../PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index cd6daa34..13b52b3e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -43,6 +43,11 @@ public class PlaywrightInstance : IDisposable } } + if (!string.IsNullOrEmpty(pattern)) + { + return null; + } + return _contexts[contextId].Pages.LastOrDefault(); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 40f617d2..7b03b57a 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -30,6 +30,13 @@ public partial class PlaywrightWebDriver includeResponseUrls: args.IncludeResponseUrls); } + if (page == null) + { + page = await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCallback, + excludeResponseUrls: args.ExcludeResponseUrls, + includeResponseUrls: args.IncludeResponseUrls); + } + var response = await page.GotoAsync(args.Url, new PageGotoOptions { Timeout = args.Timeout From f2c1777ea9dadd88a278666f66a739619e84251f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 Aug 2024 13:26:57 -0500 Subject: [PATCH 47/63] add knowledge collections --- .../Knowledges/IKnowledgeService.cs | 1 + .../Infrastructures/SettingService.cs | 1 + .../Controllers/KnowledgeBaseController.cs | 6 ++++++ .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../Functions/MemorizeKnowledgeFn.cs | 2 +- .../Services/KnowledgeService.Get.cs | 14 ++++++++++++++ src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 13 ++++++++++--- 7 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 657ac42a..d2882bdc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -2,6 +2,7 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { + Task> GetKnowledgeCollections(); Task> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options); Task FeedKnowledge(string collectionName, KnowledgeCreationModel model); Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs index fe3d9007..cbde5f44 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs @@ -31,6 +31,7 @@ public class SettingService : ISettingService var plugins = pluginService.GetPlugins(_services); var plugin = plugins.First(x => x.Module.Settings.Name == settingName); var instance = plugin.Module.GetNewSettingsInstance(); + _config.Bind(settingName, instance); if (mask) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index e20632f5..55712aaa 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -16,6 +16,12 @@ public class KnowledgeBaseController : ControllerBase _services = services; } + [HttpGet("knowledge/collections")] + public async Task> GetKnowledgeCollections() + { + return await _knowledgeService.GetKnowledgeCollections(); + } + [HttpPost("/knowledge/{collection}/search")] public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeModel model) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index ca81a4ba..62397c53 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -21,7 +21,7 @@ public class KnowledgeRetrievalFn : IFunctionCallback embedding.SetModelName(_settings.TextEmbedding.Model); var vector = await embedding.GetVectorAsync(args.Question); - var vectorDb = _services.GetRequiredService(); + var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, new List { KnowledgePayloadName.Answer }); if (!knowledges.IsNullOrEmpty()) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 218b33d4..b4b0111c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -25,7 +25,7 @@ public class MemorizeKnowledgeFn : IFunctionCallback args.Question }); - var vectorDb = _services.GetRequiredService(); + var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length); var id = Guid.NewGuid().ToString(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs index 96a5c139..d8c64cf3 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -2,6 +2,20 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { + public async Task> GetKnowledgeCollections() + { + try + { + var db = GetVectorDb(); + return await db.GetCollections(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collections. {ex.Message}\r\n{ex.InnerException}"); + return Enumerable.Empty(); + } + } + public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) { try diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index f03b4ab6..02520517 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -128,10 +128,17 @@ public class QdrantDb : IVectorDb public async Task> Search(string collectionName, float[] vector, IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { - var client = GetClient(); - var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); - var results = new List(); + + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) + { + return results; + } + + var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); + foreach (var point in points) { var data = new Dictionary(); From a7bc5fca887b9e1303848eed88cadfccecde04ae Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 Aug 2024 13:30:06 -0500 Subject: [PATCH 48/63] add try catch --- .../Services/KnowledgeService.Get.cs | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs index d8c64cf3..75824707 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -32,20 +32,28 @@ public partial class KnowledgeService public async Task> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options) { - var textEmbedding = GetTextEmbedding(); - var vector = await textEmbedding.GetVectorAsync(options.Text); - - // Vector search - var db = GetVectorDb(); - var fields = !options.Fields.IsNullOrEmpty() ? options.Fields : new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; - var found = await db.Search(collectionName, vector, fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); - - var results = found.Select(x => new KnowledgeRetrievalResult + try { - Data = x.Data, - Score = x.Score, - Vector = x.Vector - }).ToList(); - return results; + var textEmbedding = GetTextEmbedding(); + var vector = await textEmbedding.GetVectorAsync(options.Text); + + // Vector search + var db = GetVectorDb(); + var fields = !options.Fields.IsNullOrEmpty() ? options.Fields : new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + var found = await db.Search(collectionName, vector, fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); + + var results = found.Select(x => new KnowledgeRetrievalResult + { + Data = x.Data, + Score = x.Score, + Vector = x.Vector + }).ToList(); + return results; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when searching knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return new List(); + } } } From 8b86ea50d0b23cdea814f9cd2af667caba836ac5 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 Aug 2024 16:36:12 -0500 Subject: [PATCH 49/63] add agent channel instructions --- .../Agents/IAgentService.cs | 7 ++ .../Agents/Models/Agent.cs | 38 +++++----- .../Agents/Models/ChannelInstruction.cs | 7 ++ .../Models/KnowledgeSearchResult.cs | 2 +- .../Services/AgentService.CreateAgent.cs | 62 +++++++++------- .../Agents/Services/AgentService.GetAgents.cs | 24 +++++- .../Agents/Services/AgentService.LoadAgent.cs | 56 ++++++++------ .../Services/AgentService.RefreshAgents.cs | 5 +- .../Services/AgentService.UpdateAgent.cs | 8 +- .../FileRepository/FileRepository.Agent.cs | 73 ++++++++----------- .../FileRepository/FileRepository.cs | 64 ++++++++++++++-- .../Controllers/AgentController.cs | 43 +++-------- .../ViewModels/Agents/AgentCreationModel.cs | 25 ++++--- .../ViewModels/Agents/AgentUpdateModel.cs | 17 +++-- .../ViewModels/Agents/AgentViewModel.cs | 2 + .../Collections/AgentDocument.cs | 1 + .../Models/ChannelInstructionMongoElement.cs | 27 +++++++ .../Repository/MongoRepository.Agent.cs | 25 +++++-- .../Repository/MongoRepository.Transaction.cs | 6 +- 19 files changed, 310 insertions(+), 182 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Agents/Models/ChannelInstruction.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index 8405d9b5..9b65b148 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -20,6 +20,13 @@ public interface IAgentService /// Task LoadAgent(string id); + /// + /// Inherit from host agent + /// + /// + /// + Task InheritAgent(Agent agent); + string RenderedInstruction(Agent agent); string RenderedTemplate(Agent agent, string templateName); diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs index 01ec2acb..43f0f5ba 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Plugins.Models; -using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Tasks.Models; namespace BotSharp.Abstraction.Agents.Models; @@ -21,8 +20,7 @@ public class Agent /// Default LLM settings /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public AgentLlmConfig LlmConfig { get; set; } - = new AgentLlmConfig(); + public AgentLlmConfig LlmConfig { get; set; } = new(); /// /// Instruction @@ -30,33 +28,35 @@ public class Agent [JsonIgnore] public string? Instruction { get; set; } + /// + /// Channel instructions + /// + [JsonIgnore] + public List ChannelInstructions { get; set; } = new(); + /// /// Templates /// [JsonIgnore] - public List Templates { get; set; } - = new List(); + public List Templates { get; set; } = new(); /// /// Agent tasks /// [JsonIgnore] - public List Tasks { get; set; } - = new List(); + public List Tasks { get; set; } = new(); /// /// Samples /// [JsonIgnore] - public List Samples { get; set; } - = new List(); + public List Samples { get; set; } = new(); /// /// Functions /// [JsonIgnore] - public List Functions { get; set; } - = new List(); + public List Functions { get; set; } = new(); /// /// Responses @@ -93,23 +93,20 @@ public class Agent /// /// Agent utilities /// - public List Utilities { get; set; } - = new List(); + public List Utilities { get; set; } = new(); /// /// Inherit from agent /// public string? InheritAgentId { get; set; } - public List RoutingRules { get; set; } - = new List(); + public List RoutingRules { get; set; } = new(); /// /// For rendering deferral /// [JsonIgnore] - public Dictionary TemplateDict { get; set; } - = new Dictionary(); + public Dictionary TemplateDict { get; set; } = new(); public override string ToString() => $"{Name} {Id}"; @@ -124,6 +121,7 @@ public class Agent Description = agent.Description, Type = agent.Type, Instruction = agent.Instruction, + ChannelInstructions = agent.ChannelInstructions, Functions = agent.Functions, Responses = agent.Responses, Samples = agent.Samples, @@ -145,6 +143,12 @@ public class Agent return this; } + public Agent SetChannelInstructions(List instructions) + { + ChannelInstructions = instructions ?? new List(); + return this; + } + public Agent SetTemplates(List templates) { Templates = templates ?? new List(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/ChannelInstruction.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/ChannelInstruction.cs new file mode 100644 index 00000000..d4ff6c1f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/ChannelInstruction.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Agents.Models; + +public class ChannelInstruction +{ + public string Channel { get; set; } + public string Instruction { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs index b0deaa0f..2d5d9c1f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeSearchResult { - public IDictionary Data { get; set; } = new Dictionary(); + public Dictionary Data { get; set; } = new(); public double Score { get; set; } public float[]? Vector { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs index 59de3096..5fb6318f 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs @@ -41,41 +41,49 @@ public partial class AgentService }); Utilities.ClearCache(); - return await Task.FromResult(agentRecord); } - private Agent FetchAgentFileByName(string agentName, string filePath) + private (string, List) FetchInstructionsFromFile(string fileDir) { - foreach (var dir in Directory.GetDirectories(filePath)) + var defaultInstruction = string.Empty; + var channelInstructions = new List(); + + var instructionDir = Path.Combine(fileDir, "instructions"); + if (!Directory.Exists(instructionDir)) { - var agentJson = File.ReadAllText(Path.Combine(dir, "agent.json")); - var agent = JsonSerializer.Deserialize(agentJson, _options); - if (agent != null && agent.Name.IsEqualTo(agentName)) - { - var functions = FetchFunctionsFromFile(dir); - var instruction = FetchInstructionFromFile(dir); - var responses = FetchResponsesFromFile(dir); - var templates = FetchTemplatesFromFile(dir); - var samples = FetchSamplesFromFile(dir); - return agent.SetInstruction(instruction) - .SetTemplates(templates) - .SetFunctions(functions) - .SetResponses(responses) - .SetSamples(samples); - } + return (defaultInstruction, channelInstructions); } - return null; - } + foreach (var file in Directory.GetFiles(instructionDir)) + { + var extension = Path.GetExtension(file).Substring(1); + if (!extension.IsEqualTo(_agentSettings.TemplateFormat)) + { + continue; + } - private string FetchInstructionFromFile(string fileDir) - { - var file = Path.Combine(fileDir, $"instruction.{_agentSettings.TemplateFormat}"); - if (!File.Exists(file)) return null; + var segments = Path.GetFileName(file).Split(".", StringSplitOptions.RemoveEmptyEntries); + if (segments.IsNullOrEmpty() || !segments[0].IsEqualTo("instruction")) + { + continue; + } - var instruction = File.ReadAllText(file); - return instruction; + if (segments.Length == 2) + { + defaultInstruction = File.ReadAllText(file); + } + else if (segments.Length == 3) + { + var item = new ChannelInstruction + { + Channel = segments[1], + Instruction = File.ReadAllText(file) + }; + channelInstructions.Add(item); + } + } + return (defaultInstruction, channelInstructions); } private List FetchTemplatesFromFile(string fileDir) @@ -86,10 +94,10 @@ public partial class AgentService foreach (var file in Directory.GetFiles(templateDir)) { - var name = Path.GetFileNameWithoutExtension(file); var extension = Path.GetExtension(file).Substring(1); if (extension.IsEqualTo(_agentSettings.TemplateFormat)) { + var name = Path.GetFileNameWithoutExtension(file); var content = File.ReadAllText(file); templates.Add(new AgentTemplate(name, content)); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs index 15ce6eca..04b5c22d 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs @@ -53,7 +53,29 @@ public partial class AgentService } profile.Plugin = GetPlugin(profile.Id); - return profile; } + + public async Task InheritAgent(Agent agent) + { + if (string.IsNullOrWhiteSpace(agent?.InheritAgentId)) return; + + var inheritedAgent = await GetAgent(agent.InheritAgentId); + agent.Templates.AddRange(inheritedAgent.Templates + // exclude private template + .Where(x => !x.Name.StartsWith(".")) + // exclude duplicate name + .Where(x => !agent.Templates.Exists(t => t.Name == x.Name))); + + agent.Functions.AddRange(inheritedAgent.Functions + // exclude private template + .Where(x => !x.Name.StartsWith(".")) + // exclude duplicate name + .Where(x => !agent.Functions.Exists(t => t.Name == x.Name))); + + if (string.IsNullOrWhiteSpace(agent.Instruction)) + { + agent.Instruction = inheritedAgent.Instruction; + } + } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 757d9937..f0a864a2 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -34,31 +34,12 @@ public partial class AgentService return null; } - if (agent.InheritAgentId != null) - { - var inheritedAgent = await GetAgent(agent.InheritAgentId); - agent.Templates.AddRange(inheritedAgent.Templates - // exclude private template - .Where(x => !x.Name.StartsWith(".")) - // exclude duplicate name - .Where(x => !agent.Templates.Exists(t => t.Name == x.Name))); - - agent.Functions.AddRange(inheritedAgent.Functions - // exclude private template - .Where(x => !x.Name.StartsWith(".")) - // exclude duplicate name - .Where(x => !agent.Functions.Exists(t => t.Name == x.Name))); - - if (agent.Instruction == null) - { - agent.Instruction = inheritedAgent.Instruction; - } - } + await InheritAgent(agent); + OverrideInstructionByChannel(agent); AddOrUpdateParameters(agent); - agent.TemplateDict = new Dictionary(); - // Populate state into dictionary + agent.TemplateDict = new Dictionary(); PopulateState(agent.TemplateDict); // After agent is loaded @@ -94,6 +75,23 @@ public partial class AgentService return agent; } + private void OverrideInstructionByChannel(Agent agent) + { + var instructions = agent.ChannelInstructions; + if (instructions.IsNullOrEmpty()) return; + + var state = _services.GetRequiredService(); + var channel = state.GetState("channel"); + + if (string.IsNullOrWhiteSpace(channel)) + { + return; + } + + var found = instructions.FirstOrDefault(x => x.Channel.IsEqualTo(channel)); + agent.Instruction = !string.IsNullOrWhiteSpace(found?.Instruction) ? found.Instruction : agent.Instruction; + } + private void PopulateState(Dictionary dict) { var conv = _services.GetRequiredService(); @@ -114,18 +112,27 @@ public partial class AgentService private void AddOrUpdateRoutesParameters(string agentId, List routingRules) { - if(!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) parameterTypes = new(); + if(!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) + { + parameterTypes = new(); + } + foreach (var rule in routingRules.Where(x => x.Required)) { if (string.IsNullOrEmpty(rule.FieldType)) continue; parameterTypes.TryAdd(rule.Field, rule.FieldType); } + AgentParameterTypes.TryAdd(agentId, parameterTypes); } private void AddOrUpdateFunctionsParameters(string agentId, List functions) { - if (!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) parameterTypes = new(); + if (!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) + { + parameterTypes = new(); + } + var parameters = functions.Select(p => p.Parameters); foreach (var param in parameters) { @@ -139,6 +146,7 @@ public partial class AgentService } } } + AgentParameterTypes.TryAdd(agentId, parameterTypes); } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index 0b61977e..998d64c8 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -42,12 +42,13 @@ public partial class AgentService continue; } + var (defaultInstruction, channelInstructions) = FetchInstructionsFromFile(dir); var functions = FetchFunctionsFromFile(dir); - var instruction = FetchInstructionFromFile(dir); var responses = FetchResponsesFromFile(dir); var templates = FetchTemplatesFromFile(dir); var samples = FetchSamplesFromFile(dir); - agent.SetInstruction(instruction) + agent.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetTemplates(templates) .SetFunctions(functions) .SetResponses(responses) diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs index 6fd0f85e..cef56c02 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs @@ -28,6 +28,7 @@ public partial class AgentService record.Profiles = agent.Profiles ?? new List(); record.RoutingRules = agent.RoutingRules ?? new List(); record.Instruction = agent.Instruction ?? string.Empty; + record.ChannelInstructions = agent.ChannelInstructions ?? new List(); record.Functions = agent.Functions ?? new List(); record.Templates = agent.Templates ?? new List(); record.Responses = agent.Responses ?? new List(); @@ -41,7 +42,6 @@ public partial class AgentService _db.UpdateAgent(record, updateField); Utilities.ClearCache(); - await Task.CompletedTask; } @@ -90,6 +90,7 @@ public partial class AgentService .SetProfiles(foundAgent.Profiles) .SetRoutingRules(foundAgent.RoutingRules) .SetInstruction(foundAgent.Instruction) + .SetChannelInstructions(foundAgent.ChannelInstructions) .SetTemplates(foundAgent.Templates) .SetFunctions(foundAgent.Functions) .SetResponses(foundAgent.Responses) @@ -175,12 +176,13 @@ public partial class AgentService var agent = JsonSerializer.Deserialize(agentJson, _options); if (agent != null && agent.Id == agentId) { + var (defaultInstruction, channelInstructions) = FetchInstructionsFromFile(dir); var functions = FetchFunctionsFromFile(dir); - var instruction = FetchInstructionFromFile(dir); var responses = FetchResponsesFromFile(dir); var templates = FetchTemplatesFromFile(dir); var samples = FetchSamplesFromFile(dir); - return agent.SetInstruction(instruction) + return agent.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetTemplates(templates) .SetFunctions(functions) .SetResponses(responses) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index c0c59142..950a2b2d 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -1,6 +1,6 @@ +using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing.Models; using System.IO; -using System.Threading; namespace BotSharp.Core.Repository { @@ -37,7 +37,7 @@ namespace BotSharp.Core.Repository UpdateAgentRoutingRules(agent.Id, agent.RoutingRules); break; case AgentField.Instruction: - UpdateAgentInstruction(agent.Id, agent.Instruction); + UpdateAgentInstructions(agent.Id, agent.Instruction, agent.ChannelInstructions); break; case AgentField.Function: UpdateAgentFunctions(agent.Id, agent.Functions); @@ -175,17 +175,30 @@ namespace BotSharp.Core.Repository File.WriteAllText(agentFile, json); } - private void UpdateAgentInstruction(string agentId, string instruction) + private void UpdateAgentInstructions(string agentId, string instruction, List channelInstructions) { if (string.IsNullOrWhiteSpace(instruction)) return; var (agent, agentFile) = GetAgentFromFile(agentId); if (agent == null) return; - var instructionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); + var instructionDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_INSTRUCTIONS_FOLDER); + DeleteBeforeCreateDirectory(instructionDir); - File.WriteAllText(instructionFile, instruction); + // Save default instructions + var instructionFile = Path.Combine(instructionDir, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); + File.WriteAllText(instructionFile, instruction ?? string.Empty); + Thread.Sleep(100); + + // Save channel instructions + foreach (var ci in channelInstructions) + { + if (string.IsNullOrWhiteSpace(ci.Channel)) continue; + + var file = Path.Combine(instructionDir, $"{AGENT_INSTRUCTION_FILE}.{ci.Channel}.{_agentSettings.TemplateFormat}"); + File.WriteAllText(file, ci.Instruction ?? string.Empty); + Thread.Sleep(100); + } } private void UpdateAgentFunctions(string agentId, List inputFunctions) @@ -195,14 +208,8 @@ namespace BotSharp.Core.Repository var (agent, agentFile) = GetAgentFromFile(agentId); if (agent == null) return; - var functionDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, AGENT_FUNCTIONS_FOLDER); - - if (Directory.Exists(functionDir)) - { - Directory.Delete(functionDir, true); - } - Directory.CreateDirectory(functionDir); + var functionDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_FUNCTIONS_FOLDER); + DeleteBeforeCreateDirectory(functionDir); foreach (var func in inputFunctions) { @@ -211,7 +218,7 @@ namespace BotSharp.Core.Repository var text = JsonSerializer.Serialize(func, _options); var file = Path.Combine(functionDir, $"{func.Name}.json"); File.WriteAllText(file, text); - Thread.Sleep(200); + Thread.Sleep(100); } } @@ -223,16 +230,7 @@ namespace BotSharp.Core.Repository if (agent == null) return; var templateDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_TEMPLATES_FOLDER); - - if (!Directory.Exists(templateDir)) - { - Directory.CreateDirectory(templateDir); - } - - foreach (var file in Directory.GetFiles(templateDir)) - { - File.Delete(file); - } + DeleteBeforeCreateDirectory(templateDir); foreach (var template in templates) { @@ -249,15 +247,7 @@ namespace BotSharp.Core.Repository if (agent == null) return; var responseDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_RESPONSES_FOLDER); - if (!Directory.Exists(responseDir)) - { - Directory.CreateDirectory(responseDir); - } - - foreach (var file in Directory.GetFiles(responseDir)) - { - File.Delete(file); - } + DeleteBeforeCreateDirectory(responseDir); for (int i = 0; i < responses.Count; i++) { @@ -308,7 +298,7 @@ namespace BotSharp.Core.Repository var json = JsonSerializer.Serialize(agent, _options); File.WriteAllText(agentFile, json); - UpdateAgentInstruction(inputAgent.Id, inputAgent.Instruction); + UpdateAgentInstructions(inputAgent.Id, inputAgent.Instruction, agent.ChannelInstructions); UpdateAgentResponses(inputAgent.Id, inputAgent.Responses); UpdateAgentTemplates(inputAgent.Id, inputAgent.Templates); UpdateAgentFunctions(inputAgent.Id, inputAgent.Functions); @@ -348,12 +338,13 @@ namespace BotSharp.Core.Repository var record = JsonSerializer.Deserialize(json, _options); if (record == null) return null; - var instruction = FetchInstruction(dir); + var (defaultInstruction, channelInstructions) = FetchInstructions(dir); var functions = FetchFunctions(dir); var samples = FetchSamples(dir); var templates = FetchTemplates(dir); var responses = FetchResponses(dir); - return record.SetInstruction(instruction) + return record.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetFunctions(functions) .SetTemplates(templates) .SetSamples(samples) @@ -451,13 +442,9 @@ namespace BotSharp.Core.Repository return true; } - public void BulkInsertAgents(List agents) - { - } + public void BulkInsertAgents(List agents) { } - public void BulkInsertUserAgents(List userAgents) - { - } + public void BulkInsertUserAgents(List userAgents) { } public bool DeleteAgents() { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 78bd5cc0..7e5b4d9d 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -34,6 +34,7 @@ public partial class FileRepository : IBotSharpRepository private const string AGENT_TASK_PREFIX = "#metadata"; private const string AGENT_TASK_SUFFIX = "/metadata"; private const string TRANSLATION_MEMORY_FILE = "memory.json"; + private const string AGENT_INSTRUCTIONS_FOLDER = "instructions"; private const string AGENT_FUNCTIONS_FOLDER = "functions"; private const string AGENT_TEMPLATES_FOLDER = "templates"; private const string AGENT_RESPONSES_FOLDER = "responses"; @@ -123,7 +124,9 @@ public partial class FileRepository : IBotSharpRepository var agent = JsonSerializer.Deserialize(json, _options); if (agent != null) { - agent = agent.SetInstruction(FetchInstruction(d)) + var (defaultInstruction, channelInstructions) = FetchInstructions(d); + agent = agent.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetFunctions(FetchFunctions(d)) .SetTemplates(FetchTemplates(d)) .SetResponses(FetchResponses(d)) @@ -165,6 +168,17 @@ public partial class FileRepository : IBotSharpRepository #region Private methods + private void DeleteBeforeCreateDirectory(string dir) + { + if (string.IsNullOrWhiteSpace(dir)) return; + + if (Directory.Exists(dir)) + { + Directory.Delete(dir, true); + } + Directory.CreateDirectory(dir); + } + private string GetAgentDataDir(string agentId) { var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId); @@ -186,13 +200,46 @@ public partial class FileRepository : IBotSharpRepository return (agent, agentFile); } - private string? FetchInstruction(string fileDir) + private (string, List) FetchInstructions(string fileDir) { - var file = Path.Combine(fileDir, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); - if (!File.Exists(file)) return null; + var defaultInstruction = string.Empty; + var channelInstructions = new List(); - var instruction = File.ReadAllText(file); - return instruction; + var instructionDir = Path.Combine(fileDir, AGENT_INSTRUCTIONS_FOLDER); + if (!Directory.Exists(instructionDir)) + { + return (defaultInstruction, channelInstructions); + } + + foreach (var file in Directory.GetFiles(instructionDir)) + { + var extension = Path.GetExtension(file).Substring(1); + if (!extension.IsEqualTo(_agentSettings.TemplateFormat)) + { + continue; + } + + var segments = Path.GetFileName(file).Split(".", StringSplitOptions.RemoveEmptyEntries); + if (segments.IsNullOrEmpty() || !segments[0].IsEqualTo(AGENT_INSTRUCTION_FILE)) + { + continue; + } + + if (segments.Length == 2) + { + defaultInstruction = File.ReadAllText(file); + } + else if (segments.Length == 3) + { + var item = new ChannelInstruction + { + Channel = segments[1], + Instruction = File.ReadAllText(file) + }; + channelInstructions.Add(item); + } + } + return (defaultInstruction, channelInstructions); } private List FetchFunctions(string fileDir) @@ -298,13 +345,14 @@ public partial class FileRepository : IBotSharpRepository var agent = JsonSerializer.Deserialize(agentJson, _options); if (agent == null) return null; - var instruction = FetchInstruction(agentDir); + var (defaultInstruction, channelInstructions) = FetchInstructions(agentDir); var functions = FetchFunctions(agentDir); var samples = FetchSamples(agentDir); var templates = FetchTemplates(agentDir); var responses = FetchResponses(agentDir); - return agent.SetInstruction(instruction) + return agent.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetFunctions(functions) .SetTemplates(templates) .SetSamples(samples) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index f2bbc2eb..15c83c89 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Users.Enums; namespace BotSharp.OpenAPI.Controllers; @@ -28,13 +27,18 @@ public class AgentController : ControllerBase [HttpGet("/agent/{id}")] public async Task GetAgent([FromRoute] string id) { - var agents = await GetAgents(new AgentFilter + var pagedAgents = await _agentService.GetAgents(new AgentFilter { AgentIds = new List { id } - }, useHook: true); + }); - var targetAgent = agents.Items.FirstOrDefault(); - if (targetAgent == null) return null; + var foundAgent = pagedAgents.Items.FirstOrDefault(); + if (foundAgent == null) return null; + + await _agentService.InheritAgent(foundAgent); + var targetAgent = AgentViewModel.FromAgent(foundAgent); + var agentSetting = _services.GetRequiredService(); + targetAgent.IsHost = targetAgent.Id == agentSetting.HostAgentId; var redirectAgentIds = targetAgent.RoutingRules .Where(x => !string.IsNullOrEmpty(x.RedirectTo)) @@ -65,39 +69,16 @@ public class AgentController : ControllerBase } [HttpGet("/agents")] - public async Task> GetAgents([FromQuery] AgentFilter filter, [FromQuery] bool useHook = false) + public async Task> GetAgents([FromQuery] AgentFilter filter) { var agentSetting = _services.GetRequiredService(); var pagedAgents = await _agentService.GetAgents(filter); - - var items = new List(); - var agents = new List(); - if (useHook) - { - // prerender agent - foreach (var agent in pagedAgents.Items) - { - var renderedAgent = await _agentService.LoadAgent(agent.Id); - items.Add(renderedAgent); - } - - // Set IsHost - agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList(); - foreach (var agent in agents) - { - agent.IsHost = agentSetting.HostAgentId == agent.Id; - } - } - else - { - items = pagedAgents.Items.ToList(); - agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList(); - } + var agents = pagedAgents?.Items?.Select(x => AgentViewModel.FromAgent(x))?.ToList() ?? new List(); return new PagedItems { Items = agents, - Count = pagedAgents.Count + Count = pagedAgents?.Count ?? 0 }; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs index c0fa8320..78f75833 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing.Models; -using BotSharp.Core.Infrastructures; namespace BotSharp.OpenAPI.ViewModels.Agents; @@ -16,21 +15,26 @@ public class AgentCreationModel /// public string Instruction { get; set; } = string.Empty; + /// + /// + /// + public List ChannelInstructions { get; set; } = new(); + /// /// LLM extensible Instructions in addition to the default Instructions /// - public List Templates { get; set; } = new List(); + public List Templates { get; set; } = new(); /// /// LLM callable function definition /// - public List Functions { get; set; } = new List(); + public List Functions { get; set; } = new(); /// /// Response template /// - public List Responses { get; set; } = new List(); - public List Samples { get; set; } = new List(); + public List Responses { get; set; } = new(); + public List Samples { get; set; } = new(); public bool IsPublic { get; set; } @@ -43,9 +47,9 @@ public class AgentCreationModel /// /// Combine different Agents together to form a Profile. /// - public List Profiles { get; set; } = new List(); - public List Utilities { get; set; } = new List(); - public List RoutingRules { get; set; } = new List(); + public List Profiles { get; set; } = new(); + public List Utilities { get; set; } = new(); + public List RoutingRules { get; set; } = new(); public AgentLlmConfig? LlmConfig { get; set; } public Agent ToAgent() @@ -55,6 +59,7 @@ public class AgentCreationModel Name = Name, Description = Description, Instruction = Instruction, + ChannelInstructions = ChannelInstructions, Templates = Templates, Functions = Functions, Responses = Responses, @@ -64,9 +69,7 @@ public class AgentCreationModel Type = Type, Disabled = Disabled, Profiles = Profiles, - RoutingRules = RoutingRules? - .Select(x => RoutingRuleUpdateModel.ToDomainElement(x))? - .ToList() ?? new List(), + RoutingRules = RoutingRules?.Select(x => RoutingRuleUpdateModel.ToDomainElement(x))?.ToList() ?? new List(), LlmConfig = LlmConfig }; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs index 96cc7278..d64530c6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs @@ -15,6 +15,12 @@ public class AgentUpdateModel /// public string Instruction { get; set; } = string.Empty; + /// + /// Channel instructions + /// + [JsonPropertyName("channel_instructions")] + public List? ChannelInstructions { get; set; } + /// /// Templates /// @@ -39,11 +45,11 @@ public class AgentUpdateModel /// Routes /// public List? Responses { get; set; } + [JsonPropertyName("is_public")] - public bool IsPublic { get; set; } - [JsonPropertyName("allow_routing")] + [JsonPropertyName("allow_routing")] public bool AllowRouting { get; set; } public bool Disabled { get; set; } @@ -52,8 +58,8 @@ public class AgentUpdateModel /// Profile by channel /// public List? Profiles { get; set; } - [JsonPropertyName("routing_rules")] + [JsonPropertyName("routing_rules")] public List? RoutingRules { get; set; } [JsonPropertyName("llm_config")] @@ -69,10 +75,9 @@ public class AgentUpdateModel Disabled = Disabled, Type = Type, Profiles = Profiles ?? new List(), - RoutingRules = RoutingRules? - .Select(x => RoutingRuleUpdateModel.ToDomainElement(x))? - .ToList() ?? new List(), + RoutingRules = RoutingRules?.Select(x => RoutingRuleUpdateModel.ToDomainElement(x))?.ToList() ?? new List(), Instruction = Instruction ?? string.Empty, + ChannelInstructions = ChannelInstructions ?? new List(), Templates = Templates ?? new List(), Functions = Functions ?? new List(), Responses = Responses ?? new List(), diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs index a88b3c72..93440075 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs @@ -13,6 +13,7 @@ public class AgentViewModel public string Description { get; set; } public string Type { get; set; } = AgentType.Task; public string Instruction { get; set; } + public List ChannelInstructions { get; set; } public List Templates { get; set; } public List Functions { get; set; } public List Responses { get; set; } @@ -60,6 +61,7 @@ public class AgentViewModel Description = agent.Description, Type = agent.Type, Instruction = agent.Instruction, + ChannelInstructions = agent.ChannelInstructions, Templates = agent.Templates, Functions = agent.Functions, Responses = agent.Responses, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs index 4f1c5194..baa0729f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs @@ -8,6 +8,7 @@ public class AgentDocument : MongoBase public string? InheritAgentId { get; set; } public string? IconUrl { get; set; } public string Instruction { get; set; } + public List ChannelInstructions { get; set; } public List Templates { get; set; } public List Functions { get; set; } public List Responses { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs new file mode 100644 index 00000000..884c638c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Agents.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +public class ChannelInstructionMongoElement +{ + public string Channel { get; set; } + public string Instruction { get; set; } + + public static ChannelInstructionMongoElement ToMongoElement(ChannelInstruction instruction) + { + return new ChannelInstructionMongoElement + { + Channel = instruction.Channel, + Instruction = instruction.Instruction + }; + } + + public static ChannelInstruction ToDomainElement(ChannelInstructionMongoElement instruction) + { + return new ChannelInstruction + { + Channel = instruction.Channel, + Instruction = instruction.Instruction + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 60dcd6c3..49c8616e 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -38,7 +38,7 @@ public partial class MongoRepository UpdateAgentRoutingRules(agent.Id, agent.RoutingRules); break; case AgentField.Instruction: - UpdateAgentInstruction(agent.Id, agent.Instruction); + UpdateAgentInstructions(agent.Id, agent.Instruction, agent.ChannelInstructions); break; case AgentField.Function: UpdateAgentFunctions(agent.Id, agent.Functions); @@ -156,13 +156,17 @@ public partial class MongoRepository _dc.Agents.UpdateOne(filter, update); } - private void UpdateAgentInstruction(string agentId, string instruction) + private void UpdateAgentInstructions(string agentId, string instruction, List? channelInstructions) { - if (string.IsNullOrWhiteSpace(instruction)) return; + if (string.IsNullOrWhiteSpace(agentId)) return; + + var instructionElements = channelInstructions?.Select(x => ChannelInstructionMongoElement.ToMongoElement(x))? + .ToList() ?? new List(); var filter = Builders.Filter.Eq(x => x.Id, agentId); var update = Builders.Update .Set(x => x.Instruction, instruction) + .Set(x => x.ChannelInstructions, instructionElements) .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.Agents.UpdateOne(filter, update); @@ -253,6 +257,7 @@ public partial class MongoRepository .Set(x => x.Profiles, agent.Profiles) .Set(x => x.RoutingRules, agent.RoutingRules.Select(r => RoutingRuleMongoElement.ToMongoElement(r)).ToList()) .Set(x => x.Instruction, agent.Instruction) + .Set(x => x.ChannelInstructions, agent.ChannelInstructions.Select(i => ChannelInstructionMongoElement.ToMongoElement(i)).ToList()) .Set(x => x.Templates, agent.Templates.Select(t => AgentTemplateMongoElement.ToMongoElement(t)).ToList()) .Set(x => x.Functions, agent.Functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList()) .Set(x => x.Responses, agent.Responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList()) @@ -373,6 +378,9 @@ public partial class MongoRepository IconUrl = x.IconUrl, Description = x.Description, Instruction = x.Instruction, + ChannelInstructions = x.ChannelInstructions? + .Select(i => ChannelInstructionMongoElement.ToMongoElement(i))? + .ToList() ?? new List(), Templates = x.Templates? .Select(t => AgentTemplateMongoElement.ToMongoElement(t))? .ToList() ?? new List(), @@ -463,6 +471,9 @@ public partial class MongoRepository IconUrl = agentDoc.IconUrl, Description = agentDoc.Description, Instruction = agentDoc.Instruction, + ChannelInstructions = !agentDoc.ChannelInstructions.IsNullOrEmpty() ? agentDoc.ChannelInstructions + .Select(i => ChannelInstructionMongoElement.ToDomainElement(i)) + .ToList() : new List(), Templates = !agentDoc.Templates.IsNullOrEmpty() ? agentDoc.Templates .Select(t => AgentTemplateMongoElement.ToDomainElement(t)) .ToList() : new List(), @@ -472,6 +483,10 @@ public partial class MongoRepository Responses = !agentDoc.Responses.IsNullOrEmpty() ? agentDoc.Responses .Select(r => AgentResponseMongoElement.ToDomainElement(r)) .ToList() : new List(), + RoutingRules = !agentDoc.RoutingRules.IsNullOrEmpty() ? agentDoc.RoutingRules + .Select(r => RoutingRuleMongoElement.ToDomainElement(agentDoc.Id, agentDoc.Name, r)) + .ToList() : new List(), + LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agentDoc.LlmConfig), Samples = agentDoc.Samples ?? new List(), Utilities = agentDoc.Utilities ?? new List(), IsPublic = agentDoc.IsPublic, @@ -479,10 +494,6 @@ public partial class MongoRepository Type = agentDoc.Type, InheritAgentId = agentDoc.InheritAgentId, Profiles = agentDoc.Profiles, - RoutingRules = !agentDoc.RoutingRules.IsNullOrEmpty() ? agentDoc.RoutingRules - .Select(r => RoutingRuleMongoElement.ToDomainElement(agentDoc.Id, agentDoc.Name, r)) - .ToList() : new List(), - LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agentDoc.LlmConfig) }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs index 4a4542ed..e2ffbb0e 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs @@ -37,9 +37,12 @@ public partial class MongoRepository { Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), Name = x.Name, + IconUrl = x.IconUrl, Description = x.Description, Instruction = x.Instruction, - IconUrl = x.IconUrl, + ChannelInstructions = x.ChannelInstructions? + .Select(i => ChannelInstructionMongoElement.ToMongoElement(i))? + .ToList() ?? new List(), Templates = x.Templates? .Select(t => AgentTemplateMongoElement.ToMongoElement(t))? .ToList() ?? new List(), @@ -71,6 +74,7 @@ public partial class MongoRepository .Set(x => x.Name, agent.Name) .Set(x => x.Description, agent.Description) .Set(x => x.Instruction, agent.Instruction) + .Set(x => x.ChannelInstructions, agent.ChannelInstructions) .Set(x => x.Templates, agent.Templates) .Set(x => x.Functions, agent.Functions) .Set(x => x.Responses, agent.Responses) From 16795a13fd456b397b81cbc8575aabd4c0e2497f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 Aug 2024 16:54:32 -0500 Subject: [PATCH 50/63] relocate instructions --- .../BotSharp.Core/BotSharp.Core.csproj | 26 +++++++++---------- .../{ => instructions}/instruction.liquid | 0 .../{ => instructions}/instruction.liquid | 0 .../{ => instructions}/instruction.liquid | 0 .../{ => instructions}/instruction.liquid | 0 .../{ => instructions}/instruction.liquid | 0 .../{ => instructions}/instruction.liquid | 0 .../BotSharp.Plugin.HttpHandler.csproj | 4 +-- .../{ => instructions}/instruction.liquid | 0 .../BotSharp.Plugin.KnowledgeBase.csproj | 4 +-- .../{ => instructions}/instruction.liquid | 0 .../BotSharp.Plugin.SqlDriver.csproj | 4 +-- .../{ => instructions}/instruction.liquid | 0 .../BotSharp.Plugin.WebDriver.csproj | 4 +-- .../{ => instructions}/instruction.liquid | 0 15 files changed, 21 insertions(+), 21 deletions(-) rename src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/{ => instructions}/instruction.liquid (100%) rename src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/{ => instructions}/instruction.liquid (100%) rename src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/{ => instructions}/instruction.liquid (100%) rename src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/{ => instructions}/instruction.liquid (100%) rename src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/{ => instructions}/instruction.liquid (100%) rename src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/{ => instructions}/instruction.liquid (100%) rename src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/{ => instructions}/instruction.liquid (100%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/{ => instructions}/instruction.liquid (100%) rename src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/{ => instructions}/instruction.liquid (100%) rename src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/{ => instructions}/instruction.liquid (100%) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index ad46fb56..73bf035c 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -56,17 +56,17 @@ - + - + - + - + - + @@ -80,7 +80,7 @@ - + @@ -90,7 +90,7 @@ PreserveNewest - + PreserveNewest @@ -99,19 +99,19 @@ PreserveNewest - + PreserveNewest PreserveNewest - + PreserveNewest PreserveNewest - + PreserveNewest @@ -147,7 +147,7 @@ PreserveNewest - + PreserveNewest @@ -162,7 +162,7 @@ PreserveNewest - + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instructions/instruction.liquid diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj b/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj index 2911dc2b..8664c654 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj @@ -15,14 +15,14 @@ - + PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instruction.liquid b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instructions/instruction.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instruction.liquid rename to src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instructions/instruction.liquid diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index ecf15fe2..42663445 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -20,7 +20,7 @@ - + @@ -34,7 +34,7 @@ PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/instruction.liquid b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/instructions/instruction.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/instruction.liquid rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/instructions/instruction.liquid diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 8d861d68..6d2b4b05 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -18,7 +18,7 @@ - + @@ -26,7 +26,7 @@ PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instructions/instruction.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid rename to src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instructions/instruction.liquid diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index ce32616a..d6074b19 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -40,7 +40,7 @@ - + @@ -50,7 +50,7 @@ PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instructions/instruction.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid rename to src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instructions/instruction.liquid From 281ace86585cd059a36a68a0b27090bd1e509dd0 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 Aug 2024 17:08:30 -0500 Subject: [PATCH 51/63] add json display name --- .../BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs index 93440075..5c23c4d4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs @@ -13,6 +13,8 @@ public class AgentViewModel public string Description { get; set; } public string Type { get; set; } = AgentType.Task; public string Instruction { get; set; } + + [JsonPropertyName("channel_instructions")] public List ChannelInstructions { get; set; } public List Templates { get; set; } public List Functions { get; set; } From 81f2b32761c81f2597092f7c961ca5435a20243f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 Aug 2024 17:16:55 -0500 Subject: [PATCH 52/63] rename --- .../Agents/Services/AgentService.CreateAgent.cs | 12 ++++++------ .../Agents/Services/AgentService.RefreshAgents.cs | 12 ++++++------ .../Agents/Services/AgentService.UpdateAgent.cs | 14 +++++++------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs index 5fb6318f..9e2e1d5f 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs @@ -44,7 +44,7 @@ public partial class AgentService return await Task.FromResult(agentRecord); } - private (string, List) FetchInstructionsFromFile(string fileDir) + private (string, List) GetInstructionsFromFile(string fileDir) { var defaultInstruction = string.Empty; var channelInstructions = new List(); @@ -86,7 +86,7 @@ public partial class AgentService return (defaultInstruction, channelInstructions); } - private List FetchTemplatesFromFile(string fileDir) + private List GetTemplatesFromFile(string fileDir) { var templates = new List(); var templateDir = Path.Combine(fileDir, "templates"); @@ -106,7 +106,7 @@ public partial class AgentService return templates; } - private List FetchFunctionsFromFile(string fileDir) + private List GetFunctionsFromFile(string fileDir) { var functions = new List(); var functionDir = Path.Combine(fileDir, "functions"); @@ -133,7 +133,7 @@ public partial class AgentService return functions; } - private List FetchResponsesFromFile(string fileDir) + private List GetResponsesFromFile(string fileDir) { var responses = new List(); var responseDir = Path.Combine(fileDir, "responses"); @@ -151,7 +151,7 @@ public partial class AgentService return responses; } - private List FetchSamplesFromFile(string fileDir) + private List GetSamplesFromFile(string fileDir) { var file = Path.Combine(fileDir, "samples.txt"); if (!File.Exists(file)) return new List(); @@ -160,7 +160,7 @@ public partial class AgentService return samples?.ToList() ?? new List(); } - private List FetchTasksFromFile(string fileDir) + private List GetTasksFromFile(string fileDir) { var tasks = new List(); var taskDir = Path.Combine(fileDir, "tasks"); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index 998d64c8..10598d4e 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -42,11 +42,11 @@ public partial class AgentService continue; } - var (defaultInstruction, channelInstructions) = FetchInstructionsFromFile(dir); - var functions = FetchFunctionsFromFile(dir); - var responses = FetchResponsesFromFile(dir); - var templates = FetchTemplatesFromFile(dir); - var samples = FetchSamplesFromFile(dir); + var (defaultInstruction, channelInstructions) = GetInstructionsFromFile(dir); + var functions = GetFunctionsFromFile(dir); + var responses = GetResponsesFromFile(dir); + var templates = GetTemplatesFromFile(dir); + var samples = GetSamplesFromFile(dir); agent.SetInstruction(defaultInstruction) .SetChannelInstructions(channelInstructions) .SetTemplates(templates) @@ -55,7 +55,7 @@ public partial class AgentService .SetSamples(samples); var userAgent = BuildUserAgent(agent.Id, user.Id); - var tasks = FetchTasksFromFile(dir); + var tasks = GetTasksFromFile(dir); var isAgentDeleted = _db.DeleteAgent(agent.Id); if (isAgentDeleted) diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs index cef56c02..fe532e28 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs @@ -71,7 +71,7 @@ public partial class AgentService agentSettings.DataDir); var clonedAgent = Agent.Clone(agent); - var foundAgent = FetchAgentFileById(agent.Id, filePath); + var foundAgent = GetAgentFileById(agent.Id, filePath); if (foundAgent == null) { updateResult = $"Cannot find agent {agent.Name} in file directory: {filePath}"; @@ -166,7 +166,7 @@ public partial class AgentService return patchResult; } - private Agent? FetchAgentFileById(string agentId, string filePath) + private Agent? GetAgentFileById(string agentId, string filePath) { if (!Directory.Exists(filePath)) return null; @@ -176,11 +176,11 @@ public partial class AgentService var agent = JsonSerializer.Deserialize(agentJson, _options); if (agent != null && agent.Id == agentId) { - var (defaultInstruction, channelInstructions) = FetchInstructionsFromFile(dir); - var functions = FetchFunctionsFromFile(dir); - var responses = FetchResponsesFromFile(dir); - var templates = FetchTemplatesFromFile(dir); - var samples = FetchSamplesFromFile(dir); + var (defaultInstruction, channelInstructions) = GetInstructionsFromFile(dir); + var functions = GetFunctionsFromFile(dir); + var responses = GetResponsesFromFile(dir); + var templates = GetTemplatesFromFile(dir); + var samples = GetSamplesFromFile(dir); return agent.SetInstruction(defaultInstruction) .SetChannelInstructions(channelInstructions) .SetTemplates(templates) From d631c3cf8f615506beab4a5c94e08fffaa3937fb Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 Aug 2024 17:37:47 -0500 Subject: [PATCH 53/63] add channels in function def --- .../Functions/Models/FunctionDef.cs | 5 +++++ .../Agents/Services/AgentService.Rendering.cs | 21 ++++++++++++++++--- .../Controllers/AgentController.cs | 4 +++- .../Models/FunctionDefMongoElement.cs | 19 ++++++++++------- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs index 4366d323..d5dabb3c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs @@ -8,7 +8,12 @@ public class FunctionDef [JsonPropertyName("description")] public string Description { get; set; } = null!; + [JsonPropertyName("channels")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Channels { get; set; } + [JsonPropertyName("visibility_expression")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? VisibilityExpression { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs index a0b2f130..e9617d17 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs @@ -20,17 +20,32 @@ public partial class AgentService public bool RenderFunction(Agent agent, FunctionDef def) { - if (!string.IsNullOrEmpty(def.VisibilityExpression)) + var isRender = true; + + var channels = def.Channels; + if (channels != null) + { + var state = _services.GetRequiredService(); + var channel = state.GetState("channel"); + if (!string.IsNullOrWhiteSpace(channel)) + { + isRender = isRender && channels.Contains(channel); + } + } + + if (!isRender) return false; + + if (!string.IsNullOrWhiteSpace(def.VisibilityExpression)) { var render = _services.GetRequiredService(); var result = render.Render(def.VisibilityExpression, new Dictionary { { "states", agent.TemplateDict } }); - return result == "visible"; + isRender = isRender && result == "visible"; } - return true; + return isRender; } public FunctionParametersDef? RenderFunctionProperty(Agent agent, FunctionDef def) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 15c83c89..f033e31c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -42,7 +42,9 @@ public class AgentController : ControllerBase var redirectAgentIds = targetAgent.RoutingRules .Where(x => !string.IsNullOrEmpty(x.RedirectTo)) - .Select(x => x.RedirectTo).ToList(); + .Select(x => x.RedirectTo) + .ToList(); + var redirectAgents = await _agentService.GetAgents(new AgentFilter { AgentIds = redirectAgentIds diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs index dac77d69..6f72c517 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs @@ -8,6 +8,7 @@ public class FunctionDefMongoElement { public string Name { get; set; } public string Description { get; set; } + public List? Channels { get; set; } public string? VisibilityExpression { get; set; } public string? Impact { get; set; } public FunctionParametersDefMongoElement Parameters { get; set; } = new FunctionParametersDefMongoElement(); @@ -23,6 +24,7 @@ public class FunctionDefMongoElement { Name = function.Name, Description = function.Description, + Channels = function.Channels, VisibilityExpression = function.VisibilityExpression, Impact = function.Impact, Parameters = new FunctionParametersDefMongoElement @@ -34,19 +36,20 @@ public class FunctionDefMongoElement }; } - public static FunctionDef ToDomainElement(FunctionDefMongoElement mongoFunction) + public static FunctionDef ToDomainElement(FunctionDefMongoElement function) { return new FunctionDef { - Name = mongoFunction.Name, - Description = mongoFunction.Description, - VisibilityExpression = mongoFunction.VisibilityExpression, - Impact = mongoFunction.Impact, + Name = function.Name, + Description = function.Description, + Channels = function.Channels, + VisibilityExpression = function.VisibilityExpression, + Impact = function.Impact, Parameters = new FunctionParametersDef { - Type = mongoFunction.Parameters.Type, - Properties = JsonSerializer.Deserialize(mongoFunction.Parameters.Properties.IfNullOrEmptyAs("{}")), - Required = mongoFunction.Parameters.Required, + Type = function.Parameters.Type, + Properties = JsonSerializer.Deserialize(function.Parameters.Properties.IfNullOrEmptyAs("{}")), + Required = function.Parameters.Required, } }; } From b7dadd3159c9f71586e91ec5b6b4355fb467ccc5 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 13 Aug 2024 22:44:53 -0500 Subject: [PATCH 54/63] ResponseInMemory --- .../Browsing/IWebPageResponseHook.cs | 4 ++-- .../Browsing/Models/PageActionArgs.cs | 6 +++++ .../Browsing/Models/WebPageResponseData.cs | 10 +++++++++ .../Browsing/Models/WebPageResponseFilter.cs | 7 ++++++ .../PlaywrightDriver/PlaywrightInstance.cs | 22 +++++++++++++++++-- .../PlaywrightWebDriver.GoToPage.cs | 12 ++++++++-- 6 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebPageResponseHook.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebPageResponseHook.cs index a7e33b22..4e32117b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebPageResponseHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebPageResponseHook.cs @@ -4,6 +4,6 @@ namespace BotSharp.Abstraction.Browsing; public interface IWebPageResponseHook { - void OnDataFetched(MessageInfo message, string url, string postData, string responsData); - T? GetResponse(MessageInfo message, string url, string? queryParameter = null); + void OnDataFetched(MessageInfo message, WebPageResponseData response); + T? GetResponse(MessageInfo message, WebPageResponseFilter filter); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index 098b9538..d260a09d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -28,6 +28,12 @@ public class PageActionArgs /// public string[]? IncludeResponseUrls { get; set; } + /// + /// If set to true, the response will be stored in memory + /// + public bool ResponseInMemory { get; set; } = false; + public List? ResponseContainer { get; set; } + public bool UseExistingPage { get; set; } = false; public bool WaitForNetworkIdle { get; set; } = true; diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs new file mode 100644 index 00000000..1d03b0b7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class WebPageResponseData +{ + public string Url { get; set; } = null!; + public string PostData { get; set; } = null!; + public string ResponseData { get; set; } = null!; + public bool ResponseInMemory { get; set; } + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs new file mode 100644 index 00000000..cd97a01d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class WebPageResponseFilter +{ + public string Url { get; set; } = null!; + public string[]? QueryParameters { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 13b52b3e..6fb84817 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -122,7 +122,12 @@ public class PlaywrightInstance : IDisposable return _contexts[ctxId]; } - public async Task NewPage(MessageInfo message, bool enableResponseCallback = false, string[]? excludeResponseUrls = null, string[]? includeResponseUrls = null) + public async Task NewPage(MessageInfo message, + bool enableResponseCallback = false, + bool responseInMemory = false, + List? responseContainer = null, + string[]? excludeResponseUrls = null, + string[]? includeResponseUrls = null) { var context = await GetContext(message.ContextId); var page = await context.NewPageAsync(); @@ -162,7 +167,20 @@ public class PlaywrightInstance : IDisposable var webPageResponseHooks = _services.GetServices(); foreach (var hook in webPageResponseHooks) { - hook.OnDataFetched(message, e.Url.ToLower(), e.Request?.PostData ?? string.Empty, JsonSerializer.Serialize(json)); + var result = new WebPageResponseData + { + Url = e.Url.ToLower(), + PostData = e.Request?.PostData ?? string.Empty, + ResponseData = JsonSerializer.Serialize(json), + ResponseInMemory = responseInMemory + }; + + if (responseContainer != null && responseInMemory) + { + responseContainer.Add(result); + } + + hook.OnDataFetched(message, result); } } catch (ObjectDisposedException ex) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 7b03b57a..a7e5f27f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -11,6 +11,8 @@ public partial class PlaywrightWebDriver var page = args.UseExistingPage ? _instance.GetPage(message.ContextId, pattern: args.Url) : await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCallback, + responseInMemory: args.ResponseInMemory, + responseContainer: args.ResponseContainer, excludeResponseUrls: args.ExcludeResponseUrls, includeResponseUrls: args.IncludeResponseUrls); @@ -25,14 +27,20 @@ public partial class PlaywrightWebDriver if (args.UseExistingPage && args.OpenNewTab && page != null && page.Url == "about:blank") { - page = await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCallback, + page = await _instance.NewPage(message, + enableResponseCallback: args.EnableResponseCallback, + responseInMemory: args.ResponseInMemory, + responseContainer: args.ResponseContainer, excludeResponseUrls: args.ExcludeResponseUrls, includeResponseUrls: args.IncludeResponseUrls); } if (page == null) { - page = await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCallback, + page = await _instance.NewPage(message, + enableResponseCallback: args.EnableResponseCallback, + responseInMemory: args.ResponseInMemory, + responseContainer: args.ResponseContainer, excludeResponseUrls: args.ExcludeResponseUrls, includeResponseUrls: args.IncludeResponseUrls); } From 4cc3ae68d359ec27f064ae9f5d8a1dc5184275de Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 14 Aug 2024 10:45:22 -0500 Subject: [PATCH 55/63] change comment --- src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index 9b65b148..1fd7a38c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -21,7 +21,7 @@ public interface IAgentService Task LoadAgent(string id); /// - /// Inherit from host agent + /// Inherit from an agent /// /// /// From c29d2ac5518be7dea5f494ab4c8e663555555a25 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 14 Aug 2024 11:03:58 -0500 Subject: [PATCH 56/63] relocate instructions --- .../BotSharp.Plugin.PizzaBot.csproj | 12 ++++++------ .../{ => instructions}/instruction.liquid | 0 .../{ => instructions}/instruction.liquid | 0 .../{ => instructions}/instruction.liquid | 0 4 files changed, 6 insertions(+), 6 deletions(-) rename tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/{ => instructions}/instruction.liquid (100%) rename tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/{ => instructions}/instruction.liquid (100%) rename tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/{ => instructions}/instruction.liquid (100%) diff --git a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj index e448b9e0..716f1f4d 100644 --- a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj +++ b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj @@ -24,16 +24,16 @@ - + - + - + @@ -50,13 +50,13 @@ PreserveNewest - + PreserveNewest PreserveNewest - + PreserveNewest @@ -65,7 +65,7 @@ PreserveNewest - + PreserveNewest diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/instruction.liquid b/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/instructions/instruction.liquid similarity index 100% rename from tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/instruction.liquid rename to tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/instructions/instruction.liquid diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/instruction.liquid b/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/instructions/instruction.liquid similarity index 100% rename from tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/instruction.liquid rename to tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/instructions/instruction.liquid diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/instruction.liquid b/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/instructions/instruction.liquid similarity index 100% rename from tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/instruction.liquid rename to tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/instructions/instruction.liquid From 694fcbfcaeb0e7130462307dbbcf81775091c502 Mon Sep 17 00:00:00 2001 From: Bo Yin <103488@smsassist.com> Date: Wed, 14 Aug 2024 16:44:47 -0500 Subject: [PATCH 57/63] update twilioPlugin --- .../Storage/LocalFileStorageService.Audio.cs | 8 +- .../Controllers/TwilioVoiceController.cs | 127 ++++++++---------- .../Models/AssistantMessage.cs | 8 ++ .../Models/CallerMessage.cs | 5 +- .../Services/ITwilioSessionManager.cs | 11 +- .../Services/TwilioMessageQueueService.cs | 42 ++++-- .../Services/TwilioService.cs | 6 +- .../Services/TwilioSessionManager.cs | 37 +++-- .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 1 - 9 files changed, 138 insertions(+), 107 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs index 9788631e..e90c7812 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs @@ -6,19 +6,21 @@ namespace BotSharp.Core.Files.Services { public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) { - var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, TEXT_TO_SPEECH_FOLDER, conversationId); + var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } - using var file = File.Create(Path.Combine(dir, fileName)); + var filePath = Path.Combine(dir, fileName); + if (File.Exists(filePath)) return; + using var file = File.Create(filePath); using var input = data.ToStream(); await input.CopyToAsync(file); } public async Task RetrieveSpeechFileAsync(string conversationId, string fileName) { - var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, TEXT_TO_SPEECH_FOLDER, conversationId, fileName); + var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); using var file = new FileStream(path, FileMode.Open, FileAccess.Read); return await BinaryData.FromStreamAsync(file); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 6e163ce8..859e34ff 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Files; -using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Models; using BotSharp.Plugin.Twilio.Services; @@ -10,7 +9,7 @@ using System.IdentityModel.Tokens.Jwt; namespace BotSharp.Plugin.Twilio.Controllers; [AllowAnonymous] -[Route("[controller]")] +[Route("twilio/voice")] public class TwilioVoiceController : TwilioController { private readonly TwilioSetting _settings; @@ -38,73 +37,25 @@ public class TwilioVoiceController : TwilioController }; } - [HttpPost("/twilio/voice/welcome")] - public async Task StartConversation(VoiceRequest request) - { - string sessionId = $"TwilioVoice_{request.CallSid}"; - var twilio = _services.GetRequiredService(); - var response = twilio.ReturnInstructions("Hello, how may I help you?"); - return TwiML(response); - } - - [HttpPost("/twilio/voice/{agentId}")] - public async Task ReceivedVoiceMessage([FromRoute] string agentId, VoiceRequest input) - { - string sessionId = $"TwilioVoice_{input.CallSid}"; - - var inputMsg = new RoleDialogModel(AgentRole.User, input.SpeechResult); - var conv = _services.GetRequiredService(); - var routing = _services.GetRequiredService(); - routing.Context.SetMessageId(sessionId, inputMsg.MessageId); - - conv.SetConversationId(sessionId, new List - { - new MessageState("channel", ConversationChannel.Phone), - new MessageState("calling_phone", input.DialCallSid) - }); - - var twilio = _services.GetRequiredService(); - VoiceResponse response = default; - - var result = await conv.SendMessage(agentId, - inputMsg, - replyMessage: null, - async msg => - { - response = twilio.ReturnInstructions(msg.Content); - if (msg.FunctionName == "conversation_end") - { - response = twilio.HangUp(msg.Content); - } - }, async functionExecuting => - { - }, async functionExecuted => - { - }); - - return TwiML(response); - } - - - [HttpPost("start")] - public TwiMLResult InitiateConversation(VoiceRequest request) + [HttpPost("welcome")] + public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states) { if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); - string sessionId = $"TwilioVoice_{request.CallSid}"; + string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); - var url = $"twiliovoice/{sessionId}/send/0"; - var response = twilio.ReturnInstructions("twilio/welcome.mp3", url, false); + var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; + var response = twilio.ReturnInstructions("twilio/welcome.mp3", url, true); return TwiML(response); } - [HttpPost("{sessionId}/send/{seqNum}")] - public async Task SendCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request) + [HttpPost("{conversationId}/receive/{seqNum}")] + public async Task ReceiveCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, VoiceRequest request) { var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); - var url = $"twiliovoice/{sessionId}/reply/{seqNum}"; - var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(sessionId, seqNum); + var url = $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}"; + var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum); if (!string.IsNullOrWhiteSpace(request.SpeechResult)) { messages.Add(request.SpeechResult); @@ -113,48 +64,78 @@ public class TwilioVoiceController : TwilioController VoiceResponse response; if (!string.IsNullOrWhiteSpace(messageContent)) { + var callerMessage = new CallerMessage() { - SessionId = sessionId, + ConversationId = conversationId, SeqNumber = seqNum, Content = messageContent, From = request.From }; + if (!string.IsNullOrEmpty(states)) + { + var kvp = states.Split(':'); + if (kvp.Length == 2) + { + callerMessage.States.Add(kvp[0], kvp[1]); + } + } await messageQueue.EnqueueAsync(callerMessage); - response = twilio.ReturnInstructions("twilio/holdon.mp3", url, true); + response = twilio.ReturnInstructions(null, url, true, 1); } else { - response = twilio.HangUp("twilio/holdon.mp3"); + var speechPath = seqNum > 0 ? $"twilio/voice/speeches/{conversationId}/{seqNum - 1}.mp3" : "twilio/welcome.mp3"; + response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/receive/{seqNum}?states={states}", true); } return TwiML(response); } - [HttpPost("{sessionId}/reply/{seqNum}")] - public async Task ReplyCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request) + [HttpPost("{conversationId}/reply/{seqNum}")] + public async Task ReplyCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, VoiceRequest request) { var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); if (request.SpeechResult != null) { - await sessionManager.StageCallerMessageAsync(sessionId, nextSeqNum, request.SpeechResult); + await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult); } - var reply = await sessionManager.GetAssistantReplyAsync(sessionId, seqNum); + var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); VoiceResponse response; - if (string.IsNullOrEmpty(reply)) + if (reply == null) { - response = twilio.ReturnInstructions(null, $"twiliovoice/{sessionId}/reply/{seqNum}", true); + var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum); + if (indication != null) + { + var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); + var fileService = _services.GetRequiredService(); + var data = await textToSpeechService.GenerateSpeechFromTextAsync(indication); + var fileName = $"indication_{seqNum}.mp3"; + await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{fileName}", $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2); + } + else + { + response = twilio.ReturnInstructions(null, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 1); + } } else { - var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); var fileService = _services.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply); - var fileName = $"{seqNum}.mp3"; - await fileService.SaveSpeechFileAsync(sessionId, fileName, data); - response = twilio.ReturnInstructions($"twiliovoice/speeches/{sessionId}/{fileName}", $"twiliovoice/{sessionId}/send/{nextSeqNum}", true); + var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply.Content); + var fileName = $"reply_{seqNum}.mp3"; + await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + if (reply.ConversationEnd) + { + response = twilio.HangUp($"twilio/voice/speeches/{conversationId}/{fileName}"); + } + else + { + response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{fileName}", $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); + } + } return TwiML(response); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs new file mode 100644 index 00000000..f9a83613 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Plugin.Twilio.Models +{ + public class AssistantMessage + { + public bool ConversationEnd { get; set; } + public string Content { get; set; } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs index e0f4463a..a6339c7b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs @@ -2,14 +2,15 @@ namespace BotSharp.Plugin.Twilio.Models { public class CallerMessage { - public string SessionId { get; set; } + public string ConversationId { get; set; } public int SeqNumber { get; set; } public string Content { get; set; } public string From { get; set; } + public Dictionary States { get; set; } = new(); public override string ToString() { - return $"{SessionId}-{SeqNumber}"; + return $"{ConversationId}-{SeqNumber}"; } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs index 3ad027f7..b1acd298 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs @@ -1,12 +1,15 @@ +using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Services { public interface ITwilioSessionManager { - Task SetAssistantReplyAsync(string sessionId, int seqNum, string message); - Task GetAssistantReplyAsync(string sessionId, int seqNum); - Task StageCallerMessageAsync(string sessionId, int seqNum, string message); - Task> RetrieveStagedCallerMessagesAsync(string sessionId, int seqNum); + Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message); + Task GetAssistantReplyAsync(string conversationId, int seqNum); + Task StageCallerMessageAsync(string conversationId, int seqNum, string message); + Task> RetrieveStagedCallerMessagesAsync(string conversationId, int seqNum); + Task SetReplyIndicationAsync(string conversationId, int seqNum, string indication); + Task GetReplyIndicationAsync(string conversationId, int seqNum); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index dab06c4b..a84ce5d9 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -55,35 +55,53 @@ namespace BotSharp.Plugin.Twilio.Services { using var scope = _serviceProvider.CreateScope(); var sp = scope.ServiceProvider; - string reply = null; + AssistantMessage reply = null; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService(); var routing = sp.GetRequiredService(); var config = sp.GetRequiredService(); - routing.Context.SetMessageId(message.SessionId, inputMsg.MessageId); - conv.SetConversationId(message.SessionId, new List + routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); + var states = new List { new MessageState("channel", ConversationChannel.Phone), new MessageState("calling_phone", message.From) - }); + }; + foreach (var kvp in message.States) + { + states.Add(new MessageState(kvp.Key, kvp.Value)); + } + conv.SetConversationId(message.ConversationId, states); + var sessionManager = sp.GetRequiredService(); var result = await conv.SendMessage(config.AgentId, inputMsg, replyMessage: null, async msg => { - reply = msg.Content; + reply = new AssistantMessage() + { + ConversationEnd = msg.Instruction.ConversationEnd, + Content = msg.Content + }; + }, + async msg => + { + if (!string.IsNullOrEmpty(msg.Indication)) + { + await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); + } }, - async functionExecuting => - { }, async functionExecuted => { } ); - if (string.IsNullOrWhiteSpace(reply)) + if (reply == null || string.IsNullOrWhiteSpace(reply.Content)) { - reply = "Sorry, something was wrong."; - } - var sessionManager = sp.GetRequiredService(); - await sessionManager.SetAssistantReplyAsync(message.SessionId, message.SeqNumber, reply); + reply = new AssistantMessage() + { + ConversationEnd = true, + Content = "Sorry, something was wrong." + }; + } + await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 09405817..a4e9d4ab 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -63,7 +63,7 @@ public class TwilioService return response; } - public VoiceResponse ReturnInstructions(string speechPath, string callbackPath, bool actionOnEmptyResult) + public VoiceResponse ReturnInstructions(string speechPath, string callbackPath, bool actionOnEmptyResult, int timeout = 3) { var response = new VoiceResponse(); var gather = new Gather() @@ -73,7 +73,9 @@ public class TwilioService Gather.InputEnum.Speech }, Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), - SpeechTimeout = "3", + SpeechModel = Gather.SpeechModelEnum.PhoneCall, + SpeechTimeout = timeout > 0 ? timeout.ToString() : "3", + Timeout = timeout > 0 ? timeout : 3, ActionOnEmptyResult = actionOnEmptyResult }; if (!string.IsNullOrEmpty(speechPath)) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs index daefbdcc..eae0b238 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -1,3 +1,4 @@ +using BotSharp.Plugin.Twilio.Models; using StackExchange.Redis; using Task = System.Threading.Tasks.Task; @@ -12,35 +13,51 @@ namespace BotSharp.Plugin.Twilio.Services _redis = redis; } - public async Task GetAssistantReplyAsync(string sessionId, int seqNum) + public async Task GetAssistantReplyAsync(string conversationId, int seqNum) { var db = _redis.GetDatabase(); - var key = $"{sessionId}:Assisist:{seqNum}"; - return await db.StringGetAsync(key); + var key = $"{conversationId}:Assisist:{seqNum}"; + var jsonStr = await db.StringGetAsync(key); + return jsonStr.IsNull ? null : JsonSerializer.Deserialize(jsonStr); } - public async Task> RetrieveStagedCallerMessagesAsync(string sessionId, int seqNum) + public async Task> RetrieveStagedCallerMessagesAsync(string conversationId, int seqNum) { var db = _redis.GetDatabase(); - var key = $"{sessionId}:Caller:{seqNum}"; + var key = $"{conversationId}:Caller:{seqNum}"; return (await db.ListRangeAsync(key)) .Select(x => (string)x) .ToList(); } - public async Task SetAssistantReplyAsync(string sessionId, int seqNum, string message) + public async Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message) { + var jsonStr = JsonSerializer.Serialize(message); var db = _redis.GetDatabase(); - var key = $"{sessionId}:Assisist:{seqNum}"; - await db.StringSetAsync(key, message, TimeSpan.FromMinutes(5)); + var key = $"{conversationId}:Assisist:{seqNum}"; + await db.StringSetAsync(key, jsonStr, TimeSpan.FromMinutes(5)); } - public async Task StageCallerMessageAsync(string sessionId, int seqNum, string message) + public async Task StageCallerMessageAsync(string conversationId, int seqNum, string message) { var db = _redis.GetDatabase(); - var key = $"{sessionId}:Caller:{seqNum}"; + var key = $"{conversationId}:Caller:{seqNum}"; await db.ListRightPushAsync(key, message); await db.KeyExpireAsync(key, DateTime.UtcNow.AddMinutes(10)); } + + public async Task SetReplyIndicationAsync(string conversationId, int seqNum, string indication) + { + var db = _redis.GetDatabase(); + var key = $"{conversationId}:Indication:{seqNum}"; + await db.StringSetAsync(key, indication, TimeSpan.FromMinutes(5)); + } + + public async Task GetReplyIndicationAsync(string conversationId, int seqNum) + { + var db = _redis.GetDatabase(); + var key = $"{conversationId}:Indication:{seqNum}"; + return await db.StringGetAsync(key); + } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index 38d278a6..1247d0d4 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -17,7 +17,6 @@ public class TwilioPlugin : IBotSharpPlugin var settingService = provider.GetRequiredService(); return settingService.Bind("Twilio"); }); - services.AddScoped(); var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]); var sessionManager = new TwilioSessionManager(conn); From 548cd558c0366a986697f5508499068e0a42832c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 14 Aug 2024 16:56:32 -0500 Subject: [PATCH 58/63] add default collection --- .../Knowledges/Settings/KnowledgeBaseSettings.cs | 3 +++ .../Repository/FileRepository/FileRepository.Agent.cs | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 3 ++- .../Functions/MemorizeKnowledgeFn.cs | 5 +++-- src/WebStarter/appsettings.json | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index ac3f0500..f2c62fd0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -1,8 +1,11 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { public string VectorDb { get; set; } + public string DefaultCollection { get; set; } = KnowledgeCollectionName.BotSharp; public KnowledgeModelSetting TextEmbedding { get; set; } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index 950a2b2d..bb137279 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -298,7 +298,7 @@ namespace BotSharp.Core.Repository var json = JsonSerializer.Serialize(agent, _options); File.WriteAllText(agentFile, json); - UpdateAgentInstructions(inputAgent.Id, inputAgent.Instruction, agent.ChannelInstructions); + UpdateAgentInstructions(inputAgent.Id, inputAgent.Instruction, inputAgent.ChannelInstructions); UpdateAgentResponses(inputAgent.Id, inputAgent.Responses); UpdateAgentTemplates(inputAgent.Id, inputAgent.Templates); UpdateAgentFunctions(inputAgent.Id, inputAgent.Functions); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 62397c53..52e45e7e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -22,7 +22,8 @@ public class KnowledgeRetrievalFn : IFunctionCallback var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); - var knowledges = await vectorDb.Search(KnowledgeCollectionName.BotSharp, vector, new List { KnowledgePayloadName.Answer }); + var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp; + var knowledges = await vectorDb.Search(collectionName, vector, new List { KnowledgePayloadName.Answer }); if (!knowledges.IsNullOrEmpty()) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index b4b0111c..237f7385 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -26,10 +26,11 @@ public class MemorizeKnowledgeFn : IFunctionCallback }); var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); - await vectorDb.CreateCollection(KnowledgeCollectionName.BotSharp, vector[0].Length); + var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp; + await vectorDb.CreateCollection(collectionName, vector[0].Length); var id = Guid.NewGuid().ToString(); - var result = await vectorDb.Upsert(KnowledgeCollectionName.BotSharp, id, vector[0], + var result = await vectorDb.Upsert(collectionName, id, vector[0], args.Question, new Dictionary { diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 155dae1b..d8e29023 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -259,6 +259,7 @@ "KnowledgeBase": { "VectorDb": "Qdrant", + "DefaultCollection": "BotSharp", "TextEmbedding": { "Provider": "openai", "Model": "text-embedding-3-small" From d99d89783b920163d6593a79842865526bf0e41b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 14 Aug 2024 22:46:01 -0500 Subject: [PATCH 59/63] unite knowledge search model --- .../Knowledges/IKnowledgeService.cs | 4 +- .../Models/KnowledgeCollectionData.cs | 6 +-- ...alOptions.cs => KnowledgeSearchOptions.cs} | 2 +- .../Models/KnowledgeSearchResult.cs | 22 +++++++---- .../VectorStorage/IVectorDb.cs | 4 +- .../Controllers/KnowledgeBaseController.cs | 24 ++++++------ .../KnowledgeCollectionDataViewModel.cs | 33 ---------------- .../Knowledges/KnowledgeRetrivalViewModel.cs | 27 ------------- .../KnowledgeSearchResultViewModel.cs | 33 ++++++++++++++++ ...edgeModel.cs => SearchKnowledgeRequest.cs} | 2 +- .../MemVecDb/MemoryVectorDb.cs | 12 +++--- .../Services/KnowledgeService.Get.cs | 26 ++++++------- .../Providers/FaissDb.cs | 4 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 38 +++++++++++-------- .../SemanticKernelMemoryStoreProvider.cs | 9 ++--- 15 files changed, 117 insertions(+), 129 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/{KnowledgeRetrievalOptions.cs => KnowledgeSearchOptions.cs} (91%) delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchResultViewModel.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{SearchKnowledgeModel.cs => SearchKnowledgeRequest.cs} (93%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index d2882bdc..02afb420 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -3,8 +3,8 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { Task> GetKnowledgeCollections(); - Task> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options); + Task> SearchKnowledge(string collectionName, KnowledgeSearchOptions options); Task FeedKnowledge(string collectionName, KnowledgeCreationModel model); - Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); + Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); Task DeleteKnowledgeCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs index d013529f..6a8faff4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeCollectionData { public string Id { get; set; } - public string Question { get; set; } - public string Answer { get; set; } + public Dictionary Data { get; set; } = new(); + public double? Score { get; set; } public float[]? Vector { get; set; } -} +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs similarity index 91% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalOptions.cs rename to src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs index 8a608ddc..4ac1b77b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs @@ -2,7 +2,7 @@ using BotSharp.Abstraction.Knowledges.Enums; namespace BotSharp.Abstraction.Knowledges.Models; -public class KnowledgeRetrievalOptions +public class KnowledgeSearchOptions { public string Text { get; set; } = string.Empty; public IEnumerable? Fields { get; set; } = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs index 2d5d9c1f..661578c2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs @@ -1,12 +1,20 @@ namespace BotSharp.Abstraction.Knowledges.Models; -public class KnowledgeSearchResult +public class KnowledgeSearchResult : KnowledgeCollectionData { - public Dictionary Data { get; set; } = new(); - public double Score { get; set; } - public float[]? Vector { get; set; } -} + public KnowledgeSearchResult() + { + + } -public class KnowledgeRetrievalResult : KnowledgeSearchResult -{ + public static KnowledgeSearchResult CopyFrom(KnowledgeCollectionData data) + { + return new KnowledgeSearchResult + { + Id = data.Id, + Data = data.Data, + Score = data.Score, + Vector = data.Vector + }; + } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index a079aaef..ff4ec26a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { string Name { get; } - + Task> GetCollections(); Task> GetCollectionData(string collectionName, KnowledgeFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false); + Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 55712aaa..b01c4950 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -23,29 +23,29 @@ public class KnowledgeBaseController : ControllerBase } [HttpPost("/knowledge/{collection}/search")] - public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeModel model) + public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeRequest request) { - var options = new KnowledgeRetrievalOptions + var options = new KnowledgeSearchOptions { - Text = model.Text, - Fields = model.Fields, - Limit = model.Limit ?? 5, - Confidence = model.Confidence ?? 0.5f, - WithVector = model.WithVector + Text = request.Text, + Fields = request.Fields, + Limit = request.Limit ?? 5, + Confidence = request.Confidence ?? 0.5f, + WithVector = request.WithVector }; var results = await _knowledgeService.SearchKnowledge(collection, options); - return results.Select(x => KnowledgeRetrivalViewModel.From(x)).ToList(); + return results.Select(x => KnowledgeSearchResultViewModel.From(x)).ToList(); } [HttpPost("/knowledge/{collection}/data")] - public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) + public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) { var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); - var items = data.Items?.Select(x => KnowledgeCollectionDataViewModel.From(x))? - .ToList() ?? new List(); + var items = data.Items?.Select(x => KnowledgeSearchResultViewModel.From(x))? + .ToList() ?? new List(); - return new StringIdPagedItems + return new StringIdPagedItems { Count = data.Count, NextId = data.NextId, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs deleted file mode 100644 index f77b8777..00000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeCollectionDataViewModel.cs +++ /dev/null @@ -1,33 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Models; -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class KnowledgeCollectionDataViewModel -{ - [JsonPropertyName("id")] - public string Id { get; set; } - - [JsonPropertyName("question")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string Question { get; set; } - - [JsonPropertyName("answer")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string Answer { get; set; } - - [JsonPropertyName("vector")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public float[]? Vector { get; set; } - - public static KnowledgeCollectionDataViewModel From(KnowledgeCollectionData data) - { - return new KnowledgeCollectionDataViewModel - { - Id = data.Id, - Question = data.Question, - Answer = data.Answer, - Vector = data.Vector - }; - } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs deleted file mode 100644 index 2e2b9e08..00000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeRetrivalViewModel.cs +++ /dev/null @@ -1,27 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Models; -using System.Text.Json.Serialization; - -namespace BotSharp.OpenAPI.ViewModels.Knowledges; - -public class KnowledgeRetrivalViewModel -{ - [JsonPropertyName("data")] - public IDictionary Data { get; set; } - - [JsonPropertyName("score")] - public double Score { get; set; } - - [JsonPropertyName("vector")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public float[]? Vector { get; set; } - - public static KnowledgeRetrivalViewModel From(KnowledgeRetrievalResult model) - { - return new KnowledgeRetrivalViewModel - { - Data = model.Data, - Score = model.Score, - Vector = model.Vector - }; - } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchResultViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchResultViewModel.cs new file mode 100644 index 00000000..f322bc7d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchResultViewModel.cs @@ -0,0 +1,33 @@ +using BotSharp.Abstraction.Knowledges.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeSearchResultViewModel +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("data")] + public IDictionary Data { get; set; } + + [JsonPropertyName("score")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public double? Score { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + + public static KnowledgeSearchResultViewModel From(KnowledgeSearchResult result) + { + return new KnowledgeSearchResultViewModel + { + Id = result.Id, + Data = result.Data, + Score = result.Score, + Vector = result.Vector + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs similarity index 93% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs index 9a91e004..cacb133a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class SearchKnowledgeModel +public class SearchKnowledgeRequest { [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index ba5e6383..f8f31ca2 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -27,12 +27,12 @@ public class MemoryVectorDb : IVectorDb throw new NotImplementedException(); } - public async Task> Search(string collectionName, float[] vector, - IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + public async Task> Search(string collectionName, float[] vector, + IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { if (!_vectors.ContainsKey(collectionName)) { - return new List(); + return new List(); } var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); @@ -41,7 +41,7 @@ public class MemoryVectorDb : IVectorDb var results = np.argsort(similarities).ToArray() .Reverse() .Take(limit) - .Select(i => new KnowledgeSearchResult + .Select(i => new KnowledgeCollectionData { Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, Score = similarities[i], @@ -64,8 +64,8 @@ public class MemoryVectorDb : IVectorDb return true; } - public Task DeleteCollectionData(string collectionName, string id) + public async Task DeleteCollectionData(string collectionName, string id) { - throw new NotImplementedException(); + return await Task.FromResult(false); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs index 75824707..243eca02 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -16,21 +16,27 @@ public partial class KnowledgeService } } - public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) + public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) { try { var db = GetVectorDb(); - return await db.GetCollectionData(collectionName, filter); + var pagedResult = await db.GetCollectionData(collectionName, filter); + return new StringIdPagedItems + { + Count = pagedResult.Count, + Items = pagedResult.Items.Select(x => KnowledgeSearchResult.CopyFrom(x)), + NextId = pagedResult.NextId, + }; } catch (Exception ex) { _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new StringIdPagedItems(); + return new StringIdPagedItems(); } } - public async Task> SearchKnowledge(string collectionName, KnowledgeRetrievalOptions options) + public async Task> SearchKnowledge(string collectionName, KnowledgeSearchOptions options) { try { @@ -39,21 +45,15 @@ public partial class KnowledgeService // Vector search var db = GetVectorDb(); - var fields = !options.Fields.IsNullOrEmpty() ? options.Fields : new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; - var found = await db.Search(collectionName, vector, fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); + var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); - var results = found.Select(x => new KnowledgeRetrievalResult - { - Data = x.Data, - Score = x.Score, - Vector = x.Vector - }).ToList(); + var results = found.Select(x => KnowledgeSearchResult.CopyFrom(x)).ToList(); return results; } catch (Exception ex) { _logger.LogWarning($"Error when searching knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new List(); + return new List(); } } } diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 785acd4d..225c0705 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -26,8 +26,8 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, - IEnumerable fields, int limit = 10, float confidence = 0.5f, bool withVector = false) + public Task> Search(string collectionName, float[] vector, + IEnumerable? fields, int limit = 10, float confidence = 0.5f, bool withVector = false) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 02520517..0e56f680 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -57,8 +57,7 @@ public class QdrantDb : IVectorDb var points = response?.Result?.Select(x => new KnowledgeCollectionData { Id = x.Id?.Uuid ?? string.Empty, - Question = x.Payload.ContainsKey(KnowledgePayloadName.Text) ? x.Payload[KnowledgePayloadName.Text].StringValue : string.Empty, - Answer = x.Payload.ContainsKey(KnowledgePayloadName.Answer) ? x.Payload[KnowledgePayloadName.Answer].StringValue : string.Empty, + Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null })?.ToList() ?? new List(); @@ -125,10 +124,10 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, - IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + public async Task> Search(string collectionName, float[] vector, + IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { - var results = new List(); + var results = new List(); var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); @@ -138,24 +137,33 @@ public class QdrantDb : IVectorDb } var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); - + + var pickFields = fields != null; foreach (var point in points) { var data = new Dictionary(); - foreach (var field in fields) + if (pickFields) { - if (point.Payload.ContainsKey(field)) + foreach (var field in fields) { - data[field] = point.Payload[field].StringValue; - } - else - { - data[field] = ""; + if (point.Payload.ContainsKey(field)) + { + data[field] = point.Payload[field].StringValue; + } + else + { + data[field] = ""; + } } } - - results.Add(new KnowledgeSearchResult + else { + data = point.Payload.ToDictionary(k => k.Key, v => v.Value.StringValue); + } + + results.Add(new KnowledgeCollectionData + { + Id = point.Id.Uuid, Data = data, Score = point.Score, Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index a5db6de8..487b8ffd 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -2,7 +2,6 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; using Microsoft.SemanticKernel.Memory; -using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -44,15 +43,15 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, - IEnumerable fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + public async Task> Search(string collectionName, float[] vector, + IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); - var resultTexts = new List(); + var resultTexts = new List(); await foreach (var (record, score) in results) { - resultTexts.Add(new KnowledgeSearchResult + resultTexts.Add(new KnowledgeCollectionData { Data = new Dictionary { { "text", record.Metadata.Text } }, Score = score, From acff2f24a91557e494b5a33dc6f6ab61115f1dcf Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 15 Aug 2024 18:19:10 -0500 Subject: [PATCH 60/63] add graph search --- BotSharp.sln | 16 +++- .../BotSharp.Abstraction.csproj | 2 +- .../BotSharp.Abstraction/Graph/IGraphDb.cs | 10 ++ .../Graph/Models/GraphSearchData.cs | 6 ++ .../Graph/Models/GraphSearchOptions.cs | 6 ++ .../Graph/Models/GraphSearchResult.cs | 6 ++ .../Knowledges/IKnowledgeService.cs | 15 ++- .../Knowledges/Models/KnowledgeFilter.cs | 7 -- .../Models/KnowledgeSearchResult.cs | 24 ++--- .../Settings/KnowledgeBaseSettings.cs | 3 +- .../VectorStorage/IVectorDb.cs | 6 +- .../Models/VectorCollectionData.cs} | 4 +- .../VectorStorage/Models/VectorFilter.cs | 7 ++ .../Models/VectorSearchOptions.cs} | 5 +- .../Models/VectorSearchResult.cs | 20 ++++ .../Controllers/KnowledgeBaseController.cs | 82 ++++++++++++---- .../Knowledges/GraphKnowledgeViewModel.cs | 9 ++ .../Knowledges/KnowledgeSearchViewModel.cs | 12 +++ .../Knowledges/SearchGraphKnowledgeRequest.cs | 12 +++ .../Knowledges/SearchKnowledgeRequest.cs | 25 ++++- .../SearchVectorKnowledgeRequest.cs | 21 ++++ ...ewModel.cs => VectorKnowledgeViewModel.cs} | 8 +- .../BotSharp.Plugin.Graph.csproj | 17 ++++ src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 95 +++++++++++++++++++ .../BotSharp.Plugin.Graph/GraphDbSettings.cs | 6 ++ .../BotSharp.Plugin.Graph/GraphPlugin.cs | 24 +++++ .../Models/GraphQueryRequest.cs | 13 +++ src/Plugins/BotSharp.Plugin.Graph/Using.cs | 7 ++ .../MemVecDb/MemoryVectorDb.cs | 9 +- .../Services/KnowledgeService.Create.cs | 2 +- .../Services/KnowledgeService.Delete.cs | 2 +- .../Services/KnowledgeService.Get.cs | 65 +++++++++++-- .../Services/KnowledgeService.cs | 6 ++ .../BotSharp.Plugin.KnowledgeBase/Using.cs | 2 +- .../Providers/FaissDb.cs | 6 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 17 ++-- .../SemanticKernelMemoryStoreProvider.cs | 10 +- src/WebStarter/appsettings.json | 6 ++ 38 files changed, 495 insertions(+), 98 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchData.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchResult.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs rename src/Infrastructure/BotSharp.Abstraction/{Knowledges/Models/KnowledgeCollectionData.cs => VectorStorage/Models/VectorCollectionData.cs} (68%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs rename src/Infrastructure/BotSharp.Abstraction/{Knowledges/Models/KnowledgeSearchOptions.cs => VectorStorage/Models/VectorSearchOptions.cs} (69%) create mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs rename src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/{KnowledgeSearchResultViewModel.cs => VectorKnowledgeViewModel.cs} (75%) create mode 100644 src/Plugins/BotSharp.Plugin.Graph/BotSharp.Plugin.Graph.csproj create mode 100644 src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs create mode 100644 src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs create mode 100644 src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.Graph/Models/GraphQueryRequest.cs create mode 100644 src/Plugins/BotSharp.Plugin.Graph/Using.cs diff --git a/BotSharp.sln b/BotSharp.sln index 8c1030f9..8dda48e8 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -107,7 +107,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.TencentCos" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Interpreters", "Interpreters", "{C4C59872-3C8A-450D-83D5-2BE402D610D5}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.PythonInterpreter", "src\Plugins\BotSharp.Plugin.PythonInterpreter\BotSharp.Plugin.PythonInterpreter.csproj", "{05E6E405-5021-406E-8A5E-0A7CEC881F6D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInterpreter", "src\Plugins\BotSharp.Plugin.PythonInterpreter\BotSharp.Plugin.PythonInterpreter.csproj", "{05E6E405-5021-406E-8A5E-0A7CEC881F6D}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -437,6 +441,14 @@ Global {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|Any CPU.Build.0 = Release|Any CPU {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|x64.ActiveCfg = Release|Any CPU {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|x64.Build.0 = Release|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Debug|x64.ActiveCfg = Debug|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Debug|x64.Build.0 = Debug|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -489,6 +501,8 @@ Global {BF029B0A-768B-43A1-8D91-E70B95505716} = {38B37C0D-1930-4D47-BCBF-E358EC1096B1} {C4C59872-3C8A-450D-83D5-2BE402D610D5} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} {05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5} + {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index ea456184..2e3aa24c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs new file mode 100644 index 00000000..64c5388b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Graph.Models; + +namespace BotSharp.Abstraction.Graph; + +public interface IGraphDb +{ + public string Name { get; } + + Task Search(string query, GraphSearchOptions options); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchData.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchData.cs new file mode 100644 index 00000000..b6a9b714 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchData.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Graph.Models; + +public class GraphSearchData +{ + public string Result { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchOptions.cs new file mode 100644 index 00000000..294aeea7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchOptions.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Graph.Models; + +public class GraphSearchOptions +{ + public string Method { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchResult.cs new file mode 100644 index 00000000..e9985447 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchResult.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Graph.Models; + +public class GraphSearchResult +{ + public string Result { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 02afb420..5bf7aced 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -1,10 +1,15 @@ +using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.VectorStorage.Models; + namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { - Task> GetKnowledgeCollections(); - Task> SearchKnowledge(string collectionName, KnowledgeSearchOptions options); - Task FeedKnowledge(string collectionName, KnowledgeCreationModel model); - Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter); - Task DeleteKnowledgeCollectionData(string collectionName, string id); + Task> GetVectorCollections(); + Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); + Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); + Task> GetVectorCollectionData(string collectionName, VectorFilter filter); + Task DeleteVectorCollectionData(string collectionName, string id); + Task SearchGraphKnowledge(string query, GraphSearchOptions options); + Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs deleted file mode 100644 index d2d9c490..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFilter.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeFilter : StringIdPagination -{ - [JsonPropertyName("with_vector")] - public bool WithVector { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs index 661578c2..771b070b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs @@ -1,20 +1,10 @@ +using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.VectorStorage.Models; + namespace BotSharp.Abstraction.Knowledges.Models; -public class KnowledgeSearchResult : KnowledgeCollectionData +public class KnowledgeSearchResult { - public KnowledgeSearchResult() - { - - } - - public static KnowledgeSearchResult CopyFrom(KnowledgeCollectionData data) - { - return new KnowledgeSearchResult - { - Id = data.Id, - Data = data.Data, - Score = data.Score, - Vector = data.Vector - }; - } -} \ No newline at end of file + public IEnumerable VectorResult { get; set; } + public GraphSearchResult GraphResult { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index f2c62fd0..65bf7637 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -4,8 +4,9 @@ namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { - public string VectorDb { get; set; } public string DefaultCollection { get; set; } = KnowledgeCollectionName.BotSharp; + public string VectorDb { get; set; } + public string GraphDb { get; set; } public KnowledgeModelSetting TextEmbedding { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index ff4ec26a..8aeb5ee9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.VectorStorage.Models; + namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb @@ -5,9 +7,9 @@ public interface IVectorDb string Name { get; } Task> GetCollections(); - Task> GetCollectionData(string collectionName, KnowledgeFilter filter); + Task> GetCollectionData(string collectionName, VectorFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); + Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs similarity index 68% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs rename to src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs index 6a8faff4..e623532a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeCollectionData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs @@ -1,6 +1,6 @@ -namespace BotSharp.Abstraction.Knowledges.Models; +namespace BotSharp.Abstraction.VectorStorage.Models; -public class KnowledgeCollectionData +public class VectorCollectionData { public string Id { get; set; } public Dictionary Data { get; set; } = new(); diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs new file mode 100644 index 00000000..91abcc25 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorFilter : StringIdPagination +{ + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs similarity index 69% rename from src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs rename to src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs index 4ac1b77b..64943dd1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs @@ -1,10 +1,9 @@ using BotSharp.Abstraction.Knowledges.Enums; -namespace BotSharp.Abstraction.Knowledges.Models; +namespace BotSharp.Abstraction.VectorStorage.Models; -public class KnowledgeSearchOptions +public class VectorSearchOptions { - public string Text { get; set; } = string.Empty; public IEnumerable? Fields { get; set; } = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; public int? Limit { get; set; } = 5; public float? Confidence { get; set; } = 0.5f; diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs new file mode 100644 index 00000000..ce39edbf --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs @@ -0,0 +1,20 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorSearchResult : VectorCollectionData +{ + public VectorSearchResult() + { + + } + + public static VectorSearchResult CopyFrom(VectorCollectionData data) + { + return new VectorSearchResult + { + Id = data.Id, + Data = data.Data, + Score = data.Score, + Vector = data.Vector + }; + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index b01c4950..90773dfd 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Graph.Models; using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; using BotSharp.OpenAPI.ViewModels.Knowledges; namespace BotSharp.OpenAPI.Controllers; @@ -16,36 +18,35 @@ public class KnowledgeBaseController : ControllerBase _services = services; } - [HttpGet("knowledge/collections")] - public async Task> GetKnowledgeCollections() + [HttpGet("knowledge/vector/collections")] + public async Task> GetVectorCollections() { - return await _knowledgeService.GetKnowledgeCollections(); + return await _knowledgeService.GetVectorCollections(); } - [HttpPost("/knowledge/{collection}/search")] - public async Task> SearchKnowledge([FromRoute] string collection, [FromBody] SearchKnowledgeRequest request) + [HttpPost("/knowledge/vector/{collection}/search")] + public async Task> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) { - var options = new KnowledgeSearchOptions + var options = new VectorSearchOptions { - Text = request.Text, Fields = request.Fields, Limit = request.Limit ?? 5, Confidence = request.Confidence ?? 0.5f, WithVector = request.WithVector }; - var results = await _knowledgeService.SearchKnowledge(collection, options); - return results.Select(x => KnowledgeSearchResultViewModel.From(x)).ToList(); + var results = await _knowledgeService.SearchVectorKnowledge(request.Text, collection, options); + return results.Select(x => VectorKnowledgeViewModel.From(x)).ToList(); } - [HttpPost("/knowledge/{collection}/data")] - public async Task> GetKnowledgeCollectionData([FromRoute] string collection, [FromBody] KnowledgeFilter filter) + [HttpPost("/knowledge/vector/{collection}/data")] + public async Task> GetVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter) { - var data = await _knowledgeService.GetKnowledgeCollectionData(collection, filter); - var items = data.Items?.Select(x => KnowledgeSearchResultViewModel.From(x))? - .ToList() ?? new List(); + var data = await _knowledgeService.GetVectorCollectionData(collection, filter); + var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))? + .ToList() ?? new List(); - return new StringIdPagedItems + return new StringIdPagedItems { Count = data.Count, NextId = data.NextId, @@ -53,14 +54,14 @@ public class KnowledgeBaseController : ControllerBase }; } - [HttpDelete("/knowledge/{collection}/data/{id}")] - public async Task DeleteKnowledgeCollectionData([FromRoute] string collection, [FromRoute] string id) + [HttpDelete("/knowledge/vector/{collection}/data/{id}")] + public async Task DeleteVectorCollectionData([FromRoute] string collection, [FromRoute] string id) { - return await _knowledgeService.DeleteKnowledgeCollectionData(collection, id); + return await _knowledgeService.DeleteVectorCollectionData(collection, id); } - [HttpPost("/knowledge/{collection}/upload")] - public async Task UploadKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) + [HttpPost("/knowledge/vector/{collection}/upload")] + public async Task UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) { var setttings = _services.GetRequiredService(); var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); @@ -73,7 +74,7 @@ public class KnowledgeBaseController : ControllerBase } var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - await _knowledgeService.FeedKnowledge(collection, new KnowledgeCreationModel + await _knowledgeService.FeedVectorKnowledge(collection, new KnowledgeCreationModel { Content = content }); @@ -81,4 +82,43 @@ public class KnowledgeBaseController : ControllerBase System.IO.File.Delete(filePath); return Ok(new { count = 1, file.Length }); } + + [HttpPost("/knowledge/graph/search")] + public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) + { + var options = new GraphSearchOptions + { + Method = request.Method + }; + + var result = await _knowledgeService.SearchGraphKnowledge(request.Query, options); + return new GraphKnowledgeViewModel + { + Result = result.Result + }; + } + + [HttpPost("/knowledge/search")] + public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) + { + var vectorOptions = new VectorSearchOptions + { + Fields = request.VectorParams.Fields, + Limit = request.VectorParams.Limit ?? 5, + Confidence = request.VectorParams.Confidence ?? 0.5f, + WithVector = request.VectorParams.WithVector + }; + + var graphOptions = new GraphSearchOptions + { + Method = request.GraphParams.Method + }; + + var result = await _knowledgeService.SearchKnowledge(request.Text, request.VectorParams.Collection, vectorOptions, graphOptions); + return new KnowledgeSearchViewModel + { + VectorResult = result?.VectorResult?.Select(x => VectorKnowledgeViewModel.From(x)), + GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null + }; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs new file mode 100644 index 00000000..360fce1c --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class GraphKnowledgeViewModel +{ + [JsonPropertyName("result")] + public string Result { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs new file mode 100644 index 00000000..f862f184 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeSearchViewModel +{ + [JsonPropertyName("vector_result")] + public IEnumerable? VectorResult { get; set; } + + [JsonPropertyName("graph_result")] + public GraphKnowledgeViewModel? GraphResult { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs new file mode 100644 index 00000000..119e729e --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class SearchGraphKnowledgeRequest +{ + [JsonPropertyName("query")] + public string Query { get; set; } = string.Empty; + + [JsonPropertyName("method")] + public string Method { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs index cacb133a..20f80fec 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Enums; using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; @@ -8,6 +7,22 @@ public class SearchKnowledgeRequest [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; + #region Vector + [JsonPropertyName("vector_params")] + public VectorParam VectorParams { get; set; } + #endregion + + #region Graph + [JsonPropertyName("graph_params")] + public GraphParam GraphParams { get; set; } + #endregion +} + +public class VectorParam +{ + [JsonPropertyName("collection")] + public string Collection { get; set; } + [JsonPropertyName("fields")] public IEnumerable? Fields { get; set; } @@ -19,4 +34,10 @@ public class SearchKnowledgeRequest [JsonPropertyName("with_vector")] public bool WithVector { get; set; } -} +} + +public class GraphParam +{ + [JsonPropertyName("method")] + public string Method { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs new file mode 100644 index 00000000..3f11a486 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs @@ -0,0 +1,21 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class SearchVectorKnowledgeRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("fields")] + public IEnumerable? Fields { get; set; } + + [JsonPropertyName("limit")] + public int? Limit { get; set; } = 5; + + [JsonPropertyName("confidence")] + public float? Confidence { get; set; } = 0.5f; + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchResultViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs similarity index 75% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchResultViewModel.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs index f322bc7d..dcdf2e57 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchResultViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs @@ -1,9 +1,9 @@ -using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class KnowledgeSearchResultViewModel +public class VectorKnowledgeViewModel { [JsonPropertyName("id")] public string Id { get; set; } @@ -20,9 +20,9 @@ public class KnowledgeSearchResultViewModel public float[]? Vector { get; set; } - public static KnowledgeSearchResultViewModel From(KnowledgeSearchResult result) + public static VectorKnowledgeViewModel From(VectorSearchResult result) { - return new KnowledgeSearchResultViewModel + return new VectorKnowledgeViewModel { Id = result.Id, Data = result.Data, diff --git a/src/Plugins/BotSharp.Plugin.Graph/BotSharp.Plugin.Graph.csproj b/src/Plugins/BotSharp.Plugin.Graph/BotSharp.Plugin.Graph.csproj new file mode 100644 index 00000000..c03f1163 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/BotSharp.Plugin.Graph.csproj @@ -0,0 +1,17 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs new file mode 100644 index 00000000..9e0ceaee --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -0,0 +1,95 @@ +using BotSharp.Plugin.Graph.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using System.Net.Http; +using System.Net.Mime; +using System.Text; +using System.Text.Json; + +namespace BotSharp.Plugin.Graph; + +public class GraphDb : IGraphDb +{ + private readonly IServiceProvider _services; + private readonly IHttpContextAccessor _context; + private readonly GraphDbSettings _settings; + private readonly ILogger _logger; + + private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + WriteIndented = true, + AllowTrailingCommas = true, + }; + + public GraphDb( + IServiceProvider services, + IHttpContextAccessor context, + ILogger logger, + GraphDbSettings settings) + { + _services = services; + _context = context; + _logger = logger; + _settings = settings; + } + + public string Name => "Default"; + + public async Task Search(string query, GraphSearchOptions options) + { + if (string.IsNullOrWhiteSpace(_settings.BaseUrl)) + { + return new GraphSearchData(); + } + + var url = $"{_settings.BaseUrl}/query"; + var request = new GraphQueryRequest + { + Query = query, + Method = options.Method + }; + return await SendRequest(url, request); + } + + private async Task SendRequest(string url, GraphQueryRequest request) + { + var result = new GraphSearchData(); + var http = _services.GetRequiredService(); + + using (var client = http.CreateClient()) + { + var uri = new Uri(url); + try + { + var data = JsonSerializer.Serialize(request, _jsonOptions); + var message = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = uri, + Content = new StringContent(data, Encoding.UTF8, MediaTypeNames.Application.Json) + }; + + AddHeaders(client); + var rawResponse = await client.SendAsync(message); + rawResponse.EnsureSuccessStatusCode(); + + var responseStr = await rawResponse.Content.ReadAsStringAsync(); + result = JsonSerializer.Deserialize(responseStr, _jsonOptions); + return result; + } + catch (Exception ex) + { + _logger.LogError($"Error when fetching Lessen GLM response (Endpoint: {url}). {ex.Message}\r\n{ex.InnerException}"); + return result; + } + } + } + + private void AddHeaders(HttpClient client) + { + client.DefaultRequestHeaders.Add("Authorization", $"{_context.HttpContext.Request.Headers["Authorization"]}"); + client.DefaultRequestHeaders.Add("Origin", $"{_context.HttpContext.Request.Headers["Origin"]}"); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs new file mode 100644 index 00000000..89351398 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.Graph; + +public class GraphDbSettings +{ + public string BaseUrl { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs new file mode 100644 index 00000000..f05b39c1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs @@ -0,0 +1,24 @@ +using BotSharp.Abstraction.Plugins; +using BotSharp.Abstraction.Settings; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace BotSharp.Plugin.Graph; + +public class GraphPlugin : IBotSharpPlugin +{ + public string Id => "74497c25-5e8d-4ee9-b6a8-ce8fe4dabea9"; + public string Name => "Graph"; + public string Description => "Graph Database"; + public string IconUrl => "https://www.microsoft.com/en-us/research/uploads/prodnew/2024/06/GraphRag2024-BlogHeroFeature-1400x788-1.png"; + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("GraphDb"); + }); + + services.AddScoped(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Graph/Models/GraphQueryRequest.cs b/src/Plugins/BotSharp.Plugin.Graph/Models/GraphQueryRequest.cs new file mode 100644 index 00000000..ac520642 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/Models/GraphQueryRequest.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Graph.Models; + +public class GraphQueryRequest +{ + [JsonPropertyName("query")] + public string Query { get; set; } + + [JsonPropertyName("method")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Method { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Graph/Using.cs b/src/Plugins/BotSharp.Plugin.Graph/Using.cs new file mode 100644 index 00000000..02c0e3d4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/Using.cs @@ -0,0 +1,7 @@ +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Threading.Tasks; +global using Microsoft.Extensions.Logging; +global using BotSharp.Abstraction.Graph; +global using BotSharp.Abstraction.Graph.Models; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index f8f31ca2..27863dd3 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.VectorStorage.Models; using BotSharp.Plugin.KnowledgeBase.Utilities; using Tensorflow.NumPy; @@ -22,17 +23,17 @@ public class MemoryVectorDb : IVectorDb return _collections.Select(x => x.Key).ToList(); } - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, VectorFilter filter) { throw new NotImplementedException(); } - public async Task> Search(string collectionName, float[] vector, + public async Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { if (!_vectors.ContainsKey(collectionName)) { - return new List(); + return new List(); } var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); @@ -41,7 +42,7 @@ public class MemoryVectorDb : IVectorDb var results = np.argsort(similarities).ToArray() .Reverse() .Take(limit) - .Select(i => new KnowledgeCollectionData + .Select(i => new VectorCollectionData { Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, Score = similarities[i], diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs index f68d550b..69a56519 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task FeedKnowledge(string collectionName, KnowledgeCreationModel knowledge) + public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; var lines = _textChopper.Chop(knowledge.Content, new ChunkOption diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index 5020c529..e7ed1b1b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task DeleteKnowledgeCollectionData(string collectionName, string id) + public async Task DeleteVectorCollectionData(string collectionName, string id) { try { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs index 243eca02..4e001954 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -1,8 +1,11 @@ +using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.VectorStorage.Models; + namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { - public async Task> GetKnowledgeCollections() + public async Task> GetVectorCollections() { try { @@ -16,44 +19,88 @@ public partial class KnowledgeService } } - public async Task> GetKnowledgeCollectionData(string collectionName, KnowledgeFilter filter) + public async Task> GetVectorCollectionData(string collectionName, VectorFilter filter) { try { var db = GetVectorDb(); var pagedResult = await db.GetCollectionData(collectionName, filter); - return new StringIdPagedItems + return new StringIdPagedItems { Count = pagedResult.Count, - Items = pagedResult.Items.Select(x => KnowledgeSearchResult.CopyFrom(x)), + Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)), NextId = pagedResult.NextId, }; } catch (Exception ex) { _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new StringIdPagedItems(); + return new StringIdPagedItems(); } } - public async Task> SearchKnowledge(string collectionName, KnowledgeSearchOptions options) + public async Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options) { try { var textEmbedding = GetTextEmbedding(); - var vector = await textEmbedding.GetVectorAsync(options.Text); + var vector = await textEmbedding.GetVectorAsync(query); // Vector search var db = GetVectorDb(); var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); - var results = found.Select(x => KnowledgeSearchResult.CopyFrom(x)).ToList(); + var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList(); return results; } catch (Exception ex) { _logger.LogWarning($"Error when searching knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); - return new List(); + return new List(); + } + } + + public async Task SearchGraphKnowledge(string query, GraphSearchOptions options) + { + try + { + var db = GetGraphDb(); + var found = await db.Search(query, options); + return new GraphSearchResult + { + Result = found.Result + }; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when searching graph {query}. {ex.Message}\r\n{ex.InnerException}"); + return new GraphSearchResult(); + } + } + + public async Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions) + { + try + { + var textEmbedding = GetTextEmbedding(); + var vector = await textEmbedding.GetVectorAsync(query); + + var vectorDb = GetVectorDb(); + var vectorRes = await vectorDb.Search(collectionName, vector, vectorOptions.Fields, limit: vectorOptions.Limit ?? 5, + confidence: vectorOptions.Confidence ?? 0.5f, withVector: vectorOptions.WithVector); + + var graphDb = GetGraphDb(); + var graphRes = await graphDb.Search(query, graphOptions); + return new KnowledgeSearchResult + { + VectorResult = vectorRes.Select(x => VectorSearchResult.CopyFrom(x)), + GraphResult = new GraphSearchResult { Result = graphRes.Result } + }; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when searching knowledge (vector collection: {collectionName}) {query}. {ex.Message}\r\n{ex.InnerException}"); + return new KnowledgeSearchResult(); } } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index dff80134..1f96042b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -25,6 +25,12 @@ public partial class KnowledgeService : IKnowledgeService return db; } + private IGraphDb GetGraphDb() + { + var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.GraphDb); + return db; + } + private ITextEmbedding GetTextEmbedding() { var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index b92f919f..296d56b5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -15,7 +15,7 @@ global using BotSharp.Abstraction.Users; global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Agents.Settings; -global using BotSharp.Abstraction.Conversations.Settings; +global using BotSharp.Abstraction.Graph; global using BotSharp.Abstraction.Knowledges.Settings; global using BotSharp.Abstraction.Knowledges.Enums; global using BotSharp.Abstraction.VectorStorage; diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 225c0705..f5b35eaf 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -1,6 +1,6 @@ -using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; +using BotSharp.Abstraction.VectorStorage.Models; using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -16,7 +16,7 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, VectorFilter filter) { throw new NotImplementedException(); } @@ -26,7 +26,7 @@ public class FaissDb : IVectorDb throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, + public Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 10, float confidence = 0.5f, bool withVector = false) { throw new NotImplementedException(); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 0e56f680..3dd064b9 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Utilities; +using BotSharp.Abstraction.VectorStorage.Models; using Qdrant.Client; using Qdrant.Client.Grpc; @@ -41,27 +42,27 @@ public class QdrantDb : IVectorDb return collections.ToList(); } - public async Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + public async Task> GetCollectionData(string collectionName, VectorFilter filter) { var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); if (!exist) { - return new StringIdPagedItems(); + return new StringIdPagedItems(); } var totalPointCount = await client.CountAsync(collectionName); var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, vectorsSelector: filter.WithVector); - var points = response?.Result?.Select(x => new KnowledgeCollectionData + var points = response?.Result?.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null - })?.ToList() ?? new List(); + })?.ToList() ?? new List(); - return new StringIdPagedItems + return new StringIdPagedItems { Count = totalPointCount, NextId = response?.NextPageOffset?.Uuid, @@ -124,10 +125,10 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, + public async Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { - var results = new List(); + var results = new List(); var client = GetClient(); var exist = await DoesCollectionExist(client, collectionName); @@ -161,7 +162,7 @@ public class QdrantDb : IVectorDb data = point.Payload.ToDictionary(k => k.Key, v => v.Value.StringValue); } - results.Add(new KnowledgeCollectionData + results.Add(new VectorCollectionData { Id = point.Id.Uuid, Data = data, diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 487b8ffd..e5c32bd2 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -1,6 +1,6 @@ -using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; +using BotSharp.Abstraction.VectorStorage.Models; using Microsoft.SemanticKernel.Memory; using System.Collections.Generic; using System.Threading.Tasks; @@ -28,7 +28,7 @@ namespace BotSharp.Plugin.SemanticKernel await _memoryStore.CreateCollectionAsync(collectionName); } - public Task> GetCollectionData(string collectionName, KnowledgeFilter filter) + public Task> GetCollectionData(string collectionName, VectorFilter filter) { throw new System.NotImplementedException(); } @@ -43,15 +43,15 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, + public async Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); - var resultTexts = new List(); + var resultTexts = new List(); await foreach (var (record, score) in results) { - resultTexts.Add(new KnowledgeCollectionData + resultTexts.Add(new VectorCollectionData { Data = new Dictionary { { "text", record.Metadata.Text } }, Score = score, diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 3b170fa0..0720a441 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -249,6 +249,10 @@ "ApiKey": "" }, + "GraphDb": { + "BaseUrl": "" + }, + "WeChat": { "AgentId": "437bed34-1169-4833-95ce-c24b8b56154a", "Token": "#{Token}#", @@ -259,6 +263,7 @@ "KnowledgeBase": { "VectorDb": "Qdrant", + "GraphDb": "Default", "DefaultCollection": "BotSharp", "TextEmbedding": { "Provider": "openai", @@ -309,6 +314,7 @@ "BotSharp.Plugin.HuggingFace", "BotSharp.Plugin.KnowledgeBase", "BotSharp.Plugin.Planner", + "BotSharp.Plugin.Graph", "BotSharp.Plugin.Qdrant", "BotSharp.Plugin.ChatHub", "BotSharp.Plugin.WeChat", From 5c754d6b28001a8be5bf2ae162131b3265b0e2f2 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 15 Aug 2024 18:21:51 -0500 Subject: [PATCH 61/63] add proj ref --- src/WebStarter/WebStarter.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index c7443dc1..4d7a31ac 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -48,6 +48,7 @@ + From 310370c096ff434f5ce70fd0ebd73dc386daa590 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 15 Aug 2024 18:23:02 -0500 Subject: [PATCH 62/63] update setting name --- src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs | 2 +- src/WebStarter/appsettings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs index f05b39c1..3d5a2187 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs @@ -16,7 +16,7 @@ public class GraphPlugin : IBotSharpPlugin services.AddScoped(provider => { var settingService = provider.GetRequiredService(); - return settingService.Bind("GraphDb"); + return settingService.Bind("Graph"); }); services.AddScoped(); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 0720a441..68548aa5 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -249,7 +249,7 @@ "ApiKey": "" }, - "GraphDb": { + "Graph": { "BaseUrl": "" }, From a926e0dc622694b9ea42d07d69936a105dd14e55 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Wed, 21 Aug 2024 17:20:20 +0800 Subject: [PATCH 63/63] =?UTF-8?q?Revert=20"=E4=BF=AE=E6=94=B9Locator?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=E5=88=B0=E5=A4=9A=E4=B8=AA=E6=97=B6=E5=8F=96?= =?UTF-8?q?=E7=AC=AC=E4=B8=80=E4=B8=AA=E5=80=BC"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 0dcb9f8c6e4d4be38198941e4968e8fb8d387b49. --- .../Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs index 08ce88e8..91766736 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs @@ -11,7 +11,7 @@ public partial class PlaywrightWebDriver return; } - ILocator locator = page.Locator(result.Selector).First;// 匹配到多个时取第一个，否则当await locator.ClickAsync();匹配到多个就会抛异常。 + ILocator locator = page.Locator(result.Selector); var count = await locator.CountAsync(); if (count == 0) {