diff --git a/BotSharp.sln b/BotSharp.sln index b7545cdc..8dda48e8 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -105,6 +105,14 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FileStorages", "FileStorage EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.TencentCos", "src\Plugins\BotSharp.Plugin.TencentCos\BotSharp.Plugin.TencentCos.csproj", "{BF029B0A-768B-43A1-8D91-E70B95505716}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Interpreters", "Interpreters", "{C4C59872-3C8A-450D-83D5-2BE402D610D5}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInterpreter", "src\Plugins\BotSharp.Plugin.PythonInterpreter\BotSharp.Plugin.PythonInterpreter.csproj", "{05E6E405-5021-406E-8A5E-0A7CEC881F6D}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -425,6 +433,22 @@ Global {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|Any CPU.Build.0 = Release|Any CPU {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|x64.ActiveCfg = Release|Any CPU {BF029B0A-768B-43A1-8D91-E70B95505716}.Release|x64.Build.0 = Release|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Debug|x64.ActiveCfg = Debug|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Debug|x64.Build.0 = Debug|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|Any CPU.Build.0 = Release|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|x64.ActiveCfg = Release|Any CPU + {05E6E405-5021-406E-8A5E-0A7CEC881F6D}.Release|x64.Build.0 = Release|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Debug|x64.ActiveCfg = Debug|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Debug|x64.Build.0 = Debug|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -475,6 +499,10 @@ Global {54E83C6F-54EE-4ADC-8D72-93C009CC4FB4} = {51AFE054-AE99-497D-A593-69BAEFB5106F} {38B37C0D-1930-4D47-BCBF-E358EC1096B1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} {BF029B0A-768B-43A1-8D91-E70B95505716} = {38B37C0D-1930-4D47-BCBF-E358EC1096B1} + {C4C59872-3C8A-450D-83D5-2BE402D610D5} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} + {05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5} + {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} + {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index 8405d9b5..1fd7a38c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -20,6 +20,13 @@ public interface IAgentService /// Task LoadAgent(string id); + /// + /// Inherit from an agent + /// + /// + /// + Task InheritAgent(Agent agent); + string RenderedInstruction(Agent agent); string RenderedTemplate(Agent agent, string templateName); diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs index 01ec2acb..43f0f5ba 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Plugins.Models; -using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Tasks.Models; namespace BotSharp.Abstraction.Agents.Models; @@ -21,8 +20,7 @@ public class Agent /// Default LLM settings /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public AgentLlmConfig LlmConfig { get; set; } - = new AgentLlmConfig(); + public AgentLlmConfig LlmConfig { get; set; } = new(); /// /// Instruction @@ -30,33 +28,35 @@ public class Agent [JsonIgnore] public string? Instruction { get; set; } + /// + /// Channel instructions + /// + [JsonIgnore] + public List ChannelInstructions { get; set; } = new(); + /// /// Templates /// [JsonIgnore] - public List Templates { get; set; } - = new List(); + public List Templates { get; set; } = new(); /// /// Agent tasks /// [JsonIgnore] - public List Tasks { get; set; } - = new List(); + public List Tasks { get; set; } = new(); /// /// Samples /// [JsonIgnore] - public List Samples { get; set; } - = new List(); + public List Samples { get; set; } = new(); /// /// Functions /// [JsonIgnore] - public List Functions { get; set; } - = new List(); + public List Functions { get; set; } = new(); /// /// Responses @@ -93,23 +93,20 @@ public class Agent /// /// Agent utilities /// - public List Utilities { get; set; } - = new List(); + public List Utilities { get; set; } = new(); /// /// Inherit from agent /// public string? InheritAgentId { get; set; } - public List RoutingRules { get; set; } - = new List(); + public List RoutingRules { get; set; } = new(); /// /// For rendering deferral /// [JsonIgnore] - public Dictionary TemplateDict { get; set; } - = new Dictionary(); + public Dictionary TemplateDict { get; set; } = new(); public override string ToString() => $"{Name} {Id}"; @@ -124,6 +121,7 @@ public class Agent Description = agent.Description, Type = agent.Type, Instruction = agent.Instruction, + ChannelInstructions = agent.ChannelInstructions, Functions = agent.Functions, Responses = agent.Responses, Samples = agent.Samples, @@ -145,6 +143,12 @@ public class Agent return this; } + public Agent SetChannelInstructions(List instructions) + { + ChannelInstructions = instructions ?? new List(); + return this; + } + public Agent SetTemplates(List templates) { Templates = templates ?? new List(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/ChannelInstruction.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/ChannelInstruction.cs new file mode 100644 index 00000000..d4ff6c1f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/ChannelInstruction.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Agents.Models; + +public class ChannelInstruction +{ + public string Channel { get; set; } + public string Instruction { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 446b8869..2e3aa24c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -25,11 +25,14 @@ + + + diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebPageResponseHook.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebPageResponseHook.cs index a7e33b22..4e32117b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebPageResponseHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/IWebPageResponseHook.cs @@ -4,6 +4,6 @@ namespace BotSharp.Abstraction.Browsing; public interface IWebPageResponseHook { - void OnDataFetched(MessageInfo message, string url, string postData, string responsData); - T? GetResponse(MessageInfo message, string url, string? queryParameter = null); + void OnDataFetched(MessageInfo message, WebPageResponseData response); + T? GetResponse(MessageInfo message, WebPageResponseFilter filter); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs index 661b0da4..d260a09d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs @@ -15,6 +15,25 @@ public class PageActionArgs /// This value has to be set to true if you want to get the page XHR/ Fetch responses /// public bool OpenNewTab { get; set; } = false; + + public bool EnableResponseCallback { get; set; } = false; + + /// + /// Exclude urls for XHR/ Fetch responses + /// + public string[]? ExcludeResponseUrls { get; set; } + + /// + /// Only include urls for XHR/ Fetch responses + /// + public string[]? IncludeResponseUrls { get; set; } + + /// + /// If set to true, the response will be stored in memory + /// + public bool ResponseInMemory { get; set; } = false; + public List? ResponseContainer { get; set; } + public bool UseExistingPage { get; set; } = false; public bool WaitForNetworkIdle { get; set; } = true; diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs new file mode 100644 index 00000000..1d03b0b7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class WebPageResponseData +{ + public string Url { get; set; } = null!; + public string PostData { get; set; } = null!; + public string ResponseData { get; set; } = null!; + public bool ResponseInMemory { get; set; } + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs new file mode 100644 index 00000000..cd97a01d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseFilter.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Browsing.Models; + +public class WebPageResponseFilter +{ + public string Url { get; set; } = null!; + public string[]? QueryParameters { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs new file mode 100644 index 00000000..dab5bc81 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Constants/FileConstants.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Files.Constants; + +public class FileConstants +{ + public static readonly IEnumerable AudioExtensions = new List + { + ".mp3", ".wav", ".flac", ".aac", ".ogg", ".wma" + }; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs index 54ad3a6d..87df6137 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs @@ -2,6 +2,8 @@ namespace BotSharp.Abstraction.Files.Converters; public interface IPdf2ImageConverter { + public string Name { get; } + /// /// Convert pdf pages to images, and return a list of image file paths /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs new file mode 100644 index 00000000..10ccd1a0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Repositories.Enums; + +namespace BotSharp.Abstraction.Files; + +public class FileCoreSettings +{ + public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; + public string Pdf2TextConverter { get; set; } + public string Pdf2ImageConverter { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs deleted file mode 100644 index 23ba12c6..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileStorageSettings.cs +++ /dev/null @@ -1,8 +0,0 @@ -using BotSharp.Abstraction.Repositories.Enums; - -namespace BotSharp.Abstraction.Files; - -public class FileStorageSettings -{ - public string Default { get; set; } = FileStorageEnum.LocalFileStorage; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs deleted file mode 100644 index dd91a2bb..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System.IO; - -namespace BotSharp.Abstraction.Files; - -public interface IBotSharpFileService -{ - #region Conversation - /// - /// Get the files that have been uploaded in the chat. - /// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file. - /// - /// - /// - /// - /// - /// - /// - /// - Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, - bool includeScreenShot = false, int? offset = null); - - /// - /// Get the files that have been uploaded in the chat. No screenshot images are included. - /// - /// - /// - /// - /// - /// - IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, bool imageOnly = false); - string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); - IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); - bool SaveMessageFiles(string conversationId, string messageId, string source, List files); - - /// - /// Delete files under messages - /// - /// Conversation Id - /// Files in these messages will be deleted - /// The starting message to delete - /// If not null, delete messages while input a new message; otherwise, delete messages only - /// - bool DeleteMessageFiles(string conversationId, IEnumerable messageIds, string targetMessageId, string? newMessageId = null); - bool DeleteConversationFiles(IEnumerable conversationIds); - #endregion - - #region Image - Task GenerateImage(string? provider, string? model, string text); - Task VaryImage(string? provider, string? model, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); - #endregion - - #region Pdf - /// - /// Take screenshots of pdf pages and get response from llm - /// - /// - /// Pdf files - /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); - #endregion - - #region User - string GetUserAvatar(); - bool SaveUserAvatar(BotSharpFile file); - #endregion - - #region Common - /// - /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" - /// - /// - /// - (string, byte[]) GetFileInfoFromData(string data); - string GetDirectory(string conversationId); - string GetFileContentType(string filePath); - byte[] GetFileBytes(string fileStorageUrl); - bool SavefileToPath(string filePath, Stream stream); - #endregion -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs new file mode 100644 index 00000000..78a400e1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -0,0 +1,26 @@ +namespace BotSharp.Abstraction.Files; + +public interface IFileInstructService +{ + #region Image + Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task GenerateImage(string? provider, string? model, string text); + Task VaryImage(string? provider, string? model, BotSharpFile image); + Task EditImage(string? provider, string? model, string text, BotSharpFile image); + Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); + #endregion + + #region Pdf + /// + /// Take screenshots of pdf pages and get response from llm + /// + /// + /// Pdf files + /// + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + #endregion + + #region Select file + Task> SelectMessageFiles(string conversationId, SelectFileOptions options); + #endregion +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs new file mode 100644 index 00000000..a3e5a32d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -0,0 +1,63 @@ +using System.IO; + +namespace BotSharp.Abstraction.Files; + +public interface IFileStorageService +{ + #region Common + string GetDirectory(string conversationId); + byte[] GetFileBytes(string fileStorageUrl); + bool SaveFileStreamToPath(string filePath, Stream stream); + bool SaveFileBytesToPath(string filePath, byte[] bytes); + string GetParentDir(string dir, int level = 1); + bool ExistDirectory(string? dir); + void CreateDirectory(string dir); + void DeleteDirectory(string dir); + string BuildDirectory(params string[] segments); + #endregion + + + #region Conversation + /// + /// Get the message file screenshots for specific content types, e.g., pdf + /// + /// + /// + /// + Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); + + /// + /// Get the files that have been uploaded in the chat. No screenshot images are included. + /// + /// + /// + /// + /// + /// + IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); + string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); + IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); + bool SaveMessageFiles(string conversationId, string messageId, string source, List files); + + /// + /// Delete files under messages + /// + /// Conversation Id + /// Files in these messages will be deleted + /// The starting message to delete + /// If not null, delete messages while input a new message; otherwise, delete messages only + /// + bool DeleteMessageFiles(string conversationId, IEnumerable messageIds, string targetMessageId, string? newMessageId = null); + bool DeleteConversationFiles(IEnumerable conversationIds); + #endregion + + + #region User + string GetUserAvatar(); + bool SaveUserAvatar(BotSharpFile file); + #endregion + #region Speech + Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data); + Task RetrieveSpeechFileAsync(string conversationId, string fileName); + #endregion +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 7e556e67..11a11e46 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,23 +1,7 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile +public class BotSharpFile : FileBase { - [JsonPropertyName("file_name")] - public string FileName { get; set; } = string.Empty; - - /// - /// File data, e.g., "data:image/png;base64,aaaaaaaa" - /// - [JsonPropertyName("file_data")] - public string FileData { get; set; } = string.Empty; - - [JsonPropertyName("file_url")] - public string FileUrl { get; set; } = string.Empty; - - [JsonPropertyName("content_type")] - public string ContentType { get; set; } = string.Empty; - - [JsonPropertyName("file_storage_url")] - public string FileStorageUrl { get; set; } = string.Empty; + } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs new file mode 100644 index 00000000..3483921a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -0,0 +1,46 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileBase +{ + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; + + /// + /// Internal file storage url + /// + [JsonPropertyName("file_storage_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileStorageUrl { get; set; } = string.Empty; + + /// + /// File name without extension + /// + [JsonPropertyName("file_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileName { get; set; } = string.Empty; + + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileData { get; set; } = string.Empty; + + /// + /// File content type + /// + [JsonPropertyName("content_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ContentType { get; set; } = string.Empty; + + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs new file mode 100644 index 00000000..d13b4f1e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileSelectContext +{ + [JsonPropertyName("selected_ids")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? Selecteds { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 7cd93269..2a0128b6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,34 +1,10 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel +public class MessageFileModel : FileBase { [JsonPropertyName("message_id")] public string MessageId { get; set; } - /// - /// External file url - /// - [JsonPropertyName("file_url")] - public string FileUrl { get; set; } - - /// - /// Internal file storage url - /// - [JsonPropertyName("file_storage_url")] - public string FileStorageUrl { get; set; } - - /// - /// File name without extension - /// - [JsonPropertyName("file_name")] - public string FileName { get; set; } - - [JsonPropertyName("file_type")] - public string FileType { get; set; } - - [JsonPropertyName("content_type")] - public string ContentType { get; set; } - [JsonPropertyName("file_source")] public string FileSource { get; set; } = FileSourceType.User; @@ -39,6 +15,6 @@ public class MessageFileModel public override string ToString() { - return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}"; + return $"File name: {FileName}, File extension: {FileExtension}, Content type: {ContentType}, Source: {FileSource}"; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs new file mode 100644 index 00000000..d61c1b7c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -0,0 +1,49 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class SelectFileOptions +{ + /// + /// Llm provider + /// + public string? Provider { get; set; } + + /// + /// Llm model id + /// + public string? ModelId { get; set; } + + /// + /// Agent id + /// + public string? AgentId { get; set; } + + /// + /// Template (prompt) name + /// + public string? Template { get; set; } + + /// + /// Description that user provides to select files + /// + public string? Description { get; set; } + + /// + /// Whether include bot generated files + /// + public bool IncludeBotFile { get; set; } + + /// + /// Conversation breakpoint + /// + public bool FromBreakpoint { get; set; } + + /// + /// Message offset from last + /// + public int? Offset { get; set; } + + /// + /// File content types. If null, all types of files will be retrived + /// + public IEnumerable? ContentTypes { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs new file mode 100644 index 00000000..df33906d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -0,0 +1,40 @@ +using Microsoft.AspNetCore.StaticFiles; + +namespace BotSharp.Abstraction.Files.Utilities; + +public static class FileUtility +{ + /// + /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa" + /// + /// + /// + public static (string, byte[]) GetFileInfoFromData(string data) + { + if (string.IsNullOrEmpty(data)) + { + return (string.Empty, new byte[0]); + } + + var typeStartIdx = data.IndexOf(':'); + var typeEndIdx = data.IndexOf(';'); + var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); + + var base64startIdx = data.IndexOf(','); + var base64Str = data.Substring(base64startIdx + 1); + + return (contentType, Convert.FromBase64String(base64Str)); + } + + public static string GetFileContentType(string filePath) + { + string contentType; + var provider = new FileExtensionContentTypeProvider(); + if (!provider.TryGetContentType(filePath, out contentType)) + { + contentType = string.Empty; + } + + return contentType; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs index 4366d323..d5dabb3c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs @@ -8,7 +8,12 @@ public class FunctionDef [JsonPropertyName("description")] public string Description { get; set; } = null!; + [JsonPropertyName("channels")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Channels { get; set; } + [JsonPropertyName("visibility_expression")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? VisibilityExpression { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs new file mode 100644 index 00000000..64c5388b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Graph.Models; + +namespace BotSharp.Abstraction.Graph; + +public interface IGraphDb +{ + public string Name { get; } + + Task Search(string query, GraphSearchOptions options); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchData.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchData.cs new file mode 100644 index 00000000..b6a9b714 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchData.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Graph.Models; + +public class GraphSearchData +{ + public string Result { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchOptions.cs new file mode 100644 index 00000000..294aeea7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchOptions.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Graph.Models; + +public class GraphSearchOptions +{ + public string Method { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchResult.cs new file mode 100644 index 00000000..e9985447 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/Models/GraphSearchResult.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Graph.Models; + +public class GraphSearchResult +{ + public string Result { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Interpreters/Models/InterpretationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Interpreters/Models/InterpretationRequest.cs new file mode 100644 index 00000000..5a8f7552 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Interpreters/Models/InterpretationRequest.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Interpreters.Models; + +public class InterpretationRequest +{ + [JsonPropertyName("script")] + public string Script { get; set; } = null!; + + [JsonPropertyName("language")] + public string Language { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs new file mode 100644 index 00000000..7d92e504 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgeCollectionName.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Knowledges.Enums; + +public static class KnowledgeCollectionName +{ + public static string BotSharp = nameof(BotSharp); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs new file mode 100644 index 00000000..9d95967f --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Enums/KnowledgePayloadName.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Knowledges.Enums; + +public static class KnowledgePayloadName +{ + public static string Text = "text"; + public static string Question = "question"; + public static string Answer = "answer"; + public static string Request = "request"; + public static string Response = "response"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs index aefcdec6..3a5ab788 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Knowledges.Models; - namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeHook diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 5b2b795b..5bf7aced 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -1,14 +1,15 @@ -using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { - Task> CollectChunkedKnowledge(); - Task EmbedKnowledge(List chunks); - - Task Feed(KnowledgeFeedModel knowledge); - Task EmbedKnowledge(KnowledgeCreationModel knowledge); - Task GetKnowledges(KnowledgeRetrievalModel retrievalModel); - Task> GetAnswer(KnowledgeRetrievalModel retrievalModel); + Task> GetVectorCollections(); + Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); + Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); + Task> GetVectorCollectionData(string collectionName, VectorFilter filter); + Task DeleteVectorCollectionData(string collectionName, string id); + Task SearchGraphKnowledge(string query, GraphSearchOptions options); + Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index d2ca2940..b8f2d47b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -1,12 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Microsoft.AspNetCore.Http; - namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { + public string Name { get; } Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs deleted file mode 100644 index 7e3a315e..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeFeedModel -{ - public string AgentId { get; set; } = string.Empty; - public string Content { get; set; } = string.Empty; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs deleted file mode 100644 index 03f66eb5..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Knowledges.Models; - -public class KnowledgeRetrievalModel -{ - public string AgentId { get; set; } = string.Empty; - public string Question { get; set; } = string.Empty; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs new file mode 100644 index 00000000..771b070b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs @@ -0,0 +1,10 @@ +using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeSearchResult +{ + public IEnumerable VectorResult { get; set; } + public GraphSearchResult GraphResult { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs deleted file mode 100644 index 296e1ed6..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/RetrievedResult.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.Text.Json.Serialization; - -namespace BotSharp.Abstraction.Knowledges.Models; - -public class RetrievedResult -{ - public int Paragraph { get; set; } - - [JsonPropertyName("cite_source")] - public string CiteSource { get; set; } = "related text"; - - [JsonPropertyName("reasoning")] - public string Reasoning { get; set; } = ""; -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 97f7f55c..65bf7637 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -1,9 +1,17 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { + public string DefaultCollection { get; set; } = KnowledgeCollectionName.BotSharp; public string VectorDb { get; set; } - public string TextEmbedding { get; set; } - public string TextCompletion { get; set; } - public string Pdf2TextConverter { get; set; } + public string GraphDb { get; set; } + public KnowledgeModelSetting TextEmbedding { get; set; } } + +public class KnowledgeModelSetting +{ + public string Provider { get; set; } + public string Model { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs new file mode 100644 index 00000000..344fad0e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs @@ -0,0 +1,23 @@ +namespace BotSharp.Abstraction.MLTasks +{ + public interface ITextToSpeech + { + /// + /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI + /// + string Provider { get; } + + /// + /// Set model name, one provider can consume different model or version(s) + /// + /// deployment name + void SetModelName(string model); + + Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null); + } + + public interface ITextToSpeechOptions + { + + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index 89c9f7db..825f7d8f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -17,4 +17,5 @@ global using BotSharp.Abstraction.Templating; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; global using BotSharp.Abstraction.Files.Models; -global using BotSharp.Abstraction.Files.Enums; \ No newline at end of file +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs new file mode 100644 index 00000000..d8e4d355 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringIdPagination.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Abstraction.Utilities; + +public class StringIdPagination : Pagination +{ + [JsonPropertyName("start_id")] + public string? StartId { get; set; } +} + +public class StringIdPagedItems : PagedItems +{ + public new ulong Count { get; set; } + + [JsonPropertyName("next_id")] + public string? NextId { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 68aaaa78..8aeb5ee9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -1,9 +1,15 @@ +using BotSharp.Abstraction.VectorStorage.Models; + namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { - Task> GetCollections(); + string Name { get; } + + Task> GetCollections(); + Task> GetCollectionData(string collectionName, VectorFilter filter); Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null); - Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f); + Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); + Task DeleteCollectionData(string collectionName, string id); } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs new file mode 100644 index 00000000..e623532a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorCollectionData +{ + public string Id { get; set; } + public Dictionary Data { get; set; } = new(); + public double? Score { get; set; } + public float[]? Vector { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs new file mode 100644 index 00000000..91abcc25 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorFilter : StringIdPagination +{ + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs new file mode 100644 index 00000000..64943dd1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs @@ -0,0 +1,11 @@ +using BotSharp.Abstraction.Knowledges.Enums; + +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorSearchOptions +{ + public IEnumerable? Fields { get; set; } = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }; + public int? Limit { get; set; } = 5; + public float? Confidence { get; set; } = 0.5f; + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs new file mode 100644 index 00000000..ce39edbf --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs @@ -0,0 +1,20 @@ +namespace BotSharp.Abstraction.VectorStorage.Models; + +public class VectorSearchResult : VectorCollectionData +{ + public VectorSearchResult() + { + + } + + public static VectorSearchResult CopyFrom(VectorCollectionData data) + { + return new VectorSearchResult + { + Id = data.Id, + Data = data.Data, + Score = data.Score, + Vector = data.Vector + }; + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs index 59de3096..9e2e1d5f 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs @@ -41,44 +41,52 @@ public partial class AgentService }); Utilities.ClearCache(); - return await Task.FromResult(agentRecord); } - private Agent FetchAgentFileByName(string agentName, string filePath) + private (string, List) GetInstructionsFromFile(string fileDir) { - foreach (var dir in Directory.GetDirectories(filePath)) + var defaultInstruction = string.Empty; + var channelInstructions = new List(); + + var instructionDir = Path.Combine(fileDir, "instructions"); + if (!Directory.Exists(instructionDir)) { - var agentJson = File.ReadAllText(Path.Combine(dir, "agent.json")); - var agent = JsonSerializer.Deserialize(agentJson, _options); - if (agent != null && agent.Name.IsEqualTo(agentName)) - { - var functions = FetchFunctionsFromFile(dir); - var instruction = FetchInstructionFromFile(dir); - var responses = FetchResponsesFromFile(dir); - var templates = FetchTemplatesFromFile(dir); - var samples = FetchSamplesFromFile(dir); - return agent.SetInstruction(instruction) - .SetTemplates(templates) - .SetFunctions(functions) - .SetResponses(responses) - .SetSamples(samples); - } + return (defaultInstruction, channelInstructions); } - return null; + foreach (var file in Directory.GetFiles(instructionDir)) + { + var extension = Path.GetExtension(file).Substring(1); + if (!extension.IsEqualTo(_agentSettings.TemplateFormat)) + { + continue; + } + + var segments = Path.GetFileName(file).Split(".", StringSplitOptions.RemoveEmptyEntries); + if (segments.IsNullOrEmpty() || !segments[0].IsEqualTo("instruction")) + { + continue; + } + + if (segments.Length == 2) + { + defaultInstruction = File.ReadAllText(file); + } + else if (segments.Length == 3) + { + var item = new ChannelInstruction + { + Channel = segments[1], + Instruction = File.ReadAllText(file) + }; + channelInstructions.Add(item); + } + } + return (defaultInstruction, channelInstructions); } - private string FetchInstructionFromFile(string fileDir) - { - var file = Path.Combine(fileDir, $"instruction.{_agentSettings.TemplateFormat}"); - if (!File.Exists(file)) return null; - - var instruction = File.ReadAllText(file); - return instruction; - } - - private List FetchTemplatesFromFile(string fileDir) + private List GetTemplatesFromFile(string fileDir) { var templates = new List(); var templateDir = Path.Combine(fileDir, "templates"); @@ -86,10 +94,10 @@ public partial class AgentService foreach (var file in Directory.GetFiles(templateDir)) { - var name = Path.GetFileNameWithoutExtension(file); var extension = Path.GetExtension(file).Substring(1); if (extension.IsEqualTo(_agentSettings.TemplateFormat)) { + var name = Path.GetFileNameWithoutExtension(file); var content = File.ReadAllText(file); templates.Add(new AgentTemplate(name, content)); } @@ -98,7 +106,7 @@ public partial class AgentService return templates; } - private List FetchFunctionsFromFile(string fileDir) + private List GetFunctionsFromFile(string fileDir) { var functions = new List(); var functionDir = Path.Combine(fileDir, "functions"); @@ -125,7 +133,7 @@ public partial class AgentService return functions; } - private List FetchResponsesFromFile(string fileDir) + private List GetResponsesFromFile(string fileDir) { var responses = new List(); var responseDir = Path.Combine(fileDir, "responses"); @@ -143,7 +151,7 @@ public partial class AgentService return responses; } - private List FetchSamplesFromFile(string fileDir) + private List GetSamplesFromFile(string fileDir) { var file = Path.Combine(fileDir, "samples.txt"); if (!File.Exists(file)) return new List(); @@ -152,7 +160,7 @@ public partial class AgentService return samples?.ToList() ?? new List(); } - private List FetchTasksFromFile(string fileDir) + private List GetTasksFromFile(string fileDir) { var tasks = new List(); var taskDir = Path.Combine(fileDir, "tasks"); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs index 15ce6eca..04b5c22d 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs @@ -53,7 +53,29 @@ public partial class AgentService } profile.Plugin = GetPlugin(profile.Id); - return profile; } + + public async Task InheritAgent(Agent agent) + { + if (string.IsNullOrWhiteSpace(agent?.InheritAgentId)) return; + + var inheritedAgent = await GetAgent(agent.InheritAgentId); + agent.Templates.AddRange(inheritedAgent.Templates + // exclude private template + .Where(x => !x.Name.StartsWith(".")) + // exclude duplicate name + .Where(x => !agent.Templates.Exists(t => t.Name == x.Name))); + + agent.Functions.AddRange(inheritedAgent.Functions + // exclude private template + .Where(x => !x.Name.StartsWith(".")) + // exclude duplicate name + .Where(x => !agent.Functions.Exists(t => t.Name == x.Name))); + + if (string.IsNullOrWhiteSpace(agent.Instruction)) + { + agent.Instruction = inheritedAgent.Instruction; + } + } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 89c45510..f0a864a2 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -1,7 +1,12 @@ +using BotSharp.Abstraction.Routing.Models; +using System.Collections.Concurrent; + namespace BotSharp.Core.Agents.Services; public partial class AgentService { + public static ConcurrentDictionary> AgentParameterTypes = new(); + [MemoryCache(10 * 60, perInstanceCache: true)] public async Task LoadAgent(string id) { @@ -29,30 +34,12 @@ public partial class AgentService return null; } - if (agent.InheritAgentId != null) - { - var inheritedAgent = await GetAgent(agent.InheritAgentId); - agent.Templates.AddRange(inheritedAgent.Templates - // exclude private template - .Where(x => !x.Name.StartsWith(".")) - // exclude duplicate name - .Where(x => !agent.Templates.Exists(t => t.Name == x.Name))); - - agent.Functions.AddRange(inheritedAgent.Functions - // exclude private template - .Where(x => !x.Name.StartsWith(".")) - // exclude duplicate name - .Where(x => !agent.Functions.Exists(t => t.Name == x.Name))); - - if (agent.Instruction == null) - { - agent.Instruction = inheritedAgent.Instruction; - } - } - - agent.TemplateDict = new Dictionary(); + await InheritAgent(agent); + OverrideInstructionByChannel(agent); + AddOrUpdateParameters(agent); // Populate state into dictionary + agent.TemplateDict = new Dictionary(); PopulateState(agent.TemplateDict); // After agent is loaded @@ -88,6 +75,23 @@ public partial class AgentService return agent; } + private void OverrideInstructionByChannel(Agent agent) + { + var instructions = agent.ChannelInstructions; + if (instructions.IsNullOrEmpty()) return; + + var state = _services.GetRequiredService(); + var channel = state.GetState("channel"); + + if (string.IsNullOrWhiteSpace(channel)) + { + return; + } + + var found = instructions.FirstOrDefault(x => x.Channel.IsEqualTo(channel)); + agent.Instruction = !string.IsNullOrWhiteSpace(found?.Instruction) ? found.Instruction : agent.Instruction; + } + private void PopulateState(Dictionary dict) { var conv = _services.GetRequiredService(); @@ -96,4 +100,53 @@ public partial class AgentService dict[t.Key] = t.Value; } } + + private void AddOrUpdateParameters(Agent agent) + { + var agentId = agent.Id ?? agent.Name; + if (AgentParameterTypes.ContainsKey(agentId)) return; + + AddOrUpdateRoutesParameters(agentId, agent.RoutingRules); + AddOrUpdateFunctionsParameters(agentId, agent.Functions); + } + + private void AddOrUpdateRoutesParameters(string agentId, List routingRules) + { + if(!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) + { + parameterTypes = new(); + } + + foreach (var rule in routingRules.Where(x => x.Required)) + { + if (string.IsNullOrEmpty(rule.FieldType)) continue; + parameterTypes.TryAdd(rule.Field, rule.FieldType); + } + + AgentParameterTypes.TryAdd(agentId, parameterTypes); + } + + private void AddOrUpdateFunctionsParameters(string agentId, List functions) + { + if (!AgentParameterTypes.TryGetValue(agentId, out var parameterTypes)) + { + parameterTypes = new(); + } + + var parameters = functions.Select(p => p.Parameters); + foreach (var param in parameters) + { + foreach (JsonProperty prop in param.Properties.RootElement.EnumerateObject()) + { + var name = prop.Name; + var node = prop.Value; + if (node.TryGetProperty("type", out var type)) + { + parameterTypes.TryAdd(name, type.GetString()); + } + } + } + + AgentParameterTypes.TryAdd(agentId, parameterTypes); + } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index 0b61977e..10598d4e 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -42,19 +42,20 @@ public partial class AgentService continue; } - var functions = FetchFunctionsFromFile(dir); - var instruction = FetchInstructionFromFile(dir); - var responses = FetchResponsesFromFile(dir); - var templates = FetchTemplatesFromFile(dir); - var samples = FetchSamplesFromFile(dir); - agent.SetInstruction(instruction) + var (defaultInstruction, channelInstructions) = GetInstructionsFromFile(dir); + var functions = GetFunctionsFromFile(dir); + var responses = GetResponsesFromFile(dir); + var templates = GetTemplatesFromFile(dir); + var samples = GetSamplesFromFile(dir); + agent.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetTemplates(templates) .SetFunctions(functions) .SetResponses(responses) .SetSamples(samples); var userAgent = BuildUserAgent(agent.Id, user.Id); - var tasks = FetchTasksFromFile(dir); + var tasks = GetTasksFromFile(dir); var isAgentDeleted = _db.DeleteAgent(agent.Id); if (isAgentDeleted) diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs index a0b2f130..e9617d17 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs @@ -20,17 +20,32 @@ public partial class AgentService public bool RenderFunction(Agent agent, FunctionDef def) { - if (!string.IsNullOrEmpty(def.VisibilityExpression)) + var isRender = true; + + var channels = def.Channels; + if (channels != null) + { + var state = _services.GetRequiredService(); + var channel = state.GetState("channel"); + if (!string.IsNullOrWhiteSpace(channel)) + { + isRender = isRender && channels.Contains(channel); + } + } + + if (!isRender) return false; + + if (!string.IsNullOrWhiteSpace(def.VisibilityExpression)) { var render = _services.GetRequiredService(); var result = render.Render(def.VisibilityExpression, new Dictionary { { "states", agent.TemplateDict } }); - return result == "visible"; + isRender = isRender && result == "visible"; } - return true; + return isRender; } public FunctionParametersDef? RenderFunctionProperty(Agent agent, FunctionDef def) diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs index 6fd0f85e..fe532e28 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs @@ -28,6 +28,7 @@ public partial class AgentService record.Profiles = agent.Profiles ?? new List(); record.RoutingRules = agent.RoutingRules ?? new List(); record.Instruction = agent.Instruction ?? string.Empty; + record.ChannelInstructions = agent.ChannelInstructions ?? new List(); record.Functions = agent.Functions ?? new List(); record.Templates = agent.Templates ?? new List(); record.Responses = agent.Responses ?? new List(); @@ -41,7 +42,6 @@ public partial class AgentService _db.UpdateAgent(record, updateField); Utilities.ClearCache(); - await Task.CompletedTask; } @@ -71,7 +71,7 @@ public partial class AgentService agentSettings.DataDir); var clonedAgent = Agent.Clone(agent); - var foundAgent = FetchAgentFileById(agent.Id, filePath); + var foundAgent = GetAgentFileById(agent.Id, filePath); if (foundAgent == null) { updateResult = $"Cannot find agent {agent.Name} in file directory: {filePath}"; @@ -90,6 +90,7 @@ public partial class AgentService .SetProfiles(foundAgent.Profiles) .SetRoutingRules(foundAgent.RoutingRules) .SetInstruction(foundAgent.Instruction) + .SetChannelInstructions(foundAgent.ChannelInstructions) .SetTemplates(foundAgent.Templates) .SetFunctions(foundAgent.Functions) .SetResponses(foundAgent.Responses) @@ -165,7 +166,7 @@ public partial class AgentService return patchResult; } - private Agent? FetchAgentFileById(string agentId, string filePath) + private Agent? GetAgentFileById(string agentId, string filePath) { if (!Directory.Exists(filePath)) return null; @@ -175,12 +176,13 @@ public partial class AgentService var agent = JsonSerializer.Deserialize(agentJson, _options); if (agent != null && agent.Id == agentId) { - var functions = FetchFunctionsFromFile(dir); - var instruction = FetchInstructionFromFile(dir); - var responses = FetchResponsesFromFile(dir); - var templates = FetchTemplatesFromFile(dir); - var samples = FetchSamplesFromFile(dir); - return agent.SetInstruction(instruction) + var (defaultInstruction, channelInstructions) = GetInstructionsFromFile(dir); + var functions = GetFunctionsFromFile(dir); + var responses = GetResponsesFromFile(dir); + var templates = GetTemplatesFromFile(dir); + var samples = GetSamplesFromFile(dir); + return agent.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetTemplates(templates) .SetFunctions(functions) .SetResponses(responses) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 1b58a984..73bf035c 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -45,19 +45,28 @@ 1701;1702 + + + + + + + + + - + - + - + - + - + @@ -69,8 +78,9 @@ + - + @@ -80,7 +90,7 @@ PreserveNewest - + PreserveNewest @@ -89,19 +99,19 @@ PreserveNewest - + PreserveNewest PreserveNewest - + PreserveNewest PreserveNewest - + PreserveNewest @@ -137,7 +147,7 @@ PreserveNewest - + PreserveNewest @@ -152,7 +162,10 @@ PreserveNewest - + + PreserveNewest + + PreserveNewest @@ -172,7 +185,6 @@ - @@ -181,9 +193,4 @@ - - - - - diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 26c62b62..36b588ae 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -150,6 +150,7 @@ public partial class ConversationService await HookEmitter.Emit(_services, async hook => await hook.OnConversationEnding(response) ); + response.FunctionName = "conversation_end"; } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs index 451cdeed..ccc074e8 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs @@ -5,10 +5,9 @@ public partial class ConversationService : IConversationService public async Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true); - - fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId); + fileStorage.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId); var hooks = _services.GetServices().ToList(); foreach (var hook in hooks) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 87beba41..f2a93b3f 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -37,9 +37,9 @@ public partial class ConversationService : IConversationService public async Task DeleteConversations(IEnumerable ids) { var db = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var isDeleted = db.DeleteConversations(ids); - fileService.DeleteConversationFiles(ids); + fileStorage.DeleteConversationFiles(ids); return await Task.FromResult(isDeleted); } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 57f09c71..aa518ee7 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -360,9 +360,28 @@ public class ConversationStateService : IConversationStateService, IDisposable stateValue = stateValue?.ToLower(); } - SetState(property.Name, stateValue, source: StateSource.Application); + if (CheckArgType(property.Name, stateValue)) + { + SetState(property.Name, stateValue, source: StateSource.Application); + } } } } } + + private bool CheckArgType(string name, string value) + { + var agentTypes = AgentService.AgentParameterTypes.SelectMany(p => p.Value).ToList(); + var filed = agentTypes.FirstOrDefault(t => t.Key == name); + if (filed.Key != null) + { + return filed.Value switch + { + "boolean" => bool.TryParse(value, out _), + "number" => long.TryParse(value, out _), + _ => true, + }; + } + return true; + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs new file mode 100644 index 00000000..0b09d6aa --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/FileCorePlugin.cs @@ -0,0 +1,28 @@ +using BotSharp.Abstraction.Repositories.Enums; +using BotSharp.Core.Files.Services; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Files; + +public class FileCorePlugin : IBotSharpPlugin +{ + public string Id => "6a8473c0-04eb-4346-be32-24755ce5973d"; + + public string Name => "File Core"; + + public string Description => "Provides file storage and analysis."; + + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); + services.AddSingleton(fileCoreSettings); + + if (fileCoreSettings.Storage == FileStorageEnum.LocalFileStorage) + { + services.AddScoped(); + } + services.AddScoped(); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs b/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs deleted file mode 100644 index 90397b93..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs +++ /dev/null @@ -1,26 +0,0 @@ -using BotSharp.Abstraction.Repositories.Enums; -using BotSharp.Core.Files.Services; -using Microsoft.Extensions.Configuration; - -namespace BotSharp.Core.Files; - -public class FilePlugin : IBotSharpPlugin -{ - public string Id => "6a8473c0-04eb-4346-be32-24755ce5973d"; - - public string Name => "File"; - - public string Description => "Provides file analysis."; - - - public void RegisterDI(IServiceCollection services, IConfiguration config) - { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); - - if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage) - { - services.AddScoped(); - } - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs deleted file mode 100644 index be5f3180..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Microsoft.AspNetCore.StaticFiles; -using System.IO; - -namespace BotSharp.Core.Files.Services; - -public partial class BotSharpFileService -{ - public string GetDirectory(string conversationId) - { - var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments"); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - return dir; - } - - public (string, byte[]) GetFileInfoFromData(string data) - { - if (string.IsNullOrEmpty(data)) - { - return (string.Empty, new byte[0]); - } - - var typeStartIdx = data.IndexOf(':'); - var typeEndIdx = data.IndexOf(';'); - var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); - - var base64startIdx = data.IndexOf(','); - var base64Str = data.Substring(base64startIdx + 1); - - return (contentType, Convert.FromBase64String(base64Str)); - } - - public string GetFileContentType(string filePath) - { - string contentType; - var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) - { - contentType = string.Empty; - } - - return contentType; - } - - public byte[] GetFileBytes(string fileStorageUrl) - { - using var stream = File.OpenRead(fileStorageUrl); - var bytes = new byte[stream.Length]; - stream.Read(bytes, 0, (int)stream.Length); - return bytes; - } - - public bool SavefileToPath(string filePath, Stream stream) - { - using (var fileStream = new FileStream(filePath, FileMode.Create)) - { - stream.CopyTo(fileStream); - } - return true; - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs deleted file mode 100644 index daca7711..00000000 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs +++ /dev/null @@ -1,143 +0,0 @@ -using System.IO; - -namespace BotSharp.Core.Files.Services; - -public partial class BotSharpFileService -{ - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) - { - var content = string.Empty; - - if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty()) - { - return content; - } - - var guid = Guid.NewGuid().ToString(); - var sessionDir = GetSessionDirectory(guid); - if (!ExistDirectory(sessionDir)) - { - Directory.CreateDirectory(sessionDir); - } - - try - { - var pdfFiles = await DownloadFiles(sessionDir, files); - var images = await ConvertPdfToImages(pdfFiles); - if (images.IsNullOrEmpty()) return content; - - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", - model: model, modelId: modelId ?? "gpt-4", multiModal: true); - var message = await completion.GetChatCompletions(new Agent() - { - Id = Guid.Empty.ToString(), - }, new List - { - new RoleDialogModel(AgentRole.User, prompt) - { - Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() - } - }); - - content = message.Content; - return content; - } - catch (Exception ex) - { - _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}\r\n{ex.InnerException}"); - return content; - } - finally - { - Directory.Delete(sessionDir, true); - } - } - - #region Private methods - private string GetSessionDirectory(string id) - { - var dir = Path.Combine(_baseDir, SESSION_FOLDER, id); - return dir; - } - - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") - { - if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) - { - return Enumerable.Empty(); - } - - var locs = new List(); - foreach (var file in files) - { - try - { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, bytes) = GetFileInfoFromData(file.FileData); - } - - if (!bytes.IsNullOrEmpty()) - { - var guid = Guid.NewGuid().ToString(); - var fileDir = Path.Combine(dir, guid); - if (!ExistDirectory(fileDir)) - { - Directory.CreateDirectory(fileDir); - } - - var pdfDir = Path.Combine(fileDir, $"{guid}.{extension}"); - using (var fs = new FileStream(pdfDir, FileMode.Create)) - { - fs.Write(bytes, 0, bytes.Length); - fs.Close(); - locs.Add(pdfDir); - Thread.Sleep(100); - } - } - } - catch (Exception ex) - { - _logger.LogWarning($"Error when saving pdf file: {ex.Message}\r\n{ex.InnerException}"); - continue; - } - } - return locs; - } - - private async Task> ConvertPdfToImages(IEnumerable files) - { - var images = new List(); - var converter = GetPdf2ImageConverter(); - if (converter == null || files.IsNullOrEmpty()) - { - return images; - } - - foreach (var file in files) - { - try - { - var segs = file.Split(Path.DirectorySeparatorChar); - var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); - var folder = Path.Combine(dir, "screenshots"); - var urls = await converter.ConvertPdfToImages(file, folder); - images.AddRange(urls); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}\r\n{ex.InnerException}"); - continue; - } - } - return images; - } - #endregion -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs similarity index 84% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 619360d2..c9d35cb7 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -2,8 +2,24 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class FileInstructService { + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + { + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); + var message = await completion.GetChatCompletions(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, text) + { + Files = images?.ToList() ?? new List() + } + }); + return message; + } + public async Task GenerateImage(string? provider, string? model, string text) { var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); @@ -31,7 +47,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); - + stream.Close(); return message; } @@ -53,7 +69,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); - + stream.Close(); return message; } @@ -82,7 +98,7 @@ public partial class BotSharpFileService { Id = Guid.Empty.ToString() }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); - + imageStream.Close(); maskStream.Close(); return message; @@ -100,7 +116,7 @@ public partial class BotSharpFileService } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = GetFileInfoFromData(file.FileData); + (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); } return bytes; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs similarity index 76% rename from src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 1efbad6d..2aec257b 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,6 +1,8 @@ -namespace BotSharp.Plugin.TencentCos.Services; +using BotSharp.Abstraction.Files.Converters; -public partial class TencentCosService +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService { public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) { @@ -12,7 +14,9 @@ public partial class TencentCosService } var guid = Guid.NewGuid().ToString(); - var sessionDir = GetSessionDirectory(guid); + + var sessionDir = _fileStorage.BuildDirectory(SESSION_FOLDER, guid); + DeleteIfExistDirectory(sessionDir, true); try { @@ -32,9 +36,7 @@ public partial class TencentCosService Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList() } }); - - content = message.Content; - return content; + return message.Content; } catch (Exception ex) { @@ -43,17 +45,11 @@ public partial class TencentCosService } finally { - Directory.Delete(sessionDir, true); + _fileStorage.DeleteDirectory(sessionDir); } } #region Private methods - private string GetSessionDirectory(string id) - { - var dir = $"{SESSION_FOLDER}/{id}"; - return dir; - } - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) @@ -75,18 +71,17 @@ public partial class TencentCosService } else if (!string.IsNullOrEmpty(file.FileData)) { - (_, bytes) = GetFileInfoFromData(file.FileData); + (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); } if (!bytes.IsNullOrEmpty()) { var guid = Guid.NewGuid().ToString(); - var fileDir = $"{dir}/{guid}"; + var fileDir = _fileStorage.BuildDirectory(dir, guid); + DeleteIfExistDirectory(fileDir, true); - var pdfDir = $"{fileDir}/{guid}.{extension}"; - - - _cosClient.BucketClient.UploadBytes(pdfDir, bytes); + var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}"); + _fileStorage.SaveFileBytesToPath(pdfDir, bytes); locs.Add(pdfDir); } } @@ -102,7 +97,8 @@ public partial class TencentCosService private async Task> ConvertPdfToImages(IEnumerable files) { var images = new List(); - var converter = GetPdf2ImageConverter(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); if (converter == null || files.IsNullOrEmpty()) { return images; @@ -112,9 +108,8 @@ public partial class TencentCosService { try { - var segs = file.Split(Path.DirectorySeparatorChar); - var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1)); - var folder = Path.Combine(dir, "screenshots"); + var dir = _fileStorage.GetParentDir(file); + var folder = _fileStorage.BuildDirectory(dir, "screenshots"); var urls = await converter.ConvertPdfToImages(file, folder); images.AddRange(urls); } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs new file mode 100644 index 00000000..41e0becc --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -0,0 +1,138 @@ +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Templating; + +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService +{ + public async Task> SelectMessageFiles(string conversationId, SelectFileOptions options) + { + if (string.IsNullOrEmpty(conversationId)) + { + return Enumerable.Empty(); + } + + var convService = _services.GetRequiredService(); + var dialogs = convService.GetDialogHistory(fromBreakpoint: options.FromBreakpoint); + var messageIds = GetMessageIds(dialogs, options.Offset); + + var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes); + if (options.IncludeBotFile) + { + var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); + files = MergeMessageFiles(messageIds, files, botFiles); + } + + if (files.IsNullOrEmpty()) + { + return Enumerable.Empty(); + } + + return await SelectFiles(files, dialogs, options); + } + + private IEnumerable MergeMessageFiles(IEnumerable messageIds, IEnumerable userFiles, IEnumerable botFiles) + { + var files = new List(); + + if (messageIds.IsNullOrEmpty()) return files; + + foreach (var messageId in messageIds) + { + var users = userFiles.Where(x => x.MessageId == messageId).ToList(); + var bots = botFiles.Where(x => x.MessageId == messageId).ToList(); + + if (!users.IsNullOrEmpty()) files.AddRange(users); + if (!bots.IsNullOrEmpty()) files.AddRange(bots); + } + + return files; + } + + private async Task> SelectFiles(IEnumerable files, IEnumerable dialogs, SelectFileOptions options) + { + if (files.IsNullOrEmpty()) return new List(); + + var llmProviderService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + var db = _services.GetRequiredService(); + + try + { + var promptFiles = files.Select((x, idx) => + { + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}"; + }).ToList(); + + var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant; + var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "select_file_prompt"; + + var foundAgent = db.GetAgent(agentId); + var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template); + prompt = render.Render(prompt, new Dictionary + { + { "file_list", promptFiles } + }); + + var agent = new Agent + { + Id = foundAgent?.Id ?? BuiltInAgentId.UtilityAssistant, + Name = foundAgent?.Name ?? "Utility Assistant", + Instruction = prompt + }; + + var message = dialogs.LastOrDefault(); + var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : message?.Content; + if (message == null) + { + message = new RoleDialogModel(AgentRole.User, text); + } + else + { + message = RoleDialogModel.From(message, AgentRole.User, text); + } + + var providerName = options.Provider ?? "openai"; + var modelId = options?.ModelId ?? "gpt-4"; + var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName); + var model = llmProviderService.GetProviderModel(provider: provider, id: modelId); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); + + var response = await completion.GetChatCompletions(agent, new List { message }); + var content = response?.Content ?? string.Empty; + var selecteds = JsonSerializer.Deserialize(content, new JsonSerializerOptions + { + AllowTrailingCommas = true + }); + var fids = selecteds?.Selecteds ?? new List(); + return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when selecting files. {ex.Message}\r\n{ex.InnerException}"); + return new List(); + } + } + + private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + { + if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + + if (offset.HasValue && offset < 1) + { + offset = 1; + } + + var messageIds = new List(); + if (offset.HasValue) + { + messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + } + else + { + messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + } + + return messageIds; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs new file mode 100644 index 00000000..acd0ddaa --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -0,0 +1,32 @@ +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService : IFileInstructService +{ + private readonly IFileStorageService _fileStorage; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + private const string SESSION_FOLDER = "sessions"; + + public FileInstructService( + IFileStorageService fileStorate, + ILogger logger, + IServiceProvider services) + { + _fileStorage = fileStorate; + _logger = logger; + _services = services; + } + + private void DeleteIfExistDirectory(string? dir, bool createNew = false) + { + if (_fileStorage.ExistDirectory(dir)) + { + _fileStorage.DeleteDirectory(dir); + } + else if (createNew) + { + _fileStorage.CreateDirectory(dir); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs new file mode 100644 index 00000000..e90c7812 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs @@ -0,0 +1,28 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services +{ + public partial class LocalFileStorageService + { + public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + { + var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + var filePath = Path.Combine(dir, fileName); + if (File.Exists(filePath)) return; + using var file = File.Create(filePath); + using var input = data.ToStream(); + await input.CopyToAsync(file); + } + + public async Task RetrieveSpeechFileAsync(string conversationId, string fileName) + { + var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); + using var file = new FileStream(path, FileMode.Open, FileAccess.Read); + return await BinaryData.FromStreamAsync(file); + } + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs new file mode 100644 index 00000000..c49edbce --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -0,0 +1,73 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class LocalFileStorageService +{ + public string GetDirectory(string conversationId) + { + var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, "attachments"); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + return dir; + } + + public byte[] GetFileBytes(string fileStorageUrl) + { + using var stream = File.OpenRead(fileStorageUrl); + var bytes = new byte[stream.Length]; + stream.Read(bytes, 0, (int)stream.Length); + return bytes; + } + + public bool SaveFileStreamToPath(string filePath, Stream stream) + { + if (string.IsNullOrEmpty(filePath)) return false; + + using (var fileStream = new FileStream(filePath, FileMode.Create)) + { + stream.CopyTo(fileStream); + } + return true; + } + + public bool SaveFileBytesToPath(string filePath, byte[] bytes) + { + using (var fs = new FileStream(filePath, FileMode.Create)) + { + fs.Write(bytes, 0, bytes.Length); + fs.Flush(); + fs.Close(); + } + return true; + } + + public string GetParentDir(string dir, int level = 1) + { + var segs = dir.Split(Path.DirectorySeparatorChar); + return string.Join(Path.DirectorySeparatorChar, segs.SkipLast(level)); + } + + public string BuildDirectory(params string[] segments) + { + var relativePath = Path.Combine(segments); + return Path.Combine(_baseDir, relativePath); + } + + public void CreateDirectory(string dir) + { + Directory.CreateDirectory(dir); + } + + public bool ExistDirectory(string? dir) + { + return !string.IsNullOrEmpty(dir) && Directory.Exists(dir); + } + + public void DeleteDirectory(string dir) + { + Directory.Delete(dir, true); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs similarity index 63% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 1b98f0d9..8c2ed831 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -4,24 +4,22 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class LocalFileStorageService { - public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(conversations, offset); + var source = FileSourceType.User; var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); foreach (var messageId in messageIds) { - var dir = Path.Combine(pathPrefix, messageId, source); + var dir = Path.Combine(pathPrefix, messageId, FileSourceType.User); if (!ExistDirectory(dir)) continue; foreach (var subDir in Directory.GetDirectories(dir)) @@ -29,21 +27,18 @@ public partial class BotSharpFileService var file = Directory.GetFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = GetFileContentType(file); - if (contentTypes?.Contains(contentType) != true) continue; + var screenshots = await GetScreenshots(file, subDir, messageId, source); + if (screenshots.IsNullOrEmpty()) continue; - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; - - files.AddRange(foundFiles); + files.AddRange(screenshots); } } - return files; } + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, bool imageOnly = false) + string source, IEnumerable? contentTypes = null) { var files = new List(); if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; @@ -62,21 +57,21 @@ public partial class BotSharpFileService foreach (var file in Directory.GetFiles(subDir)) { - var contentType = GetFileContentType(file); - if (imageOnly && !_imageTypes.Contains(contentType)) + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) { continue; } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}", FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -84,7 +79,6 @@ public partial class BotSharpFileService } } } - return files; } @@ -141,7 +135,7 @@ public partial class BotSharpFileService try { - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = Path.Combine(dir, source, $"{i + 1}"); if (!ExistDirectory(subDir)) { @@ -180,7 +174,7 @@ public partial class BotSharpFileService { if (ExistDirectory(newDir)) { - Directory.Delete(newDir, true); + DeleteDirectory(newDir); } Directory.Move(prevDir, newDir); @@ -189,7 +183,7 @@ public partial class BotSharpFileService var botDir = Path.Combine(newDir, BOT_FILE_FOLDER); if (ExistDirectory(botDir)) { - Directory.Delete(botDir, true); + DeleteDirectory(botDir); } } } @@ -199,8 +193,8 @@ public partial class BotSharpFileService var dir = GetConversationFileDirectory(conversationId, messageId); if (!ExistDirectory(dir)) continue; + DeleteDirectory(dir); Thread.Sleep(100); - Directory.Delete(dir, true); } return true; @@ -215,7 +209,7 @@ public partial class BotSharpFileService var convDir = GetConversationDirectory(conversationId); if (!ExistDirectory(convDir)) continue; - Directory.Delete(convDir, true); + DeleteDirectory(convDir); } return true; } @@ -244,111 +238,28 @@ public partial class BotSharpFileService return dir; } - private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) { - if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + if (dialogs.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 0) + if (offset.HasValue && offset < 1) { - offset = MIN_OFFSET; - } - else if (offset > MAX_OFFSET) - { - offset = MAX_OFFSET; + offset = 1; } var messageIds = new List(); if (offset.HasValue) { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); } else { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); } return messageIds; } - - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) - { - var files = new List(); - - try - { - if (!_imageTypes.Contains(contentType) && includeScreenShot) - { - var screenShotDir = Path.Combine(fileDir, SCREENSHOT_FILE_FOLDER); - if (ExistDirectory(screenShotDir) && !Directory.GetFiles(screenShotDir).IsNullOrEmpty()) - { - foreach (var screenShot in Directory.GetFiles(screenShotDir)) - { - contentType = GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) - { - contentType = GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - } - else - { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - - return files; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); - return files; - } - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); @@ -364,8 +275,66 @@ public partial class BotSharpFileService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; + } + + private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + { + var files = new List(); + + try + { + var contentType = FileUtility.GetFileContentType(file); + var screenshotDir = Path.Combine(parentDir, SCREENSHOT_FILE_FOLDER); + + if (ExistDirectory(screenshotDir) && !Directory.GetFiles(screenshotDir).IsNullOrEmpty()) + { + foreach (var screenshot in Directory.GetFiles(screenshotDir)) + { + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = screenshot, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = image, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } + } + return files; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + return files; + } } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs similarity index 91% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs index fa99b13f..43ff9eed 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs @@ -2,7 +2,7 @@ using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService +public partial class LocalFileStorageService { public string GetUserAvatar() { @@ -30,11 +30,11 @@ public partial class BotSharpFileService if (Directory.Exists(dir)) { - Directory.Delete(dir, true); + DeleteDirectory(dir); } dir = GetUserAvatarDir(user?.Id, createNewDir: true); - var (_, bytes) = GetFileInfoFromData(file.FileData); + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); File.WriteAllBytes(Path.Combine(dir, file.FileName), bytes); return true; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs similarity index 61% rename from src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index b7c9a946..750803c4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -1,20 +1,14 @@ -using Microsoft.AspNetCore.StaticFiles; using System.IO; namespace BotSharp.Core.Files.Services; -public partial class BotSharpFileService : IBotSharpFileService +public partial class LocalFileStorageService : IFileStorageService { private readonly BotSharpDatabaseSettings _dbSettings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _imageTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg - }; private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; @@ -24,14 +18,12 @@ public partial class BotSharpFileService : IBotSharpFileService private const string USERS_FOLDER = "users"; private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; + private const string TEXT_TO_SPEECH_FOLDER = "speeches"; - private const int MIN_OFFSET = 1; - private const int MAX_OFFSET = 5; - - public BotSharpFileService( + public LocalFileStorageService( BotSharpDatabaseSettings dbSettings, IUserIdentity user, - ILogger logger, + ILogger logger, IServiceProvider services) { _dbSettings = dbSettings; @@ -40,11 +32,4 @@ public partial class BotSharpFileService : IBotSharpFileService _services = services; _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository); } - - #region Private methods - private bool ExistDirectory(string? dir) - { - return !string.IsNullOrEmpty(dir) && Directory.Exists(dir); - } - #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index dd6b99af..f6a12188 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -115,6 +115,22 @@ public class CompletionProvider return completer; } + public static ITextToSpeech GetTextToSpeech( + IServiceProvider services, + string provider, + string model) + { + var completions = services.GetServices(); + var completer = completions.FirstOrDefault(x => x.Provider == provider); + if (completer == null) + { + var logger = services.GetRequiredService>(); + logger.LogError($"Can't resolve text2speech provider by {provider}"); + } + completer.SetModelName(model); + return completer; + } + private static (string, string) GetProviderAndModel(IServiceProvider services, string? provider = null, string? model = null, diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs index fe3d9007..cbde5f44 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs @@ -31,6 +31,7 @@ public class SettingService : ISettingService var plugins = pluginService.GetPlugins(_services); var plugin = plugins.First(x => x.Module.Settings.Name == settingName); var instance = plugin.Module.GetNewSettingsInstance(); + _config.Bind(settingName, instance); if (mask) { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index c0c59142..bb137279 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -1,6 +1,6 @@ +using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing.Models; using System.IO; -using System.Threading; namespace BotSharp.Core.Repository { @@ -37,7 +37,7 @@ namespace BotSharp.Core.Repository UpdateAgentRoutingRules(agent.Id, agent.RoutingRules); break; case AgentField.Instruction: - UpdateAgentInstruction(agent.Id, agent.Instruction); + UpdateAgentInstructions(agent.Id, agent.Instruction, agent.ChannelInstructions); break; case AgentField.Function: UpdateAgentFunctions(agent.Id, agent.Functions); @@ -175,17 +175,30 @@ namespace BotSharp.Core.Repository File.WriteAllText(agentFile, json); } - private void UpdateAgentInstruction(string agentId, string instruction) + private void UpdateAgentInstructions(string agentId, string instruction, List channelInstructions) { if (string.IsNullOrWhiteSpace(instruction)) return; var (agent, agentFile) = GetAgentFromFile(agentId); if (agent == null) return; - var instructionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); + var instructionDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_INSTRUCTIONS_FOLDER); + DeleteBeforeCreateDirectory(instructionDir); - File.WriteAllText(instructionFile, instruction); + // Save default instructions + var instructionFile = Path.Combine(instructionDir, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); + File.WriteAllText(instructionFile, instruction ?? string.Empty); + Thread.Sleep(100); + + // Save channel instructions + foreach (var ci in channelInstructions) + { + if (string.IsNullOrWhiteSpace(ci.Channel)) continue; + + var file = Path.Combine(instructionDir, $"{AGENT_INSTRUCTION_FILE}.{ci.Channel}.{_agentSettings.TemplateFormat}"); + File.WriteAllText(file, ci.Instruction ?? string.Empty); + Thread.Sleep(100); + } } private void UpdateAgentFunctions(string agentId, List inputFunctions) @@ -195,14 +208,8 @@ namespace BotSharp.Core.Repository var (agent, agentFile) = GetAgentFromFile(agentId); if (agent == null) return; - var functionDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, - agentId, AGENT_FUNCTIONS_FOLDER); - - if (Directory.Exists(functionDir)) - { - Directory.Delete(functionDir, true); - } - Directory.CreateDirectory(functionDir); + var functionDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_FUNCTIONS_FOLDER); + DeleteBeforeCreateDirectory(functionDir); foreach (var func in inputFunctions) { @@ -211,7 +218,7 @@ namespace BotSharp.Core.Repository var text = JsonSerializer.Serialize(func, _options); var file = Path.Combine(functionDir, $"{func.Name}.json"); File.WriteAllText(file, text); - Thread.Sleep(200); + Thread.Sleep(100); } } @@ -223,16 +230,7 @@ namespace BotSharp.Core.Repository if (agent == null) return; var templateDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_TEMPLATES_FOLDER); - - if (!Directory.Exists(templateDir)) - { - Directory.CreateDirectory(templateDir); - } - - foreach (var file in Directory.GetFiles(templateDir)) - { - File.Delete(file); - } + DeleteBeforeCreateDirectory(templateDir); foreach (var template in templates) { @@ -249,15 +247,7 @@ namespace BotSharp.Core.Repository if (agent == null) return; var responseDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_RESPONSES_FOLDER); - if (!Directory.Exists(responseDir)) - { - Directory.CreateDirectory(responseDir); - } - - foreach (var file in Directory.GetFiles(responseDir)) - { - File.Delete(file); - } + DeleteBeforeCreateDirectory(responseDir); for (int i = 0; i < responses.Count; i++) { @@ -308,7 +298,7 @@ namespace BotSharp.Core.Repository var json = JsonSerializer.Serialize(agent, _options); File.WriteAllText(agentFile, json); - UpdateAgentInstruction(inputAgent.Id, inputAgent.Instruction); + UpdateAgentInstructions(inputAgent.Id, inputAgent.Instruction, inputAgent.ChannelInstructions); UpdateAgentResponses(inputAgent.Id, inputAgent.Responses); UpdateAgentTemplates(inputAgent.Id, inputAgent.Templates); UpdateAgentFunctions(inputAgent.Id, inputAgent.Functions); @@ -348,12 +338,13 @@ namespace BotSharp.Core.Repository var record = JsonSerializer.Deserialize(json, _options); if (record == null) return null; - var instruction = FetchInstruction(dir); + var (defaultInstruction, channelInstructions) = FetchInstructions(dir); var functions = FetchFunctions(dir); var samples = FetchSamples(dir); var templates = FetchTemplates(dir); var responses = FetchResponses(dir); - return record.SetInstruction(instruction) + return record.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetFunctions(functions) .SetTemplates(templates) .SetSamples(samples) @@ -451,13 +442,9 @@ namespace BotSharp.Core.Repository return true; } - public void BulkInsertAgents(List agents) - { - } + public void BulkInsertAgents(List agents) { } - public void BulkInsertUserAgents(List userAgents) - { - } + public void BulkInsertUserAgents(List userAgents) { } public bool DeleteAgents() { diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 78bd5cc0..7e5b4d9d 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -34,6 +34,7 @@ public partial class FileRepository : IBotSharpRepository private const string AGENT_TASK_PREFIX = "#metadata"; private const string AGENT_TASK_SUFFIX = "/metadata"; private const string TRANSLATION_MEMORY_FILE = "memory.json"; + private const string AGENT_INSTRUCTIONS_FOLDER = "instructions"; private const string AGENT_FUNCTIONS_FOLDER = "functions"; private const string AGENT_TEMPLATES_FOLDER = "templates"; private const string AGENT_RESPONSES_FOLDER = "responses"; @@ -123,7 +124,9 @@ public partial class FileRepository : IBotSharpRepository var agent = JsonSerializer.Deserialize(json, _options); if (agent != null) { - agent = agent.SetInstruction(FetchInstruction(d)) + var (defaultInstruction, channelInstructions) = FetchInstructions(d); + agent = agent.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetFunctions(FetchFunctions(d)) .SetTemplates(FetchTemplates(d)) .SetResponses(FetchResponses(d)) @@ -165,6 +168,17 @@ public partial class FileRepository : IBotSharpRepository #region Private methods + private void DeleteBeforeCreateDirectory(string dir) + { + if (string.IsNullOrWhiteSpace(dir)) return; + + if (Directory.Exists(dir)) + { + Directory.Delete(dir, true); + } + Directory.CreateDirectory(dir); + } + private string GetAgentDataDir(string agentId) { var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId); @@ -186,13 +200,46 @@ public partial class FileRepository : IBotSharpRepository return (agent, agentFile); } - private string? FetchInstruction(string fileDir) + private (string, List) FetchInstructions(string fileDir) { - var file = Path.Combine(fileDir, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}"); - if (!File.Exists(file)) return null; + var defaultInstruction = string.Empty; + var channelInstructions = new List(); - var instruction = File.ReadAllText(file); - return instruction; + var instructionDir = Path.Combine(fileDir, AGENT_INSTRUCTIONS_FOLDER); + if (!Directory.Exists(instructionDir)) + { + return (defaultInstruction, channelInstructions); + } + + foreach (var file in Directory.GetFiles(instructionDir)) + { + var extension = Path.GetExtension(file).Substring(1); + if (!extension.IsEqualTo(_agentSettings.TemplateFormat)) + { + continue; + } + + var segments = Path.GetFileName(file).Split(".", StringSplitOptions.RemoveEmptyEntries); + if (segments.IsNullOrEmpty() || !segments[0].IsEqualTo(AGENT_INSTRUCTION_FILE)) + { + continue; + } + + if (segments.Length == 2) + { + defaultInstruction = File.ReadAllText(file); + } + else if (segments.Length == 3) + { + var item = new ChannelInstruction + { + Channel = segments[1], + Instruction = File.ReadAllText(file) + }; + channelInstructions.Add(item); + } + } + return (defaultInstruction, channelInstructions); } private List FetchFunctions(string fileDir) @@ -298,13 +345,14 @@ public partial class FileRepository : IBotSharpRepository var agent = JsonSerializer.Deserialize(agentJson, _options); if (agent == null) return null; - var instruction = FetchInstruction(agentDir); + var (defaultInstruction, channelInstructions) = FetchInstructions(agentDir); var functions = FetchFunctions(agentDir); var samples = FetchSamples(agentDir); var templates = FetchTemplates(agentDir); var responses = FetchResponses(agentDir); - return agent.SetInstruction(instruction) + return agent.SetInstruction(defaultInstruction) + .SetChannelInstructions(channelInstructions) .SetFunctions(functions) .SetTemplates(templates) .SetSamples(samples) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index 4a686c6b..e8431cce 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -26,6 +26,10 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler new ParameterPropertyDef("user_goal_agent", "agent who can acheive user initial task, must align with user_goal_description.", required: true), + new ParameterPropertyDef("conversation_end", + "user is ending the conversation.", + type: "boolean", + required: true), new ParameterPropertyDef("is_new_task", "whether the user is requesting a new task that is different from the previous topic.", type: "boolean") diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index bf833c8a..2fa38215 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -205,11 +205,12 @@ public class UserService : IUserService var config = _services.GetRequiredService(); var issuer = config["Jwt:Issuer"]; var audience = config["Jwt:Audience"]; + var expireInMinutes = int.Parse(config["Jwt:ExpireInMinutes"] ?? "120"); var key = Encoding.ASCII.GetBytes(config["Jwt:Key"]); var tokenDescriptor = new SecurityTokenDescriptor { Subject = new ClaimsIdentity(claims), - Expires = DateTime.UtcNow.AddHours(2), + Expires = DateTime.UtcNow.AddMinutes(expireInMinutes), Issuer = issuer, Audience = audience, SigningCredentials = new SigningCredentials(new SymmetricSecurityKey(key), diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 88293d97..fa3e94d2 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -29,6 +29,7 @@ global using BotSharp.Abstraction.Translation; global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Utilities; global using BotSharp.Abstraction.Translation.Attributes; global using BotSharp.Abstraction.Messaging.Enums; global using BotSharp.Core.Repository; @@ -37,4 +38,4 @@ global using BotSharp.Core.Agents.Services; global using BotSharp.Core.Conversations.Services; global using BotSharp.Core.Infrastructures; global using BotSharp.Core.Users.Services; -global using Aspects.Cache; +global using Aspects.Cache; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json index 536ca340..9fc0c389 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -7,5 +7,11 @@ "updatedDateTime": "2024-01-15T14:39:32Z", "iconUrl": "/images/logo.png", "disabled": false, - "isPublic": true + "isPublic": true, + "llmConfig": { + "is_inherit": false, + "provider": "openai", + "model": "gpt-4o-mini", + "max_recursion_depth": 3 + } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/instructions/instruction.liquid diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid similarity index 97% rename from src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid index f4295baa..57f9895c 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_file_prompt.liquid @@ -2,7 +2,7 @@ Please take a look at the files in the [FILES] section from the conversation and ** Ensure the output is only in JSON format without any additional text. ** If no files are selected, you must output an empty list []. -** You may need to look at the file_name as a reference to find the correct file id. +** You may need to look at the file_name as a reference to find the correct file id or ids. Here is the JSON format to use: { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instructions/instruction.liquid similarity index 100% rename from src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instruction.liquid rename to src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/instructions/instruction.liquid diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index f2bbc2eb..f033e31c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Users.Enums; namespace BotSharp.OpenAPI.Controllers; @@ -28,17 +27,24 @@ public class AgentController : ControllerBase [HttpGet("/agent/{id}")] public async Task GetAgent([FromRoute] string id) { - var agents = await GetAgents(new AgentFilter + var pagedAgents = await _agentService.GetAgents(new AgentFilter { AgentIds = new List { id } - }, useHook: true); + }); - var targetAgent = agents.Items.FirstOrDefault(); - if (targetAgent == null) return null; + var foundAgent = pagedAgents.Items.FirstOrDefault(); + if (foundAgent == null) return null; + + await _agentService.InheritAgent(foundAgent); + var targetAgent = AgentViewModel.FromAgent(foundAgent); + var agentSetting = _services.GetRequiredService(); + targetAgent.IsHost = targetAgent.Id == agentSetting.HostAgentId; var redirectAgentIds = targetAgent.RoutingRules .Where(x => !string.IsNullOrEmpty(x.RedirectTo)) - .Select(x => x.RedirectTo).ToList(); + .Select(x => x.RedirectTo) + .ToList(); + var redirectAgents = await _agentService.GetAgents(new AgentFilter { AgentIds = redirectAgentIds @@ -65,39 +71,16 @@ public class AgentController : ControllerBase } [HttpGet("/agents")] - public async Task> GetAgents([FromQuery] AgentFilter filter, [FromQuery] bool useHook = false) + public async Task> GetAgents([FromQuery] AgentFilter filter) { var agentSetting = _services.GetRequiredService(); var pagedAgents = await _agentService.GetAgents(filter); - - var items = new List(); - var agents = new List(); - if (useHook) - { - // prerender agent - foreach (var agent in pagedAgents.Items) - { - var renderedAgent = await _agentService.LoadAgent(agent.Id); - items.Add(renderedAgent); - } - - // Set IsHost - agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList(); - foreach (var agent in agents) - { - agent.IsHost = agentSetting.HostAgentId == agent.Id; - } - } - else - { - items = pagedAgents.Items.ToList(); - agents = items.Select(x => AgentViewModel.FromAgent(x)).ToList(); - } + var agents = pagedAgents?.Items?.Select(x => AgentViewModel.FromAgent(x))?.ToList() ?? new List(); return new PagedItems { Items = agents, - Count = pagedAgents.Count + Count = pagedAgents?.Count ?? 0 }; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 22649d72..89cdc6e9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Constants; using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; @@ -80,10 +81,10 @@ public class ConversationController : ControllerBase var userService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var messageIds = history.Select(x => x.MessageId).Distinct().ToList(); - var fileMessages = fileService.GetMessagesWithFile(conversationId, messageIds); + var fileMessages = fileStorage.GetMessagesWithFile(conversationId, messageIds); var dialogs = new List(); foreach (var message in history) @@ -348,15 +349,15 @@ public class ConversationController : ControllerBase { if (files != null && files.Length > 0) { - var fileService = _services.GetRequiredService(); - var dir = fileService.GetDirectory(conversationId); + var fileStorage = _services.GetRequiredService(); + var dir = fileStorage.GetDirectory(conversationId); foreach (var file in files) { // Save the file, process it, etc. var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"'); var filePath = Path.Combine(dir, fileName); - fileService.SavefileToPath(filePath, file.OpenReadStream()); + fileStorage.SaveFileStreamToPath(filePath, file.OpenReadStream()); } return Ok(new { message = "File uploaded successfully." }); @@ -371,25 +372,25 @@ public class ConversationController : ControllerBase var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); var conv = await convService.GetConversationRecordOrCreateNew(agentId); - var fileService = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var messageId = Guid.NewGuid().ToString(); - var isSaved = fileService.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); + var isSaved = fileStorage.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); return isSaved ? messageId : string.Empty; } [HttpGet("/conversation/{conversationId}/files/{messageId}/{source}")] public IEnumerable GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source) { - var fileService = _services.GetRequiredService(); - var files = fileService.GetMessageFiles(conversationId, new List { messageId }, source, imageOnly: false); + var fileStorage = _services.GetRequiredService(); + var files = fileStorage.GetMessageFiles(conversationId, new List { messageId }, source); return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List(); } [HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")] public IActionResult GetMessageFile([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source, [FromRoute] string index, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); - var file = fileService.GetMessageFile(conversationId, messageId, source, index, fileName); + var fileStorage = _services.GetRequiredService(); + var file = fileStorage.GetMessageFile(conversationId, messageId, source, index, fileName); if (string.IsNullOrEmpty(file)) { return NotFound(); @@ -413,7 +414,9 @@ public class ConversationController : ControllerBase using Stream stream = System.IO.File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read); var bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); - return File(bytes, "application/octet-stream", Path.GetFileName(file)); + var fileExtension = Path.GetExtension(file).ToLower(); + var enableRangeProcessing = FileConstants.AudioExtensions.Contains(fileExtension); + return File(bytes, "application/octet-stream", Path.GetFileName(file), enableRangeProcessing: enableRangeProcessing); } private async Task OnChunkReceived(HttpResponse response, ChatResponseModel message) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 2189af2f..48fdafe1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; -using NetTopologySuite.IO; namespace BotSharp.OpenAPI.Controllers; @@ -87,18 +86,8 @@ public class InstructModeController : ControllerBase try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", - model: input.Model ?? "gpt-4o", multiModal: true); - var message = await completion.GetChatCompletions(new Agent() - { - Id = Guid.Empty.ToString(), - }, new List - { - new RoleDialogModel(AgentRole.User, input.Text) - { - Files = input.Files - } - }); + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); return message.Content; } catch (Exception ex) @@ -114,14 +103,14 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-generation")] public async Task ImageGeneration([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); try { - var message = await fileService.GenerateImage(input.Provider, input.Model, input.Text); + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.GenerateImage(input.Provider, input.Model, input.Text); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -140,7 +129,6 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-variation")] public async Task ImageVariation([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -152,7 +140,9 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } - var message = await fileService.VaryImage(input.Provider, input.Model, image); + + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.VaryImage(input.Provider, input.Model, image); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -169,7 +159,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-edit")] public async Task ImageEdit([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); + var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -181,7 +171,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } - var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -198,7 +188,7 @@ public class InstructModeController : ControllerBase [HttpPost("/instruct/image-mask-edit")] public async Task ImageMaskEdit([FromBody] IncomingMessageModel input) { - var fileService = _services.GetRequiredService(); + var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); var imageViewModel = new ImageGenerationViewModel(); @@ -211,7 +201,7 @@ public class InstructModeController : ControllerBase { return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" }; } - var message = await fileService.EditImage(input.Provider, input.Model, input.Text, image, mask); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -236,8 +226,8 @@ public class InstructModeController : ControllerBase try { - var fileService = _services.GetRequiredService(); - var content = await fileService.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadPdf(input.Provider, input.Model, input.ModelId, input.Text, input.Files); viewModel.Content = content; return viewModel; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 6ad463b3..90773dfd 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -1,6 +1,7 @@ +using BotSharp.Abstraction.Graph.Models; using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.Knowledges.Settings; -using Microsoft.AspNetCore.Http; +using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.OpenAPI.ViewModels.Knowledges; namespace BotSharp.OpenAPI.Controllers; @@ -17,77 +18,107 @@ public class KnowledgeBaseController : ControllerBase _services = services; } - [HttpGet("/knowledge/{agentId}")] - public async Task> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) + [HttpGet("knowledge/vector/collections")] + public async Task> GetVectorCollections() { - return await _knowledgeService.GetAnswer(new KnowledgeRetrievalModel - { - AgentId = agentId, - Question = question - }); + return await _knowledgeService.GetVectorCollections(); } - [HttpPost("/knowledge-base/upload")] - public async Task UploadKnowledge(IFormFile file, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum) + [HttpPost("/knowledge/vector/{collection}/search")] + public async Task> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); + var options = new VectorSearchOptions + { + Fields = request.Fields, + Limit = request.Limit ?? 5, + Confidence = request.Confidence ?? 0.5f, + WithVector = request.WithVector + }; + + var results = await _knowledgeService.SearchVectorKnowledge(request.Text, collection, options); + return results.Select(x => VectorKnowledgeViewModel.From(x)).ToList(); + } + + [HttpPost("/knowledge/vector/{collection}/data")] + public async Task> GetVectorCollectionData([FromRoute] string collection, [FromBody] VectorFilter filter) + { + var data = await _knowledgeService.GetVectorCollectionData(collection, filter); + var items = data.Items?.Select(x => VectorKnowledgeViewModel.From(x))? + .ToList() ?? new List(); + + return new StringIdPagedItems + { + Count = data.Count, + NextId = data.NextId, + Items = items + }; + } + + [HttpDelete("/knowledge/vector/{collection}/data/{id}")] + public async Task DeleteVectorCollectionData([FromRoute] string collection, [FromRoute] string id) + { + return await _knowledgeService.DeleteVectorCollectionData(collection, id); + } + + [HttpPost("/knowledge/vector/{collection}/upload")] + public async Task UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) + { + var setttings = _services.GetRequiredService(); + var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); var filePath = Path.GetTempFileName(); - using (var stream = System.IO.File.Create(filePath)) + using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) { await file.CopyToAsync(stream); + await stream.FlushAsync(); } var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); - - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.EmbedKnowledge(new KnowledgeCreationModel + await _knowledgeService.FeedVectorKnowledge(collection, new KnowledgeCreationModel { Content = content }); + System.IO.File.Delete(filePath); return Ok(new { count = 1, file.Length }); } - [HttpPost("/knowledge/{agentId}")] - public async Task FeedKnowledge([FromRoute] string agentId, List files, [FromQuery] int? startPageNum, [FromQuery] int? endPageNum, [FromQuery] bool? paddleModel) + [HttpPost("/knowledge/graph/search")] + public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) { - var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().First(x => x.GetType().FullName.EndsWith(setttings.Pdf2TextConverter)); - long size = files.Sum(f => f.Length); - - foreach (var formFile in files) + var options = new GraphSearchOptions { - var filePath = Path.GetTempFileName(); + Method = request.Method + }; + var result = await _knowledgeService.SearchGraphKnowledge(request.Query, options); + return new GraphKnowledgeViewModel + { + Result = result.Result + }; + } - using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) - { - await formFile.CopyToAsync(stream); - await stream.FlushAsync(); // Ensure all data is written to the file - } + [HttpPost("/knowledge/search")] + public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) + { + var vectorOptions = new VectorSearchOptions + { + Fields = request.VectorParams.Fields, + Limit = request.VectorParams.Limit ?? 5, + Confidence = request.VectorParams.Confidence ?? 0.5f, + WithVector = request.VectorParams.WithVector + }; - var content = await textConverter.ConvertPdfToText(filePath, startPageNum, endPageNum); + var graphOptions = new GraphSearchOptions + { + Method = request.GraphParams.Method + }; - // Process uploaded files - // Don't rely on or trust the FileName property without validation. - - // Add FeedWithMetaData - await _knowledgeService.Feed(new KnowledgeFeedModel - { - AgentId = agentId, - Content = content - }); - - // Delete the temp file after processing to clean up - System.IO.File.Delete(filePath); - } - - return Ok(new { count = files.Count, size }); + var result = await _knowledgeService.SearchKnowledge(request.Text, request.VectorParams.Collection, vectorOptions, graphOptions); + return new KnowledgeSearchViewModel + { + VectorResult = result?.VectorResult?.Select(x => VectorKnowledgeViewModel.From(x)), + GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null + }; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 966f51c9..a7d4c449 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -137,15 +137,15 @@ public class UserController : ControllerBase [HttpPost("/user/avatar")] public bool UploadUserAvatar([FromBody] BotSharpFile file) { - var fileService = _services.GetRequiredService(); - return fileService.SaveUserAvatar(file); + var fileStorage = _services.GetRequiredService(); + return fileStorage.SaveUserAvatar(file); } [HttpGet("/user/avatar")] public IActionResult GetUserAvatar() { - var fileService = _services.GetRequiredService(); - var file = fileService.GetUserAvatar(); + var fileStorage = _services.GetRequiredService(); + var file = fileStorage.GetUserAvatar(); if (string.IsNullOrEmpty(file)) { return NotFound(); @@ -158,8 +158,8 @@ public class UserController : ControllerBase #region Private methods private FileContentResult BuildFileResult(string file) { - var fileService = _services.GetRequiredService(); - var bytes = fileService.GetFileBytes(file); + var fileStorage = _services.GetRequiredService(); + var bytes = fileStorage.GetFileBytes(file); return File(bytes, "application/octet-stream", Path.GetFileName(file)); } #endregion diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs index c0fa8320..78f75833 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing.Models; -using BotSharp.Core.Infrastructures; namespace BotSharp.OpenAPI.ViewModels.Agents; @@ -16,21 +15,26 @@ public class AgentCreationModel /// public string Instruction { get; set; } = string.Empty; + /// + /// + /// + public List ChannelInstructions { get; set; } = new(); + /// /// LLM extensible Instructions in addition to the default Instructions /// - public List Templates { get; set; } = new List(); + public List Templates { get; set; } = new(); /// /// LLM callable function definition /// - public List Functions { get; set; } = new List(); + public List Functions { get; set; } = new(); /// /// Response template /// - public List Responses { get; set; } = new List(); - public List Samples { get; set; } = new List(); + public List Responses { get; set; } = new(); + public List Samples { get; set; } = new(); public bool IsPublic { get; set; } @@ -43,9 +47,9 @@ public class AgentCreationModel /// /// Combine different Agents together to form a Profile. /// - public List Profiles { get; set; } = new List(); - public List Utilities { get; set; } = new List(); - public List RoutingRules { get; set; } = new List(); + public List Profiles { get; set; } = new(); + public List Utilities { get; set; } = new(); + public List RoutingRules { get; set; } = new(); public AgentLlmConfig? LlmConfig { get; set; } public Agent ToAgent() @@ -55,6 +59,7 @@ public class AgentCreationModel Name = Name, Description = Description, Instruction = Instruction, + ChannelInstructions = ChannelInstructions, Templates = Templates, Functions = Functions, Responses = Responses, @@ -64,9 +69,7 @@ public class AgentCreationModel Type = Type, Disabled = Disabled, Profiles = Profiles, - RoutingRules = RoutingRules? - .Select(x => RoutingRuleUpdateModel.ToDomainElement(x))? - .ToList() ?? new List(), + RoutingRules = RoutingRules?.Select(x => RoutingRuleUpdateModel.ToDomainElement(x))?.ToList() ?? new List(), LlmConfig = LlmConfig }; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs index 96cc7278..d64530c6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs @@ -15,6 +15,12 @@ public class AgentUpdateModel /// public string Instruction { get; set; } = string.Empty; + /// + /// Channel instructions + /// + [JsonPropertyName("channel_instructions")] + public List? ChannelInstructions { get; set; } + /// /// Templates /// @@ -39,11 +45,11 @@ public class AgentUpdateModel /// Routes /// public List? Responses { get; set; } + [JsonPropertyName("is_public")] - public bool IsPublic { get; set; } - [JsonPropertyName("allow_routing")] + [JsonPropertyName("allow_routing")] public bool AllowRouting { get; set; } public bool Disabled { get; set; } @@ -52,8 +58,8 @@ public class AgentUpdateModel /// Profile by channel /// public List? Profiles { get; set; } - [JsonPropertyName("routing_rules")] + [JsonPropertyName("routing_rules")] public List? RoutingRules { get; set; } [JsonPropertyName("llm_config")] @@ -69,10 +75,9 @@ public class AgentUpdateModel Disabled = Disabled, Type = Type, Profiles = Profiles ?? new List(), - RoutingRules = RoutingRules? - .Select(x => RoutingRuleUpdateModel.ToDomainElement(x))? - .ToList() ?? new List(), + RoutingRules = RoutingRules?.Select(x => RoutingRuleUpdateModel.ToDomainElement(x))?.ToList() ?? new List(), Instruction = Instruction ?? string.Empty, + ChannelInstructions = ChannelInstructions ?? new List(), Templates = Templates ?? new List(), Functions = Functions ?? new List(), Responses = Responses ?? new List(), diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs index a88b3c72..5c23c4d4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs @@ -13,6 +13,9 @@ public class AgentViewModel public string Description { get; set; } public string Type { get; set; } = AgentType.Task; public string Instruction { get; set; } + + [JsonPropertyName("channel_instructions")] + public List ChannelInstructions { get; set; } public List Templates { get; set; } public List Functions { get; set; } public List Responses { get; set; } @@ -60,6 +63,7 @@ public class AgentViewModel Description = agent.Description, Type = agent.Type, Instruction = agent.Instruction, + ChannelInstructions = agent.ChannelInstructions, Templates = agent.Templates, Functions = agent.Functions, Responses = agent.Responses, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs index 131a9baf..787ab147 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs @@ -10,8 +10,8 @@ public class MessageFileViewModel [JsonPropertyName("file_name")] public string FileName { get; set; } - [JsonPropertyName("file_type")] - public string FileType { get; set; } + [JsonPropertyName("file_extension")] + public string FileExtension { get; set; } [JsonPropertyName("content_type")] public string ContentType { get; set; } @@ -30,7 +30,7 @@ public class MessageFileViewModel { FileUrl = model.FileUrl, FileName = model.FileName, - FileType = model.FileType, + FileExtension = model.FileExtension, ContentType = model.ContentType, FileSource = model.FileSource }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs new file mode 100644 index 00000000..360fce1c --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/GraphKnowledgeViewModel.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class GraphKnowledgeViewModel +{ + [JsonPropertyName("result")] + public string Result { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs new file mode 100644 index 00000000..f862f184 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class KnowledgeSearchViewModel +{ + [JsonPropertyName("vector_result")] + public IEnumerable? VectorResult { get; set; } + + [JsonPropertyName("graph_result")] + public GraphKnowledgeViewModel? GraphResult { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs new file mode 100644 index 00000000..119e729e --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchGraphKnowledgeRequest.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class SearchGraphKnowledgeRequest +{ + [JsonPropertyName("query")] + public string Query { get; set; } = string.Empty; + + [JsonPropertyName("method")] + public string Method { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs new file mode 100644 index 00000000..20f80fec --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs @@ -0,0 +1,43 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class SearchKnowledgeRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + #region Vector + [JsonPropertyName("vector_params")] + public VectorParam VectorParams { get; set; } + #endregion + + #region Graph + [JsonPropertyName("graph_params")] + public GraphParam GraphParams { get; set; } + #endregion +} + +public class VectorParam +{ + [JsonPropertyName("collection")] + public string Collection { get; set; } + + [JsonPropertyName("fields")] + public IEnumerable? Fields { get; set; } + + [JsonPropertyName("limit")] + public int? Limit { get; set; } = 5; + + [JsonPropertyName("confidence")] + public float? Confidence { get; set; } = 0.5f; + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} + +public class GraphParam +{ + [JsonPropertyName("method")] + public string Method { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs new file mode 100644 index 00000000..3f11a486 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchVectorKnowledgeRequest.cs @@ -0,0 +1,21 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class SearchVectorKnowledgeRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("fields")] + public IEnumerable? Fields { get; set; } + + [JsonPropertyName("limit")] + public int? Limit { get; set; } = 5; + + [JsonPropertyName("confidence")] + public float? Confidence { get; set; } = 0.5f; + + [JsonPropertyName("with_vector")] + public bool WithVector { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs new file mode 100644 index 00000000..dcdf2e57 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs @@ -0,0 +1,33 @@ +using BotSharp.Abstraction.VectorStorage.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Knowledges; + +public class VectorKnowledgeViewModel +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("data")] + public IDictionary Data { get; set; } + + [JsonPropertyName("score")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public double? Score { get; set; } + + [JsonPropertyName("vector")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public float[]? Vector { get; set; } + + + public static VectorKnowledgeViewModel From(VectorSearchResult result) + { + return new VectorKnowledgeViewModel + { + Id = result.Id, + Data = result.Data, + Score = result.Score, + Vector = result.Vector + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 16e2841a..af82b037 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; namespace BotSharp.Plugin.AzureOpenAI.Providers.Chat; @@ -196,8 +197,8 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -262,23 +263,23 @@ public class ChatCompletionProvider : IChatCompletion { foreach (var file in message.Files) { - if (!string.IsNullOrEmpty(file.FileUrl)) + if (!string.IsNullOrEmpty(file.FileData)) { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj index f5926a53..3aa65e97 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj @@ -28,9 +28,6 @@ PreserveNewest - - PreserveNewest - PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs index 121638e7..d0f11466 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs @@ -1,17 +1,7 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Messaging.Models.RichContent.Template; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.EmailHandler.Models; using BotSharp.Plugin.EmailHandler.Providers; using MailKit; -using MailKit.Net.Imap; -using MailKit.Search; -using MailKit.Security; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using MimeKit; namespace BotSharp.Plugin.EmailReader.Functions; @@ -31,13 +21,14 @@ public class HandleEmailReaderFn : IFunctionCallback private readonly IConversationStateService _state; private readonly IEmailReader _emailProvider; - public HandleEmailReaderFn(IServiceProvider services, - ILogger logger, - IHttpContextAccessor context, - BotSharpOptions options, - EmailReaderSettings emailPluginSettings, - IConversationStateService state, - IEmailReader emailProvider) + public HandleEmailReaderFn( + IServiceProvider services, + ILogger logger, + IHttpContextAccessor context, + BotSharpOptions options, + EmailReaderSettings emailPluginSettings, + IConversationStateService state, + IEmailReader emailProvider) { _services = services; _logger = logger; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 45dc65be..b4f9a6e4 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -1,7 +1,6 @@ using MailKit.Net.Smtp; using MailKit.Security; using MimeKit; -using System.IO; namespace BotSharp.Plugin.EmailHandler.Functions; @@ -65,7 +64,7 @@ public class HandleEmailSenderFn : IFunctionCallback catch (Exception ex) { var msg = $"Failed to send the email. {ex.Message}"; - _logger.LogError($"{msg}\n(Error: {ex.Message})"); + _logger.LogError($"{msg}\n(Error: {ex.Message}\r\n{ex.InnerException})"); message.Content = msg; return false; } @@ -74,57 +73,11 @@ public class HandleEmailSenderFn : IFunctionCallback private async Task> GetConversationFiles() { var convService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var conversationId = convService.ConversationId; - var dialogs = convService.GetDialogHistory(fromBreakpoint: false); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var userFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User); - var botFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot); - return await SelectFiles(userFiles.Concat(botFiles), dialogs); - } - private async Task> SelectFiles(IEnumerable files, List dialogs) - { - if (files.IsNullOrEmpty()) return new List(); - - var llmProviderService = _services.GetRequiredService(); - var render = _services.GetRequiredService(); - var db = _services.GetRequiredService(); - - try - { - var promptFiles = files.Select((x, idx) => - { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; - }).ToList(); - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt"); - prompt = render.Render(prompt, new Dictionary - { - { "file_list", promptFiles } - }); - - var agent = new Agent - { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant", - Instruction = prompt - }; - - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var latest = dialogs.LastOrDefault(); - var response = await completion.GetChatCompletions(agent, new List { latest }); - var content = response?.Content ?? string.Empty; - var selecteds = JsonSerializer.Deserialize(content); - var fids = selecteds?.Selecteds ?? new List(); - return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}"); - return new List(); - } + var fileInstruct = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(conversationId, new SelectFileOptions { IncludeBotFile = true }); + return selecteds; } private void BuildEmailAttachments(BodyBuilder builder, IEnumerable files) @@ -135,10 +88,9 @@ public class HandleEmailSenderFn : IFunctionCallback { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - using var fs = File.OpenRead(file.FileStorageUrl); - var binary = BinaryData.FromStream(fs); - builder.Attachments.Add($"{file.FileName}.{file.FileType}", binary.ToArray(), ContentType.Parse(file.ContentType)); - fs.Close(); + var fileStorage = _services.GetRequiredService(); + var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); + builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 78d77ac4..9b097fed 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -47,9 +47,6 @@ PreserveNewest - - PreserveNewest - diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 63938c5e..759f94b6 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Templating; using System.IO; namespace BotSharp.Plugin.FileHandler.Functions; @@ -28,7 +27,7 @@ public class EditImageFn : IFunctionCallback Init(message); SetImageOptions(); - var image = await SelectConversationImage(descrpition); + var image = await SelectImage(descrpition); var response = await GetImageEditGeneration(message, descrpition, image); message.Content = response; return true; @@ -48,64 +47,15 @@ public class EditImageFn : IFunctionCallback state.SetState("image_count", "1"); } - private async Task SelectConversationImage(string? description) + private async Task SelectImage(string? description) { - var convService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(); - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var userImages = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true); - return await SelectImage(userImages, dialogs.LastOrDefault(), description); - } - - private async Task SelectImage(IEnumerable images, RoleDialogModel message, string? description) - { - if (images.IsNullOrEmpty()) return null; - - var llmProviderService = _services.GetRequiredService(); - var render = _services.GetRequiredService(); - var db = _services.GetRequiredService(); - - try + var fileInstruct = _services.GetRequiredService(); + var selecteds = await fileInstruct.SelectMessageFiles(_conversationId, new SelectFileOptions { - var promptImages = images.Where(x => x.ContentType == MediaTypeNames.Image.Png).Select((x, idx) => - { - return $"id: {idx + 1}, image_name: {x.FileName}.{x.FileType}"; - }).ToList(); - - if (promptImages.IsNullOrEmpty()) return null; - - var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_edit_image_prompt"); - prompt = render.Render(prompt, new Dictionary - { - { "image_list", promptImages } - }); - - var agent = new Agent - { - Id = BuiltInAgentId.UtilityAssistant, - Name = "Utility Assistant", - Instruction = prompt - }; - - var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); - var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - - var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; - var dialog = RoleDialogModel.From(message, AgentRole.User, text); - - var response = await completion.GetChatCompletions(agent, new List { dialog }); - var content = response?.Content ?? string.Empty; - var selected = JsonSerializer.Deserialize(content); - var fid = selected?.Selected ?? -1; - return fid > 0 ? images.Where((x, idx) => idx == fid - 1).FirstOrDefault() : null; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting the image edit response. {ex.Message}\r\n{ex.InnerException}"); - return null; - } + Description = description, + ContentTypes = new List { MediaTypeNames.Image.Png } + }); + return selecteds?.FirstOrDefault(); } private async Task GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image) @@ -126,12 +76,16 @@ public class EditImageFn : IFunctionCallback Name = "Utility Assistant" }; - using var stream = File.OpenRead(image.FileStorageUrl); + var fileStorage = _services.GetRequiredService(); + var fileBytes = fileStorage.GetFileBytes(image.FileStorageUrl); + using var stream = new MemoryStream(); + stream.Write(fileBytes); + stream.Position = 0; var result = await completion.GetImageEdits(agent, dialog, stream, image.FileName ?? string.Empty); stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); - return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted."; + return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted."; } catch (Exception ex) { @@ -154,7 +108,7 @@ public class EditImageFn : IFunctionCallback } }; - var fileService = _services.GetRequiredService(); - fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + var fileStorage = _services.GetRequiredService(); + fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4d53f880..4102ff7b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -83,7 +83,7 @@ public class GenerateImageFn : IFunctionCallback FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" }).ToList(); - var fileService = _services.GetRequiredService(); - fileService.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + var fileStorage = _services.GetRequiredService(); + fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 66f69353..a415207e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -8,12 +8,6 @@ public class ReadImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly IEnumerable _imageContentTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg, - }; - public ReadImageFn( IServiceProvider services, ILogger logger) @@ -29,7 +23,7 @@ public class ReadImageFn : IFunctionCallback var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); var fileAgent = new Agent { @@ -44,15 +38,20 @@ public class ReadImageFn : IFunctionCallback return true; } - private async Task> AssembleFiles(string conversationId, List dialogs) + private List AssembleFiles(string conversationId, List dialogs) { if (dialogs.IsNullOrEmpty()) { return new List(); } - var fileService = _services.GetRequiredService(); - var images = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); + var fileStorage = _services.GetRequiredService(); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List + { + MediaTypeNames.Image.Png, + MediaTypeNames.Image.Jpeg + }); foreach (var dialog in dialogs) { @@ -62,6 +61,7 @@ public class ReadImageFn : IFunctionCallback dialog.Files = found.Select(x => new BotSharpFile { ContentType = x.ContentType, + FileUrl = x.FileUrl, FileStorageUrl = x.FileStorageUrl }).ToList(); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index 85c2afbc..e2b465c8 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -50,17 +50,21 @@ public class ReadPdfFn : IFunctionCallback return new List(); } - var fileService = _services.GetRequiredService(); - var files = await fileService.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); + var fileStorage = _services.GetRequiredService(); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + + if (screenshots.IsNullOrEmpty()) return dialogs; foreach (var dialog in dialogs) { - var found = files.Where(x => x.MessageId == dialog.MessageId).ToList(); + var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile { ContentType = x.ContentType, + FileUrl = x.FileUrl, FileStorageUrl = x.FileStorageUrl }).ToList(); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid deleted file mode 100644 index 9e67faad..00000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid +++ /dev/null @@ -1,41 +0,0 @@ -Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user. - -** Ensure the output is only in JSON format without any additional text. -** You may need to look at the image_name as a reference to find the correct image id. - -Here is the JSON format to use: -{ - "selected_id": the id selected from the [IMAGES] section -} - - -Suppose there are four images: - -id: 1, image_name: example_image_a.png -id: 2, image_name: example_image_b.png -id: 3, image_name: example_image_c.png -id: 4, image_name: example_image_d.png - -===== -Example 1: -USER: I want to add a dog in the first file. -OUTPUT: { "selected_id": 1 } - -Example 2: -USER: Add a coffee cup in the second image I uploaded. -OUTPUT: { "selected_id": 2 } - -Example 3: -USER: Please remove the left tree in the third and the first images. -OUTPUT: { "selected_id": 3 } - -Example 4: -USER: Circle the head of the dog in example_image_b.png. -OUTPUT: { "selected_id": 4 } -===== - - -[IMAGES] -{% for image in image_list -%} -{{ image }}{{ "\r\n" }} -{%- endfor %} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Graph/BotSharp.Plugin.Graph.csproj b/src/Plugins/BotSharp.Plugin.Graph/BotSharp.Plugin.Graph.csproj new file mode 100644 index 00000000..c03f1163 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/BotSharp.Plugin.Graph.csproj @@ -0,0 +1,17 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs new file mode 100644 index 00000000..9e0ceaee --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -0,0 +1,95 @@ +using BotSharp.Plugin.Graph.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using System.Net.Http; +using System.Net.Mime; +using System.Text; +using System.Text.Json; + +namespace BotSharp.Plugin.Graph; + +public class GraphDb : IGraphDb +{ + private readonly IServiceProvider _services; + private readonly IHttpContextAccessor _context; + private readonly GraphDbSettings _settings; + private readonly ILogger _logger; + + private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + WriteIndented = true, + AllowTrailingCommas = true, + }; + + public GraphDb( + IServiceProvider services, + IHttpContextAccessor context, + ILogger logger, + GraphDbSettings settings) + { + _services = services; + _context = context; + _logger = logger; + _settings = settings; + } + + public string Name => "Default"; + + public async Task Search(string query, GraphSearchOptions options) + { + if (string.IsNullOrWhiteSpace(_settings.BaseUrl)) + { + return new GraphSearchData(); + } + + var url = $"{_settings.BaseUrl}/query"; + var request = new GraphQueryRequest + { + Query = query, + Method = options.Method + }; + return await SendRequest(url, request); + } + + private async Task SendRequest(string url, GraphQueryRequest request) + { + var result = new GraphSearchData(); + var http = _services.GetRequiredService(); + + using (var client = http.CreateClient()) + { + var uri = new Uri(url); + try + { + var data = JsonSerializer.Serialize(request, _jsonOptions); + var message = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = uri, + Content = new StringContent(data, Encoding.UTF8, MediaTypeNames.Application.Json) + }; + + AddHeaders(client); + var rawResponse = await client.SendAsync(message); + rawResponse.EnsureSuccessStatusCode(); + + var responseStr = await rawResponse.Content.ReadAsStringAsync(); + result = JsonSerializer.Deserialize(responseStr, _jsonOptions); + return result; + } + catch (Exception ex) + { + _logger.LogError($"Error when fetching Lessen GLM response (Endpoint: {url}). {ex.Message}\r\n{ex.InnerException}"); + return result; + } + } + } + + private void AddHeaders(HttpClient client) + { + client.DefaultRequestHeaders.Add("Authorization", $"{_context.HttpContext.Request.Headers["Authorization"]}"); + client.DefaultRequestHeaders.Add("Origin", $"{_context.HttpContext.Request.Headers["Origin"]}"); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs new file mode 100644 index 00000000..89351398 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.Graph; + +public class GraphDbSettings +{ + public string BaseUrl { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs new file mode 100644 index 00000000..3d5a2187 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphPlugin.cs @@ -0,0 +1,24 @@ +using BotSharp.Abstraction.Plugins; +using BotSharp.Abstraction.Settings; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace BotSharp.Plugin.Graph; + +public class GraphPlugin : IBotSharpPlugin +{ + public string Id => "74497c25-5e8d-4ee9-b6a8-ce8fe4dabea9"; + public string Name => "Graph"; + public string Description => "Graph Database"; + public string IconUrl => "https://www.microsoft.com/en-us/research/uploads/prodnew/2024/06/GraphRag2024-BlogHeroFeature-1400x788-1.png"; + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("Graph"); + }); + + services.AddScoped(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Graph/Models/GraphQueryRequest.cs b/src/Plugins/BotSharp.Plugin.Graph/Models/GraphQueryRequest.cs new file mode 100644 index 00000000..ac520642 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/Models/GraphQueryRequest.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Graph.Models; + +public class GraphQueryRequest +{ + [JsonPropertyName("query")] + public string Query { get; set; } + + [JsonPropertyName("method")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Method { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Graph/Using.cs b/src/Plugins/BotSharp.Plugin.Graph/Using.cs new file mode 100644 index 00000000..02c0e3d4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Graph/Using.cs @@ -0,0 +1,7 @@ +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Threading.Tasks; +global using Microsoft.Extensions.Logging; +global using BotSharp.Abstraction.Graph; +global using BotSharp.Abstraction.Graph.Models; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj b/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj index 2911dc2b..8664c654 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj @@ -15,14 +15,14 @@ - + PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instruction.liquid b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instructions/instruction.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instruction.liquid rename to src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instructions/instruction.liquid diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index ecf15fe2..42663445 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -20,7 +20,7 @@ - + @@ -34,7 +34,7 @@ PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs index 521dc221..bac82d2e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/ConfirmKnowledgePersistenceFn.cs @@ -1,9 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Messaging.Enums; -using BotSharp.Abstraction.Messaging.Models.RichContent.Template; -using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.Messaging; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class ConfirmKnowledgePersistenceFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 3eaa72f5..52e45e7e 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback @@ -20,22 +17,18 @@ public class KnowledgeRetrievalFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); - var vector = await embedding.GetVectorsAsync(new List + var vector = await embedding.GetVectorAsync(args.Question); + var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp; + var knowledges = await vectorDb.Search(collectionName, vector, new List { KnowledgePayloadName.Answer }); + + if (!knowledges.IsNullOrEmpty()) { - args.Question - }); - - var vectorDb = _services.GetRequiredService(); - - var id = Utilities.HashTextMd5(args.Question); - var knowledges = await vectorDb.Search("lessen", vector[0], "answer"); - - if (knowledges.Count > 0) - { - message.Content = string.Join("\r\n\r\n=====\r\n", knowledges); + var answers = knowledges.Select(x => x.Data[KnowledgePayloadName.Answer]).ToList(); + message.Content = string.Join("\r\n\r\n=====\r\n", answers); } else { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 23b49b94..237f7385 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class MemorizeKnowledgeFn : IFunctionCallback @@ -20,28 +17,27 @@ public class MemorizeKnowledgeFn : IFunctionCallback { var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); - var embedding = _services.GetServices() - .First(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + embedding.SetModelName(_settings.TextEmbedding.Model); var vector = await embedding.GetVectorsAsync(new List { args.Question }); - var vectorDb = _services.GetRequiredService(); + var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var collectionName = !string.IsNullOrWhiteSpace(_settings.DefaultCollection) ? _settings.DefaultCollection : KnowledgeCollectionName.BotSharp; + await vectorDb.CreateCollection(collectionName, vector[0].Length); - await vectorDb.CreateCollection("lessen", vector[0].Length); - - var id = Utilities.HashTextMd5(args.Question); - var result = await vectorDb.Upsert("lessen", id, vector[0], + var id = Guid.NewGuid().ToString(); + var result = await vectorDb.Upsert(collectionName, id, vector[0], args.Question, new Dictionary { - { "answer", args.Answer } + { KnowledgePayloadName.Answer, args.Answer } }); message.Content = result ? "Saved to my brain" : "I forgot it"; - return true; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs index 2eebf75c..f8296520 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseAgentHook.cs @@ -1,9 +1,3 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; -using BotSharp.Plugin.KnowledgeBase.Enum; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseAgentHook : AgentHookBase, IAgentHook diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs index fd163cbf..cd428136 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Hooks/KnowledgeBaseUtilityHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.KnowledgeBase.Enum; - namespace BotSharp.Plugin.KnowledgeBase.Hooks; public class KnowledgeBaseUtilityHook : IAgentUtilityHook diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs index 4710faea..df97bc34 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVecDbPlugin.cs @@ -9,6 +9,6 @@ public class MemVecDbPlugin : IBotSharpPlugin public string Description => "Store text embedding, search similar text from memory."; public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddSingleton(); + services.AddSingleton(); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs new file mode 100644 index 00000000..27863dd3 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -0,0 +1,72 @@ +using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Plugin.KnowledgeBase.Utilities; +using Tensorflow.NumPy; + +namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; + +public class MemoryVectorDb : IVectorDb +{ + private readonly Dictionary _collections = new Dictionary(); + private readonly Dictionary> _vectors = new Dictionary>(); + + + public string Name => "MemoryVector"; + + public async Task CreateCollection(string collectionName, int dim) + { + _collections[collectionName] = dim; + _vectors[collectionName] = new List(); + } + + public async Task> GetCollections() + { + return _collections.Select(x => x.Key).ToList(); + } + + public Task> GetCollectionData(string collectionName, VectorFilter filter) + { + throw new NotImplementedException(); + } + + public async Task> Search(string collectionName, float[] vector, + IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) + { + if (!_vectors.ContainsKey(collectionName)) + { + return new List(); + } + + var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); + // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); + + var results = np.argsort(similarities).ToArray() + .Reverse() + .Take(limit) + .Select(i => new VectorCollectionData + { + Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, + Score = similarities[i], + Vector = withVector ? _vectors[collectionName][i].Vector : null, + }) + .ToList(); + + return await Task.FromResult(results); + } + + public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) + { + _vectors[collectionName].Add(new VecRecord + { + Id = id, + Vector = vector, + Text = text + }); + + return true; + } + + public async Task DeleteCollectionData(string collectionName, string id) + { + return await Task.FromResult(false); + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs new file mode 100644 index 00000000..69a56519 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -0,0 +1,28 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) + { + var index = 0; + var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + { + Size = 1024, + Conjunction = 32, + SplitByWord = true, + }); + + var db = GetVectorDb(); + var textEmbedding = GetTextEmbedding(); + + await db.CreateCollection(collectionName, textEmbedding.Dimension); + foreach (var line in lines) + { + var vec = await textEmbedding.GetVectorAsync(line); + var id = Guid.NewGuid().ToString(); + await db.Upsert(collectionName, id, vec, line); + index++; + Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n"); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs new file mode 100644 index 00000000..e7ed1b1b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task DeleteVectorCollectionData(string collectionName, string id) + { + try + { + var db = GetVectorDb(); + return await db.DeleteCollectionData(collectionName, id); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting knowledge collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs new file mode 100644 index 00000000..4e001954 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -0,0 +1,106 @@ +using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.KnowledgeBase.Services; + +public partial class KnowledgeService +{ + public async Task> GetVectorCollections() + { + try + { + var db = GetVectorDb(); + return await db.GetCollections(); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collections. {ex.Message}\r\n{ex.InnerException}"); + return Enumerable.Empty(); + } + } + + public async Task> GetVectorCollectionData(string collectionName, VectorFilter filter) + { + try + { + var db = GetVectorDb(); + var pagedResult = await db.GetCollectionData(collectionName, filter); + return new StringIdPagedItems + { + Count = pagedResult.Count, + Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)), + NextId = pagedResult.NextId, + }; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return new StringIdPagedItems(); + } + } + + public async Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options) + { + try + { + var textEmbedding = GetTextEmbedding(); + var vector = await textEmbedding.GetVectorAsync(query); + + // Vector search + var db = GetVectorDb(); + var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector); + + var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList(); + return results; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when searching knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return new List(); + } + } + + public async Task SearchGraphKnowledge(string query, GraphSearchOptions options) + { + try + { + var db = GetGraphDb(); + var found = await db.Search(query, options); + return new GraphSearchResult + { + Result = found.Result + }; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when searching graph {query}. {ex.Message}\r\n{ex.InnerException}"); + return new GraphSearchResult(); + } + } + + public async Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions) + { + try + { + var textEmbedding = GetTextEmbedding(); + var vector = await textEmbedding.GetVectorAsync(query); + + var vectorDb = GetVectorDb(); + var vectorRes = await vectorDb.Search(collectionName, vector, vectorOptions.Fields, limit: vectorOptions.Limit ?? 5, + confidence: vectorOptions.Confidence ?? 0.5f, withVector: vectorOptions.WithVector); + + var graphDb = GetGraphDb(); + var graphRes = await graphDb.Search(query, graphOptions); + return new KnowledgeSearchResult + { + VectorResult = vectorRes.Select(x => VectorSearchResult.CopyFrom(x)), + GraphResult = new GraphSearchResult { Result = graphRes.Result } + }; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when searching knowledge (vector collection: {collectionName}) {query}. {ex.Message}\r\n{ex.InnerException}"); + return new KnowledgeSearchResult(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 7ac5cf4c..1f96042b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -5,114 +5,39 @@ public partial class KnowledgeService : IKnowledgeService private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; private readonly ITextChopper _textChopper; + private readonly ILogger _logger; - public KnowledgeService(IServiceProvider services, + public KnowledgeService( + IServiceProvider services, KnowledgeBaseSettings settings, - ITextChopper textChopper) + ITextChopper textChopper, + ILogger logger) { _services = services; _settings = settings; _textChopper = textChopper; + _logger = logger; } - public async Task EmbedKnowledge(KnowledgeCreationModel knowledge) + private IVectorDb GetVectorDb() { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection("shared", textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert("shared", idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task Feed(KnowledgeFeedModel knowledge) - { - var idStart = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption - { - Size = 1024, - Conjunction = 32, - SplitByWord = true, - }); - - var db = GetVectorDb(); - var textEmbedding = GetTextEmbedding(); - - await db.CreateCollection(knowledge.AgentId, textEmbedding.Dimension); - foreach (var line in lines) - { - var vec = await textEmbedding.GetVectorAsync(line); - await db.Upsert(knowledge.AgentId, idStart.ToString(), vec, line); - idStart++; - Console.WriteLine($"Saved vector {idStart}/{lines.Count}: {line}\n"); - } - } - - public async Task GetKnowledges(KnowledgeRetrievalModel retrievalModel) - { - var textEmbedding = GetTextEmbedding(); - var vector = await textEmbedding.GetVectorAsync(retrievalModel.Question); - - // Vector search - var db = GetVectorDb(); - var result = await db.Search("shared", vector, "answer", limit: 10); - - // Restore - return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}")); - } - - public async Task> GetAnswer(KnowledgeRetrievalModel retrievalModel) - { - // Restore - var prompt = await GetKnowledges(retrievalModel); - - var sb = new StringBuilder(prompt); - sb.AppendLine(); - sb.AppendLine(); - sb.AppendLine("------"); - sb.AppendLine("Answer question based on the given information above. Keep your answers concise. Please response with paragraph number, cite sources and reasoning in JSON format, if multiple paragraphs are found, put them in a JSON array. make sure the paragraph number is real. If you don't know the answer just output empty."); - sb.AppendLine("[" + JsonSerializer.Serialize(new RetrievedResult()) + "]"); - sb.AppendLine("------"); - sb.AppendLine($"QUESTION: \"{retrievalModel.Question}\""); - sb.AppendLine("Which paragraphs are relevant in order to answer the above question?"); - sb.AppendLine("ANSWER: "); - prompt = sb.ToString().Trim(); - - var completion = await GetTextCompletion().GetCompletion(prompt, Guid.Empty.ToString(), Guid.Empty.ToString()); - return JsonSerializer.Deserialize>(completion); - } - - public IVectorDb GetVectorDb() - { - var db = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.VectorDb)); + var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); return db; } - public ITextEmbedding GetTextEmbedding() + private IGraphDb GetGraphDb() { - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding)); - return embedding; + var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.GraphDb); + return db; } - public ITextCompletion GetTextCompletion() + private ITextEmbedding GetTextEmbedding() { - var textCompletion = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextCompletion)); - return textCompletion; + var embedding = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.TextEmbedding.Provider); + if (embedding != null) + { + embedding.SetModelName(_settings.TextEmbedding.Model); + } + return embedding; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs deleted file mode 100644 index eccfed78..00000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.i.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; - -public partial class KnowledgeService -{ - public async Task> CollectChunkedKnowledge() - { - throw new NotImplementedException(); - } - - public async Task EmbedKnowledge(List chunks) - { - throw new NotImplementedException(); - } -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs index 17706c13..09401f20 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs @@ -5,6 +5,8 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public class PigPdf2TextConverter : IPdf2TextConverter { + public string Name => "Pig"; + public Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { // since PdfDocument.Open is not async, we dont need to make this method async diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs index 96dc021d..88c77641 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs @@ -16,7 +16,7 @@ public class TextChopperService : ITextChopper var chunks = new List(); var words = content.Split(' ') - .Where(x => !string.IsNullOrEmpty(x)) + .Where(x => !string.IsNullOrWhiteSpace(x)) .ToList(); var chunk = ""; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 60db13a6..296d56b5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -15,9 +15,20 @@ global using BotSharp.Abstraction.Users; global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.Abstraction.Agents.Settings; -global using BotSharp.Abstraction.Conversations.Settings; +global using BotSharp.Abstraction.Graph; global using BotSharp.Abstraction.Knowledges.Settings; +global using BotSharp.Abstraction.Knowledges.Enums; global using BotSharp.Abstraction.VectorStorage; global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.Abstraction.MLTasks; -global using BotSharp.Plugin.KnowledgeBase.Services; \ No newline at end of file +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Abstraction.Messaging.Models.RichContent.Template; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Plugin.KnowledgeBase.Services; +global using BotSharp.Plugin.KnowledgeBase.Enum; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs similarity index 53% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs index c46ccdf5..f0538af0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemVectorDatabase.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs @@ -1,55 +1,12 @@ +using BotSharp.Plugin.KnowledgeBase.MemVecDb; using Tensorflow.NumPy; using static Tensorflow.Binding; -namespace BotSharp.Plugin.KnowledgeBase.MemVecDb; +namespace BotSharp.Plugin.KnowledgeBase.Utilities; -public class MemVectorDatabase : IVectorDb +public static class VectorUtility { - private readonly Dictionary _collections = new Dictionary(); - private readonly Dictionary> _vectors = new Dictionary>(); - public async Task CreateCollection(string collectionName, int dim) - { - _collections[collectionName] = dim; - _vectors[collectionName] = new List(); - } - - public async Task> GetCollections() - { - return _collections.Select(x => x.Key).ToList(); - } - - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) - { - if (!_vectors.ContainsKey(collectionName)) - { - return new List(); - } - - var similarities = CalCosineSimilarity(vector, _vectors[collectionName]); - // var similarities2 = CalEuclideanDistance(vector, _vectors[collectionName]); - - var texts = np.argsort(similarities).ToArray() - .Reverse() - .Take(limit) - .Select(i => _vectors[collectionName][i].Text) - .ToList(); - - return texts; - } - - public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload = null) - { - _vectors[collectionName].Add(new VecRecord - { - Id = id, - Vector = vector, - Text = text - }); - - return true; - } - - private float[] CalEuclideanDistance(float[] vec, List records) + public static float[] CalEuclideanDistance(float[] vec, List records) { var a = np.zeros((records.Count, vec.Length), np.float32); var b = np.zeros((records.Count, vec.Length), np.float32); @@ -64,7 +21,7 @@ public class MemVectorDatabase : IVectorDb return c.ToArray(); } - public NDArray CalCosineSimilarity(float[] vec, List records) + public static NDArray CalCosineSimilarity(float[] vec, List records) { var recordsArray = np.zeros((records.Count, records[0].Vector.Length), dtype: np.float32); @@ -85,7 +42,7 @@ public class MemVectorDatabase : IVectorDb return simiMatix; } - public (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) + public static (int, float)[] CalCosineSimilarityTopK(float[] vec, List records, int topK = 10, float filterProb = 0.75f) { var simiMatix = CalCosineSimilarity(vec, records); @@ -108,7 +65,7 @@ public class MemVectorDatabase : IVectorDb return resIndex.ToArray(); } - public (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) + private static (NDArray, NDArray) SafeNormalize(NDArray x, double eps = 2.223E-15) { var squaredX = np.sum(np.multiply(x, x), axis: 1); var normX = np.sqrt(squaredX); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/instruction.liquid b/src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/instructions/instruction.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/instruction.liquid rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/instructions/instruction.liquid diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 48e2be17..f5b35eaf 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; +using BotSharp.Abstraction.VectorStorage.Models; using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -7,17 +9,25 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { + public string Name => "Faiss"; + public Task CreateCollection(string collectionName, int dim) { throw new NotImplementedException(); } - public Task> GetCollections() + public Task> GetCollectionData(string collectionName, VectorFilter filter) { throw new NotImplementedException(); } - public Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f) + public Task> GetCollections() + { + throw new NotImplementedException(); + } + + public Task> Search(string collectionName, float[] vector, + IEnumerable? fields, int limit = 10, float confidence = 0.5f, bool withVector = false) { throw new NotImplementedException(); } @@ -26,4 +36,9 @@ public class FaissDb : IVectorDb { throw new NotImplementedException(); } + + public Task DeleteCollectionData(string collectionName, string id) + { + throw new NotImplementedException(); + } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs index 4f1c5194..baa0729f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs @@ -8,6 +8,7 @@ public class AgentDocument : MongoBase public string? InheritAgentId { get; set; } public string? IconUrl { get; set; } public string Instruction { get; set; } + public List ChannelInstructions { get; set; } public List Templates { get; set; } public List Functions { get; set; } public List Responses { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs new file mode 100644 index 00000000..884c638c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs @@ -0,0 +1,27 @@ +using BotSharp.Abstraction.Agents.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +public class ChannelInstructionMongoElement +{ + public string Channel { get; set; } + public string Instruction { get; set; } + + public static ChannelInstructionMongoElement ToMongoElement(ChannelInstruction instruction) + { + return new ChannelInstructionMongoElement + { + Channel = instruction.Channel, + Instruction = instruction.Instruction + }; + } + + public static ChannelInstruction ToDomainElement(ChannelInstructionMongoElement instruction) + { + return new ChannelInstruction + { + Channel = instruction.Channel, + Instruction = instruction.Instruction + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs index dac77d69..6f72c517 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs @@ -8,6 +8,7 @@ public class FunctionDefMongoElement { public string Name { get; set; } public string Description { get; set; } + public List? Channels { get; set; } public string? VisibilityExpression { get; set; } public string? Impact { get; set; } public FunctionParametersDefMongoElement Parameters { get; set; } = new FunctionParametersDefMongoElement(); @@ -23,6 +24,7 @@ public class FunctionDefMongoElement { Name = function.Name, Description = function.Description, + Channels = function.Channels, VisibilityExpression = function.VisibilityExpression, Impact = function.Impact, Parameters = new FunctionParametersDefMongoElement @@ -34,19 +36,20 @@ public class FunctionDefMongoElement }; } - public static FunctionDef ToDomainElement(FunctionDefMongoElement mongoFunction) + public static FunctionDef ToDomainElement(FunctionDefMongoElement function) { return new FunctionDef { - Name = mongoFunction.Name, - Description = mongoFunction.Description, - VisibilityExpression = mongoFunction.VisibilityExpression, - Impact = mongoFunction.Impact, + Name = function.Name, + Description = function.Description, + Channels = function.Channels, + VisibilityExpression = function.VisibilityExpression, + Impact = function.Impact, Parameters = new FunctionParametersDef { - Type = mongoFunction.Parameters.Type, - Properties = JsonSerializer.Deserialize(mongoFunction.Parameters.Properties.IfNullOrEmptyAs("{}")), - Required = mongoFunction.Parameters.Required, + Type = function.Parameters.Type, + Properties = JsonSerializer.Deserialize(function.Parameters.Properties.IfNullOrEmptyAs("{}")), + Required = function.Parameters.Required, } }; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 60dcd6c3..49c8616e 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -38,7 +38,7 @@ public partial class MongoRepository UpdateAgentRoutingRules(agent.Id, agent.RoutingRules); break; case AgentField.Instruction: - UpdateAgentInstruction(agent.Id, agent.Instruction); + UpdateAgentInstructions(agent.Id, agent.Instruction, agent.ChannelInstructions); break; case AgentField.Function: UpdateAgentFunctions(agent.Id, agent.Functions); @@ -156,13 +156,17 @@ public partial class MongoRepository _dc.Agents.UpdateOne(filter, update); } - private void UpdateAgentInstruction(string agentId, string instruction) + private void UpdateAgentInstructions(string agentId, string instruction, List? channelInstructions) { - if (string.IsNullOrWhiteSpace(instruction)) return; + if (string.IsNullOrWhiteSpace(agentId)) return; + + var instructionElements = channelInstructions?.Select(x => ChannelInstructionMongoElement.ToMongoElement(x))? + .ToList() ?? new List(); var filter = Builders.Filter.Eq(x => x.Id, agentId); var update = Builders.Update .Set(x => x.Instruction, instruction) + .Set(x => x.ChannelInstructions, instructionElements) .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.Agents.UpdateOne(filter, update); @@ -253,6 +257,7 @@ public partial class MongoRepository .Set(x => x.Profiles, agent.Profiles) .Set(x => x.RoutingRules, agent.RoutingRules.Select(r => RoutingRuleMongoElement.ToMongoElement(r)).ToList()) .Set(x => x.Instruction, agent.Instruction) + .Set(x => x.ChannelInstructions, agent.ChannelInstructions.Select(i => ChannelInstructionMongoElement.ToMongoElement(i)).ToList()) .Set(x => x.Templates, agent.Templates.Select(t => AgentTemplateMongoElement.ToMongoElement(t)).ToList()) .Set(x => x.Functions, agent.Functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList()) .Set(x => x.Responses, agent.Responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList()) @@ -373,6 +378,9 @@ public partial class MongoRepository IconUrl = x.IconUrl, Description = x.Description, Instruction = x.Instruction, + ChannelInstructions = x.ChannelInstructions? + .Select(i => ChannelInstructionMongoElement.ToMongoElement(i))? + .ToList() ?? new List(), Templates = x.Templates? .Select(t => AgentTemplateMongoElement.ToMongoElement(t))? .ToList() ?? new List(), @@ -463,6 +471,9 @@ public partial class MongoRepository IconUrl = agentDoc.IconUrl, Description = agentDoc.Description, Instruction = agentDoc.Instruction, + ChannelInstructions = !agentDoc.ChannelInstructions.IsNullOrEmpty() ? agentDoc.ChannelInstructions + .Select(i => ChannelInstructionMongoElement.ToDomainElement(i)) + .ToList() : new List(), Templates = !agentDoc.Templates.IsNullOrEmpty() ? agentDoc.Templates .Select(t => AgentTemplateMongoElement.ToDomainElement(t)) .ToList() : new List(), @@ -472,6 +483,10 @@ public partial class MongoRepository Responses = !agentDoc.Responses.IsNullOrEmpty() ? agentDoc.Responses .Select(r => AgentResponseMongoElement.ToDomainElement(r)) .ToList() : new List(), + RoutingRules = !agentDoc.RoutingRules.IsNullOrEmpty() ? agentDoc.RoutingRules + .Select(r => RoutingRuleMongoElement.ToDomainElement(agentDoc.Id, agentDoc.Name, r)) + .ToList() : new List(), + LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agentDoc.LlmConfig), Samples = agentDoc.Samples ?? new List(), Utilities = agentDoc.Utilities ?? new List(), IsPublic = agentDoc.IsPublic, @@ -479,10 +494,6 @@ public partial class MongoRepository Type = agentDoc.Type, InheritAgentId = agentDoc.InheritAgentId, Profiles = agentDoc.Profiles, - RoutingRules = !agentDoc.RoutingRules.IsNullOrEmpty() ? agentDoc.RoutingRules - .Select(r => RoutingRuleMongoElement.ToDomainElement(agentDoc.Id, agentDoc.Name, r)) - .ToList() : new List(), - LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agentDoc.LlmConfig) }; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs index 4a4542ed..e2ffbb0e 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs @@ -37,9 +37,12 @@ public partial class MongoRepository { Id = !string.IsNullOrEmpty(x.Id) ? x.Id : Guid.NewGuid().ToString(), Name = x.Name, + IconUrl = x.IconUrl, Description = x.Description, Instruction = x.Instruction, - IconUrl = x.IconUrl, + ChannelInstructions = x.ChannelInstructions? + .Select(i => ChannelInstructionMongoElement.ToMongoElement(i))? + .ToList() ?? new List(), Templates = x.Templates? .Select(t => AgentTemplateMongoElement.ToMongoElement(t))? .ToList() ?? new List(), @@ -71,6 +74,7 @@ public partial class MongoRepository .Set(x => x.Name, agent.Name) .Set(x => x.Description, agent.Description) .Set(x => x.Instruction, agent.Instruction) + .Set(x => x.ChannelInstructions, agent.ChannelInstructions) .Set(x => x.Templates, agent.Templates) .Set(x => x.Functions, agent.Functions) .Set(x => x.Responses, agent.Responses) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index 1bc69aaf..1fdb5bfb 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -5,6 +5,7 @@ using BotSharp.Plugin.OpenAI.Providers.Image; using BotSharp.Plugin.OpenAI.Providers.Text; using BotSharp.Plugin.OpenAI.Providers.Chat; using Microsoft.Extensions.Configuration; +using BotSharp.Plugin.OpenAI.Providers.Audio; namespace BotSharp.Plugin.OpenAI; @@ -30,5 +31,6 @@ public class OpenAiPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs new file mode 100644 index 00000000..e559e109 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs @@ -0,0 +1,30 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio +{ + public partial class TextToSpeechProvider : ITextToSpeech + { + public string Provider => "openai"; + private readonly IServiceProvider _services; + private string? _model; + + public TextToSpeechProvider( + IServiceProvider services) + { + _services = services; + } + + public void SetModelName(string model) + { + _model = model; + } + + public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) + { + var client = ProviderHelper + .GetClient(Provider, _model, _services) + .GetAudioClient(_model); + return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 23084ead..00f47149 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -197,8 +198,8 @@ public class ChatCompletionProvider : IChatCompletion protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); - var fileService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -263,23 +264,23 @@ public class ChatCompletionProvider : IChatCompletion { foreach (var file in message.Files) { - if (!string.IsNullOrEmpty(file.FileUrl)) + if (!string.IsNullOrEmpty(file.FileData)) { - var uri = new Uri(file.FileUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); - contentParts.Add(contentPart); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData); + var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } else if (!string.IsNullOrEmpty(file.FileStorageUrl)) { - var contentType = fileService.GetFileContentType(file.FileStorageUrl); - using var stream = File.OpenRead(file.FileStorageUrl); - var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low); + var contentType = FileUtility.GetFileContentType(file.FileStorageUrl); + var bytes = fileStorage.GetFileBytes(file.FileStorageUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low); + contentParts.Add(contentPart); + } + else if (!string.IsNullOrEmpty(file.FileUrl)) + { + var uri = new Uri(file.FileUrl); + var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low); contentParts.Add(contentPart); } } diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index a66cb8f9..81c53f83 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -19,15 +19,18 @@ using BotSharp.Plugin.PaddleSharp.Settings; namespace BotSharp.Plugin.PaddleSharp.Providers; public class Pdf2TextConverter : IPdf2TextConverter -{ +{ private Dictionary _mappings = new Dictionary(); private FullOcrModel _model; private PaddleSharpSettings _paddleSharpSettings; + public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) { _paddleSharpSettings = paddleSharpSettings; } + public string Name => "Paddle"; + public async Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { await ConvertPdfToLocalImagesAsync(filePath, startPageNum, endPageNum); diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj new file mode 100644 index 00000000..b3293a0f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/BotSharp.Plugin.PythonInterpreter.csproj @@ -0,0 +1,35 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Enums/UtilityName.cs new file mode 100644 index 00000000..95a5b8a2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Enums/UtilityName.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.PythonInterpreter.Enums; + +public class UtilityName +{ + public const string PythonInterpreter = "python-interpreter"; +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/InterpretationFn.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/InterpretationFn.cs new file mode 100644 index 00000000..b12cb739 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/InterpretationFn.cs @@ -0,0 +1,46 @@ +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Interpreters.Models; +using Microsoft.Extensions.Logging; +using Python.Runtime; +using System.Text.Json; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.PythonInterpreter.Functions; + +public class InterpretationFn : IFunctionCallback +{ + public string Name => "python_interpreter"; + public string Indication => "Interpreting python code"; + + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + using (Py.GIL()) + { + // Import necessary Python modules + dynamic sys = Py.Import("sys"); + dynamic io = Py.Import("io"); + + // Redirect standard output to capture it + dynamic stringIO = io.StringIO(); + sys.stdout = stringIO; + + // Execute a simple Python script + using var locals = new PyDict(); + PythonEngine.Exec(args.Script, null, locals); + + // Console.WriteLine($"Result from Python: {result}"); + message.Content = stringIO.getvalue(); + + // Restore the original stdout + sys.stdout = sys.__stdout__; + } + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterAgentHook.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterAgentHook.cs new file mode 100644 index 00000000..64e0ec27 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterAgentHook.cs @@ -0,0 +1,51 @@ +namespace BotSharp.Plugin.PythonInterpreter.Hooks; + +public class InterpreterAgentHook : AgentHookBase +{ + private static string FUNCTION_NAME = "python_interpreter"; + + public override string SelfId => string.Empty; + + public InterpreterAgentHook(IServiceProvider services, AgentSettings settings) + : base(services, settings) + { + } + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.PythonInterpreter); + + if (isConvMode && isEnabled) + { + var (prompt, fn) = GetPromptAndFunction(); + if (fn != null) + { + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); + } + } + } + + base.OnAgentLoaded(agent); + } + + private (string, FunctionDef?) GetPromptAndFunction() + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{FUNCTION_NAME}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(FUNCTION_NAME)); + return (prompt, loadAttachmentFn); + } +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterUtilityHook.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterUtilityHook.cs new file mode 100644 index 00000000..be37bfa1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Hooks/InterpreterUtilityHook.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Plugin.PythonInterpreter.Hooks; + +public class InterpreterUtilityHook : IAgentUtilityHook +{ + public void AddUtilities(List utilities) + { + utilities.Add(UtilityName.PythonInterpreter); + } +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/InterpreterPlugin.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/InterpreterPlugin.cs new file mode 100644 index 00000000..075038e1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/InterpreterPlugin.cs @@ -0,0 +1,17 @@ +using BotSharp.Plugin.PythonInterpreter.Hooks; + +namespace BotSharp.Plugin.PythonInterpreter; + +public class InterpreterPlugin : IBotSharpPlugin +{ + public string Id => "23174e08-e866-4173-824a-cf1d97afa8d0"; + public string Name => "Python Interpreter"; + public string Description => "Python Interpreter enables AI to write and execute Python code within a secure, sandboxed environment."; + public string? IconUrl => "https://static.vecteezy.com/system/resources/previews/012/697/295/non_2x/3d-python-programming-language-logo-free-png.png"; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + services.AddScoped(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Using.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Using.cs new file mode 100644 index 00000000..8cc31aa1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Using.cs @@ -0,0 +1,18 @@ +global using System; +global using System.Linq; +global using System.Collections.Generic; + +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; + +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Agents.Settings; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; + +global using BotSharp.Plugin.PythonInterpreter.Enums; diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/python_interpreter.json b/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/python_interpreter.json new file mode 100644 index 00000000..ac2b2a91 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/python_interpreter.json @@ -0,0 +1,19 @@ +{ + "name": "python_interpreter", + "description": "write and execute python code, print the result in Console", + "parameters": { + "type": "object", + "properties": { + "script": { + "type": "string", + "description": "python code" + }, + "language": { + "type": "string", + "enum": [ "python" ], + "description": "python code" + } + }, + "required": [ "language", "script" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/python_interpreter.fn.liquid b/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/python_interpreter.fn.liquid new file mode 100644 index 00000000..8dd2425f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/python_interpreter.fn.liquid @@ -0,0 +1 @@ +Write and execute Python script in python_interpreter function, and use python function print(a) to output the result in stand output. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index e698d5e3..3dd064b9 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -1,13 +1,7 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.VectorStorage; -using Microsoft.Extensions.DependencyInjection; +using BotSharp.Abstraction.Utilities; +using BotSharp.Abstraction.VectorStorage.Models; using Qdrant.Client; using Qdrant.Client.Grpc; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; namespace BotSharp.Plugin.Qdrant; @@ -17,14 +11,16 @@ public class QdrantDb : IVectorDb private readonly QdrantSetting _setting; private readonly IServiceProvider _services; - public QdrantDb(QdrantSetting setting, + public QdrantDb( + QdrantSetting setting, IServiceProvider services) { _setting = setting; _services = services; - } + public string Name => "Qdrant"; + private QdrantClient GetClient() { if (_client == null) @@ -39,20 +35,49 @@ public class QdrantDb : IVectorDb return _client; } - public async Task> GetCollections() + public async Task> GetCollections() { // List all the collections var collections = await GetClient().ListCollectionsAsync(); return collections.ToList(); } + public async Task> GetCollectionData(string collectionName, VectorFilter filter) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) + { + return new StringIdPagedItems(); + } + + var totalPointCount = await client.CountAsync(collectionName); + var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, + offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, + vectorsSelector: filter.WithVector); + var points = response?.Result?.Select(x => new VectorCollectionData + { + Id = x.Id?.Uuid ?? string.Empty, + Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), + Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null + })?.ToList() ?? new List(); + + return new StringIdPagedItems + { + Count = totalPointCount, + NextId = response?.NextPageOffset?.Uuid, + Items = points + }; + } + public async Task CreateCollection(string collectionName, int dim) { - var collections = await GetCollections(); - if (!collections.Contains(collectionName)) + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) { // Create a new collection - await GetClient().CreateCollectionAsync(collectionName, new VectorParams() + await client.CreateCollectionAsync(collectionName, new VectorParams() { Size = (ulong)dim, Distance = Distance.Cosine @@ -60,7 +85,7 @@ public class QdrantDb : IVectorDb } // Get collection info - var collectionInfo = await _client.GetCollectionInfoAsync(collectionName); + var collectionInfo = await client.GetCollectionInfoAsync(collectionName); if (collectionInfo == null) { throw new Exception($"Create {collectionName} failed."); @@ -77,10 +102,9 @@ public class QdrantDb : IVectorDb Uuid = id }, Vectors = vector, - - Payload = + Payload = { - { "text", text } + { KnowledgePayloadName.Text, text } } }; @@ -93,7 +117,6 @@ public class QdrantDb : IVectorDb } var client = GetClient(); - var result = await client.UpsertAsync(collectionName, points: new List { point @@ -102,13 +125,70 @@ public class QdrantDb : IVectorDb return result.Status == UpdateStatus.Completed; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { - var client = GetClient(); - var points = await client.SearchAsync(collectionName, vector, - limit: (ulong)limit, - scoreThreshold: confidence); + var results = new List(); - return points.Select(x => x.Payload[returnFieldName].StringValue).ToList(); + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + if (!exist) + { + return results; + } + + var points = await client.SearchAsync(collectionName, vector, limit: (ulong)limit, scoreThreshold: confidence); + + var pickFields = fields != null; + foreach (var point in points) + { + var data = new Dictionary(); + if (pickFields) + { + foreach (var field in fields) + { + if (point.Payload.ContainsKey(field)) + { + data[field] = point.Payload[field].StringValue; + } + else + { + data[field] = ""; + } + } + } + else + { + data = point.Payload.ToDictionary(k => k.Key, v => v.Value.StringValue); + } + + results.Add(new VectorCollectionData + { + Id = point.Id.Uuid, + Data = data, + Score = point.Score, + Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null + }); + } + + return results; + } + + public async Task DeleteCollectionData(string collectionName, string id) + { + if (!Guid.TryParse(id, out var guid)) + { + return false; + } + + var client = GetClient(); + var result = await client.DeleteAsync(collectionName, guid); + return result.Status == UpdateStatus.Completed; + } + + + private async Task DoesCollectionExist(QdrantClient client, string collectionName) + { + return await client.CollectionExistsAsync(collectionName); } } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs index 8cfd137b..a3bc116a 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantPlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; -using BotSharp.Abstraction.VectorStorage; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs new file mode 100644 index 00000000..dc7e9747 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Qdrant/Using.cs @@ -0,0 +1,7 @@ +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Threading.Tasks; +global using BotSharp.Abstraction.VectorStorage; +global using BotSharp.Abstraction.Knowledges.Enums; +global using BotSharp.Abstraction.Knowledges.Models; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs index 089b5168..c52baf59 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs @@ -57,8 +57,8 @@ public class IntentClassifier return; } - var vector = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding)); + var vector = _services.GetServices().FirstOrDefault(x => x.Provider == _knowledgeBaseSettings.TextEmbedding.Provider); + vector.SetModelName(_knowledgeBaseSettings.TextEmbedding.Model); var layers = new List { @@ -136,8 +136,8 @@ public class IntentClassifier public NDArray GetTextEmbedding(string text) { var knowledgeSettings = _services.GetRequiredService(); - var embedding = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(knowledgeSettings.TextEmbedding)); + var embedding = _services.GetServices() .FirstOrDefault(x => x.Provider == knowledgeSettings.TextEmbedding.Provider); + embedding.SetModelName(knowledgeSettings.TextEmbedding.Model); var x = np.zeros((1, embedding.Dimension), dtype: np.float32); x[0] = embedding.GetVectorAsync(text).GetAwaiter().GetResult(); diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 20fdeecc..e5c32bd2 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -1,8 +1,8 @@ +using BotSharp.Abstraction.Utilities; using BotSharp.Abstraction.VectorStorage; +using BotSharp.Abstraction.VectorStorage.Models; using Microsoft.SemanticKernel.Memory; -using System; using System.Collections.Generic; -using System.Text; using System.Threading.Tasks; namespace BotSharp.Plugin.SemanticKernel @@ -19,12 +19,21 @@ namespace BotSharp.Plugin.SemanticKernel { this._memoryStore = memoryStore; } + + + public string Name => "SemanticKernel"; + public async Task CreateCollection(string collectionName, int dim) { await _memoryStore.CreateCollectionAsync(collectionName); } - public async Task> GetCollections() + public Task> GetCollectionData(string collectionName, VectorFilter filter) + { + throw new System.NotImplementedException(); + } + + public async Task> GetCollections() { var result = new List(); await foreach (var collection in _memoryStore.GetCollectionsAsync()) @@ -34,18 +43,23 @@ namespace BotSharp.Plugin.SemanticKernel return result; } - public async Task> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f) + public async Task> Search(string collectionName, float[] vector, + IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false) { var results = _memoryStore.GetNearestMatchesAsync(collectionName, vector, limit); - var resultTexts = new List(); - await foreach (var (record, _) in results) + var resultTexts = new List(); + await foreach (var (record, score) in results) { - resultTexts.Add(record.Metadata.Text); + resultTexts.Add(new VectorCollectionData + { + Data = new Dictionary { { "text", record.Metadata.Text } }, + Score = score, + Vector = withVector ? record.Embedding.ToArray() : null + }); } return resultTexts; - } public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary? payload) @@ -55,5 +69,17 @@ namespace BotSharp.Plugin.SemanticKernel #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. return true; } + + public async Task DeleteCollectionData(string collectionName, string id) + { + var exist = await _memoryStore.DoesCollectionExistAsync(collectionName); + + if (exist) + { + await _memoryStore.RemoveAsync(collectionName, id); + return true; + } + return false; + } } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 8d861d68..6d2b4b05 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -18,7 +18,7 @@ - + @@ -26,7 +26,7 @@ PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instructions/instruction.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid rename to src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instructions/instruction.liquid diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs new file mode 100644 index 00000000..4d803628 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.TencentCos.Services; + +public partial class TencentCosService +{ + public Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + { + throw new NotImplementedException(); + } + + public Task RetrieveSpeechFileAsync(string conversationId, string fileName) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index e9631786..e94ab6ce 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -1,5 +1,3 @@ -using Microsoft.AspNetCore.StaticFiles; - namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -9,42 +7,11 @@ public partial class TencentCosService return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } - public (string, byte[]) GetFileInfoFromData(string data) - { - if (string.IsNullOrEmpty(data)) - { - return (string.Empty, new byte[0]); - } - - var typeStartIdx = data.IndexOf(':'); - var typeEndIdx = data.IndexOf(';'); - var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1); - - var base64startIdx = data.IndexOf(','); - var base64Str = data.Substring(base64startIdx + 1); - - return (contentType, Convert.FromBase64String(base64Str)); - } - - public string GetFileContentType(string filePath) - { - string contentType; - var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) - { - contentType = string.Empty; - } - - return contentType; - } - public byte[] GetFileBytes(string fileStorageUrl) { try { - var fileData = _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); - - return fileData; + return _cosClient.BucketClient.DownloadFileBytes(fileStorageUrl); } catch (Exception ex) { @@ -53,7 +20,7 @@ public partial class TencentCosService return Array.Empty(); } - public bool SavefileToPath(string filePath, Stream stream) + public bool SaveFileStreamToPath(string filePath, Stream stream) { if (string.IsNullOrEmpty(filePath)) return false; @@ -63,8 +30,49 @@ public partial class TencentCosService } catch (Exception ex) { - _logger.LogWarning($"Error when saving file to path: {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when saving file stream to path: {ex.Message}\r\n{ex.InnerException}"); return false; } } + + public bool SaveFileBytesToPath(string filePath, byte[] bytes) + { + if (string.IsNullOrEmpty(filePath)) return false; + + try + { + return _cosClient.BucketClient.UploadBytes(filePath, bytes); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving file bytes to path: {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + + public string GetParentDir(string dir, int level = 1) + { + var segs = dir.Split("/"); + return string.Join("/", segs.SkipLast(level)); + } + + public string BuildDirectory(params string[] segments) + { + return string.Join("/", segments); + } + + public void CreateDirectory(string dir) + { + + } + + public bool ExistDirectory(string? dir) + { + return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir); + } + + public void DeleteDirectory(string dir) + { + _cosClient.BucketClient.DeleteDir(dir); + } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index df39e0a9..7e2462de 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,40 +1,35 @@ +using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Files.Enums; +using BotSharp.Abstraction.Files.Utilities; using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetChatFiles(string conversationId, string source, - IEnumerable conversations, IEnumerable contentTypes, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(conversations, offset); + var source = FileSourceType.User; var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; - foreach (var messageId in messageIds) { var dir = $"{pathPrefix}/{messageId}/{source}"; - foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) { var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = GetFileContentType(file); - if (contentTypes?.Contains(contentType) != true) continue; + var screenshots = await GetScreenshots(file, subDir, messageId, source); + if (screenshots.IsNullOrEmpty()) continue; - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; - - files.AddRange(foundFiles); + files.AddRange(screenshots); } } @@ -42,7 +37,7 @@ public partial class TencentCosService } public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, bool imageOnly = false) + string source, IEnumerable? contentTypes = null) { var files = new List(); if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; @@ -59,21 +54,21 @@ public partial class TencentCosService { foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir)) { - var contentType = GetFileContentType(file); - if (imageOnly && !_imageTypes.Contains(contentType)) + var contentType = FileUtility.GetFileContentType(file); + if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) { continue; } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, - FileUrl = $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}", + FileUrl = BuilFileUrl(file), FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -85,12 +80,13 @@ public partial class TencentCosService return files; } + + public string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName) { var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/"; var fileList = _cosClient.BucketClient.GetDirFiles(dir); - var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName)); return found; } @@ -135,10 +131,8 @@ public partial class TencentCosService try { - var (_, bytes) = GetFileInfoFromData(file.FileData); - + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var subDir = $"{dir}/{source}/{i + 1}"; - _cosClient.BucketClient.UploadBytes($"{subDir}/{file.FileName}", bytes); } catch (Exception ex) @@ -221,113 +215,29 @@ public partial class TencentCosService return dir; } - private IEnumerable GetMessageIds(IEnumerable conversations, int? offset = null) + private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) { - if (conversations.IsNullOrEmpty()) return Enumerable.Empty(); + if (dialogs.IsNullOrEmpty()) return Enumerable.Empty(); - if (offset <= 0) + if (offset.HasValue && offset < 1) { - offset = MIN_OFFSET; - } - else if (offset > MAX_OFFSET) - { - offset = MAX_OFFSET; + offset = 1; } var messageIds = new List(); if (offset.HasValue) { - messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); } else { - messageIds = conversations.Select(x => x.MessageId).Distinct().ToList(); + messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); } return messageIds; } - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) - { - var files = new List(); - try - { - if (!_imageTypes.Contains(contentType) && includeScreenShot) - { - var screenShotDir = $"{fileDir}/{SCREENSHOT_FILE_FOLDER}/"; - - var fileList = _cosClient.BucketClient.GetDirFiles(screenShotDir); - - if (!fileList.IsNullOrEmpty()) - { - foreach (var screenShot in fileList) - { - contentType = GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) - { - contentType = GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - } - else - { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - - return files; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); - return files; - } - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); @@ -343,8 +253,73 @@ public partial class TencentCosService private IPdf2ImageConverter? GetPdf2ImageConverter() { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); + var settings = _services.GetRequiredService(); + var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + return converter; + } + + private string BuilFileUrl(string file) + { + return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}"; + } + + private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + { + var files = new List(); + + try + { + var contentType = FileUtility.GetFileContentType(file); + var screenshotDir = $"{parentDir}/{SCREENSHOT_FILE_FOLDER}/"; + var screenshots = _cosClient.BucketClient.GetDirFiles(screenshotDir); + if (!screenshots.IsNullOrEmpty()) + { + foreach (var screenshot in screenshots) + { + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(screenshot), + FileStorageUrl = screenshot, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(image), + FileStorageUrl = image, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + return files; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + return files; + } } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs deleted file mode 100644 index e8628ce3..00000000 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Image.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace BotSharp.Plugin.TencentCos.Services; - -public partial class TencentCosService -{ - public async Task GenerateImage(string? provider, string? model, string text) - { - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-3"); - var message = await completion.GetImageGeneration(new Agent() - { - Id = Guid.Empty.ToString(), - }, new RoleDialogModel(AgentRole.User, text)); - return message; - } - - public async Task VaryImage(string? provider, string? model, BotSharpFile image) - { - if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) - { - throw new ArgumentException($"Cannot find image url or data!"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); - stream.Position = 0; - - var message = await completion.GetImageVariation(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); - - stream.Close(); - return message; - } - - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image) - { - if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) - { - throw new ArgumentException($"Cannot find image url or data!"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var bytes = await DownloadFile(image); - using var stream = new MemoryStream(); - stream.Write(bytes, 0, bytes.Length); - stream.Position = 0; - - var message = await completion.GetImageEdits(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); - - stream.Close(); - return message; - } - - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask) - { - if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || - (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) - { - throw new ArgumentException($"Cannot find image/mask url or data"); - } - - var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); - var imageBytes = await DownloadFile(image); - var maskBytes = await DownloadFile(mask); - - using var imageStream = new MemoryStream(); - imageStream.Write(imageBytes, 0, imageBytes.Length); - imageStream.Position = 0; - - using var maskStream = new MemoryStream(); - maskStream.Write(maskBytes, 0, maskBytes.Length); - maskStream.Position = 0; - - var message = await completion.GetImageEdits(new Agent() - { - Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); - - imageStream.Close(); - maskStream.Close(); - return message; - } - - #region Private methods - private async Task DownloadFile(BotSharpFile file) - { - var bytes = new byte[0]; - if (!string.IsNullOrEmpty(file.FileUrl)) - { - var http = _services.GetRequiredService(); - using var client = http.CreateClient(); - bytes = await client.GetByteArrayAsync(file.FileUrl); - } - else if (!string.IsNullOrEmpty(file.FileData)) - { - (_, bytes) = GetFileInfoFromData(file.FileData); - } - - return bytes; - } - #endregion -} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index 23ce68c0..de222c43 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Files.Utilities; + namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService @@ -26,10 +28,8 @@ public partial class TencentCosService if (string.IsNullOrEmpty(dir)) return false; - var (_, bytes) = GetFileInfoFromData(file.FileData); - + var (_, bytes) = FileUtility.GetFileInfoFromData(file.FileData); var extension = Path.GetExtension(file.FileName); - var fileName = user?.Id == null ? file.FileName : $"{user?.Id}{extension}"; return _cosClient.BucketClient.UploadBytes($"{dir}/{fileName}", bytes); @@ -51,7 +51,6 @@ public partial class TencentCosService } var dir = $"{USERS_FOLDER}/{userId}/{USER_AVATAR_FOLDER}/"; - return dir; } #endregion diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index 80b8fd78..ddb5e031 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -5,8 +5,9 @@ using System.Net.Mime; namespace BotSharp.Plugin.TencentCos.Services; -public partial class TencentCosService : IBotSharpFileService +public partial class TencentCosService : IFileStorageService { + private readonly TencentCosClient _cosClient; private readonly TencentCosSettings _settings; private readonly IServiceProvider _services; private readonly IUserIdentity _user; @@ -27,10 +28,6 @@ public partial class TencentCosService : IBotSharpFileService private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - private const int MIN_OFFSET = 1; - private const int MAX_OFFSET = 5; - - private readonly TencentCosClient _cosClient; public TencentCosService( TencentCosSettings settings, @@ -43,14 +40,7 @@ public partial class TencentCosService : IBotSharpFileService _user = user; _logger = logger; _services = services; - _fullBuketName = $"{_settings.BucketName}-{_settings.AppId}"; + _fullBuketName = $"{settings.BucketName}-{settings.AppId}"; _cosClient = cosClient; } - - #region Private methods - private bool ExistDirectory(string? dir) - { - return !string.IsNullOrEmpty(dir) && _cosClient.BucketClient.DirExists(dir); - } - #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs index b7e600f3..527be651 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosClient.cs @@ -19,7 +19,6 @@ namespace BotSharp.Plugin.TencentCos settings.SecretId, settings.SecretKey, settings.KeyDurationSecond); var cosXml = new CosXmlServer(cosXmlConfig, cosCredentialProvider); - BucketClient = new BucketClient(cosXml, $"{settings.BucketName}-{settings.AppId}", settings.AppId, settings.Region); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs index 93a99c14..791e7f98 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/TencentCosPlugin.cs @@ -18,10 +18,10 @@ public class TencentCosPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - var myFileStorageSettings = new FileStorageSettings(); - config.Bind("FileStorage", myFileStorageSettings); + var fileCoreSettings = new FileCoreSettings(); + config.Bind("FileCore", fileCoreSettings); - if (myFileStorageSettings.Default == FileStorageEnum.TencentCosStorage) + if (fileCoreSettings.Storage == FileStorageEnum.TencentCosStorage) { services.AddScoped(provider => { @@ -30,8 +30,7 @@ public class TencentCosPlugin : IBotSharpPlugin }); services.AddScoped(); - - services.AddScoped(); + services.AddScoped(); } } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index ed99a926..0ad04dcd 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -9,6 +9,7 @@ + @@ -16,6 +17,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 30facac9..859e34ff 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -1,12 +1,15 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Services; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using System.IdentityModel.Tokens.Jwt; -using BotSharp.Plugin.Twilio.Services; -using BotSharp.Abstraction.Routing; namespace BotSharp.Plugin.Twilio.Controllers; [AllowAnonymous] +[Route("twilio/voice")] public class TwilioVoiceController : TwilioController { private readonly TwilioSetting _settings; @@ -34,50 +37,116 @@ public class TwilioVoiceController : TwilioController }; } - [HttpPost("/twilio/voice/welcome")] - public async Task StartConversation(VoiceRequest request) + [HttpPost("welcome")] + public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states) { - string sessionId = $"TwilioVoice_{request.CallSid}"; + if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); - var response = twilio.ReturnInstructions("Hello, how may I help you?"); + var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; + var response = twilio.ReturnInstructions("twilio/welcome.mp3", url, true); return TwiML(response); } - [HttpPost("/twilio/voice/{agentId}")] - public async Task ReceivedVoiceMessage([FromRoute] string agentId, VoiceRequest input) + [HttpPost("{conversationId}/receive/{seqNum}")] + public async Task ReceiveCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, VoiceRequest request) { - string sessionId = $"TwilioVoice_{input.CallSid}"; - - var inputMsg = new RoleDialogModel(AgentRole.User, input.SpeechResult); - var conv = _services.GetRequiredService(); - var routing = _services.GetRequiredService(); - routing.Context.SetMessageId(sessionId, inputMsg.MessageId); - - conv.SetConversationId(sessionId, new List - { - new MessageState("channel", ConversationChannel.Phone), - new MessageState("calling_phone", input.DialCallSid) - }); - var twilio = _services.GetRequiredService(); - VoiceResponse response = default; + var messageQueue = _services.GetRequiredService(); + var sessionManager = _services.GetRequiredService(); + var url = $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}"; + var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum); + if (!string.IsNullOrWhiteSpace(request.SpeechResult)) + { + messages.Add(request.SpeechResult); + } + var messageContent = string.Join("\r\n", messages); + VoiceResponse response; + if (!string.IsNullOrWhiteSpace(messageContent)) + { - var result = await conv.SendMessage(agentId, - inputMsg, - replyMessage: null, - async msg => + var callerMessage = new CallerMessage() { - response = twilio.ReturnInstructions(msg.Content); - if (msg.FunctionName == "conversation_end") + ConversationId = conversationId, + SeqNumber = seqNum, + Content = messageContent, + From = request.From + }; + if (!string.IsNullOrEmpty(states)) + { + var kvp = states.Split(':'); + if (kvp.Length == 2) { - response = twilio.HangUp(msg.Content); + callerMessage.States.Add(kvp[0], kvp[1]); } - }, async functionExecuting => - { - }, async functionExecuted => - { - }); - + } + await messageQueue.EnqueueAsync(callerMessage); + response = twilio.ReturnInstructions(null, url, true, 1); + } + else + { + var speechPath = seqNum > 0 ? $"twilio/voice/speeches/{conversationId}/{seqNum - 1}.mp3" : "twilio/welcome.mp3"; + response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/receive/{seqNum}?states={states}", true); + } return TwiML(response); } + + [HttpPost("{conversationId}/reply/{seqNum}")] + public async Task ReplyCallerMessage([FromRoute] string conversationId, [FromRoute] int seqNum, [FromQuery] string states, VoiceRequest request) + { + var nextSeqNum = seqNum + 1; + var sessionManager = _services.GetRequiredService(); + var twilio = _services.GetRequiredService(); + if (request.SpeechResult != null) + { + await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult); + } + var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); + VoiceResponse response; + if (reply == null) + { + var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum); + if (indication != null) + { + var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); + var fileService = _services.GetRequiredService(); + var data = await textToSpeechService.GenerateSpeechFromTextAsync(indication); + var fileName = $"indication_{seqNum}.mp3"; + await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{fileName}", $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2); + } + else + { + response = twilio.ReturnInstructions(null, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 1); + } + } + else + { + var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); + var fileService = _services.GetRequiredService(); + var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply.Content); + var fileName = $"reply_{seqNum}.mp3"; + await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + if (reply.ConversationEnd) + { + response = twilio.HangUp($"twilio/voice/speeches/{conversationId}/{fileName}"); + } + else + { + response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{fileName}", $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); + } + + } + return TwiML(response); + } + + [HttpGet("speeches/{conversationId}/{fileName}")] + public async Task RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) + { + var fileService = _services.GetRequiredService(); + var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName); + var result = new FileContentResult(data.ToArray(), "audio/mpeg"); + result.FileDownloadName = fileName; + return result; + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs new file mode 100644 index 00000000..f9a83613 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/AssistantMessage.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Plugin.Twilio.Models +{ + public class AssistantMessage + { + public bool ConversationEnd { get; set; } + public string Content { get; set; } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs new file mode 100644 index 00000000..a6339c7b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs @@ -0,0 +1,16 @@ +namespace BotSharp.Plugin.Twilio.Models +{ + public class CallerMessage + { + public string ConversationId { get; set; } + public int SeqNumber { get; set; } + public string Content { get; set; } + public string From { get; set; } + public Dictionary States { get; set; } = new(); + + public override string ToString() + { + return $"{ConversationId}-{SeqNumber}"; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs new file mode 100644 index 00000000..b1acd298 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/ITwilioSessionManager.cs @@ -0,0 +1,15 @@ +using BotSharp.Plugin.Twilio.Models; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services +{ + public interface ITwilioSessionManager + { + Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message); + Task GetAssistantReplyAsync(string conversationId, int seqNum); + Task StageCallerMessageAsync(string conversationId, int seqNum, string message); + Task> RetrieveStagedCallerMessagesAsync(string conversationId, int seqNum); + Task SetReplyIndicationAsync(string conversationId, int seqNum, string indication); + Task GetReplyIndicationAsync(string conversationId, int seqNum); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs new file mode 100644 index 00000000..455e314c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs @@ -0,0 +1,32 @@ +using BotSharp.Plugin.Twilio.Models; +using System.Threading.Channels; + +namespace BotSharp.Plugin.Twilio.Services +{ + public class TwilioMessageQueue + { + private readonly Channel _queue; + internal ChannelReader Reader => _queue.Reader; + public TwilioMessageQueue() + { + BoundedChannelOptions options = new(100) + { + FullMode = BoundedChannelFullMode.Wait + }; + _queue = Channel.CreateBounded(options); + } + + public async ValueTask EnqueueAsync(CallerMessage request) + { + if (request == null) throw new ArgumentNullException(nameof(request)); + Console.WriteLine($"[{DateTime.UtcNow}] Enqueue {request}"); + await _queue.Writer.WriteAsync(request); + } + + internal void Stop() + { + Console.WriteLine($"[{DateTime.UtcNow}] Complete queue"); + _queue.Writer.TryComplete(); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs new file mode 100644 index 00000000..a84ce5d9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -0,0 +1,107 @@ +using BotSharp.Abstraction.Routing; +using BotSharp.Plugin.Twilio.Models; +using Microsoft.Extensions.Hosting; +using System.Threading; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services +{ + public class TwilioMessageQueueService : BackgroundService + { + private readonly TwilioMessageQueue _queue; + private readonly IServiceProvider _serviceProvider; + private readonly SemaphoreSlim _throttler; + + public TwilioMessageQueueService( + TwilioMessageQueue queue, + IServiceProvider serviceProvider) + { + _queue = queue; + _serviceProvider = serviceProvider; + _throttler = new SemaphoreSlim(4, 4); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await foreach (var message in _queue.Reader.ReadAllAsync(stoppingToken)) + { + await _throttler.WaitAsync(stoppingToken); + _ = Task.Run(async () => + { + try + { + Console.WriteLine($"Start processing {message}."); + await ProcessUserMessageAsync(message); + } + catch (Exception ex) + { + Console.WriteLine($"Processing {message} failed due to {ex.Message}."); + } + finally + { + _throttler.Release(); + } + }); + } + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + _queue.Stop(); + await base.StopAsync(cancellationToken); + } + + private async Task ProcessUserMessageAsync(CallerMessage message) + { + using var scope = _serviceProvider.CreateScope(); + var sp = scope.ServiceProvider; + AssistantMessage reply = null; + var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); + var conv = sp.GetRequiredService(); + var routing = sp.GetRequiredService(); + var config = sp.GetRequiredService(); + routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); + var states = new List + { + new MessageState("channel", ConversationChannel.Phone), + new MessageState("calling_phone", message.From) + }; + foreach (var kvp in message.States) + { + states.Add(new MessageState(kvp.Key, kvp.Value)); + } + conv.SetConversationId(message.ConversationId, states); + var sessionManager = sp.GetRequiredService(); + var result = await conv.SendMessage(config.AgentId, + inputMsg, + replyMessage: null, + async msg => + { + reply = new AssistantMessage() + { + ConversationEnd = msg.Instruction.ConversationEnd, + Content = msg.Content + }; + }, + async msg => + { + if (!string.IsNullOrEmpty(msg.Indication)) + { + await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); + } + }, + async functionExecuted => + { } + ); + if (reply == null || string.IsNullOrWhiteSpace(reply.Content)) + { + reply = new AssistantMessage() + { + ConversationEnd = true, + Content = "Sorry, something was wrong." + }; + } + await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index f01a7e29..a4e9d4ab 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,4 +1,3 @@ -using BotSharp.Plugin.Twilio.Settings; using Twilio.Jwt.AccessToken; using Token = Twilio.Jwt.AccessToken.Token; @@ -64,12 +63,35 @@ public class TwilioService return response; } - public VoiceResponse HangUp(string message) + public VoiceResponse ReturnInstructions(string speechPath, string callbackPath, bool actionOnEmptyResult, int timeout = 3) { var response = new VoiceResponse(); - if (!string.IsNullOrEmpty(message)) + var gather = new Gather() { - response.Say(message); + Input = new List() + { + Gather.InputEnum.Speech + }, + Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"), + SpeechModel = Gather.SpeechModelEnum.PhoneCall, + SpeechTimeout = timeout > 0 ? timeout.ToString() : "3", + Timeout = timeout > 0 ? timeout : 3, + ActionOnEmptyResult = actionOnEmptyResult + }; + if (!string.IsNullOrEmpty(speechPath)) + { + gather.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); + } + response.Append(gather); + return response; + } + + public VoiceResponse HangUp(string speechPath) + { + var response = new VoiceResponse(); + if (!string.IsNullOrEmpty(speechPath)) + { + response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); } response.Hangup(); return response; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs new file mode 100644 index 00000000..eae0b238 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -0,0 +1,63 @@ +using BotSharp.Plugin.Twilio.Models; +using StackExchange.Redis; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services +{ + public class TwilioSessionManager : ITwilioSessionManager + { + private readonly ConnectionMultiplexer _redis; + + public TwilioSessionManager(ConnectionMultiplexer redis) + { + _redis = redis; + } + + public async Task GetAssistantReplyAsync(string conversationId, int seqNum) + { + var db = _redis.GetDatabase(); + var key = $"{conversationId}:Assisist:{seqNum}"; + var jsonStr = await db.StringGetAsync(key); + return jsonStr.IsNull ? null : JsonSerializer.Deserialize(jsonStr); + } + + public async Task> RetrieveStagedCallerMessagesAsync(string conversationId, int seqNum) + { + var db = _redis.GetDatabase(); + var key = $"{conversationId}:Caller:{seqNum}"; + return (await db.ListRangeAsync(key)) + .Select(x => (string)x) + .ToList(); + } + + public async Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message) + { + var jsonStr = JsonSerializer.Serialize(message); + var db = _redis.GetDatabase(); + var key = $"{conversationId}:Assisist:{seqNum}"; + await db.StringSetAsync(key, jsonStr, TimeSpan.FromMinutes(5)); + } + + public async Task StageCallerMessageAsync(string conversationId, int seqNum, string message) + { + var db = _redis.GetDatabase(); + var key = $"{conversationId}:Caller:{seqNum}"; + await db.ListRightPushAsync(key, message); + await db.KeyExpireAsync(key, DateTime.UtcNow.AddMinutes(10)); + } + + public async Task SetReplyIndicationAsync(string conversationId, int seqNum, string indication) + { + var db = _redis.GetDatabase(); + var key = $"{conversationId}:Indication:{seqNum}"; + await db.StringSetAsync(key, indication, TimeSpan.FromMinutes(5)); + } + + public async Task GetReplyIndicationAsync(string conversationId, int seqNum) + { + var db = _redis.GetDatabase(); + var key = $"{conversationId}:Indication:{seqNum}"; + return await db.StringGetAsync(key); + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index 1a2065ca..1247d0d4 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Settings; using BotSharp.Plugin.Twilio.Services; +using StackExchange.Redis; namespace BotSharp.Plugin.Twilio; @@ -11,12 +12,17 @@ public class TwilioPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddSingleton(provider => + services.AddScoped(provider => { var settingService = provider.GetRequiredService(); return settingService.Bind("Twilio"); }); - services.AddScoped(); + var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]); + var sessionManager = new TwilioSessionManager(conn); + services.AddSingleton(sessionManager); + services.AddSingleton(); + services.AddHostedService(); + } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj index ce32616a..d6074b19 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj @@ -40,7 +40,7 @@ - + @@ -50,7 +50,7 @@ PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index 79eec038..6fb84817 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -9,6 +9,7 @@ public class PlaywrightInstance : IDisposable public IServiceProvider Services => _services; Dictionary _contexts = new Dictionary(); Dictionary> _pages = new Dictionary>(); + IPage? _activePage = null; /// /// ContextId and BrowserContext @@ -25,21 +26,28 @@ public class PlaywrightInstance : IDisposable _services = services; } - public IPage GetPage(string contextId, string? pattern = null) + public IPage? GetPage(string contextId, string? pattern = null) { if (string.IsNullOrEmpty(pattern)) { - return _contexts[contextId].Pages.LastOrDefault(); + return _activePage ?? _contexts[contextId].Pages.LastOrDefault(); } foreach (var page in _contexts[contextId].Pages) { if (page.Url.ToLower() == pattern.ToLower()) { + _activePage = page; + page.BringToFrontAsync().Wait(); return page; } } + if (!string.IsNullOrEmpty(pattern)) + { + return null; + } + return _contexts[contextId].Pages.LastOrDefault(); } @@ -84,6 +92,7 @@ public class PlaywrightInstance : IDisposable _contexts[ctxId].Page += async (sender, page) => { + _activePage = page; _pages[ctxId].Add(page); page.Close += async (sender, e) => { @@ -113,7 +122,12 @@ public class PlaywrightInstance : IDisposable return _contexts[ctxId]; } - public async Task NewPage(MessageInfo message) + public async Task NewPage(MessageInfo message, + bool enableResponseCallback = false, + bool responseInMemory = false, + List? responseContainer = null, + string[]? excludeResponseUrls = null, + string[]? includeResponseUrls = null) { var context = await GetContext(message.ContextId); var page = await context.NewPageAsync(); @@ -123,12 +137,19 @@ public class PlaywrightInstance : IDisposable var js = @"Object.defineProperties(navigator, {webdriver:{get:()=>false}});"; await page.AddInitScriptAsync(js); + if (!enableResponseCallback) + { + return page; + } + page.Response += async (sender, e) => { if (e.Status != 204 && e.Headers.ContainsKey("content-type") && e.Headers["content-type"].Contains("application/json") && - (e.Request.ResourceType == "fetch" || e.Request.ResourceType == "xhr")) + (e.Request.ResourceType == "fetch" || e.Request.ResourceType == "xhr") && + (excludeResponseUrls == null || !excludeResponseUrls.Any(url => e.Url.ToLower().Contains(url))) && + (includeResponseUrls == null || includeResponseUrls.Any(url => e.Url.ToLower().Contains(url)))) { Serilog.Log.Information($"{e.Request.Method}: {e.Url}"); JsonElement? json = null; @@ -146,7 +167,20 @@ public class PlaywrightInstance : IDisposable var webPageResponseHooks = _services.GetServices(); foreach (var hook in webPageResponseHooks) { - hook.OnDataFetched(message, e.Url.ToLower(), e.Request?.PostData ?? string.Empty, JsonSerializer.Serialize(json)); + var result = new WebPageResponseData + { + Url = e.Url.ToLower(), + PostData = e.Request?.PostData ?? string.Empty, + ResponseData = JsonSerializer.Serialize(json), + ResponseInMemory = responseInMemory + }; + + if (responseContainer != null && responseInMemory) + { + responseContainer.Add(result); + } + + hook.OnDataFetched(message, result); } } catch (ObjectDisposedException ex) @@ -155,7 +189,7 @@ public class PlaywrightInstance : IDisposable } catch (Exception ex) { - Serilog.Log.Error(ex.ToString()); + Serilog.Log.Error($"{e.Url}\r\n" + ex.ToString()); } } }; @@ -199,6 +233,7 @@ public class PlaywrightInstance : IDisposable if (page != null) { await page.CloseAsync(); + _activePage = _pages[ctxId].LastOrDefault(); } } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 76e63171..a7e5f27f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -10,9 +10,13 @@ public partial class PlaywrightWebDriver { var page = args.UseExistingPage ? _instance.GetPage(message.ContextId, pattern: args.Url) : - await _instance.NewPage(message); + await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCallback, + responseInMemory: args.ResponseInMemory, + responseContainer: args.ResponseContainer, + excludeResponseUrls: args.ExcludeResponseUrls, + includeResponseUrls: args.IncludeResponseUrls); - if (args.UseExistingPage && page != null && page.Url != "about:blank") + if (args.UseExistingPage && page != null && page.Url == args.Url) { Serilog.Log.Information($"goto existing page: {args.Url}"); result.IsSuccess = true; @@ -23,7 +27,22 @@ public partial class PlaywrightWebDriver if (args.UseExistingPage && args.OpenNewTab && page != null && page.Url == "about:blank") { - page = await _instance.NewPage(message); + page = await _instance.NewPage(message, + enableResponseCallback: args.EnableResponseCallback, + responseInMemory: args.ResponseInMemory, + responseContainer: args.ResponseContainer, + excludeResponseUrls: args.ExcludeResponseUrls, + includeResponseUrls: args.IncludeResponseUrls); + } + + if (page == null) + { + page = await _instance.NewPage(message, + enableResponseCallback: args.EnableResponseCallback, + responseInMemory: args.ResponseInMemory, + responseContainer: args.ResponseContainer, + excludeResponseUrls: args.ExcludeResponseUrls, + includeResponseUrls: args.IncludeResponseUrls); } var response = await page.GotoAsync(args.Url, new PageGotoOptions diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instructions/instruction.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid rename to src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instructions/instruction.liquid diff --git a/src/WebStarter/Program.cs b/src/WebStarter/Program.cs index ed0f8baf..a13d9d0f 100644 --- a/src/WebStarter/Program.cs +++ b/src/WebStarter/Program.cs @@ -4,6 +4,7 @@ using BotSharp.Logger; using BotSharp.Plugin.ChatHub; using Serilog; using BotSharp.Abstraction.Messaging.JsonConverters; +using Python.Runtime; var builder = WebApplication.CreateBuilder(args); @@ -41,4 +42,11 @@ app.UseBotSharp() .UseBotSharpOpenAPI(app.Environment) .UseBotSharpUI(); +Runtime.PythonDLL = @"C:\Users\xxx\AppData\Local\Programs\Python\Python311\python311.dll"; +PythonEngine.Initialize(); +PythonEngine.BeginAllowThreads(); + app.Run(); + +// Shut down the Python engine +PythonEngine.Shutdown(); \ No newline at end of file diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 5a5b1ced..4d7a31ac 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -30,6 +30,7 @@ + @@ -47,6 +48,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index b10e2160..68548aa5 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -230,9 +230,13 @@ "FileRepository": "data", "Assemblies": [ "BotSharp.Core" ] }, - "FileStorage": { - "Default": "LocalFileStorage" + + "FileCore": { + "Storage": "LocalFileStorage", + "Pdf2TextConverter": "", + "Pdf2ImageConverter": "" }, + "TencentCos": { "AppId": "", "SecretId": "", @@ -245,6 +249,10 @@ "ApiKey": "" }, + "Graph": { + "BaseUrl": "" + }, + "WeChat": { "AgentId": "437bed34-1169-4833-95ce-c24b8b56154a", "Token": "#{Token}#", @@ -254,10 +262,13 @@ }, "KnowledgeBase": { - "VectorDb": "MemVectorDatabase", - "TextEmbedding": "fastTextEmbeddingProvider", - "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", - "Pdf2TextConverter": "PigPdf2TextConverter" + "VectorDb": "Qdrant", + "GraphDb": "Default", + "DefaultCollection": "BotSharp", + "TextEmbedding": { + "Provider": "openai", + "Model": "text-embedding-3-small" + } }, "SparkDesk": { @@ -303,6 +314,7 @@ "BotSharp.Plugin.HuggingFace", "BotSharp.Plugin.KnowledgeBase", "BotSharp.Plugin.Planner", + "BotSharp.Plugin.Graph", "BotSharp.Plugin.Qdrant", "BotSharp.Plugin.ChatHub", "BotSharp.Plugin.WeChat", @@ -314,7 +326,8 @@ "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", "BotSharp.Plugin.EmailHandler", - "BotSharp.Plugin.TencentCos" + "BotSharp.Plugin.TencentCos", + "BotSharp.Plugin.PythonInterpreter" ] } } diff --git a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj index e448b9e0..716f1f4d 100644 --- a/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj +++ b/tests/BotSharp.Plugin.PizzaBot/BotSharp.Plugin.PizzaBot.csproj @@ -24,16 +24,16 @@ - + - + - + @@ -50,13 +50,13 @@ PreserveNewest - + PreserveNewest PreserveNewest - + PreserveNewest @@ -65,7 +65,7 @@ PreserveNewest - + PreserveNewest diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/instruction.liquid b/tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/instructions/instruction.liquid similarity index 100% rename from tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/instruction.liquid rename to tests/BotSharp.Plugin.PizzaBot/data/agents/b284db86-e9c2-4c25-a59e-4649797dd130/instructions/instruction.liquid diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/instruction.liquid b/tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/instructions/instruction.liquid similarity index 100% rename from tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/instruction.liquid rename to tests/BotSharp.Plugin.PizzaBot/data/agents/c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd/instructions/instruction.liquid diff --git a/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/instruction.liquid b/tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/instructions/instruction.liquid similarity index 100% rename from tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/instruction.liquid rename to tests/BotSharp.Plugin.PizzaBot/data/agents/fe8c60aa-b114-4ef3-93cb-a8efeac80f75/instructions/instruction.liquid