From 57b522fe42aaefc48a3c7c4275ca8fe330116203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E7=A3=8A?= Date: Thu, 13 Jun 2024 15:59:13 +0800 Subject: [PATCH 1/9] Update TranslationService.cs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复空白字符串AI不翻译返回导致数组越界的问题 --- .../BotSharp.Core/Translation/TranslationService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index 97e997c7..ec33f609 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -121,7 +121,7 @@ public class TranslationService : ITranslationService if (data == null) return; var dataType = data.GetType(); - if (IsStringType(dataType)) + if (IsStringType(dataType) && !string.IsNullOrWhiteSpace(data.ToString())) { res.Add(data.ToString()); return; @@ -139,7 +139,7 @@ public class TranslationService : ITranslationService { foreach (var item in (data as IEnumerable)) { - if (item == null) continue; + if (string.IsNullOrWhiteSpace(item)) continue; res.Add(item); } } From 3de77403d72ae3d99f377be4cbd661b3e191d283 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 18 Jun 2024 13:18:24 -0500 Subject: [PATCH 2/9] append image analysis prompt --- .../Files/Hooks/AttachmentProcessingHook.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs index 0a1d74b4..ebe35041 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs @@ -1,3 +1,4 @@ + namespace BotSharp.Core.Files.Hooks; public class AttachmentProcessingHook : AgentHookBase @@ -14,6 +15,20 @@ public class AttachmentProcessingHook : AgentHookBase _agentSettings = settings; } + public override void OnAgentLoaded(Agent agent) + { + var fileService = _services.GetRequiredService(); + var conv = _services.GetRequiredService(); + var hasConvFiles = fileService.HasConversationUserFiles(conv.ConversationId); + + if (hasConvFiles) + { + agent.Instruction += "\r\n\r\nIf user wants to describe images or pdf files, please call load_attachment."; + } + + base.OnAgentLoaded(agent); + } + public override bool OnFunctionsLoaded(List functions) { var fileService = _services.GetRequiredService(); From ff5a553296d34c07d50bf06279aa5ed0abea330f Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 19 Jun 2024 04:24:49 -0500 Subject: [PATCH 3/9] refine load attachment --- .../Files/Functions/LoadAttachmentFn.cs | 2 +- .../Files/Hooks/AttachmentProcessingHook.cs | 58 ++++++++----------- .../Providers/ChatCompletionProvider.cs | 2 - 3 files changed, 26 insertions(+), 36 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs index db3abff3..a4f04149 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs @@ -31,7 +31,7 @@ public class LoadAttachmentFn : IFunctionCallback var wholeDialogs = conv.GetDialogHistory(); var fileTypes = args?.FileTypes?.Split(",")?.ToList() ?? new List(); var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes); - var agent = await agentService.LoadAgent(AIAssistant); + var agent = await agentService.LoadAgent(!string.IsNullOrEmpty(message.CurrentAgentId) ? message.CurrentAgentId : AIAssistant); var fileAgent = new Agent { Id = agent.Id, diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs index ebe35041..558657b8 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs @@ -1,10 +1,11 @@ +using Microsoft.EntityFrameworkCore; + namespace BotSharp.Core.Files.Hooks; public class AttachmentProcessingHook : AgentHookBase { private readonly IServiceProvider _services; - private readonly AgentSettings _agentSettings; public override string SelfId => string.Empty; @@ -12,7 +13,6 @@ public class AttachmentProcessingHook : AgentHookBase : base(services, settings) { _services = services; - _agentSettings = settings; } public override void OnAgentLoaded(Agent agent) @@ -23,39 +23,29 @@ public class AttachmentProcessingHook : AgentHookBase if (hasConvFiles) { - agent.Instruction += "\r\n\r\nIf user wants to describe images or pdf files, please call load_attachment."; - } + agent.Instruction += "\r\n\r\nPlease call load_attachment if user wants to describe files, such as images, pdf.\r\n\r\n"; - base.OnAgentLoaded(agent); - } - - public override bool OnFunctionsLoaded(List functions) - { - var fileService = _services.GetRequiredService(); - var conv = _services.GetRequiredService(); - var hasConvFiles = fileService.HasConversationUserFiles(conv.ConversationId); - - if (hasConvFiles) - { - var json = JsonSerializer.Serialize(new + if (agent.Functions != null) { - user_request = new + var json = JsonSerializer.Serialize(new { - type = "string", - description = "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." - }, - file_types = new - { - type = "string", - description = "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'" - } - }); + user_request = new + { + type = "string", + description = "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." + }, + file_types = new + { + type = "string", + description = "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'" + } + }); - functions.Add(new FunctionDef - { - Name = "load_attachment", - Description = "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", - Parameters = + agent.Functions.Add(new FunctionDef + { + Name = "load_attachment", + Description = "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", + Parameters = { Properties = JsonSerializer.Deserialize(json), Required = new List @@ -64,8 +54,10 @@ public class AttachmentProcessingHook : AgentHookBase "file_types" } } - }); + }); + } } - return base.OnFunctionsLoaded(functions); ; + + base.OnAgentLoaded(agent); } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 2fcbf4c7..50fd33ce 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -5,7 +5,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Files; -using BotSharp.Abstraction.Files.Models; using BotSharp.Abstraction.Loggers; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Utilities; @@ -16,7 +15,6 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.InteropServices.ComTypes; using System.Threading.Tasks; namespace BotSharp.Plugin.AzureOpenAI.Providers; From eb679701bd0a205213faf3632ebe541c713169f1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 19 Jun 2024 11:58:27 -0500 Subject: [PATCH 4/9] fix file save --- .../Services/ConversationService.SendMessage.cs | 10 +++++----- .../Files/BotSharpFileService.Conversation.cs | 7 +++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index e4d65452..3489d260 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -16,6 +16,11 @@ public partial class ConversationService { var conversation = await GetConversationRecord(agentId); + // Save message files + var fileService = _services.GetRequiredService(); + fileService.SaveMessageFiles(_conversationId, message.MessageId, FileSourceType.User, message.Files); + message.Files?.Clear(); + var agentService = _services.GetRequiredService(); Agent agent = await agentService.LoadAgent(agentId); @@ -46,11 +51,6 @@ public partial class ConversationService routing.Context.SetMessageId(_conversationId, message.MessageId); routing.Context.Push(agent.Id); - // Save message files - var fileService = _services.GetRequiredService(); - fileService.SaveMessageFiles(_conversationId, message.MessageId, FileSourceType.User, message.Files); - message.Files?.Clear(); - // Save payload if (replyMessage != null && !string.IsNullOrEmpty(replyMessage.Payload)) { diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs index d8f88b5c..92f4107e 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs @@ -1,3 +1,4 @@ +using Amazon.Runtime.Internal.Auth; using BotSharp.Abstraction.Browsing; using BotSharp.Abstraction.Browsing.Models; using Microsoft.EntityFrameworkCore; @@ -221,14 +222,16 @@ public partial class BotSharpFileService } var (_, bytes) = GetFileInfoFromData(file.FileData); - Thread.Sleep(100); var subDir = Path.Combine(dir, source, $"{i + 1}"); if (!ExistDirectory(subDir)) { Directory.CreateDirectory(subDir); } - File.WriteAllBytes(Path.Combine(subDir, file.FileName), bytes); + using var fs = new FileStream(Path.Combine(subDir, file.FileName), FileMode.OpenOrCreate); + fs.Write(bytes, 0, bytes.Length); + fs.Flush(); + Thread.Sleep(100); } return true; From bab05fe21e2fcec0b72614ef3cbb5f0da8266d81 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 19 Jun 2024 11:59:31 -0500 Subject: [PATCH 5/9] minor change --- .../BotSharp.Core/Files/BotSharpFileService.Conversation.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs index 92f4107e..76eda14d 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs @@ -1,4 +1,3 @@ -using Amazon.Runtime.Internal.Auth; using BotSharp.Abstraction.Browsing; using BotSharp.Abstraction.Browsing.Models; using Microsoft.EntityFrameworkCore; From a6e5c9342ae5440a55753fcab0f6fda5d26552d5 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 19 Jun 2024 12:29:24 -0500 Subject: [PATCH 6/9] sleep --- .../BotSharp.Core/Files/BotSharpFileService.Conversation.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs index 76eda14d..c58300bd 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs @@ -230,7 +230,8 @@ public partial class BotSharpFileService using var fs = new FileStream(Path.Combine(subDir, file.FileName), FileMode.OpenOrCreate); fs.Write(bytes, 0, bytes.Length); fs.Flush(); - Thread.Sleep(100); + fs.Close(); + Thread.Sleep(2000); } return true; From 716008c685af407ff314891da8409cceae4533e1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 19 Jun 2024 12:29:47 -0500 Subject: [PATCH 7/9] minor change --- .../BotSharp.Core/Files/BotSharpFileService.Conversation.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs index c58300bd..4ded2359 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs @@ -230,7 +230,6 @@ public partial class BotSharpFileService using var fs = new FileStream(Path.Combine(subDir, file.FileName), FileMode.OpenOrCreate); fs.Write(bytes, 0, bytes.Length); fs.Flush(); - fs.Close(); Thread.Sleep(2000); } From 743de0a8755ded87d3bc874fc82df67db471e894 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 19 Jun 2024 12:44:14 -0500 Subject: [PATCH 8/9] flush to disk --- .../BotSharp.Core/Files/BotSharpFileService.Conversation.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs index 4ded2359..61e12145 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs @@ -227,10 +227,9 @@ public partial class BotSharpFileService Directory.CreateDirectory(subDir); } - using var fs = new FileStream(Path.Combine(subDir, file.FileName), FileMode.OpenOrCreate); + using var fs = new FileStream(Path.Combine(subDir, file.FileName), FileMode.Create); fs.Write(bytes, 0, bytes.Length); - fs.Flush(); - Thread.Sleep(2000); + fs.Flush(true); } return true; From a5deaeff57adaf123b0765c4299132eae5de55a1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 19 Jun 2024 13:24:58 -0500 Subject: [PATCH 9/9] add sleep --- .../BotSharp.Core/Files/BotSharpFileService.Conversation.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs index 61e12145..99c42999 100644 --- a/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs @@ -230,6 +230,7 @@ public partial class BotSharpFileService using var fs = new FileStream(Path.Combine(subDir, file.FileName), FileMode.Create); fs.Write(bytes, 0, bytes.Length); fs.Flush(true); + Thread.Sleep(2000); } return true;