diff --git a/.gitignore b/.gitignore index ab28c2e0..4f9359c7 100644 --- a/.gitignore +++ b/.gitignore @@ -292,4 +292,5 @@ XMLs logs wwwroot appsettings.Production.json -*.csproj.user \ No newline at end of file +*.csproj.user +env/ \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index cc504b66..b7c8b78e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -96,6 +96,15 @@ The main documentation for the site is organized into the following sections: llm/few-shot-learning llm/provider +.. _llamasharp: + +.. toctree:: + :maxdepth: 2 + :caption: Use Local LLM Models + + llama-sharp/config-llamasharp + llama-sharp/use-llamasharp-in-ui + .. _architecture-docs: .. toctree:: diff --git a/docs/llama-sharp/assets/check-llamasharp-version.png b/docs/llama-sharp/assets/check-llamasharp-version.png new file mode 100644 index 00000000..b44498ea Binary files /dev/null and b/docs/llama-sharp/assets/check-llamasharp-version.png differ diff --git a/docs/llama-sharp/assets/choose-llamasharp-as-provider.png b/docs/llama-sharp/assets/choose-llamasharp-as-provider.png new file mode 100644 index 00000000..e4177ba9 Binary files /dev/null and b/docs/llama-sharp/assets/choose-llamasharp-as-provider.png differ diff --git a/docs/llama-sharp/assets/click-test-button.png b/docs/llama-sharp/assets/click-test-button.png new file mode 100644 index 00000000..b5bb123d Binary files /dev/null and b/docs/llama-sharp/assets/click-test-button.png differ diff --git a/docs/llama-sharp/assets/console-output-in-botsharp.png b/docs/llama-sharp/assets/console-output-in-botsharp.png new file mode 100644 index 00000000..6860144d Binary files /dev/null and b/docs/llama-sharp/assets/console-output-in-botsharp.png differ diff --git a/docs/llama-sharp/assets/converstaion-examples.png b/docs/llama-sharp/assets/converstaion-examples.png new file mode 100644 index 00000000..b119c61b Binary files /dev/null and b/docs/llama-sharp/assets/converstaion-examples.png differ diff --git a/docs/llama-sharp/assets/edit-agent.png b/docs/llama-sharp/assets/edit-agent.png new file mode 100644 index 00000000..5ae0d986 Binary files /dev/null and b/docs/llama-sharp/assets/edit-agent.png differ diff --git a/docs/llama-sharp/assets/install-llamasharp-plugin.png b/docs/llama-sharp/assets/install-llamasharp-plugin.png new file mode 100644 index 00000000..eb7f208c Binary files /dev/null and b/docs/llama-sharp/assets/install-llamasharp-plugin.png differ diff --git a/docs/llama-sharp/config-llamasharp.md b/docs/llama-sharp/config-llamasharp.md new file mode 100644 index 00000000..a6c66183 --- /dev/null +++ b/docs/llama-sharp/config-llamasharp.md @@ -0,0 +1,60 @@ +# Config LLamaSharp + +BotSharp contains LLamaSharp plugin that allows you to run local llm models. To use the LLamaSharp, you need to config the BotSharp project with few steps. + +## Install LLamaSharp Backend + +Before use LLamaSharp plugin, you need to install one of the LLamaSharp backend services that suits your environment. + +- [`LLamaSharp.Backend.Cpu`](https://www.nuget.org/packages/LLamaSharp.Backend.Cpu): Pure CPU for Windows & Linux. Metal for Mac. +- [`LLamaSharp.Backend.Cuda11`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda11): CUDA 11 for Windows and Linux +- [`LLamaSharp.Backend.Cuda12`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda12): CUDA 12 for Windows and Linux + +**Please install the same version of LLamaSharp Backend with the LLamaSharp in BotSharp.Plugin.LLamaSharp.csproj.** + +![Check LLamaSharp Version](assets/check-llamasharp-version.png) + +```shell +# move to the LLamaSharp Plugin Project +$ cd src/Plugins/BotSharp.Plugin.LLamaSharp +# Install the LLamaSharp Backend +$ dotnet add package LLamaSharp.Backend.Cpu --version 0.9.1 +``` + +## Download and Config Local LLM Models + +LLamaSharp supports many LLM Models like LLaMA and Alpaca. Download the `gguf` format models and save them in your machine. + +We will use a [Llama 2](https://huggingface.co/TheBloke/llama-2-7B-Guanaco-QLoRA-GGUF) model in this tutorial. + +After downloading the model, open the `src/WebStarter/appsettings.json` file to config the LLamaSharp models. Set the `LlmProviders` and `LlamaSharp` fields to correct settings as your computer. For example: + +```json +{ + ..., + "LlmProviders": [ + ..., + { + "Provider": "llama-sharp", + "Models": [ + { + "Name": "llama-2-7b.Q2_K.gguf", + "Type": "chat" + } + ] + }, + ... + ], + ..., + "LlamaSharp": { + "Interactive": true, + "ModelDir": "/Users/wenwei/Desktop/LLM", + "DefaultModel": "llama-2-7b.Q2_K.gguf", + "MaxContextLength": 1024, + "NumberOfGpuLayer": 20 + }, + ... +} +``` + +For more details about LLamaSharp, visit [LLamaSharp - GitHub](https://github.com/SciSharp/LLamaSharp). diff --git a/docs/llama-sharp/use-llamasharp-in-ui.md b/docs/llama-sharp/use-llamasharp-in-ui.md new file mode 100644 index 00000000..9dd98e07 --- /dev/null +++ b/docs/llama-sharp/use-llamasharp-in-ui.md @@ -0,0 +1,29 @@ +# Use LLamaSharp in BotSharp + +Start the BotSharp backend and frontend services, and follow this tutorial. + +## Install LLamaSharp Plugin in UI. + +Go to the Plugin page and install LLamaSharp Plugin. + +![Install LlamaSharp Plugin](assets/install-llamasharp-plugin.png) + +## Config LLamaSharp as LLM Providers for Agents + +Edit or create an agent in Agents page, and config the agent. + +![Edit Agent](assets/edit-agent.png) + +In the edit page, config the provider as llama-sharp. + +![Choose LLamaSharp as Provider](assets/choose-llamasharp-as-provider.png) + +Then test the agent. + +![Click Test Agent Button](assets/click-test-button.png) + +![Test Agent Example](assets/converstaion-examples.png) + +If run successfully, you will see log like this in BotSharp service's console. + +![Console Output](assets/console-output-in-botsharp.png) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index 1592aa8f..5bceefaf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -67,4 +67,7 @@ public abstract class ConversationHookBase : IConversationHook public virtual Task OnUserAgentConnectedInitially(Conversation conversation) => Task.CompletedTask; + + public virtual Task OnMessageDeleted(string conversationId, string messageId) + => Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index 95afaab4..8552aa20 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -84,4 +84,12 @@ public interface IConversationHook /// /// Task OnHumanInterventionNeeded(RoleDialogModel message); + + /// + /// Delete message in a conversation + /// + /// + /// + /// + Task OnMessageDeleted(string conversationId, string messageId); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs index 74845e1f..771223ef 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs @@ -31,7 +31,7 @@ public interface IRoutingService List GetHandlers(Agent router); void ResetRecursiveCounter(); Task InvokeAgent(string agentId, List dialogs); - Task InvokeFunction(string name, RoleDialogModel message); + Task InvokeFunction(string name, RoleDialogModel message, bool restoreOriginalFunctionName = true); Task InstructLoop(RoleDialogModel message); /// diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index a6910603..5d618611 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Templating; namespace BotSharp.Core.Agents.Services; @@ -96,12 +95,24 @@ public partial class AgentService // render liquid template var render = _services.GetRequiredService(); var template = agent.Templates.First(x => x.Name == templateName).Content; + // update states + var conv = _services.GetRequiredService(); + foreach (var t in conv.States.GetStates()) + { + agent.TemplateDict[t.Key] = t.Value; + } return render.Render(template, agent.TemplateDict); } public string RenderedInstruction(Agent agent) { var render = _services.GetRequiredService(); + // update states + var conv = _services.GetRequiredService(); + foreach (var t in conv.States.GetStates()) + { + agent.TemplateDict[t.Key] = t.Value; + } return render.Render(agent.Instruction, agent.TemplateDict); } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 5824384b..48fd25df 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -131,7 +131,6 @@ - diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs index 1f1cc8fd..8c82eb1f 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs @@ -8,6 +8,11 @@ public partial class ConversationService : IConversationService { var db = _services.GetRequiredService(); var isSaved = db.TruncateConversation(conversationId, messageId, true); + var hooks = _services.GetServices().ToList(); + foreach (var hook in hooks) + { + await hook.OnMessageDeleted(conversationId, messageId); + } return await Task.FromResult(isSaved); } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 49d695f0..b55fb365 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -305,7 +305,7 @@ namespace BotSharp.Core.Repository public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { var ids = new List(); - var batchLimit = 50; + var batchLimit = 100; var utcNow = DateTime.UtcNow; var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs index 34f1952a..39a50a30 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs @@ -12,7 +12,7 @@ public class HumanInterventionNeededHandler : RoutingHandlerBase, IRoutingHandle { new ParameterPropertyDef("reason", "why need customer service"), new ParameterPropertyDef("summary", "the whole conversation summary with important information"), - new ParameterPropertyDef("response", "response content to user") + new ParameterPropertyDef("response", "tell the user that you are being transferred to customer service") }; public HumanInterventionNeededHandler(IServiceProvider services, ILogger logger, RoutingSettings settings) @@ -23,13 +23,9 @@ public class HumanInterventionNeededHandler : RoutingHandlerBase, IRoutingHandle public async Task Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message) { - var response = new RoleDialogModel(AgentRole.Assistant, inst.Response) - { - CurrentAgentId = message.CurrentAgentId, - MessageId = message.MessageId, - StopCompletion = true, - FunctionName = inst.Function - }; + var response = RoleDialogModel.From(message, + role: AgentRole.Assistant, + content: inst.Response); _dialogs.Add(response); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs index b21250db..5ff9894e 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs @@ -44,13 +44,14 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler var states = _services.GetRequiredService(); var goalAgent = states.GetState("user_goal_agent"); - if (!string.IsNullOrEmpty(goalAgent)) + if (!string.IsNullOrEmpty(goalAgent) && inst.OriginalAgent != goalAgent) { inst.OriginalAgent = goalAgent; + // Emit hook + await HookEmitter.Emit(_services, async hook => + await hook.OnRoutingInstructionRevised(inst, message) + ); } - await HookEmitter.Emit(_services, async hook => - await hook.OnRoutingInstructionRevised(inst, message) - ); var agentId = routing.Context.GetCurrentAgentId(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs index 4e508662..dd156054 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs @@ -1,8 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; -using BotSharp.Abstraction.Repositories.Filters; -using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Abstraction.Templating; @@ -93,6 +88,12 @@ public class HFPlanner : IPlaner { var template = router.Templates.First(x => x.Name == "planner_prompt.hf").Content; var render = _services.GetRequiredService(); + // update states + var conv = _services.GetRequiredService(); + foreach (var t in conv.States.GetStates()) + { + router.TemplateDict[t.Key] = t.Value; + } var prompt = render.Render(template, router.TemplateDict); return prompt.Trim(); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 6ef40c58..6f70e063 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Routing; public partial class RoutingService { - public async Task InvokeFunction(string name, RoleDialogModel message) + public async Task InvokeFunction(string name, RoleDialogModel message, bool restoreOriginalFunctionName = true) { var function = _services.GetServices().FirstOrDefault(x => x.Name == name); if (function == null) @@ -56,7 +56,9 @@ public partial class RoutingService } // restore original function name - if (!message.StopCompletion && message.FunctionName != originalFunctionName) + if (!message.StopCompletion && + message.FunctionName != originalFunctionName && + restoreOriginalFunctionName) { message.FunctionName = originalFunctionName; } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 0328c409..d7f62bac 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -54,6 +54,7 @@ public class UserService : IUserService db.CreateUser(record); + _logger.LogWarning($"Created new user account: {record.Id} {record.UserName}"); Utilities.ClearCache(); return record; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 33c273f0..ba5a393a 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -240,8 +240,7 @@ public class ChatCompletionProvider : IChatCompletion } else if (message.Role == ChatRole.User) { - var userMessage = new ChatRequestUserMessage( - new ChatMessageTextContentItem(message.Content)) + var userMessage = new ChatRequestUserMessage(message.Content) { // To display Planner name in log Name = message.FunctionName, diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index 187c2bf4..045a707a 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -96,4 +96,14 @@ public class ChatHubConversationHook : ConversationHookBase await base.OnResponseGenerated(message); } + + public override async Task OnMessageDeleted(string conversationId, string messageId) + { + await _chatHub.Clients.User(_user.Id).SendAsync("OnMessageDeleted", new ChatResponseModel + { + ConversationId = conversationId, + MessageId = messageId + }); + await base.OnMessageDeleted(conversationId, messageId); + } } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index f0602080..254520e7 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -133,11 +133,6 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR log += $"\r\n```json\r\n{richContent}\r\n```"; } - if (!string.IsNullOrEmpty(message.FunctionName)) - { - log += $"\r\n\r\n**{message.FunctionName}**"; - } - var input = new ContentLogInputModel(conv.ConversationId, message) { Name = agent?.Name, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 968c6b2c..3ff1b925 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -306,7 +306,7 @@ public partial class MongoRepository public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours) { var page = 1; - var batchLimit = 50; + var batchLimit = 100; var utcNow = DateTime.UtcNow; var conversationIds = new List();