Merge pull request #341 from SciSharp/master

merge code

.gitignore

@@ -292,4 +292,5 @@ XMLs
 logs
 wwwroot
 appsettings.Production.json
-*.csproj.user
+*.csproj.user
+env/

docs/index.rst

@@ -96,6 +96,15 @@ The main documentation for the site is organized into the following sections:
    llm/few-shot-learning
    llm/provider
 
+.. _llamasharp:
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Use Local LLM Models
+
+   llama-sharp/config-llamasharp
+   llama-sharp/use-llamasharp-in-ui
+
 .. _architecture-docs:
 
 .. toctree::

docs/llama-sharp/config-llamasharp.md

@@ -0,0 +1,60 @@
+# Config LLamaSharp
+
+BotSharp contains LLamaSharp plugin that allows you to run local llm models. To use the LLamaSharp, you need to config the BotSharp project with few steps.
+
+## Install LLamaSharp Backend
+
+Before use LLamaSharp plugin, you need to install one of the LLamaSharp backend services that suits your environment.
+
+- [`LLamaSharp.Backend.Cpu`](https://www.nuget.org/packages/LLamaSharp.Backend.Cpu): Pure CPU for Windows & Linux. Metal for Mac.
+- [`LLamaSharp.Backend.Cuda11`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda11): CUDA 11 for Windows and Linux
+- [`LLamaSharp.Backend.Cuda12`](https://www.nuget.org/packages/LLamaSharp.Backend.Cuda12): CUDA 12 for Windows and Linux
+
+**Please install the same version of LLamaSharp Backend with the LLamaSharp in BotSharp.Plugin.LLamaSharp.csproj.**
+
+![Check LLamaSharp Version](assets/check-llamasharp-version.png)
+
+```shell
+# move to the LLamaSharp Plugin Project
+$ cd src/Plugins/BotSharp.Plugin.LLamaSharp
+# Install the LLamaSharp Backend
+$ dotnet add package LLamaSharp.Backend.Cpu --version 0.9.1
+```
+
+## Download and Config Local LLM Models
+
+LLamaSharp supports many LLM Models like LLaMA and Alpaca. Download the `gguf` format models and save them in your machine.
+
+We will use a [Llama 2](https://huggingface.co/TheBloke/llama-2-7B-Guanaco-QLoRA-GGUF) model in this tutorial.
+
+After downloading the model, open the `src/WebStarter/appsettings.json` file to config the LLamaSharp models. Set the `LlmProviders` and `LlamaSharp` fields to correct settings as your computer. For example:
+
+```json
+{
+    ...,
+    "LlmProviders": [
+        ...,
+        {
+            "Provider": "llama-sharp",
+            "Models": [
+                {
+                "Name": "llama-2-7b.Q2_K.gguf",
+                "Type": "chat"
+                }
+            ]
+        },
+        ...
+    ],
+    ...,
+    "LlamaSharp": {
+        "Interactive": true,
+        "ModelDir": "/Users/wenwei/Desktop/LLM",
+        "DefaultModel": "llama-2-7b.Q2_K.gguf",
+        "MaxContextLength": 1024,
+        "NumberOfGpuLayer": 20
+    },
+    ...
+}
+```
+
+For more details about LLamaSharp, visit [LLamaSharp - GitHub](https://github.com/SciSharp/LLamaSharp).

docs/llama-sharp/use-llamasharp-in-ui.md

@@ -0,0 +1,29 @@
+# Use LLamaSharp in BotSharp
+
+Start the BotSharp backend and frontend services, and follow this tutorial.
+
+## Install LLamaSharp Plugin in UI.
+
+Go to the Plugin page and install LLamaSharp Plugin.
+
+![Install LlamaSharp Plugin](assets/install-llamasharp-plugin.png)
+
+## Config LLamaSharp as LLM Providers for Agents
+
+Edit or create an agent in Agents page, and config the agent.
+
+![Edit Agent](assets/edit-agent.png)
+
+In the edit page, config the provider as llama-sharp.
+
+![Choose LLamaSharp as Provider](assets/choose-llamasharp-as-provider.png)
+
+Then test the agent.
+
+![Click Test Agent Button](assets/click-test-button.png)
+
+![Test Agent Example](assets/converstaion-examples.png)
+
+If run successfully, you will see log like this in BotSharp service's console.
+
+![Console Output](assets/console-output-in-botsharp.png)

src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs

@@ -67,4 +67,7 @@ public abstract class ConversationHookBase : IConversationHook
 
     public virtual Task OnUserAgentConnectedInitially(Conversation conversation)
         => Task.CompletedTask;
+
+    public virtual Task OnMessageDeleted(string conversationId, string messageId)
+        => Task.CompletedTask;
 }

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs

@@ -84,4 +84,12 @@ public interface IConversationHook
     /// <param name="conversation"></param>
     /// <returns></returns>
     Task OnHumanInterventionNeeded(RoleDialogModel message);
+
+    /// <summary>
+    /// Delete message in a conversation
+    /// </summary>
+    /// <param name="conversationId"></param>
+    /// <param name="messageId"></param>
+    /// <returns></returns>
+    Task OnMessageDeleted(string conversationId, string messageId);
 }

src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs

@@ -31,7 +31,7 @@ public interface IRoutingService
     List<RoutingHandlerDef> GetHandlers(Agent router);
     void ResetRecursiveCounter();
     Task<bool> InvokeAgent(string agentId, List<RoleDialogModel> dialogs);
-    Task<bool> InvokeFunction(string name, RoleDialogModel message);
+    Task<bool> InvokeFunction(string name, RoleDialogModel message, bool restoreOriginalFunctionName = true);
     Task<RoleDialogModel> InstructLoop(RoleDialogModel message);
 
     /// <summary>

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs

@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Templating;
 
 namespace BotSharp.Core.Agents.Services;
@@ -96,12 +95,24 @@ public partial class AgentService
         // render liquid template
         var render = _services.GetRequiredService<ITemplateRender>();
         var template = agent.Templates.First(x => x.Name == templateName).Content;
+        // update states
+        var conv = _services.GetRequiredService<IConversationService>();
+        foreach (var t in conv.States.GetStates())
+        {
+            agent.TemplateDict[t.Key] = t.Value;
+        }
         return render.Render(template, agent.TemplateDict);
     }
 
     public string RenderedInstruction(Agent agent)
     {
         var render = _services.GetRequiredService<ITemplateRender>();
+        // update states
+        var conv = _services.GetRequiredService<IConversationService>();
+        foreach (var t in conv.States.GetStates())
+        {
+            agent.TemplateDict[t.Key] = t.Value;
+        }
         return render.Render(agent.Instruction, agent.TemplateDict);
     }
 

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -131,7 +131,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="AspectInjector" Version="2.8.2" />
     <PackageReference Include="Aspects.Cache" Version="2.0.4" />
     <PackageReference Include="Colorful.Console" Version="1.2.15" />
     <PackageReference Include="EntityFrameworkCore.BootKit" Version="6.3.0" />

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs

@@ -8,6 +8,11 @@ public partial class ConversationService : IConversationService
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var isSaved = db.TruncateConversation(conversationId, messageId, true);
+        var hooks = _services.GetServices<IConversationHook>().ToList();
+        foreach (var hook in hooks)
+        {
+            await hook.OnMessageDeleted(conversationId, messageId);
+        }
         return await Task.FromResult(isSaved);
     }
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -305,7 +305,7 @@ namespace BotSharp.Core.Repository
         public List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
         {
             var ids = new List<string>();
-            var batchLimit = 50;
+            var batchLimit = 100;
             var utcNow = DateTime.UtcNow;
             var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir);
 

src/Infrastructure/BotSharp.Core/Routing/Handlers/HumanInterventionNeededHandler.cs

@@ -12,7 +12,7 @@ public class HumanInterventionNeededHandler : RoutingHandlerBase, IRoutingHandle
     {
         new ParameterPropertyDef("reason", "why need customer service"),
         new ParameterPropertyDef("summary", "the whole conversation summary with important information"),
-        new ParameterPropertyDef("response", "response content to user")
+        new ParameterPropertyDef("response", "tell the user that you are being transferred to customer service")
     };
 
     public HumanInterventionNeededHandler(IServiceProvider services, ILogger<HumanInterventionNeededHandler> logger, RoutingSettings settings)
@@ -23,13 +23,9 @@ public class HumanInterventionNeededHandler : RoutingHandlerBase, IRoutingHandle
 
     public async Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message)
     {
-        var response = new RoleDialogModel(AgentRole.Assistant, inst.Response)
-        {
-            CurrentAgentId = message.CurrentAgentId,
-            MessageId = message.MessageId,
-            StopCompletion = true,
-            FunctionName = inst.Function
-        };
+        var response = RoleDialogModel.From(message, 
+            role: AgentRole.Assistant, 
+            content: inst.Response);
 
         _dialogs.Add(response);
 

src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs

@@ -44,13 +44,14 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler
 
         var states = _services.GetRequiredService<IConversationStateService>();
         var goalAgent = states.GetState("user_goal_agent");
-        if (!string.IsNullOrEmpty(goalAgent))
+        if (!string.IsNullOrEmpty(goalAgent) && inst.OriginalAgent != goalAgent)
         {
             inst.OriginalAgent = goalAgent;
+            // Emit hook
+            await HookEmitter.Emit<IRoutingHook>(_services, async hook =>
+                await hook.OnRoutingInstructionRevised(inst, message)
+            );
         }
-        await HookEmitter.Emit<IRoutingHook>(_services, async hook =>
-            await hook.OnRoutingInstructionRevised(inst, message)
-        );
 
         var agentId = routing.Context.GetCurrentAgentId();
 

src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs

@@ -1,8 +1,3 @@
-using BotSharp.Abstraction.Agents.Models;
-using BotSharp.Abstraction.Functions.Models;
-using BotSharp.Abstraction.Repositories;
-using BotSharp.Abstraction.Repositories.Filters;
-using BotSharp.Abstraction.Routing;
 using BotSharp.Abstraction.Routing.Planning;
 using BotSharp.Abstraction.Templating;
 
@@ -93,6 +88,12 @@ public class HFPlanner : IPlaner
     {
         var template = router.Templates.First(x => x.Name == "planner_prompt.hf").Content;
         var render = _services.GetRequiredService<ITemplateRender>();
+        // update states
+        var conv = _services.GetRequiredService<IConversationService>();
+        foreach (var t in conv.States.GetStates())
+        {
+            router.TemplateDict[t.Key] = t.Value;
+        }
         var prompt = render.Render(template, router.TemplateDict);
         return prompt.Trim();
     }

src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs

@@ -4,7 +4,7 @@ namespace BotSharp.Core.Routing;
 
 public partial class RoutingService
 {
-    public async Task<bool> InvokeFunction(string name, RoleDialogModel message)
+    public async Task<bool> InvokeFunction(string name, RoleDialogModel message, bool restoreOriginalFunctionName = true)
     {
         var function = _services.GetServices<IFunctionCallback>().FirstOrDefault(x => x.Name == name);
         if (function == null)
@@ -56,7 +56,9 @@ public partial class RoutingService
         }
 
         // restore original function name
-        if (!message.StopCompletion && message.FunctionName != originalFunctionName)
+        if (!message.StopCompletion && 
+            message.FunctionName != originalFunctionName &&
+            restoreOriginalFunctionName)
         {
             message.FunctionName = originalFunctionName;
         }

src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

@@ -54,6 +54,7 @@ public class UserService : IUserService
 
         db.CreateUser(record);
 
+        _logger.LogWarning($"Created new user account: {record.Id} {record.UserName}");
         Utilities.ClearCache();
 
         return record;

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs

@@ -240,8 +240,7 @@ public class ChatCompletionProvider : IChatCompletion
             }
             else if (message.Role == ChatRole.User)
             {
-                var userMessage = new ChatRequestUserMessage(
-                    new ChatMessageTextContentItem(message.Content))
+                var userMessage = new ChatRequestUserMessage(message.Content)
                 {
                     // To display Planner name in log
                     Name = message.FunctionName,

src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs

@@ -96,4 +96,14 @@ public class ChatHubConversationHook : ConversationHookBase
 
         await base.OnResponseGenerated(message);
     }
+
+    public override async Task OnMessageDeleted(string conversationId, string messageId)
+    {
+        await _chatHub.Clients.User(_user.Id).SendAsync("OnMessageDeleted", new ChatResponseModel
+        {
+            ConversationId = conversationId,
+            MessageId = messageId
+        });
+        await base.OnMessageDeleted(conversationId, messageId);
+    }
 }

src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs

@@ -133,11 +133,6 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
                 log += $"\r\n```json\r\n{richContent}\r\n```";
             }
 
-            if (!string.IsNullOrEmpty(message.FunctionName))
-            {
-                log += $"\r\n\r\n**{message.FunctionName}**";
-            }
-
             var input = new ContentLogInputModel(conv.ConversationId, message)
             {
                 Name = agent?.Name,

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -306,7 +306,7 @@ public partial class MongoRepository
     public List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours)
     {
         var page = 1;
-        var batchLimit = 50;
+        var batchLimit = 100;
         var utcNow = DateTime.UtcNow;
         var conversationIds = new List<string>();