diff --git a/Directory.Build.props b/Directory.Build.props index a61058f4..249103d7 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ 10.0 ..\..\..\packages - 0.12.1 + 0.12.3 true \ No newline at end of file diff --git a/docs/architecture/assets/overview.drawio b/docs/architecture/assets/overview.drawio index bfb06fbf..5915ea63 100644 --- a/docs/architecture/assets/overview.drawio +++ b/docs/architecture/assets/overview.drawio @@ -1,11 +1,11 @@ - + - + - - + + diff --git a/docs/conf.py b/docs/conf.py index 666719bf..0f57d91c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,9 +64,9 @@ author = 'Haiping Chen' # built documents. # # The short X.Y version. -version = '0.9' +version = '0.12' # The full version, including alpha/beta/rc tags. -release = '0.9.0' +release = '0.12.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -170,7 +170,7 @@ man_pages = [ # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'BotSharp', 'BotSharp Documentation', - author, 'BotSharp', 'The LLM powered Chatbot framework.', + author, 'BotSharp', 'The LLM application framework.', 'Miscellaneous'), ] diff --git a/docs/index.rst b/docs/index.rst index 0c1f23f8..6d710b2d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,23 +3,23 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -The Open Source AI Bot Platform Builder +The Open Source LLM Application Framework ====================================================== .. image:: https://img.shields.io/discord/1106946823282761851?label=Discord :target: `discord`_ -**Build the AI chatbot platform from scratch with .NET** +**Build the AI bot from scratch with .NET** -> The LLM powered Conversational Service framework +> The LLM powered Conversational Service building blocks and best practice *"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to .NET developers using BotSharp the Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your robots and precisely control every step of the AI processing pipeline."* -**BotSharp** is an open source bot framework for AI Bot platform builders. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. +**BotSharp** is an open source AI framework for your enterprise-grade LLM applications. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in business oriented systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. It's witten in C# running on .NET which is a full cross-platform framework. C# is an enterprise-grade programming language which is widely used to code business logic in information management related system. More friendly to corporate developers. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be easier when refactoring code in system scope. -BotSharp is strictly in accordance with the components principle and decouples every part that is needed in the platform builder. So you can choose different UI/UX, or pick up a different NLP Tagger, or select a more advanced algorithm to do NER tasks. They are all modularized based on unified interfaces. +BotSharp is strictly in accordance with the components principle and decouples every part that is needed in the platform builder. So you can choose different UI/UX, or pick up a different Vector Storage, or select a more advanced algorithm to do NLU tasks. They are all modularized based on unified interfaces. .. image:: static/logos/BotSharp.png :height: 64px diff --git a/docs/quick-start/assets/overview.png b/docs/quick-start/assets/overview.png new file mode 100644 index 00000000..77411272 Binary files /dev/null and b/docs/quick-start/assets/overview.png differ diff --git a/docs/quick-start/overview.md b/docs/quick-start/overview.md index f6cebe09..12146ba3 100644 --- a/docs/quick-start/overview.md +++ b/docs/quick-start/overview.md @@ -1,10 +1,10 @@ # Overview -BotSharp is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out of the box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. +BotSharp is an open source application framework to speed up integrate LLMs into your current business system. This project involves natural language understanding and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out of the box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. BotSharp is an high compatible and high scalable platform builder. It is in accordance with components princple strictly, decouples every part that needed in the platform builder. So you can choose different UI/UX, or pick up a different NLP Tagger, or select a more advanced algrithm to do NER task. They are all modulized based on unfied interfaces. -![ ](../static/screenshots/BotSharp_arch.png "BotSharp Architecture Chart") +![ ](./assets/overview.png "BotSharp Architecture Chart") From the chart ahead we can see that based on botsharp you can launch your own chatbot platform with 3 components: - Storage module: Botsharp supports memory and redis DB 2 methods. @@ -21,7 +21,7 @@ Even with this simple question, you can see conversational experience are hard t Your code would have to handle all these different types of requests ro carry out the same logic: looking up some forecast information for a feature. For this reason, a traditional computer interface would tend to force users to input a well-known, standard request at the detriment of the user experience, because it's just easier. -However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you.When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand. +However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you. When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand. Features ------------- @@ -30,6 +30,6 @@ Features * Integrate with multiple LLMs like ChatGPT and LLaMA. * Using plug-in design, it is easy to expand functions. * Working with multiple Vector Stores for senmatic search. -* Supporting different UI providers like [Chatbot UI](https://github.com/mckaywrigley/chatbot-ui) and [HuggingChat UI](https://github.com/huggingface/chat-ui). +* Supporting different UI providers like [Chatbot UI](https://github.com/SciSharp/chatbot-ui) and [HuggingChat UI](https://github.com/huggingface/chat-ui). * Integrated with popular social platforms like Facebook Messenger, Slack and Telegram. * Providing REST APIs to work with your own UI. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs index 8818eb03..4a39c144 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs @@ -8,7 +8,7 @@ public interface IConversationStateService ConversationState Load(string conversationId); string GetState(string name, string defaultValue = ""); ConversationState GetStates(); - IConversationStateService SetState(string name, string value); + IConversationStateService SetState(string name, T value); void CleanState(); void Save(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/IncomingMessageModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/IncomingMessageModel.cs index b0186d8a..e69d94bc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/IncomingMessageModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/IncomingMessageModel.cs @@ -8,11 +8,17 @@ public class IncomingMessageModel public virtual string Channel { get; set; } = string.Empty; + /// + /// Completion Provider + /// + [JsonPropertyName("provider")] + public virtual string? Provider { get; set; } = null; + /// /// Model name /// [JsonPropertyName("model")] - public virtual string? ModelName { get; set; } = null; + public virtual string? Model { get; set; } = null; /// /// The sampling temperature to use that controls the apparent creativity of generated completions. diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index f347a8a7..15ad54aa 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.MLTasks; public interface IChatCompletion { - string ModelName { get; } + string Provider { get; } Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs index 107461cb..c51f59e6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs @@ -1,4 +1,3 @@ -using System.Linq; using System.Text.RegularExpressions; namespace BotSharp.Abstraction.Utilities; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.GetChatCompletionsAsyncRecursively.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.GetChatCompletionsAsyncRecursively.cs index f125149d..9ba5e4a5 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.GetChatCompletionsAsyncRecursively.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.GetChatCompletionsAsyncRecursively.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Templating; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 9ed62d0f..47b144f7 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -131,7 +131,7 @@ public partial class ConversationService { if (!string.IsNullOrEmpty(property.Value.ToString())) { - stateService.SetState(property.Name, property.Value.ToString()); + stateService.SetState(property.Name, property.Value); } } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index cfbf1bf6..913bf88f 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -29,13 +29,18 @@ public class ConversationStateService : IConversationStateService, IDisposable _states = new ConversationState(); } - public IConversationStateService SetState(string name, string value) + public IConversationStateService SetState(string name, T value) { + if (value == null) + { + return this; + } + + var currentValue = value.ToString(); var hooks = _services.GetServices(); string preValue = _states.ContainsKey(name) ? _states[name] : ""; - if (!_states.ContainsKey(name) || _states[name] != value) + if (!_states.ContainsKey(name) || _states[name] != currentValue) { - var currentValue = value; _states[name] = currentValue; _logger.LogInformation($"Set state: {name} = {value}"); foreach (var hook in hooks) diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index f5dd62db..9534581e 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -4,16 +4,16 @@ namespace BotSharp.Core.Infrastructures; public class CompletionProvider { - public static IChatCompletion GetChatCompletion(IServiceProvider services, string? model = null) + public static IChatCompletion GetChatCompletion(IServiceProvider services, string? provider = null) { var completions = services.GetServices(); var state = services.GetRequiredService(); - if (model == null) + if (provider == null) { - model = state.GetState("model", "gpt-3.5-turbo"); + provider = state.GetState("provider", "azure-gpt-3.5"); } - return completions.FirstOrDefault(x => x.ModelName == model); + return completions.FirstOrDefault(x => x.Provider == provider); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs b/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs index 77905af2..8bcad2f7 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Simulator.cs @@ -61,7 +61,7 @@ public class Simulator { var wholeDialogs = new List { - new RoleDialogModel(AgentRole.User, @"What's the next step, your response must be in JSON format with ""function"" and ""parameters"". ") + new RoleDialogModel(AgentRole.User, @"What's the next step? Response in JSON format with ""function"" and ""parameters"".") }; var chatCompletion = CompletionProvider.GetChatCompletion(_services); @@ -146,7 +146,7 @@ public class Simulator { if (!string.IsNullOrEmpty(property.Value.ToString())) { - stateService.SetState(property.Name, property.Value.ToString()); + stateService.SetState(property.Name, property.Value); } } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index ac64b87a..3df4a110 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -45,9 +45,10 @@ public class ConversationController : ControllerBase, IApiAdapter var conv = _services.GetRequiredService(); conv.SetConversationId(conversationId, input.States); conv.States.SetState("channel", input.Channel) - .SetState("model", input.ModelName) - .SetState("temperature", input.Temperature.ToString()) - .SetState("sampling_factor", input.SamplingFactor.ToString()); + .SetState("provider", input.Provider) + .SetState("model", input.Model) + .SetState("temperature", input.Temperature) + .SetState("sampling_factor", input.SamplingFactor); var response = new MessageResponseModel(); var stackMsg = new List(); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 65604967..ad792f7f 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -24,7 +24,7 @@ public class ChatCompletionProvider : IChatCompletion protected readonly IServiceProvider _services; protected readonly ILogger _logger; - public virtual string ModelName => "gpt-3.5-turbo"; + public virtual string Provider => "azure-gpt-3.5"; public ChatCompletionProvider(AzureOpenAiSettings settings, ILogger logger, diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/GPT4CompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/GPT4CompletionProvider.cs index bb9dd480..63c0da53 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/GPT4CompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/GPT4CompletionProvider.cs @@ -8,7 +8,7 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers; public class GPT4CompletionProvider : ChatCompletionProvider { - public override string ModelName => "gpt-4"; + public override string Provider => "azure-gpt-4"; public GPT4CompletionProvider(AzureOpenAiSettings settings, ILogger logger, diff --git a/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs b/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs index ea784148..2aa4787a 100644 --- a/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs +++ b/src/Plugins/BotSharp.Plugin.ChatbotUI/ChatbotUiController.cs @@ -76,10 +76,11 @@ public class ChatbotUiController : ControllerBase, IApiAdapter var conv = _services.GetRequiredService(); conv.SetConversationId(input.ConversationId, input.States); - conv.States.SetState("model", input.ModelName); - conv.States.SetState("channel", "webchat"); - conv.States.SetState("temperature", input.Temperature.ToString()); - conv.States.SetState("sampling_factor", input.SamplingFactor.ToString()); + conv.States.SetState("provider", input.Provider) + .SetState("model", input.Model) + .SetState("channel", "webchat") + .SetState("temperature", input.Temperature) + .SetState("sampling_factor", input.SamplingFactor); var result = await conv.SendMessage(input.AgentId, message, diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj index c76256da..7c6b51fe 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/BotSharp.Plugin.HuggingFace.csproj @@ -10,6 +10,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/FalconLlmResponse.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/FalconLlmResponse.cs new file mode 100644 index 00000000..a44a1f76 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/FalconLlmResponse.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Plugin.HuggingFace.DataModels; + +public class FalconLlmResponse +{ + [JsonPropertyName("generated_text")] + public string GeneratedText { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInput.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInput.cs new file mode 100644 index 00000000..d432590f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInput.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Plugin.HuggingFace.DataModels; + +public class InferenceInput +{ + [JsonPropertyName("inputs")] + public string Inputs { get; set; } = string.Empty; + + [JsonPropertyName("parameters")] + public InferenceInputParameters Parameters { get; set; } + = new InferenceInputParameters(); + + [JsonPropertyName("options")] + public InferenceInputOptions Options { get; set; } + = new InferenceInputOptions(); +} diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputOptions.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputOptions.cs new file mode 100644 index 00000000..960ca8a2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputOptions.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Plugin.HuggingFace.DataModels; + +public class InferenceInputOptions +{ + [JsonPropertyName("use_cache")] + public bool UseCache { get; set; } = true; + + [JsonPropertyName("wait_for_model")] + public bool WaitForModel { get; set; } = false; +} diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputParameters.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputParameters.cs new file mode 100644 index 00000000..13ec492c --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputParameters.cs @@ -0,0 +1,16 @@ +namespace BotSharp.Plugin.HuggingFace.DataModels; + +public class InferenceInputParameters +{ + [JsonPropertyName("temperature")] + public float Temperature { get; set; } = 1.0f; + + [JsonPropertyName("max_new_tokens")] + public int MaxNewTokens { get; set; } = 250; + + [JsonPropertyName("return_full_text")] + public bool ReturnFullText { get; set; } = false; + + [JsonPropertyName("num_return_sequences")] + public int NumReturnSequences { get; set; } = 1; +} diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/HuggingChat/HuggingChatController.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/HuggingChat/HuggingChatController.cs index 3944ef6f..46acb00d 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/HuggingChat/HuggingChatController.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/HuggingChat/HuggingChatController.cs @@ -3,12 +3,8 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Net.Http.Headers; using Newtonsoft.Json.Serialization; using Newtonsoft.Json; -using System; -using System.Text; -using System.Threading.Tasks; using BotSharp.Plugin.HuggingFace.HuggingChat.ViewModels; using BotSharp.Abstraction.TextGeneratives; -using System.Collections.Generic; namespace BotSharp.Plugin.HuggingFace.HuggingChat; diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/HuggingFacePlugin.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/HuggingFacePlugin.cs new file mode 100644 index 00000000..87098557 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/HuggingFacePlugin.cs @@ -0,0 +1,29 @@ +using BotSharp.Abstraction.Plugins; +using BotSharp.Plugin.HuggingFace.Providers; +using BotSharp.Plugin.HuggingFace.Services; +using BotSharp.Plugin.HuggingFace.Settings; +using Refit; + +namespace BotSharp.Plugin.HuggingFace; + +public class HuggingFacePlugin : IBotSharpPlugin +{ + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + var settings = new HuggingFaceSettings(); + config.Bind("HuggingFace", settings); + services.AddSingleton(x => + { + Console.WriteLine($"Loaded HuggingFace settings: {settings.EndPoint} ({settings.Model}) {settings.Token.SubstringMax(4)}"); + return settings; + }); + + services + .AddRefitClient() + .AddHttpMessageHandler() + .ConfigureHttpClient(c => c.BaseAddress = new Uri(settings.EndPoint)); + + services.AddTransient(); + services.AddScoped(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs new file mode 100644 index 00000000..ceac5320 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -0,0 +1,72 @@ +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Conversations.Settings; +using BotSharp.Plugin.HuggingFace.Services; +using BotSharp.Plugin.HuggingFace.Settings; +using Microsoft.Extensions.Logging; + +namespace BotSharp.Plugin.HuggingFace.Providers; + +public class ChatCompletionProvider : IChatCompletion +{ + public string Provider => "huggingface"; + + private readonly IServiceProvider _services; + private readonly HuggingFaceSettings _settings; + private readonly ILogger _logger; + + public ChatCompletionProvider(IServiceProvider services, + HuggingFaceSettings settings, + ILogger logger) + { + _services = services; + _settings = settings; + _logger = logger; + } + + public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) + { + var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim(); + content += $"\r\n{AgentRole.Assistant}: "; + + var prompt = agent.Instruction + "\r\n" + content; + + var convSetting = _services.GetRequiredService(); + if (convSetting.ShowVerboseLog) + { + _logger.LogInformation(prompt); + } + + var api = _services.GetRequiredService(); + + if (_settings.Model.Contains('/')) + { + var space = _settings.Model.Split('/')[0]; + var model = _settings.Model.Split("/")[1]; + + var response = await api.Post(space, model, new InferenceInput + { + Inputs = prompt + }); + + var falcon = JsonSerializer.Deserialize>(response); + + var message = falcon[0].GeneratedText.Trim(); + _logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}"); + + var msg = new RoleDialogModel(AgentRole.Assistant, message) + { + CurrentAgentId = agent.Id + }; + + // Text response received + await onMessageReceived(msg); + } + + return true; + } + + public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) + { + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Services/AuthHeaderHandler.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Services/AuthHeaderHandler.cs new file mode 100644 index 00000000..c350f98e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Services/AuthHeaderHandler.cs @@ -0,0 +1,22 @@ +using BotSharp.Plugin.HuggingFace.Settings; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; + +namespace BotSharp.Plugin.HuggingFace.Services; + +public class AuthHeaderHandler : DelegatingHandler +{ + private readonly HuggingFaceSettings _settings; + public AuthHeaderHandler(HuggingFaceSettings settings) + { + _settings = settings; + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _settings.Token); + + return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Services/IInferenceApi.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Services/IInferenceApi.cs new file mode 100644 index 00000000..49b93249 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Services/IInferenceApi.cs @@ -0,0 +1,12 @@ +using Refit; + +namespace BotSharp.Plugin.HuggingFace.Services; + +/// +/// https://huggingface.co/docs/api-inference/quicktour +/// +public interface IInferenceApi +{ + [Post("/models/{space}/{model}")] + Task Post(string space, string model, [Body] InferenceInput input); +} diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Settings/HuggingFaceSettings.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Settings/HuggingFaceSettings.cs new file mode 100644 index 00000000..cc8a42ba --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Settings/HuggingFaceSettings.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Plugin.HuggingFace.Settings; + +public class HuggingFaceSettings +{ + public string EndPoint { get; set; } + public string Model { get; set; } + public string Token { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Using.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Using.cs new file mode 100644 index 00000000..3f378dea --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Using.cs @@ -0,0 +1,14 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Threading.Tasks; +global using System.Linq; +global using System.Text.Json; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.MLTasks; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using System.Text.Json.Serialization; +global using BotSharp.Plugin.HuggingFace.DataModels; +global using BotSharp.Abstraction.Utilities; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/LlamaAiModel.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/LlamaAiModel.cs index 6f3e840c..7f9a7748 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/LlamaAiModel.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/LlamaAiModel.cs @@ -2,6 +2,7 @@ using BotSharp.Plugin.LLamaSharp.Settings; using LLama; using LLama.Abstractions; using LLama.Common; +using System.IO; namespace BotSharp.Plugins.LLamaSharp; @@ -22,23 +23,23 @@ public class LlamaAiModel public LlamaAiModel(LlamaSharpSettings settings) { _settings = settings; - - _params = new ModelParams(_settings.ModelPath) - { - ContextSize = _settings.MaxContextLength, - Seed = 1337, - GpuLayerCount = _settings.NumberOfGpuLayer - }; } - public void LoadModel() + public void LoadModel(string model) { if (_model != null) { return; } + _params = new ModelParams(Path.Combine(_settings.ModelDir, model)) + { + ContextSize = _settings.MaxContextLength, + Seed = 1337, + GpuLayerCount = _settings.NumberOfGpuLayer + }; + _model = LLamaWeights.LoadFromFile(_params); } diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index c40bf189..191d087f 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -1,8 +1,10 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Conversations.Settings; using BotSharp.Abstraction.MLTasks; +using BotSharp.Plugin.LLamaSharp.Settings; using BotSharp.Plugins.LLamaSharp; using LLama; using LLama.Common; @@ -19,39 +21,44 @@ public class ChatCompletionProvider : IChatCompletion { private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly LlamaSharpSettings _settings; public ChatCompletionProvider(IServiceProvider services, - ILogger logger) + ILogger logger, + LlamaSharpSettings settings) { _services = services; _logger = logger; + _settings = settings; } - public string ModelName => "llama-2"; - + public string Provider => "llama-sharp"; public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { - var content = string.Join("\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); - content += $"\n{AgentRole.Assistant}: "; + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); + content += $"\r\n{AgentRole.Assistant}: "; + + var state = _services.GetRequiredService(); + var model = state.GetState("model", _settings.DefaultModel); var llama = _services.GetRequiredService(); - llama.LoadModel(); + llama.LoadModel(model); var executor = llama.GetStatelessExecutor(); var inferenceParams = new InferenceParams() { - Temperature = 1.0f, - AntiPrompts = new List { $"{AgentRole.User}:", "\n", "?" }, + Temperature = 0.9f, + AntiPrompts = new List { $"{AgentRole.User}:" }, MaxTokens = 256 }; string totalResponse = ""; - var prompt = agent.Instruction + content; + var prompt = agent.Instruction + "\r\n" + content; var convSetting = _services.GetRequiredService(); if (convSetting.ShowVerboseLog) @@ -70,7 +77,15 @@ public class ChatCompletionProvider : IChatCompletion totalResponse = totalResponse.Replace(anti, "").Trim(); } - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, totalResponse)); + _logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {totalResponse}"); + + var msg = new RoleDialogModel(AgentRole.Assistant, totalResponse) + { + CurrentAgentId = agent.Id + }; + + // Text response received + await onMessageReceived(msg); return true; } @@ -78,11 +93,15 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) { string totalResponse = ""; - var content = string.Join("\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); - content += $"\n{AgentRole.Assistant}: "; + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); + content += $"\r\n{AgentRole.Assistant}: "; + + var state = _services.GetRequiredService(); + var model = state.GetState("model", "llama-2-7b-chat.Q8_0"); var llama = _services.GetRequiredService(); - llama.LoadModel(); + llama.LoadModel(model); + var executor = new StatelessExecutor(llama.Model, llama.Params); var inferenceParams = new InferenceParams() { Temperature = 1.0f, AntiPrompts = new List { $"{AgentRole.User}:" }, MaxTokens = 64 }; diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs index 62009ef7..aa42b7e2 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Conversations; using BotSharp.Abstraction.MLTasks; +using BotSharp.Plugin.LLamaSharp.Settings; using BotSharp.Plugins.LLamaSharp; using LLama; using LLama.Common; @@ -11,16 +13,22 @@ namespace BotSharp.Plugin.LLamaSharp.Providers; public class TextCompletionProvider : ITextCompletion { private readonly IServiceProvider _services; + private readonly LlamaSharpSettings _settings; - public TextCompletionProvider(IServiceProvider services) + public TextCompletionProvider(IServiceProvider services, + LlamaSharpSettings settings) { _services = services; + _settings = settings; } public Task GetCompletion(string text) { + var state = _services.GetRequiredService(); + var model = state.GetState("model", _settings.DefaultModel); + var llama = _services.GetRequiredService(); - llama.LoadModel(); + llama.LoadModel(model); var executor = new InstructExecutor(llama.Model.CreateContext(llama.Params)); var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 }; diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs index 0738c1a4..d5774735 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs @@ -4,6 +4,7 @@ using LLama; using LLama.Common; using System; using System.Collections.Generic; +using System.IO; namespace BotSharp.Plugin.LLamaSharp.Providers; @@ -24,7 +25,8 @@ public class TextEmbeddingProvider : ITextEmbedding { if (_embedder == null) { - _embedder = new LLamaEmbedder(new ModelParams(_settings.ModelPath)); + var path = Path.Combine(_settings.ModelDir, _settings.DefaultModel); + _embedder = new LLamaEmbedder(new ModelParams(path)); } return _embedder.GetEmbeddings(text); diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Settings/LlamaSharpSettings.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Settings/LlamaSharpSettings.cs index fc5d7d44..f2a204e2 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Settings/LlamaSharpSettings.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Settings/LlamaSharpSettings.cs @@ -2,7 +2,8 @@ namespace BotSharp.Plugin.LLamaSharp.Settings; public class LlamaSharpSettings { - public string ModelPath { get; set; } = string.Empty; + public string ModelDir { get; set; } = string.Empty; + public string DefaultModel { get; set; } = "llama-2-7b-chat.Q8_0.gguf"; public int MaxContextLength { get; set; } = 512; public float RepeatPenalty { get; set; } = 1.0f; public bool VerbosePrompt { get; set; } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index dd96fd71..23c898a7 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -31,9 +31,10 @@ "LlamaSharp": { "Interactive": true, - "ModelPath": "C:/Users/haipi/Downloads/llama-2-7b-guanaco-qlora.Q4_K_S.gguf", + "ModelDir": "C:/Users/haipi/Downloads", + "DefaultModel": "llama-2-7b-chat.Q8_0.gguf", "MaxContextLength": 1024, - "NumberOfGpuLayer": 15 + "NumberOfGpuLayer": 10 }, "AzureOpenAi": { @@ -45,6 +46,12 @@ } }, + "HuggingFace": { + "Endpoint": "https://api-inference.huggingface.co", + "Model": "tiiuae/falcon-180B-chat", + "Token": "" + }, + "MetaAi": { "fastText": { "ModelPath": "dbpedia.ftz" @@ -88,13 +95,13 @@ }, "KnowledgeBase": { - "VectorDb": "MemVectorDatabase", - // "VectorDb": "QdrantDb", - "TextEmbedding": "fastTextEmbeddingProvider", - // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider", - "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", - // "TextCompletion": "LLamaSharp.TextCompletionProvider", - "Pdf2TextConverter": "PaddleSharp.Providers.Pdf2TextConverter" + "VectorDb": "MemVectorDatabase", + // "VectorDb": "QdrantDb", + "TextEmbedding": "fastTextEmbeddingProvider", + // "TextEmbedding": "LLamaSharp.TextEmbeddingProvider", + "TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider", + // "TextCompletion": "LLamaSharp.TextCompletionProvider", + "Pdf2TextConverter": "PaddleSharp.Providers.Pdf2TextConverter" }, "PluginLoader": { @@ -103,6 +110,7 @@ "BotSharp.Core", "BotSharp.Plugin.AzureOpenAI", "BotSharp.Plugin.MetaAI", + "BotSharp.Plugin.HuggingFace", "BotSharp.Plugin.LLamaSharp", "BotSharp.Plugin.KnowledgeBase", "BotSharp.Plugin.Qdrant", diff --git a/src/WebStarter/data/agents/91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json b/src/WebStarter/data/agents/91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json deleted file mode 100644 index 96367116..00000000 --- a/src/WebStarter/data/agents/91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "Pizza Ordering Bot", - "description": "Pizza restaurant ordering AI Chatbot.", - "createdDateTime": "2023-08-14T18:14:11.6748378Z", - "updatedDateTime": "2023-08-14T18:14:11.6756088Z", - "id": "91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a" -} \ No newline at end of file diff --git a/src/WebStarter/data/agents/91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.txt b/src/WebStarter/data/agents/91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.txt deleted file mode 100644 index 890e61f7..00000000 --- a/src/WebStarter/data/agents/91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instruction.txt +++ /dev/null @@ -1 +0,0 @@ -You are now a pizza ordering chatbot, and you can help customers order a pizza according to the user's preferences. \ No newline at end of file diff --git a/src/WebStarter/data/agents/91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/samples.txt b/src/WebStarter/data/agents/91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/samples.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/src/WebStarter/data/users/456e35c5-caf0-4d45-9084-b44a8ca717e4/agents.json b/src/WebStarter/data/users/456e35c5-caf0-4d45-9084-b44a8ca717e4/agents.json index 18aee784..3bb0ecf7 100644 --- a/src/WebStarter/data/users/456e35c5-caf0-4d45-9084-b44a8ca717e4/agents.json +++ b/src/WebStarter/data/users/456e35c5-caf0-4d45-9084-b44a8ca717e4/agents.json @@ -1,9 +1,37 @@ [ { "userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4", - "agentId": "91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", + "agentId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", "updatedTime": "2023-08-14T18:14:11.6833783Z", "createdTime": "2023-08-14T18:14:11.6829767Z", "id": "1273379c-4419-460a-b0a2-5695afd097f5" + }, + { + "userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4", + "agentId": "ff431273-0c28-4647-88bf-86d82443c579", + "updatedTime": "2023-08-14T18:14:11.6833783Z", + "createdTime": "2023-08-14T18:14:11.6829767Z", + "id": "5ed47062-3dba-4b96-a7d7-5dbab48b71ad" + }, + { + "userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4", + "agentId": "c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd", + "updatedTime": "2023-08-14T18:14:11.6833783Z", + "createdTime": "2023-08-14T18:14:11.6829767Z", + "id": "91ec91bc-2854-4c21-be7e-4a6806406d56" + }, + { + "userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4", + "agentId": "b284db86-e9c2-4c25-a59e-4649797dd130", + "updatedTime": "2023-08-14T18:14:11.6833783Z", + "createdTime": "2023-08-14T18:14:11.6829767Z", + "id": "1d6e86ee-1f72-4ac3-93bc-549a65855dc6" + }, + { + "userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4", + "agentId": "03d3fb55-9ada-423b-a6b4-f9ecddf4b26e", + "updatedTime": "2023-08-14T18:14:11.6833783Z", + "createdTime": "2023-08-14T18:14:11.6829767Z", + "id": "b10c3209-6fd6-41d1-afd9-ea9543be286d" } ] \ No newline at end of file diff --git a/tests/BotSharp.Plugin.PizzaBot/Functions/GetBakingTimeFn.cs b/tests/BotSharp.Plugin.PizzaBot/Functions/GetBakingTimeFn.cs index 1d72dfbb..0ebc56e7 100644 --- a/tests/BotSharp.Plugin.PizzaBot/Functions/GetBakingTimeFn.cs +++ b/tests/BotSharp.Plugin.PizzaBot/Functions/GetBakingTimeFn.cs @@ -4,7 +4,7 @@ namespace BotSharp.Plugin.PizzaBot.Functions; public class GetBakingTimeFn : IFunctionCallback { - public string Name => "get_cooking_remaing_time"; + public string Name => "get_cooking_remaining_time"; public async Task Execute(RoleDialogModel message) {