diff --git a/docs/agent/intro.md b/docs/agent/intro.md index ededa11c..e832d537 100644 --- a/docs/agent/intro.md +++ b/docs/agent/intro.md @@ -16,14 +16,19 @@ Suppose we need to write a Pizza restaurant order AI Bot. First, specify a name BotSharp uses the latest large language model in natural language understanding, can interact with OpenAI's ChatGPT, and also supports the most widely used open source large language model [LLaMA](https://ai.meta.com/blog/large-language-model-llama-meta-ai/) and its fine-tuning model. In this example, we use [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service) as the LLM engine. ```json -"AzureOpenAi": { - "ApiKey": "", - "Endpoint": "", - "DeploymentModel": { - "ChatCompletionModel": "", - "TextCompletionModel": "" +"LlmProviders": [ + { + "Provider": "azure-openai", + "Models": [{ + "Name": "gpt-35-turbo", + "ApiKey": "", + "Endpoint": "https://gpt-35-turbo.openai.azure.com/", + "Type": "chat", + "PromptCost": 0.0015, + "CompletionCost": 0.002 + }] } -} +] ``` If you use the installation package to run, please ensure that the [BotSharp.Plugin.AzureOpenAI](https://www.nuget.org/packages/BotSharp.Plugin.AzureOpenAI) plugin package is installed. diff --git a/docs/conf.py b/docs/conf.py index 0aacd996..a69f2133 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,9 +64,9 @@ author = 'Haiping Chen' # built documents. # # The short X.Y version. -version = '0.20' +version = '0.21' # The full version, including alpha/beta/rc tags. -release = '0.20.0' +release = '0.21.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/quick-start/installation.md b/docs/quick-start/installation.md index 4f670dbe..319bdf64 100644 --- a/docs/quick-start/installation.md +++ b/docs/quick-start/installation.md @@ -19,14 +19,29 @@ PS D:\> dotnet build `BotSharp` can work with serveral LLM providers. Update `appsettings.json` in your project. Below config is tasking Azure OpenAI as the LLM backend ```json -"AzureOpenAi": { - "ApiKey": "", - "Endpoint": "https://xxx.openai.azure.com/", - "DeploymentModel": { - "ChatCompletionModel": "", - "TextCompletionModel": "" - } -} +"LlmProviders": [ + { + "Provider": "azure-openai", + "Models": [ + { + "Name": "gpt-35-turbo", + "ApiKey": "", + "Endpoint": "https://gpt-35-turbo.openai.azure.com/", + "Type": "chat", + "PromptCost": 0.0015, + "CompletionCost": 0.002 + }, + { + "Name": "gpt-35-turbo-instruct", + "ApiKey": "", + "Endpoint": "https://gpt-35-turbo-instruct.openai.azure.com/", + "Type": "text", + "PromptCost": 0.0015, + "CompletionCost": 0.002 + } + ] + } +] ``` ### Run backend web project diff --git a/docs/quick-start/overview.md b/docs/quick-start/overview.md index 12146ba3..82ddce15 100644 --- a/docs/quick-start/overview.md +++ b/docs/quick-start/overview.md @@ -21,15 +21,4 @@ Even with this simple question, you can see conversational experience are hard t Your code would have to handle all these different types of requests ro carry out the same logic: looking up some forecast information for a feature. For this reason, a traditional computer interface would tend to force users to input a well-known, standard request at the detriment of the user experience, because it's just easier. -However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you. When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand. - -Features -------------- - -* Built-in multi-Agents management, easy to build Bot as a Service platform. -* Integrate with multiple LLMs like ChatGPT and LLaMA. -* Using plug-in design, it is easy to expand functions. -* Working with multiple Vector Stores for senmatic search. -* Supporting different UI providers like [Chatbot UI](https://github.com/SciSharp/chatbot-ui) and [HuggingChat UI](https://github.com/huggingface/chat-ui). -* Integrated with popular social platforms like Facebook Messenger, Slack and Telegram. -* Providing REST APIs to work with your own UI. \ No newline at end of file +However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you. When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand. \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs index 4092f9c7..3e127125 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs @@ -11,6 +11,11 @@ public class Agent public DateTime CreatedDateTime { get; set; } public DateTime UpdatedDateTime { get; set; } + /// + /// Default LLM settings + /// + public AgentLlmConfig? LlmConfig { get; set; } + /// /// Instruction /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs new file mode 100644 index 00000000..5b075f35 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs @@ -0,0 +1,16 @@ +namespace BotSharp.Abstraction.Agents.Models; + +public class AgentLlmConfig +{ + /// + /// Completion Provider + /// + [JsonPropertyName("provider")] + public string? Provider { get; set; } + + /// + /// Model name + /// + [JsonPropertyName("model")] + public string? Model { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs index cd54eb04..23388807 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs @@ -2,18 +2,10 @@ namespace BotSharp.Abstraction.Conversations.Models; public class TokenStatsModel { + public string Provider { get; set; } public string Model { get; set; } public string Prompt { get; set; } public int PromptCount { get; set; } public int CompletionCount { get; set; } - - /// - /// Prompt cost per 1K token - /// - public float PromptCost { get; set; } - - /// - /// Completion cost per 1K token - /// - public float CompletionCost { get; set; } + public AgentLlmConfig LlmConfig { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs index c08f4f71..5bb3f850 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs @@ -2,7 +2,5 @@ namespace BotSharp.Abstraction.Evaluations.Settings; public class EvaluatorSetting { - public string EvaluatorId { get; set; } - public string Provider { get; set; } - public string Model { get; set; } + public string AgentId { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderSettingService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderSettingService.cs new file mode 100644 index 00000000..bfe5d420 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderSettingService.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.MLTasks.Settings; + +namespace BotSharp.Abstraction.MLTasks; + +public interface ILlmProviderSettingService +{ + LlmModelSetting GetSetting(string provider, string model); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/ChatCompletionSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/ChatCompletionSetting.cs deleted file mode 100644 index ba0a6a59..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/ChatCompletionSetting.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks.Settings; - -public class ChatCompletionSetting -{ - public string Provider { get; set; } - public string Model { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs new file mode 100644 index 00000000..07fbe63b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -0,0 +1,30 @@ +namespace BotSharp.Abstraction.MLTasks.Settings; + +public class LlmModelSetting +{ + public string Name { get; set; } + public string ApiKey { get; set; } + public string Endpoint { get; set; } + public LlmModelType Type { get; set; } = LlmModelType.Chat; + + /// + /// Prompt cost per 1K token + /// + public float PromptCost { get; set; } + + /// + /// Completion cost per 1K token + /// + public float CompletionCost { get; set; } + + public override string ToString() + { + return $"[{Type}] {Name} {Endpoint}"; + } +} + +public enum LlmModelType +{ + Text = 1, + Chat = 2 +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmProviderSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmProviderSetting.cs new file mode 100644 index 00000000..b3a9d28e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmProviderSetting.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Abstraction.MLTasks.Settings; + +public class LlmProviderSetting +{ + public string Provider { get; set; } + = "azure-openai"; + + public List Models { get; set; } + = new List(); + + public override string ToString() + { + return $"{Provider} with {Models.Count} models"; + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/TextCompletionSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/TextCompletionSetting.cs deleted file mode 100644 index 7fbb1a7d..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/TextCompletionSetting.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks.Settings; - -public class TextCompletionSetting -{ - public string Provider { get; set; } - public string Model { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingContext.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingContext.cs index c65ea62f..18a20713 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingContext.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingContext.cs @@ -22,14 +22,14 @@ public class RoutingContext /// Agent that can handl user original goal. /// public string OriginAgentId - => _stack.Where(x => x != _setting.RouterId).Last(); + => _stack.Where(x => x != _setting.AgentId).Last(); public bool IsEmpty => !_stack.Any(); public string GetCurrentAgentId() { if (_stack.Count == 0) { - _stack.Push(_setting.RouterId); + _stack.Push(_setting.AgentId); } return _stack.Peek(); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs index 62a6ae4a..8428ac99 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs @@ -5,10 +5,7 @@ public class RoutingSettings /// /// Router Agent Id /// - public string RouterId { get; set; } = string.Empty; + public string AgentId { get; set; } = string.Empty; public string Planner { get; set; } = string.Empty; - public string Provider { get; set; } = string.Empty; - - public string Model { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs index afa93e33..bfb8dca1 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs @@ -19,6 +19,8 @@ using BotSharp.Core.Evaluations; using BotSharp.Abstraction.MLTasks.Settings; using BotSharp.Abstraction.Planning; using BotSharp.Core.Planning; +using BotSharp.Abstraction.MLTasks; +using static Dapper.SqlMapper; namespace BotSharp.Core; @@ -27,7 +29,7 @@ public static class BotSharpCoreExtensions public static IServiceCollection AddBotSharpCore(this IServiceCollection services, IConfiguration config) { services.AddScoped(); - + services.AddScoped(); services.AddScoped(); var agentSettings = new AgentSettings(); @@ -51,13 +53,16 @@ public static class BotSharpCoreExtensions config.Bind("Database", myDatabaseSettings); services.AddSingleton((IServiceProvider x) => myDatabaseSettings); - var textCompletionSettings = new TextCompletionSetting(); - config.Bind("TextCompletion", textCompletionSettings); - services.AddSingleton((IServiceProvider x) => textCompletionSettings); - - var chatCompletionSettings = new ChatCompletionSetting(); - config.Bind("ChatCompletion", chatCompletionSettings); - services.AddSingleton((IServiceProvider x) => chatCompletionSettings); + var llmProviders = new List(); + config.Bind("LlmProviders", llmProviders); + services.AddSingleton((IServiceProvider x) => + { + foreach (var llmProvider in llmProviders) + { + Console.WriteLine($"Loaded LlmProvider {llmProvider.Provider} settings with {llmProvider.Models.Count} models."); + } + return llmProviders; + }); RegisterPlugins(services, config); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index ad1faeef..eebf4c00 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -59,7 +59,7 @@ public partial class ConversationService var routing = _services.GetRequiredService(); var settings = _services.GetRequiredService(); - response = agentId == settings.RouterId ? + response = agentId == settings.AgentId ? await routing.InstructLoop(message) : await routing.ExecuteDirectly(agent, message); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index ef5d2ba3..62b52f53 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.MLTasks; using System.Diagnostics; using System.Drawing; @@ -36,19 +37,24 @@ public class TokenStatistics : ITokenStatistics _model = stats.Model; _promptTokenCount += stats.PromptCount; _completionTokenCount += stats.CompletionCount; - _promptCost += stats.PromptCount / 1000f * stats.PromptCost; - _completionCost += stats.CompletionCount / 1000f * stats.CompletionCost; + + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(stats.Provider, _model); + + _promptCost += stats.PromptCount / 1000f * settings.PromptCost; + _completionCost += stats.CompletionCount / 1000f * settings.CompletionCost; // Accumulated Token var stat = _services.GetRequiredService(); - var count1 = int.Parse(stat.GetState("prompt_total", "0")); - stat.SetState("prompt_total", stats.PromptCount + count1); - var count2 = int.Parse(stat.GetState("completion_total", "0")); - stat.SetState("completion_total", stats.CompletionCount + count2); + var inputCount = int.Parse(stat.GetState("prompt_total", "0")); + stat.SetState("prompt_total", stats.PromptCount + inputCount); + var outputCount = int.Parse(stat.GetState("completion_total", "0")); + stat.SetState("completion_total", stats.CompletionCount + outputCount); // Total cost - var count3 = float.Parse(stat.GetState("llm_total_cost", "0")); - stat.SetState("llm_total_cost", stats.PromptCount / 1000f * stats.PromptCost + stats.CompletionCount / 1000f * stats.CompletionCost + count3); + var total_cost = float.Parse(stat.GetState("llm_total_cost", "0")); + total_cost += Cost; + stat.SetState("llm_total_cost", total_cost); } public void PrintStatistics() diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs index 75191edf..340b8f7e 100644 --- a/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs +++ b/src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs @@ -20,7 +20,7 @@ public class EvaluatingService : IEvaluatingService public async Task Execute(string task, EvaluationRequest request) { var agentService = _services.GetRequiredService(); - var evaluator = await agentService.GetAgent(_settings.EvaluatorId); + var evaluator = await agentService.GetAgent(_settings.AgentId); // Task execution mode evaluator.Instruction = evaluator.Templates.First(x => x.Name == "instruction.executor").Content; var taskPrompt = evaluator.Templates.First(x => x.Name == $"task.{task}").Content; diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 48d063d0..4dac7e68 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.MLTasks.Settings; namespace BotSharp.Core.Infrastructures; @@ -7,19 +6,18 @@ public class CompletionProvider { public static IChatCompletion GetChatCompletion(IServiceProvider services, string? provider = null, string? model = null) { - var settings = services.GetRequiredService(); var completions = services.GetServices(); var state = services.GetRequiredService(); if (string.IsNullOrEmpty(provider)) { - provider = state.GetState("provider", settings.Provider ?? "azure-openai"); + provider = state.GetState("provider", "azure-openai"); } if (string.IsNullOrEmpty(model)) { - model = state.GetState("model", settings.Model ?? "gpt-3.5-turbo"); + model = state.GetState("model", "gpt-35-turbo-4k"); } var completer = completions.FirstOrDefault(x => x.Provider == provider); @@ -36,19 +34,18 @@ public class CompletionProvider public static ITextCompletion GetTextCompletion(IServiceProvider services, string? provider = null, string? model = null) { - var settings = services.GetRequiredService(); var completions = services.GetServices(); var state = services.GetRequiredService(); if (string.IsNullOrEmpty(provider)) { - provider = state.GetState("provider", settings.Provider ?? "azure-openai"); + provider = state.GetState("provider", "azure-openai"); } if (string.IsNullOrEmpty(model)) { - model = state.GetState("model", settings.Model ?? "gpt-3.5-turbo"); + model = state.GetState("model", "gpt-35-turbo-instruct"); } var completer = completions.FirstOrDefault(x => x.Provider == provider); diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderSettingService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderSettingService.cs new file mode 100644 index 00000000..a3f8a8d3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderSettingService.cs @@ -0,0 +1,36 @@ +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.MLTasks.Settings; + +namespace BotSharp.Core.Infrastructures; + +public class LlmProviderSettingService : ILlmProviderSettingService +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public LlmProviderSettingService(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + public LlmModelSetting? GetSetting(string provider, string model) + { + var settings = _services.GetRequiredService>(); + var providerSetting = settings.FirstOrDefault(p => p.Provider.Equals(provider, StringComparison.CurrentCultureIgnoreCase)); + if (providerSetting == null) + { + _logger.LogError($"Can't find provider settings for {provider}"); + return null; + } + + var modelSetting = providerSetting.Models.FirstOrDefault(m => m.Name.Equals(model, StringComparison.CurrentCultureIgnoreCase)); + if (modelSetting == null) + { + _logger.LogError($"Can't find model settings for {provider}.{model}"); + return null; + } + + return modelSetting; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs index c5c2f13d..5a109452 100644 --- a/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs @@ -30,10 +30,9 @@ public class HFPlanner : IPlaner RoleDialogModel response = default; var inst = new FunctionCallFromLlm(); - var routerSetting = _services.GetRequiredService(); var completion = CompletionProvider.GetChatCompletion(_services, - provider: routerSetting.Provider, - model: routerSetting.Model); + provider: router?.LlmConfig?.Provider, + model: router?.LlmConfig?.Model); int retryCount = 0; while (retryCount < 3) diff --git a/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs b/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs index f69d31ae..a2de0992 100644 --- a/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs @@ -32,10 +32,9 @@ public class NaivePlanner : IPlaner var completion = CompletionProvider.GetTextCompletion(_services);*/ // chat completion - var routerSetting = _services.GetRequiredService(); var completion = CompletionProvider.GetChatCompletion(_services, - provider: routerSetting.Provider, - model: routerSetting.Model); + provider: router?.LlmConfig?.Provider, + model: router?.LlmConfig?.Model); int retryCount = 0; while (retryCount < 3) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs index 07647e10..2d3d3ab2 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs @@ -7,7 +7,7 @@ namespace BotSharp.Core.Routing.Hooks; public class RoutingAgentHook : AgentHookBase { private readonly RoutingSettings _routingSetting; - public override string SelfId => _routingSetting.RouterId; + public override string SelfId => _routingSetting.AgentId; public RoutingAgentHook(IServiceProvider services, AgentSettings settings, RoutingSettings routingSetting) : base(services, settings) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index 56699372..e218e9b7 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -21,7 +21,9 @@ public partial class RoutingService var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(agentId); - var chatCompletion = CompletionProvider.GetChatCompletion(_services); + var chatCompletion = CompletionProvider.GetChatCompletion(_services, + provider: agent?.LlmConfig?.Provider, + model: agent.LlmConfig?.Model); var message = dialogs.Last(); var response = chatCompletion.GetChatCompletions(agent, dialogs); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index a92e4a6e..8304b46e 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -64,7 +64,7 @@ public partial class RoutingService : IRoutingService public async Task InstructLoop(RoleDialogModel message) { var agentService = _services.GetRequiredService(); - _router = await agentService.LoadAgent(_settings.RouterId); + _router = await agentService.LoadAgent(_settings.AgentId); RoleDialogModel response = default; diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/TokenStatsConversationHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/TokenStatsConversationHook.cs index c7e05d15..f9bfb6e5 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/TokenStatsConversationHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/TokenStatsConversationHook.cs @@ -18,8 +18,6 @@ public class TokenStatsConversationHook : IContentGeneratingHook public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats) { _tokenStatistics.StopTimer(); - tokenStats.PromptCost = 0.0015f; - tokenStats.CompletionCost = 0.002f; _tokenStatistics.AddToken(tokenStats); await Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/NewMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/NewMessageModel.cs index 1750a303..939fa601 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/NewMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/NewMessageModel.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Conversations.Enums; -using BotSharp.Abstraction.Conversations.Models; - namespace BotSharp.OpenAPI.ViewModels.Conversations; public class NewMessageModel : IncomingMessageModel diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index 09c9d32e..624ecfe5 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -23,7 +23,7 @@ public class AzureOpenAiPlugin : IBotSharpPlugin config.Bind("AzureOpenAi", settings); services.AddSingleton(x => { - Console.WriteLine($"Loaded AzureOpenAi settings: ({settings.Endpoint}) {settings.ApiKey.SubstringMax(4)}"); + Console.WriteLine($"Loaded AzureOpenAi settings"); return settings; }); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index d2bd08a1..3953756a 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -45,7 +45,7 @@ public class ChatCompletionProvider : IChatCompletion hook.BeforeGenerating(agent, conversations).Wait(); } - var client = ProviderHelper.GetClient(_model, _settings); + var client = ProviderHelper.GetClient(_model, _services); var (prompt, chatCompletionsOptions) = PrepareOptions(agent, conversations); chatCompletionsOptions.DeploymentName = _model; var response = client.GetChatCompletions(chatCompletionsOptions); @@ -81,6 +81,7 @@ public class ChatCompletionProvider : IChatCompletion hook.AfterGenerated(responseMessage, new TokenStatsModel { Prompt = prompt, + Provider = Provider, Model = _model, PromptCount = response.Value.Usage.PromptTokens, CompletionCount = response.Value.Usage.CompletionTokens @@ -103,7 +104,7 @@ public class ChatCompletionProvider : IChatCompletion await hook.BeforeGenerating(agent, conversations); } - var client = ProviderHelper.GetClient(_model, _settings); + var client = ProviderHelper.GetClient(_model, _services); var (prompt, chatCompletionsOptions) = PrepareOptions(agent, conversations); chatCompletionsOptions.DeploymentName = _model; @@ -122,6 +123,7 @@ public class ChatCompletionProvider : IChatCompletion await hook.AfterGenerated(msg, new TokenStatsModel { Prompt = prompt, + Provider = Provider, Model = _model, PromptCount = response.Value.Usage.PromptTokens, CompletionCount = response.Value.Usage.CompletionTokens @@ -159,7 +161,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) { - var client = ProviderHelper.GetClient(_model, _settings); + var client = ProviderHelper.GetClient(_model, _services); var (prompt, chatCompletionsOptions) = PrepareOptions(agent, conversations); chatCompletionsOptions.DeploymentName = _model; var response = await client.GetChatCompletionsStreamingAsync(chatCompletionsOptions); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs index 2be17f0a..a88459f0 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs @@ -1,26 +1,21 @@ using Azure.AI.OpenAI; using Azure; using System; -using BotSharp.Plugin.AzureOpenAI.Settings; using BotSharp.Abstraction.Conversations.Models; using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using BotSharp.Abstraction.MLTasks; namespace BotSharp.Plugin.AzureOpenAI.Providers; public class ProviderHelper { - public static OpenAIClient GetClient(string model, AzureOpenAiSettings settings) + public static OpenAIClient GetClient(string model, IServiceProvider services) { - if (model.Contains("gpt-4") || model.Contains("gpt4")) - { - var client = new OpenAIClient(new Uri(settings.GPT4.Endpoint), new AzureKeyCredential(settings.GPT4.ApiKey)); - return client; - } - else - { - var client = new OpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey)); - return client; - } + var settingsService = services.GetRequiredService(); + var settings = settingsService.GetSetting("azure-openai", model); + var client = new OpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey)); + return client; } public static List GetChatSamples(List lines) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs index c24d20ef..1078a36b 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs @@ -50,7 +50,7 @@ public class TextCompletionProvider : ITextCompletion message })).ToArray()); - var client = ProviderHelper.GetClient(_model, _settings); + var client = ProviderHelper.GetClient(_model, _services); var completionsOptions = new CompletionsOptions() { @@ -87,6 +87,7 @@ public class TextCompletionProvider : ITextCompletion hook.AfterGenerated(responseMessage, new TokenStatsModel { Prompt = text, + Provider = Provider, Model = _model, PromptCount = response.Value.Usage.PromptTokens, CompletionCount = response.Value.Usage.CompletionTokens diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs index 60858a8f..d64482ff 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs @@ -2,7 +2,5 @@ namespace BotSharp.Plugin.AzureOpenAI.Settings; public class AzureOpenAiSettings { - public string ApiKey { get; set; } = string.Empty; - public string Endpoint { get; set; } = string.Empty; - public GPT4Settings GPT4 { get; set; } + } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/GPT4Settings.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/GPT4Settings.cs deleted file mode 100644 index eab41763..00000000 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/GPT4Settings.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BotSharp.Plugin.AzureOpenAI.Settings; - -public class GPT4Settings -{ - public string ApiKey { get; set; } - public string Endpoint { get; set; } - public string DeploymentModel { get; set; } -} diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs index 368f49bf..11744d0d 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs @@ -53,7 +53,7 @@ public class RoutingConversationHook: ConversationHookBase public override async Task OnResponseGenerated(RoleDialogModel message) { var routerSettings = _services.GetRequiredService(); - bool saveFlag = message.CurrentAgentId != routerSettings.RouterId; + bool saveFlag = message.CurrentAgentId != routerSettings.AgentId; if (saveFlag) { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index dcbcf30c..374a00ad 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -56,7 +56,7 @@ public class TwilioService { Gather.InputEnum.Speech }, - Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{routingSetting.RouterId}") + Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{routingSetting.AgentId}") }; gather.Say(message); response.Append(gather); @@ -82,7 +82,7 @@ public class TwilioService var gather = new Gather() { Input = new List() { Gather.InputEnum.Speech }, - Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{routingSetting.RouterId}"), + Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{routingSetting.AgentId}"), ActionOnEmptyResult = true }; if (!string.IsNullOrEmpty(message)) diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 6dbcbc2c..1558291a 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -13,17 +13,37 @@ "Key": "31ba6052aa6f4569901facc3a41fcb4a" }, + "LlmProviders": [ + { + "Provider": "azure-openai", + "Models": [ + { + "Name": "gpt-35-turbo", + "ApiKey": "", + "Endpoint": "https://gpt-35-turbo.openai.azure.com/", + "Type": "chat", + "PromptCost": 0.0015, + "CompletionCost": 0.002 + }, + { + "Name": "gpt-35-turbo-instruct", + "ApiKey": "", + "Endpoint": "https://gpt-35-turbo-instruct.openai.azure.com/", + "Type": "text", + "PromptCost": 0.0015, + "CompletionCost": 0.002 + } + ] + } + ], + "Router": { - "RouterId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", - "Planner": "NaivePlanner", - "Provider": "azure-openai", - "Model": "gpt-3.5-turbo" + "AgentId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", + "Planner": "NaivePlanner" }, "Evaluator": { - "EvaluatorId": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869", - "Provider": "azure-openai", - "Model": "gpt-3.5-turbo" + "AgentId": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869" }, "Agent": { @@ -47,19 +67,7 @@ "NumberOfGpuLayer": 10 }, - "ChatCompletion": { - "Provider": "azure-openai", - "Model": "gpt-3.5-turbo" - }, - - "TextCompletion": { - "Provider": "azure-openai", - "Model": "gpt-3.5-turbo" - }, - "AzureOpenAi": { - "ApiKey": "", - "Endpoint": "" }, "GoogleAi": { diff --git a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json index af5d7e1a..24cb1d76 100644 --- a/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json +++ b/src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json @@ -4,5 +4,9 @@ "createdDateTime": "2023-08-18T14:39:32.2349685Z", "updatedDateTime": "2023-08-18T14:39:32.2349686Z", "id": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a", - "isPublic": true + "isPublic": true, + "llmConfig": { + "provider": "azure-openai", + "model": "gpt-35-turbo" + } } \ No newline at end of file diff --git a/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json index 7b6500a5..26f15972 100644 --- a/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json +++ b/src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json @@ -3,5 +3,8 @@ "description": "Evaluate the performance of the LLM agents", "createdDateTime": "2023-08-18T00:00:00Z", "updatedDateTime": "2023-08-18T00:00:00Z", - "id": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869" + "id": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869", + "llmConfig": { + "model": "gpt-35-turbo-instruct" + } } \ No newline at end of file