Merge pull request #236 from hchen2020/llm-provider-settings

Unify the LLM Provider Settings #234

docs/agent/intro.md

@@ -16,14 +16,19 @@ Suppose we need to write a Pizza restaurant order AI Bot. First, specify a name
 BotSharp uses the latest large language model in natural language understanding, can interact with OpenAI's ChatGPT, and also supports the most widely used open source large language model [LLaMA](https://ai.meta.com/blog/large-language-model-llama-meta-ai/) and its fine-tuning model. In this example, we use [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service) as the LLM engine. 
 
 ```json
-"AzureOpenAi": {
-    "ApiKey": "",
-    "Endpoint": "",
-    "DeploymentModel": {
-        "ChatCompletionModel": "",
-        "TextCompletionModel": ""
+"LlmProviders": [
+    {
+        "Provider": "azure-openai",
+        "Models": [{
+            "Name": "gpt-35-turbo",
+            "ApiKey": "",
+            "Endpoint": "https://gpt-35-turbo.openai.azure.com/",
+            "Type": "chat",
+            "PromptCost": 0.0015,
+            "CompletionCost": 0.002
+        }]
     }
-}
+]
 ```
 
 If you use the installation package to run, please ensure that the [BotSharp.Plugin.AzureOpenAI](https://www.nuget.org/packages/BotSharp.Plugin.AzureOpenAI) plugin package is installed.

docs/conf.py

@@ -64,9 +64,9 @@ author = 'Haiping Chen'
 # built documents.
 #
 # The short X.Y version.
-version = '0.20'
+version = '0.21'
 # The full version, including alpha/beta/rc tags.
-release = '0.20.0'
+release = '0.21.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/quick-start/installation.md

@@ -19,14 +19,29 @@ PS D:\> dotnet build
 `BotSharp` can work with serveral LLM providers. Update `appsettings.json` in your project. Below config is tasking Azure OpenAI as the LLM backend
 
 ```json
-"AzureOpenAi": {
-    "ApiKey": "",
-    "Endpoint": "https://xxx.openai.azure.com/",
-    "DeploymentModel": {
-      "ChatCompletionModel": "",
-      "TextCompletionModel": ""
-    }
-}
+"LlmProviders": [
+  {
+    "Provider": "azure-openai",
+    "Models": [
+      {
+        "Name": "gpt-35-turbo",
+        "ApiKey": "",
+        "Endpoint": "https://gpt-35-turbo.openai.azure.com/",
+        "Type": "chat",
+        "PromptCost": 0.0015,
+        "CompletionCost": 0.002
+      },
+      {
+        "Name": "gpt-35-turbo-instruct",
+        "ApiKey": "",
+        "Endpoint": "https://gpt-35-turbo-instruct.openai.azure.com/",
+        "Type": "text",
+        "PromptCost": 0.0015,
+        "CompletionCost": 0.002
+      }
+    ]
+  }
+]
 ```
 
 ### Run backend web project

docs/quick-start/overview.md

@@ -21,15 +21,4 @@ Even with this simple question, you can see conversational experience are hard t
 
 Your code would have to handle all these different types of requests ro carry out the same logic: looking up some forecast information for a feature. For this reason, a traditional computer interface would tend to force users to input a well-known, standard request at the detriment of the user experience, because it's just easier.
 
-However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you. When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand.
-
-Features
--------------
-
-* Built-in multi-Agents management, easy to build Bot as a Service platform.
-* Integrate with multiple LLMs like ChatGPT and LLaMA.
-* Using plug-in design, it is easy to expand functions. 
-* Working with multiple Vector Stores for senmatic search.
-* Supporting different UI providers like [Chatbot UI](https://github.com/SciSharp/chatbot-ui) and [HuggingChat UI](https://github.com/huggingface/chat-ui).
-* Integrated with popular social platforms like Facebook Messenger, Slack and Telegram.
-* Providing REST APIs to work with your own UI.
+However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you. When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand.

src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs

@@ -11,6 +11,11 @@ public class Agent
     public DateTime CreatedDateTime { get; set; }
     public DateTime UpdatedDateTime { get; set; }
 
+    /// <summary>
+    /// Default LLM settings
+    /// </summary>
+    public AgentLlmConfig? LlmConfig { get; set; }
+
     /// <summary>
     /// Instruction
     /// </summary>

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs

@@ -0,0 +1,16 @@
+namespace BotSharp.Abstraction.Agents.Models;
+
+public class AgentLlmConfig
+{
+    /// <summary>
+    /// Completion Provider
+    /// </summary>
+    [JsonPropertyName("provider")]
+    public string? Provider { get; set; }
+
+    /// <summary>
+    /// Model name
+    /// </summary>
+    [JsonPropertyName("model")]
+    public string? Model { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs

@@ -2,18 +2,10 @@ namespace BotSharp.Abstraction.Conversations.Models;
 
 public class TokenStatsModel
 {
+    public string Provider { get; set; }
     public string Model { get; set; }
     public string Prompt { get; set; }
     public int PromptCount { get; set; }
     public int CompletionCount { get; set; }
-
-    /// <summary>
-    /// Prompt cost per 1K token
-    /// </summary>
-    public float PromptCost { get; set; }
-
-    /// <summary>
-    /// Completion cost per 1K token
-    /// </summary>
-    public float CompletionCost { get; set; }
+    public AgentLlmConfig LlmConfig { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs

@@ -2,7 +2,5 @@ namespace BotSharp.Abstraction.Evaluations.Settings;
 
 public class EvaluatorSetting
 {
-    public string EvaluatorId { get; set; }
-    public string Provider { get; set; }
-    public string Model { get; set; }
+    public string AgentId { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderSettingService.cs

@@ -0,0 +1,8 @@
+using BotSharp.Abstraction.MLTasks.Settings;
+
+namespace BotSharp.Abstraction.MLTasks;
+
+public interface ILlmProviderSettingService
+{
+    LlmModelSetting GetSetting(string provider, string model);
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/ChatCompletionSetting.cs

@@ -1,7 +0,0 @@
-namespace BotSharp.Abstraction.MLTasks.Settings;
-
-public class ChatCompletionSetting
-{
-    public string Provider { get; set; }
-    public string Model { get; set; }
-}

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -0,0 +1,30 @@
+namespace BotSharp.Abstraction.MLTasks.Settings;
+
+public class LlmModelSetting
+{
+    public string Name { get; set; }
+    public string ApiKey { get; set; }
+    public string Endpoint { get; set; }
+    public LlmModelType Type { get; set; } = LlmModelType.Chat;
+
+    /// <summary>
+    /// Prompt cost per 1K token
+    /// </summary>
+    public float PromptCost { get; set; }
+
+    /// <summary>
+    /// Completion cost per 1K token
+    /// </summary>
+    public float CompletionCost { get; set; }
+
+    public override string ToString()
+    {
+        return $"[{Type}] {Name} {Endpoint}";
+    }
+}
+
+public enum LlmModelType
+{
+    Text = 1,
+    Chat = 2
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmProviderSetting.cs

@@ -0,0 +1,15 @@
+namespace BotSharp.Abstraction.MLTasks.Settings;
+
+public class LlmProviderSetting
+{
+    public string Provider { get; set; } 
+        = "azure-openai";
+
+    public List<LlmModelSetting> Models { get; set; } 
+        = new List<LlmModelSetting>();
+
+    public override string ToString()
+    {
+        return $"{Provider} with {Models.Count} models";
+    }
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/TextCompletionSetting.cs

@@ -1,7 +0,0 @@
-namespace BotSharp.Abstraction.MLTasks.Settings;
-
-public class TextCompletionSetting
-{
-    public string Provider { get; set; }
-    public string Model { get; set; }
-}

src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingContext.cs

@@ -22,14 +22,14 @@ public class RoutingContext
     /// Agent that can handl user original goal.
     /// </summary>
     public string OriginAgentId
-        => _stack.Where(x => x != _setting.RouterId).Last();
+        => _stack.Where(x => x != _setting.AgentId).Last();
 
     public bool IsEmpty => !_stack.Any();
     public string GetCurrentAgentId()
     {
         if (_stack.Count == 0)
         {
-            _stack.Push(_setting.RouterId);
+            _stack.Push(_setting.AgentId);
         }
         return _stack.Peek();
     }

src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs

@@ -5,10 +5,7 @@ public class RoutingSettings
     /// <summary>
     /// Router Agent Id
     /// </summary>
-    public string RouterId { get; set; } = string.Empty;
+    public string AgentId { get; set; } = string.Empty;
 
     public string Planner { get; set; } = string.Empty;
-    public string Provider {  get; set; } = string.Empty;
-
-    public string Model { get; set; } = string.Empty;
 }

src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs

@@ -19,6 +19,8 @@ using BotSharp.Core.Evaluations;
 using BotSharp.Abstraction.MLTasks.Settings;
 using BotSharp.Abstraction.Planning;
 using BotSharp.Core.Planning;
+using BotSharp.Abstraction.MLTasks;
+using static Dapper.SqlMapper;
 
 namespace BotSharp.Core;
 
@@ -27,7 +29,7 @@ public static class BotSharpCoreExtensions
     public static IServiceCollection AddBotSharpCore(this IServiceCollection services, IConfiguration config)
     {
         services.AddScoped<IUserService, UserService>();
-
+        services.AddScoped<ILlmProviderSettingService, LlmProviderSettingService>();
         services.AddScoped<IAgentService, AgentService>();
 
         var agentSettings = new AgentSettings();
@@ -51,13 +53,16 @@ public static class BotSharpCoreExtensions
         config.Bind("Database", myDatabaseSettings);
         services.AddSingleton((IServiceProvider x) => myDatabaseSettings);
 
-        var textCompletionSettings = new TextCompletionSetting();
-        config.Bind("TextCompletion", textCompletionSettings);
-        services.AddSingleton((IServiceProvider x) => textCompletionSettings);
-
-        var chatCompletionSettings = new ChatCompletionSetting();
-        config.Bind("ChatCompletion", chatCompletionSettings);
-        services.AddSingleton((IServiceProvider x) => chatCompletionSettings);
+        var llmProviders = new List<LlmProviderSetting>();
+        config.Bind("LlmProviders", llmProviders);
+        services.AddSingleton((IServiceProvider x) =>
+        {
+            foreach (var llmProvider in llmProviders)
+            {
+                Console.WriteLine($"Loaded LlmProvider {llmProvider.Provider} settings with {llmProvider.Models.Count} models.");
+            }
+            return llmProviders;
+        });
 
         RegisterPlugins(services, config);
 

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs

@@ -59,7 +59,7 @@ public partial class ConversationService
             var routing = _services.GetRequiredService<IRoutingService>();
             var settings = _services.GetRequiredService<RoutingSettings>();
 
-            response = agentId == settings.RouterId ?
+            response = agentId == settings.AgentId ?
                 await routing.InstructLoop(message) :
                 await routing.ExecuteDirectly(agent, message);
 

src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.MLTasks;
 using System.Diagnostics;
 using System.Drawing;
 
@@ -36,19 +37,24 @@ public class TokenStatistics : ITokenStatistics
         _model = stats.Model;
         _promptTokenCount += stats.PromptCount;
         _completionTokenCount += stats.CompletionCount;
-        _promptCost += stats.PromptCount / 1000f * stats.PromptCost;
-        _completionCost += stats.CompletionCount / 1000f * stats.CompletionCost;
+
+        var settingsService = _services.GetRequiredService<ILlmProviderSettingService>();
+        var settings = settingsService.GetSetting(stats.Provider, _model);
+
+        _promptCost += stats.PromptCount / 1000f * settings.PromptCost;
+        _completionCost += stats.CompletionCount / 1000f * settings.CompletionCost;
 
         // Accumulated Token
         var stat = _services.GetRequiredService<IConversationStateService>();
-        var count1 = int.Parse(stat.GetState("prompt_total", "0"));
-        stat.SetState("prompt_total", stats.PromptCount + count1);
-        var count2 = int.Parse(stat.GetState("completion_total", "0"));
-        stat.SetState("completion_total", stats.CompletionCount + count2);
+        var inputCount = int.Parse(stat.GetState("prompt_total", "0"));
+        stat.SetState("prompt_total", stats.PromptCount + inputCount);
+        var outputCount = int.Parse(stat.GetState("completion_total", "0"));
+        stat.SetState("completion_total", stats.CompletionCount + outputCount);
 
         // Total cost
-        var count3 = float.Parse(stat.GetState("llm_total_cost", "0"));
-        stat.SetState("llm_total_cost", stats.PromptCount / 1000f * stats.PromptCost + stats.CompletionCount / 1000f * stats.CompletionCost + count3);
+        var total_cost = float.Parse(stat.GetState("llm_total_cost", "0"));
+        total_cost += Cost;
+        stat.SetState("llm_total_cost", total_cost);
     }
 
     public void PrintStatistics()

src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs

@@ -20,7 +20,7 @@ public class EvaluatingService : IEvaluatingService
     public async Task<Conversation> Execute(string task, EvaluationRequest request)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
-        var evaluator = await agentService.GetAgent(_settings.EvaluatorId);
+        var evaluator = await agentService.GetAgent(_settings.AgentId);
         // Task execution mode
         evaluator.Instruction = evaluator.Templates.First(x => x.Name == "instruction.executor").Content;
         var taskPrompt = evaluator.Templates.First(x => x.Name == $"task.{task}").Content;

src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.MLTasks;
-using BotSharp.Abstraction.MLTasks.Settings;
 
 namespace BotSharp.Core.Infrastructures;
 
@@ -7,19 +6,18 @@ public class CompletionProvider
 {
     public static IChatCompletion GetChatCompletion(IServiceProvider services, string? provider = null, string? model = null)
     {
-        var settings = services.GetRequiredService<ChatCompletionSetting>();
         var completions = services.GetServices<IChatCompletion>();
 
         var state = services.GetRequiredService<IConversationStateService>();
 
         if (string.IsNullOrEmpty(provider))
         {
-            provider = state.GetState("provider", settings.Provider ?? "azure-openai");
+            provider = state.GetState("provider", "azure-openai");
         }
 
         if (string.IsNullOrEmpty(model))
         {
-            model = state.GetState("model", settings.Model ?? "gpt-3.5-turbo");
+            model = state.GetState("model", "gpt-35-turbo-4k");
         }
 
         var completer = completions.FirstOrDefault(x => x.Provider == provider);
@@ -36,19 +34,18 @@ public class CompletionProvider
 
     public static ITextCompletion GetTextCompletion(IServiceProvider services, string? provider = null, string? model = null)
     {
-        var settings = services.GetRequiredService<TextCompletionSetting>();
         var completions = services.GetServices<ITextCompletion>();
 
         var state = services.GetRequiredService<IConversationStateService>();
 
         if (string.IsNullOrEmpty(provider))
         {
-            provider = state.GetState("provider", settings.Provider ?? "azure-openai");
+            provider = state.GetState("provider", "azure-openai");
         }
 
         if (string.IsNullOrEmpty(model))
         {
-            model = state.GetState("model", settings.Model ?? "gpt-3.5-turbo");
+            model = state.GetState("model", "gpt-35-turbo-instruct");
         }
 
         var completer = completions.FirstOrDefault(x => x.Provider == provider);

src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderSettingService.cs

@@ -0,0 +1,36 @@
+using BotSharp.Abstraction.MLTasks;
+using BotSharp.Abstraction.MLTasks.Settings;
+
+namespace BotSharp.Core.Infrastructures;
+
+public class LlmProviderSettingService : ILlmProviderSettingService
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public LlmProviderSettingService(IServiceProvider services, ILogger<LlmProviderSettingService> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public LlmModelSetting? GetSetting(string provider, string model)
+    {
+        var settings = _services.GetRequiredService<List<LlmProviderSetting>>();
+        var providerSetting = settings.FirstOrDefault(p => p.Provider.Equals(provider, StringComparison.CurrentCultureIgnoreCase));
+        if (providerSetting == null)
+        {
+            _logger.LogError($"Can't find provider settings for {provider}");
+            return null;
+        }
+
+        var modelSetting = providerSetting.Models.FirstOrDefault(m => m.Name.Equals(model, StringComparison.CurrentCultureIgnoreCase));
+        if (modelSetting == null)
+        {
+            _logger.LogError($"Can't find model settings for {provider}.{model}");
+            return null;
+        }
+
+        return modelSetting;
+    }
+}

src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs

@@ -30,10 +30,9 @@ public class HFPlanner : IPlaner
         RoleDialogModel response = default;
         var inst = new FunctionCallFromLlm();
 
-        var routerSetting = _services.GetRequiredService<RoutingSettings>();
         var completion = CompletionProvider.GetChatCompletion(_services,
-            provider: routerSetting.Provider,
-            model: routerSetting.Model);
+            provider: router?.LlmConfig?.Provider,
+            model: router?.LlmConfig?.Model);
 
         int retryCount = 0;
         while (retryCount < 3)

src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs

@@ -32,10 +32,9 @@ public class NaivePlanner : IPlaner
         var completion = CompletionProvider.GetTextCompletion(_services);*/
 
         // chat completion
-        var routerSetting = _services.GetRequiredService<RoutingSettings>();
         var completion = CompletionProvider.GetChatCompletion(_services, 
-            provider: routerSetting.Provider,
-            model: routerSetting.Model);
+            provider: router?.LlmConfig?.Provider,
+            model: router?.LlmConfig?.Model);
 
         int retryCount = 0;
         while (retryCount < 3)

src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingAgentHook.cs

@@ -7,7 +7,7 @@ namespace BotSharp.Core.Routing.Hooks;
 public class RoutingAgentHook : AgentHookBase
 {
     private readonly RoutingSettings _routingSetting;
-    public override string SelfId => _routingSetting.RouterId;
+    public override string SelfId => _routingSetting.AgentId;
 
     public RoutingAgentHook(IServiceProvider services, AgentSettings settings, RoutingSettings routingSetting) 
         : base(services, settings)

src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs

@@ -21,7 +21,9 @@ public partial class RoutingService
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(agentId);
 
-        var chatCompletion = CompletionProvider.GetChatCompletion(_services);
+        var chatCompletion = CompletionProvider.GetChatCompletion(_services, 
+            provider: agent?.LlmConfig?.Provider,
+            model: agent.LlmConfig?.Model);
 
         var message = dialogs.Last();
         var response = chatCompletion.GetChatCompletions(agent, dialogs);

src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs

@@ -64,7 +64,7 @@ public partial class RoutingService : IRoutingService
     public async Task<RoleDialogModel> InstructLoop(RoleDialogModel message)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
-        _router = await agentService.LoadAgent(_settings.RouterId);
+        _router = await agentService.LoadAgent(_settings.AgentId);
 
         RoleDialogModel response = default;
 

src/Infrastructure/BotSharp.Logger/Hooks/TokenStatsConversationHook.cs

@@ -18,8 +18,6 @@ public class TokenStatsConversationHook : IContentGeneratingHook
     public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats)
     {
         _tokenStatistics.StopTimer();
-        tokenStats.PromptCost = 0.0015f;
-        tokenStats.CompletionCost = 0.002f;
         _tokenStatistics.AddToken(tokenStats);
         await Task.CompletedTask;
     }

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/NewMessageModel.cs

@@ -1,6 +1,3 @@
-using BotSharp.Abstraction.Conversations.Enums;
-using BotSharp.Abstraction.Conversations.Models;
-
 namespace BotSharp.OpenAPI.ViewModels.Conversations;
 
 public class NewMessageModel : IncomingMessageModel

src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs

@@ -23,7 +23,7 @@ public class AzureOpenAiPlugin : IBotSharpPlugin
         config.Bind("AzureOpenAi", settings);
         services.AddSingleton(x =>
         {
-            Console.WriteLine($"Loaded AzureOpenAi settings: ({settings.Endpoint}) {settings.ApiKey.SubstringMax(4)}");
+            Console.WriteLine($"Loaded AzureOpenAi settings");
             return settings;
         });
 

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs

@@ -45,7 +45,7 @@ public class ChatCompletionProvider : IChatCompletion
             hook.BeforeGenerating(agent, conversations).Wait();
         }
 
-        var client = ProviderHelper.GetClient(_model, _settings);
+        var client = ProviderHelper.GetClient(_model, _services);
         var (prompt, chatCompletionsOptions) = PrepareOptions(agent, conversations);
         chatCompletionsOptions.DeploymentName = _model;
         var response = client.GetChatCompletions(chatCompletionsOptions);
@@ -81,6 +81,7 @@ public class ChatCompletionProvider : IChatCompletion
             hook.AfterGenerated(responseMessage, new TokenStatsModel
             {
                 Prompt = prompt,
+                Provider = Provider,
                 Model = _model,
                 PromptCount = response.Value.Usage.PromptTokens,
                 CompletionCount = response.Value.Usage.CompletionTokens
@@ -103,7 +104,7 @@ public class ChatCompletionProvider : IChatCompletion
             await hook.BeforeGenerating(agent, conversations);
         }
 
-        var client = ProviderHelper.GetClient(_model, _settings);
+        var client = ProviderHelper.GetClient(_model, _services);
         var (prompt, chatCompletionsOptions) = PrepareOptions(agent, conversations);
 
         chatCompletionsOptions.DeploymentName = _model;
@@ -122,6 +123,7 @@ public class ChatCompletionProvider : IChatCompletion
             await hook.AfterGenerated(msg, new TokenStatsModel
             {
                 Prompt = prompt,
+                Provider = Provider,
                 Model = _model,
                 PromptCount = response.Value.Usage.PromptTokens,
                 CompletionCount = response.Value.Usage.CompletionTokens
@@ -159,7 +161,7 @@ public class ChatCompletionProvider : IChatCompletion
 
     public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived)
     {
-        var client = ProviderHelper.GetClient(_model, _settings);
+        var client = ProviderHelper.GetClient(_model, _services);
         var (prompt, chatCompletionsOptions) = PrepareOptions(agent, conversations);
         chatCompletionsOptions.DeploymentName = _model;
         var response = await client.GetChatCompletionsStreamingAsync(chatCompletionsOptions);

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs

@@ -1,26 +1,21 @@
 using Azure.AI.OpenAI;
 using Azure;
 using System;
-using BotSharp.Plugin.AzureOpenAI.Settings;
 using BotSharp.Abstraction.Conversations.Models;
 using System.Collections.Generic;
+using Microsoft.Extensions.DependencyInjection;
+using BotSharp.Abstraction.MLTasks;
 
 namespace BotSharp.Plugin.AzureOpenAI.Providers;
 
 public class ProviderHelper
 {
-    public static OpenAIClient GetClient(string model, AzureOpenAiSettings settings)
+    public static OpenAIClient GetClient(string model, IServiceProvider services)
     {
-        if (model.Contains("gpt-4") || model.Contains("gpt4"))
-        {
-            var client = new OpenAIClient(new Uri(settings.GPT4.Endpoint), new AzureKeyCredential(settings.GPT4.ApiKey));
-            return client;
-        }
-        else
-        {
-            var client = new OpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey));
-            return client;
-        }
+        var settingsService = services.GetRequiredService<ILlmProviderSettingService>();
+        var settings = settingsService.GetSetting("azure-openai", model);
+        var client = new OpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey));
+        return client;
     }
 
     public static List<RoleDialogModel> GetChatSamples(List<string> lines)

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs

@@ -50,7 +50,7 @@ public class TextCompletionProvider : ITextCompletion
                     message
                 })).ToArray());
 
-        var client = ProviderHelper.GetClient(_model, _settings);
+        var client = ProviderHelper.GetClient(_model, _services);
 
         var completionsOptions = new CompletionsOptions()
         {
@@ -87,6 +87,7 @@ public class TextCompletionProvider : ITextCompletion
             hook.AfterGenerated(responseMessage, new TokenStatsModel
             {
                 Prompt = text,
+                Provider = Provider,
                 Model = _model,
                 PromptCount = response.Value.Usage.PromptTokens,
                 CompletionCount = response.Value.Usage.CompletionTokens

src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs

@@ -2,7 +2,5 @@ namespace BotSharp.Plugin.AzureOpenAI.Settings;
 
 public class AzureOpenAiSettings
 {
-    public string ApiKey { get; set; } = string.Empty;
-    public string Endpoint { get; set; } = string.Empty;
-    public GPT4Settings GPT4 { get; set; }
+
 }

src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/GPT4Settings.cs

@@ -1,8 +0,0 @@
-namespace BotSharp.Plugin.AzureOpenAI.Settings;
-
-public class GPT4Settings
-{
-    public string ApiKey { get; set; }
-    public string Endpoint { get; set; }
-    public string DeploymentModel { get; set; }
-}

src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs

@@ -53,7 +53,7 @@ public class RoutingConversationHook: ConversationHookBase
     public override async Task OnResponseGenerated(RoleDialogModel message)
     {
         var routerSettings = _services.GetRequiredService<RoutingSettings>();
-        bool saveFlag = message.CurrentAgentId != routerSettings.RouterId;
+        bool saveFlag = message.CurrentAgentId != routerSettings.AgentId;
 
         if (saveFlag)
         {

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -56,7 +56,7 @@ public class TwilioService
             {
                 Gather.InputEnum.Speech
             },
-            Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{routingSetting.RouterId}")
+            Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{routingSetting.AgentId}")
         };
         gather.Say(message);
         response.Append(gather);
@@ -82,7 +82,7 @@ public class TwilioService
         var gather = new Gather()
         {
             Input = new List<Gather.InputEnum>() { Gather.InputEnum.Speech },
-            Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{routingSetting.RouterId}"),
+            Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{routingSetting.AgentId}"),
             ActionOnEmptyResult = true
         };
         if (!string.IsNullOrEmpty(message))

src/WebStarter/appsettings.json

@@ -13,17 +13,37 @@
     "Key": "31ba6052aa6f4569901facc3a41fcb4a"
   },
 
+  "LlmProviders": [
+    {
+      "Provider": "azure-openai",
+      "Models": [
+        {
+          "Name": "gpt-35-turbo",
+          "ApiKey": "",
+          "Endpoint": "https://gpt-35-turbo.openai.azure.com/",
+          "Type": "chat",
+          "PromptCost": 0.0015,
+          "CompletionCost": 0.002
+        },
+        {
+          "Name": "gpt-35-turbo-instruct",
+          "ApiKey": "",
+          "Endpoint": "https://gpt-35-turbo-instruct.openai.azure.com/",
+          "Type": "text",
+          "PromptCost": 0.0015,
+          "CompletionCost": 0.002
+        }
+      ]
+    }
+  ],
+
   "Router": {
-    "RouterId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",
-    "Planner": "NaivePlanner",
-    "Provider": "azure-openai",
-    "Model": "gpt-3.5-turbo"
+    "AgentId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",
+    "Planner": "NaivePlanner"
   },
 
   "Evaluator": {
-    "EvaluatorId": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869",
-    "Provider": "azure-openai",
-    "Model": "gpt-3.5-turbo"
+    "AgentId": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869"
   },
 
   "Agent": {
@@ -47,19 +67,7 @@
     "NumberOfGpuLayer": 10
   },
 
-  "ChatCompletion": {
-    "Provider": "azure-openai",
-    "Model": "gpt-3.5-turbo"
-  },
-
-  "TextCompletion": {
-    "Provider": "azure-openai",
-    "Model": "gpt-3.5-turbo"
-  },
-
   "AzureOpenAi": {
-    "ApiKey": "",
-    "Endpoint": ""
   },
 
   "GoogleAi": {

src/WebStarter/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/agent.json

@@ -4,5 +4,9 @@
   "createdDateTime": "2023-08-18T14:39:32.2349685Z",
   "updatedDateTime": "2023-08-18T14:39:32.2349686Z",
   "id": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",
-  "isPublic": true
+  "isPublic": true,
+  "llmConfig": {
+    "provider": "azure-openai",
+    "model": "gpt-35-turbo"
+  }
 }

src/WebStarter/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/agent.json

@@ -3,5 +3,8 @@
     "description": "Evaluate the performance of the LLM agents",
     "createdDateTime": "2023-08-18T00:00:00Z",
     "updatedDateTime": "2023-08-18T00:00:00Z",
-    "id": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869"
+    "id": "dfd9b46d-d00c-40af-8a75-3fbdc2b89869",
+    "llmConfig": {
+      "model": "gpt-35-turbo-instruct"
+    }
   }