From 8c60e6ae7847d8236d771f82d96fd62aa5f98325 Mon Sep 17 00:00:00 2001 From: Haiping Chen <101423@smsassist.com> Date: Mon, 23 Oct 2023 18:20:18 -0500 Subject: [PATCH] Use text completion for instruct mode. --- .../Instructs/IInstructService.cs | 6 +- .../Instructs/Models/InstructResult.cs | 1 - .../MLTasks/Settings/ChatCompletionSetting.cs | 7 ++ .../MLTasks/Settings/TextCompletionSetting.cs | 7 ++ .../BotSharpServiceCollectionExtensions.cs | 9 ++ .../Infrastructures/CompletionProvider.cs | 11 +- .../Instructs/InstructService.cs | 101 ++---------------- .../Controllers/InstructModeController.cs | 34 +++--- .../AzureOpenAiPlugin.cs | 2 +- .../Providers/ChatCompletionProvider.cs | 12 +-- .../Providers/ProviderHelper.cs | 6 +- .../Providers/TextCompletionProvider.cs | 10 +- .../Settings/AzureOpenAiSettings.cs | 5 - .../Settings/DeploymentModelSetting.cs | 12 --- src/WebStarter/appsettings.json | 16 ++- 15 files changed, 86 insertions(+), 153 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/ChatCompletionSetting.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/TextCompletionSetting.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/DeploymentModelSetting.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs index 76346165..2162204d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs @@ -4,9 +4,5 @@ namespace BotSharp.Abstraction.Instructs; public interface IInstructService { - Task ExecuteInstruction(Agent agent, - RoleDialogModel message, - Func onMessageReceived, - Func onFunctionExecuting, - Func onFunctionExecuted); + Task Execute(Agent agent, RoleDialogModel message); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs index f89bb8e0..1fb1f628 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs @@ -3,6 +3,5 @@ namespace BotSharp.Abstraction.Instructs.Models; public class InstructResult { public string Text { get; set; } - public string Function { get; set; } public object Data { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/ChatCompletionSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/ChatCompletionSetting.cs new file mode 100644 index 00000000..ba0a6a59 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/ChatCompletionSetting.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.MLTasks.Settings; + +public class ChatCompletionSetting +{ + public string Provider { get; set; } + public string Model { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/TextCompletionSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/TextCompletionSetting.cs new file mode 100644 index 00000000..7fbb1a7d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/TextCompletionSetting.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.MLTasks.Settings; + +public class TextCompletionSetting +{ + public string Provider { get; set; } + public string Model { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index fba5e2de..9b64b127 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -16,6 +16,7 @@ using BotSharp.Abstraction.Evaluations.Settings; using BotSharp.Abstraction.Evaluations; using BotSharp.Core.Evaluatings; using BotSharp.Core.Evaluations; +using BotSharp.Abstraction.MLTasks.Settings; namespace BotSharp.Core; @@ -48,6 +49,14 @@ public static class BotSharpServiceCollectionExtensions config.Bind("Database", myDatabaseSettings); services.AddSingleton((IServiceProvider x) => myDatabaseSettings); + var textCompletionSettings = new TextCompletionSetting(); + config.Bind("TextCompletion", textCompletionSettings); + services.AddSingleton((IServiceProvider x) => textCompletionSettings); + + var chatCompletionSettings = new ChatCompletionSetting(); + config.Bind("ChatCompletion", chatCompletionSettings); + services.AddSingleton((IServiceProvider x) => chatCompletionSettings); + RegisterPlugins(services, config); // Register template render diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 34feb606..48d063d0 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.MLTasks.Settings; namespace BotSharp.Core.Infrastructures; @@ -6,18 +7,19 @@ public class CompletionProvider { public static IChatCompletion GetChatCompletion(IServiceProvider services, string? provider = null, string? model = null) { + var settings = services.GetRequiredService(); var completions = services.GetServices(); var state = services.GetRequiredService(); if (string.IsNullOrEmpty(provider)) { - provider = state.GetState("provider", "azure-openai"); + provider = state.GetState("provider", settings.Provider ?? "azure-openai"); } if (string.IsNullOrEmpty(model)) { - model = state.GetState("model", "gpt-3.5-turbo"); + model = state.GetState("model", settings.Model ?? "gpt-3.5-turbo"); } var completer = completions.FirstOrDefault(x => x.Provider == provider); @@ -34,18 +36,19 @@ public class CompletionProvider public static ITextCompletion GetTextCompletion(IServiceProvider services, string? provider = null, string? model = null) { + var settings = services.GetRequiredService(); var completions = services.GetServices(); var state = services.GetRequiredService(); if (string.IsNullOrEmpty(provider)) { - provider = state.GetState("provider", "azure-openai"); + provider = state.GetState("provider", settings.Provider ?? "azure-openai"); } if (string.IsNullOrEmpty(model)) { - model = state.GetState("model", "gpt-3.5-turbo"); + model = state.GetState("model", settings.Model ?? "gpt-3.5-turbo"); } var completer = completions.FirstOrDefault(x => x.Provider == provider); diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs b/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs index 1a10ff25..2243e5e7 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs @@ -1,10 +1,6 @@ -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Templating; -using System.IO; namespace BotSharp.Core.Instructs; @@ -19,19 +15,8 @@ public partial class InstructService : IInstructService _logger = logger; } - public async Task ExecuteInstruction(Agent agent, - RoleDialogModel message, - Func onMessageReceived, - Func onFunctionExecuting, - Func onFunctionExecuted) + public async Task Execute(Agent agent, RoleDialogModel message) { - var response = new InstructResult(); - - var wholeDialogs = new List - { - message - }; - // Trigger before completion hooks var hooks = _services.GetServices(); foreach (var hook in hooks) @@ -39,23 +24,13 @@ public partial class InstructService : IInstructService await hook.BeforeCompletion(message); } - await ExecuteInstructionRecursively(agent, - wholeDialogs, - async msg => - { - response.Text = msg.Content; - await onMessageReceived(msg); - }, - async fn => - { - response.Function = fn.FunctionName; - await onFunctionExecuting(fn); - }, - async fn => - { - response.Data = fn.Data; - await onFunctionExecuted(fn); - }); + var completer = CompletionProvider.GetTextCompletion(_services); + + var result = await completer.GetCompletion(agent.Instruction); + var response = new InstructResult + { + Text = result + }; foreach (var hook in hooks) { @@ -64,64 +39,4 @@ public partial class InstructService : IInstructService return response; } - - private async Task ExecuteInstructionRecursively(Agent agent, - List wholeDialogs, - Func onMessageReceived, - Func onFunctionExecuting, - Func onFunctionExecuted) - { - var chatCompletion = CompletionProvider.GetChatCompletion(_services); - - var result = await chatCompletion.GetChatCompletionsAsync(agent, wholeDialogs, async msg => - { - await onMessageReceived(msg); - - wholeDialogs.Add(msg); - }, async fn => - { - var preAgentId = agent.Id; - - await HandleFunctionMessage(fn, onFunctionExecuting, onFunctionExecuted); - - // Function executed has exception - if (fn.Content == null || fn.StopCompletion) - { - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, fn.Content)); - return; - } - - fn.Content = fn.FunctionArgs.Replace("\r", " ").Replace("\n", " ").Trim() + " => " + fn.Content; - - // Find response template - var templateService = _services.GetRequiredService(); - var response = await templateService.RenderFunctionResponse(agent.Id, fn); - if (!string.IsNullOrEmpty(response)) - { - await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, response)); - return; - } - - // After function is executed, pass the result to LLM to get a natural response - wholeDialogs.Add(fn); - - await ExecuteInstructionRecursively(agent, - wholeDialogs, - onMessageReceived, - onFunctionExecuting, - onFunctionExecuted); - }); - - return result; - } - - private async Task HandleFunctionMessage(RoleDialogModel msg, - Func onFunctionExecuting, - Func onFunctionExecuted) - { - // Call functions - await onFunctionExecuting(msg); - await CallFunctions(msg); - await onFunctionExecuted(msg); - } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index d51f91b0..9cb7b438 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -4,6 +4,7 @@ using BotSharp.Abstraction.ApiAdapters; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Templating; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; @@ -24,34 +25,37 @@ public class InstructModeController : ControllerBase, IApiAdapter public async Task InstructCompletion([FromRoute] string agentId, [FromBody] InstructMessageModel input) { - var instructor = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Split('=')[0], x.Split('=')[1])); + state.SetState("provider", input.Provider) + .SetState("model", input.Model) + .SetState("input_text", input.Text); + var agentService = _services.GetRequiredService(); Agent agent = await agentService.LoadAgent(agentId); // switch to different instruction template if (!string.IsNullOrEmpty(input.Template)) { - agent.Instruction = agent.Templates.First(x => x.Name == input.Template).Content; + var template = agent.Templates.First(x => x.Name == input.Template).Content; + var render = _services.GetRequiredService(); + var dict = new Dictionary(); + state.GetStates().Select(x => dict[x.Key] = x.Value).ToArray(); + var prompt = render.Render(template, dict); + agent.Instruction = prompt; } - var conv = _services.GetRequiredService(); - input.States.ForEach(x => conv.States.SetState(x.Split('=')[0], x.Split('=')[1])); - conv.States.SetState("provider", input.Provider) - .SetState("model", input.Model); - - return await instructor.ExecuteInstruction(agent, - new RoleDialogModel(AgentRole.User, input.Text), - fn => Task.CompletedTask, - fn => Task.CompletedTask, - fn => Task.CompletedTask); + var instructor = _services.GetRequiredService(); + return await instructor.Execute(agent, + new RoleDialogModel(AgentRole.User, input.Text)); } [HttpPost("/instruct/text-completion")] public async Task TextCompletion([FromBody] IncomingMessageModel input) { - var conv = _services.GetRequiredService(); - input.States.ForEach(x => conv.States.SetState(x.Split('=')[0], x.Split('=')[1])); - conv.States.SetState("provider", input.Provider) + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Split('=')[0], x.Split('=')[1])); + state.SetState("provider", input.Provider) .SetState("model", input.Model); var textCompletion = CompletionProvider.GetTextCompletion(_services); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index e18316c1..653be1b1 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -24,7 +24,7 @@ public class AzureOpenAiPlugin : IBotSharpPlugin config.Bind("AzureOpenAi", settings); services.AddSingleton(x => { - Console.WriteLine($"Loaded AzureOpenAi settings: {settings.DeploymentModel} ({settings.Endpoint}) {settings.ApiKey.SubstringMax(4)}"); + Console.WriteLine($"Loaded AzureOpenAi settings: ({settings.Endpoint}) {settings.ApiKey.SubstringMax(4)}"); return settings; }); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 98fba046..bd2933f3 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -43,10 +43,10 @@ public class ChatCompletionProvider : IChatCompletion Task.WaitAll(hooks.Select(hook => hook.BeforeGenerating(agent, conversations)).ToArray()); - var (client, deploymentModel) = ProviderHelper.GetClient(_model, _settings); + var client = ProviderHelper.GetClient(_model, _settings); var chatCompletionsOptions = PrepareOptions(agent, conversations); - var response = client.GetChatCompletions(deploymentModel, chatCompletionsOptions); + var response = client.GetChatCompletions(_model, chatCompletionsOptions); var choice = response.Value.Choices[0]; var message = choice.Message; @@ -96,10 +96,10 @@ public class ChatCompletionProvider : IChatCompletion Task.WaitAll(hooks.Select(hook => hook.BeforeGenerating(agent, conversations)).ToArray()); - var (client, deploymentModel) = ProviderHelper.GetClient(_model, _settings); + var client = ProviderHelper.GetClient(_model, _settings); var chatCompletionsOptions = PrepareOptions(agent, conversations); - var response = await client.GetChatCompletionsAsync(deploymentModel, chatCompletionsOptions); + var response = await client.GetChatCompletionsAsync(_model, chatCompletionsOptions); var choice = response.Value.Choices[0]; var message = choice.Message; @@ -148,10 +148,10 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived) { - var client = new OpenAIClient(new Uri(_settings.Endpoint), new AzureKeyCredential(_settings.ApiKey)); + var client = ProviderHelper.GetClient(_model, _settings); var chatCompletionsOptions = PrepareOptions(agent, conversations); - var response = await client.GetChatCompletionsStreamingAsync(_settings.DeploymentModel.ChatCompletionModel, chatCompletionsOptions); + var response = await client.GetChatCompletionsStreamingAsync(_model, chatCompletionsOptions); using StreamingChatCompletions streaming = response.Value; string output = ""; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs index dffc196d..e3411fcc 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs @@ -9,17 +9,17 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers; public class ProviderHelper { - public static (OpenAIClient, string) GetClient(string model, AzureOpenAiSettings settings) + public static OpenAIClient GetClient(string model, AzureOpenAiSettings settings) { if (model == "gpt-4") { var client = new OpenAIClient(new Uri(settings.GPT4.Endpoint), new AzureKeyCredential(settings.GPT4.ApiKey)); - return (client, settings.GPT4.DeploymentModel); + return client; } else { var client = new OpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey)); - return (client, settings.DeploymentModel.ChatCompletionModel); + return client; } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs index 98d17b5d..4094b762 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/TextCompletionProvider.cs @@ -37,9 +37,13 @@ public class TextCompletionProvider : ITextCompletion // Before chat completion hook Task.WaitAll(hooks.Select(hook => - hook.BeforeGenerating(new Agent(), new List { new RoleDialogModel(AgentRole.User, text) })).ToArray()); + hook.BeforeGenerating(new Agent(), + new List + { + new RoleDialogModel(AgentRole.User, text) + })).ToArray()); - var (client, _) = ProviderHelper.GetClient(_model, _settings); + var client = ProviderHelper.GetClient(_model, _settings); var completionsOptions = new CompletionsOptions() { @@ -58,7 +62,7 @@ public class TextCompletionProvider : ITextCompletion completionsOptions.NucleusSamplingFactor = samplingFactor; var response = await client.GetCompletionsAsync( - deploymentOrModelName: _settings.DeploymentModel.TextCompletionModel, + deploymentOrModelName: _model, completionsOptions); // OpenAI diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs index 510bec5a..60858a8f 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/AzureOpenAiSettings.cs @@ -1,13 +1,8 @@ -using BotSharp.Abstraction.Routing.Settings; - namespace BotSharp.Plugin.AzureOpenAI.Settings; public class AzureOpenAiSettings { public string ApiKey { get; set; } = string.Empty; public string Endpoint { get; set; } = string.Empty; - public DeploymentModelSetting DeploymentModel { get; set; } - = new DeploymentModelSetting(); - public GPT4Settings GPT4 { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/DeploymentModelSetting.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/DeploymentModelSetting.cs deleted file mode 100644 index ad634d1b..00000000 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Settings/DeploymentModelSetting.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace BotSharp.Plugin.AzureOpenAI.Settings; - -public class DeploymentModelSetting -{ - public string ChatCompletionModel { get; set; } = string.Empty; - public string? TextCompletionModel { get; set; } - - public override string ToString() - { - return $"ChatCompletion - {ChatCompletionModel}, TextCompletion - {TextCompletionModel}"; - } -} diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 8bc29acf..318c4298 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -46,13 +46,19 @@ "NumberOfGpuLayer": 10 }, + "ChatCompletion": { + "Provider": "azure-openai", + "Model": "gpt-3.5-turbo" + }, + + "TextCompletion": { + "Provider": "azure-openai", + "Model": "gpt-3.5-turbo" + }, + "AzureOpenAi": { "ApiKey": "", - "Endpoint": "", - "DeploymentModel": { - "ChatCompletionModel": "", - "TextCompletionModel": "" - } + "Endpoint": "" }, "GoogleAi": {