From 405a725dc37cc274b2632c4aea8b03d5d3a6f486 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 28 May 2023 20:06:05 -0500 Subject: [PATCH] Improve IChatCompletionHandler --- .../IChatCompletionHandler.cs | 10 +-- .../BotSharp.Abstraction/IPlatformMidware.cs | 7 +- .../BotSharp.Core/Services/PlatformMidware.cs | 14 ++-- .../AzureAiSettings.cs | 9 +-- .../Handlers/ChatCompletionHandler.cs | 57 ++++++++++++--- .../Handlers/ChatCompletionHandler.cs | 72 ++++++++++++++----- .../LlamaSharpSettings.cs | 9 ++- .../LlamaSharplServiceCollectionExtensions.cs | 2 +- src/UiAdapters/ChatbotUI/Chatbot-UI.md | 20 +----- .../ChatbotUI/ChatbotUiController.cs | 26 +++---- src/WebStarter/Program.cs | 4 +- src/WebStarter/WebStarter.csproj | 4 ++ src/WebStarter/appsettings.json | 3 + 13 files changed, 145 insertions(+), 92 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/IChatCompletionHandler.cs b/src/Infrastructure/BotSharp.Abstraction/IChatCompletionHandler.cs index 403a2643..bed4600e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/IChatCompletionHandler.cs +++ b/src/Infrastructure/BotSharp.Abstraction/IChatCompletionHandler.cs @@ -4,9 +4,9 @@ namespace BotSharp.Abstraction; public interface IChatCompletionHandler { - Task GetChatCompletionsAsync(string text, - Func GetInstruction, - Func> GetChatHistory, - Func onChunkReceived, - Func onChunkCompleted); + string GetInstruction(); + List GetChatSamples(); + + Task GetChatCompletionsAsync(List conversations, + Func onChunkReceived); } diff --git a/src/Infrastructure/BotSharp.Abstraction/IPlatformMidware.cs b/src/Infrastructure/BotSharp.Abstraction/IPlatformMidware.cs index 492d2680..9ef270f1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/IPlatformMidware.cs +++ b/src/Infrastructure/BotSharp.Abstraction/IPlatformMidware.cs @@ -4,9 +4,6 @@ namespace BotSharp.Abstraction; public interface IPlatformMidware { - Task GetChatCompletionsAsync(string text, - Func GetInstruction, - Func> GetChatHistory, - Func onChunkReceived, - Func onChunkCompleted); + Task GetChatCompletionsAsync(List conversations, + Func onChunkReceived); } diff --git a/src/Infrastructure/BotSharp.Core/Services/PlatformMidware.cs b/src/Infrastructure/BotSharp.Core/Services/PlatformMidware.cs index 1c4815ff..2bc58a21 100644 --- a/src/Infrastructure/BotSharp.Core/Services/PlatformMidware.cs +++ b/src/Infrastructure/BotSharp.Core/Services/PlatformMidware.cs @@ -11,21 +11,15 @@ public class PlatformMidware : IPlatformMidware _services = services; } - public async Task GetChatCompletionsAsync(string text, - Func GetInstruction, - Func> GetChatHistory, - Func onChunkReceived, - Func onChunkCompleted) + public async Task GetChatCompletionsAsync(List conversations, + Func onChunkReceived) { var handlers = _services.GetServices().ToList(); for (int i = 0; i < handlers.Count(); i++) { var handler = handlers[i]; - await handler.GetChatCompletionsAsync(text, - GetInstruction, - GetChatHistory, - onChunkReceived, - onChunkCompleted); + await handler.GetChatCompletionsAsync(conversations, + onChunkReceived); } } } diff --git a/src/Platforms/BotSharp.Platform.AureAi/AzureAiSettings.cs b/src/Platforms/BotSharp.Platform.AureAi/AzureAiSettings.cs index 28989dd2..98ca25bb 100644 --- a/src/Platforms/BotSharp.Platform.AureAi/AzureAiSettings.cs +++ b/src/Platforms/BotSharp.Platform.AureAi/AzureAiSettings.cs @@ -2,8 +2,9 @@ namespace BotSharp.Platform.AzureAi; public class AzureAiSettings { - public string ApiKey { get; set; } - public string Endpoint { get; set; } - public string DeploymentModel { get; set; } - public string InstructionFile { get; set; } + public string ApiKey { get; set; } = string.Empty; + public string Endpoint { get; set; } = string.Empty; + public string DeploymentModel { get; set; } = string.Empty; + public string InstructionFile { get; set; } = string.Empty; + public string ChatSampleFile { get; set; } = string.Empty; } diff --git a/src/Platforms/BotSharp.Platform.AureAi/Handlers/ChatCompletionHandler.cs b/src/Platforms/BotSharp.Platform.AureAi/Handlers/ChatCompletionHandler.cs index d17cb372..f1f42521 100644 --- a/src/Platforms/BotSharp.Platform.AureAi/Handlers/ChatCompletionHandler.cs +++ b/src/Platforms/BotSharp.Platform.AureAi/Handlers/ChatCompletionHandler.cs @@ -18,14 +18,11 @@ public class ChatCompletionHandler : IChatCompletionHandler _settings = settings; } - public async Task GetChatCompletionsAsync(string text, - Func GetInstruction, - Func> GetChatHistory, - Func onChunkReceived, - Func onChunkCompleted) + public async Task GetChatCompletionsAsync(List conversations, + Func onChunkReceived) { var client = new OpenAIClient(new Uri(_settings.Endpoint), new AzureKeyCredential(_settings.ApiKey)); - var chatCompletionsOptions = PrepareOptions(text, GetInstruction, GetChatHistory); + var chatCompletionsOptions = PrepareOptions(conversations); var response = await client.GetChatCompletionsStreamingAsync(_settings.DeploymentModel, chatCompletionsOptions); using StreamingChatCompletions streaming = response.Value; @@ -44,14 +41,42 @@ public class ChatCompletionHandler : IChatCompletionHandler } Console.WriteLine(); - await onChunkCompleted(); } - private ChatCompletionsOptions PrepareOptions(string text, - Func GetInstruction, - Func> GetChatHistory) + public List GetChatSamples() { - var prompt = File.ReadAllText(_settings.InstructionFile); + var samples = new List(); + if (!string.IsNullOrEmpty(_settings.ChatSampleFile)) + { + var lines = File.ReadAllLines(_settings.ChatSampleFile); + for (int i = 0; i < lines.Length; i++) + { + var line = lines[i]; + var role = line.Substring(0, line.IndexOf(' ') - 1); + var content = line.Substring(line.IndexOf(' ') + 1); + + samples.Add(new RoleDialogModel + { + Role = role, + Content = content + }); + } + } + return samples; + } + + public string GetInstruction() + { + if (!string.IsNullOrEmpty(_settings.InstructionFile)) + { + return File.ReadAllText(_settings.InstructionFile); + } + return string.Empty; + } + + private ChatCompletionsOptions PrepareOptions(List conversations) + { + var prompt = GetInstruction(); var chatCompletionsOptions = new ChatCompletionsOptions() { Messages = @@ -60,6 +85,16 @@ public class ChatCompletionHandler : IChatCompletionHandler } }; + foreach (var message in GetChatSamples()) + { + chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content)); + } + + foreach (var message in conversations) + { + chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content)); + } + return chatCompletionsOptions; } } diff --git a/src/Platforms/BotSharp.Platform.LlamaSharp/Handlers/ChatCompletionHandler.cs b/src/Platforms/BotSharp.Platform.LlamaSharp/Handlers/ChatCompletionHandler.cs index 84f6893b..8c1334ca 100644 --- a/src/Platforms/BotSharp.Platform.LlamaSharp/Handlers/ChatCompletionHandler.cs +++ b/src/Platforms/BotSharp.Platform.LlamaSharp/Handlers/ChatCompletionHandler.cs @@ -5,6 +5,8 @@ using LLama; using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Runtime; using System.Text; using System.Threading.Tasks; @@ -19,28 +21,24 @@ public class ChatCompletionHandler : IChatCompletionHandler { _settings = settings; _model = new LLamaModel(new LLamaParams(model: _settings.ModelPath, - n_ctx: 512, - interactive: true, - repeat_penalty: 1.0f, - verbose_prompt: false)); + n_ctx: _settings.MaxContextLength, + interactive: _settings.Interactive, + repeat_penalty: _settings.RepeatPenalty, + verbose_prompt: _settings.VerbosePrompt, + n_gpu_layers: _settings.NumberOfGpuLayer)); - if (!string.IsNullOrEmpty(settings.InstructionFile)) - { - var prompt = File.ReadAllText(settings.InstructionFile); - _model.InitChatPrompt(prompt, "UTF-8"); - } - - _model.InitChatAntiprompt(new string[] { "User:" }); + var prompt = GetInstruction(); + _model.InitChatPrompt(prompt, "UTF-8"); + _model.InitChatAntiprompt(new string[] { "user:" }); } - public async Task GetChatCompletionsAsync(string text, - Func GetInstruction, - Func> GetChatHistory, - Func onChunkReceived, - Func onChunkCompleted) + public async Task GetChatCompletionsAsync(List conversations, + Func onChunkReceived) { string totalResponse = ""; - foreach (var response in _model.Chat(text, "", "UTF-8")) + // var prompt = GetInstruction(); + // var content = string.Join("\n", conversations.Select(x => $"\n{x.Content}")); + foreach (var response in _model.Chat(conversations.Last().Content, "", "UTF-8")) { Console.Write(response); totalResponse += response; @@ -48,6 +46,44 @@ public class ChatCompletionHandler : IChatCompletionHandler } Console.WriteLine(); - await onChunkCompleted(); + } + + public List GetChatSamples() + { + var samples = new List(); + if (!string.IsNullOrEmpty(_settings.ChatSampleFile)) + { + var lines = File.ReadAllLines(_settings.ChatSampleFile); + for (int i = 0; i < lines.Length; i++) + { + var line = lines[i]; + var role = line.Substring(0, line.IndexOf(' ') - 1); + var content = line.Substring(line.IndexOf(' ') + 1); + + samples.Add(new RoleDialogModel + { + Role = role, + Content = content + }); + } + } + return samples; + } + + public string GetInstruction() + { + var instruction = ""; + if (!string.IsNullOrEmpty(_settings.InstructionFile)) + { + instruction = File.ReadAllText(_settings.InstructionFile); + } + + instruction += "\n"; + foreach (var message in GetChatSamples()) + { + instruction += $"\n{message.Role}: {message.Content}"; + } + + return instruction; } } diff --git a/src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharpSettings.cs b/src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharpSettings.cs index 5d1f5c58..42a67c05 100644 --- a/src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharpSettings.cs +++ b/src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharpSettings.cs @@ -2,7 +2,12 @@ namespace BotSharp.Platform.LlamaSharp; public class LlamaSharpSettings { - public string ModelPath { get; set; } - public string InstructionFile { get; set; } + public string ModelPath { get; set; } = string.Empty; + public string InstructionFile { get; set; } = string.Empty; + public string ChatSampleFile { get; set; } = string.Empty; public int MaxContextLength { get; set; } = 512; + public float RepeatPenalty { get; set; } = 1.0f; + public bool VerbosePrompt { get; set; } + public bool Interactive { get; set; } = true; + public int NumberOfGpuLayer { get; set; } } diff --git a/src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharplServiceCollectionExtensions.cs b/src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharplServiceCollectionExtensions.cs index 47c03838..c26275da 100644 --- a/src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharplServiceCollectionExtensions.cs +++ b/src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharplServiceCollectionExtensions.cs @@ -16,7 +16,7 @@ public static class LlamaSharplServiceCollectionExtensions config.Bind("LlamaSharp", settings); return settings; }); - services.AddScoped(); + services.AddSingleton(); return services; } } \ No newline at end of file diff --git a/src/UiAdapters/ChatbotUI/Chatbot-UI.md b/src/UiAdapters/ChatbotUI/Chatbot-UI.md index 23c8541d..9961ca37 100644 --- a/src/UiAdapters/ChatbotUI/Chatbot-UI.md +++ b/src/UiAdapters/ChatbotUI/Chatbot-UI.md @@ -9,24 +9,8 @@ npm run dev * Rename `.env.local.example` to `.env.local` -### Azure OpenAI ```shell -# Chatbot UI -OPENAI_API_TYPE=azure -OPENAI_API_VERSION=2023-03-15-preview -AZURE_DEPLOYMENT_ID= -OPENAI_API_KEY= -DEFAULT_MODEL=gpt-35-turbo -OPENAI_API_HOST= -NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT= -``` - -### OpenAI ChatGPT -```shell -# Chatbot UI -DEFAULT_MODEL=gpt-3.5-turbo -NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT= -OPENAI_API_KEY=YOUR_KEY -OPENAI_API_HOST=https://api.openai.com +# Change host to BotSharp service +OPENAI_API_HOST=http://localhost:5500 ``` diff --git a/src/UiAdapters/ChatbotUI/ChatbotUiController.cs b/src/UiAdapters/ChatbotUI/ChatbotUiController.cs index 26547f5e..32d26de8 100644 --- a/src/UiAdapters/ChatbotUI/ChatbotUiController.cs +++ b/src/UiAdapters/ChatbotUI/ChatbotUiController.cs @@ -30,7 +30,6 @@ public class ChatbotUiController : ControllerBase, IBotUiAdapter } [HttpGet("/v1/models")] - [HttpGet("/openai/deployments")] public OpenAiModels GetOpenAiModels() { return new OpenAiModels @@ -50,8 +49,7 @@ public class ChatbotUiController : ControllerBase, IBotUiAdapter } [HttpPost("/v1/chat/completions")] - [HttpPost("/openai/deployments/one/chat/completions")] - public async Task SendMessage([FromBody] OpenAiMessageInput input, [FromQuery(Name = "api-version")] string apiVersion = "2023-03-15-preview") + public async Task SendMessage([FromBody] OpenAiMessageInput input) { Response.StatusCode = 200; Response.Headers.Add(HeaderNames.ContentType, "text/event-stream"); @@ -59,23 +57,19 @@ public class ChatbotUiController : ControllerBase, IBotUiAdapter Response.Headers.Add(HeaderNames.Connection, "keep-alive"); var outputStream = Response.Body; - await _platform.GetChatCompletionsAsync(input.Messages.Last().Content, - delegate - { - return ""; - }, - delegate - { - return new List(); - }, + var conversations = input.Messages.Skip(1).Select(x => new RoleDialogModel + { + Role = x.Role, + Content = x.Content + }).ToList(); + + await _platform.GetChatCompletionsAsync(conversations, async content => { await OnChunkReceived(outputStream, content); - }, - async () => - { - await OnEventCompleted(outputStream); }); + + await OnEventCompleted(outputStream); } private async Task OnChunkReceived(Stream outputStream, string content) diff --git a/src/WebStarter/Program.cs b/src/WebStarter/Program.cs index 7b8d942c..0ceaeabb 100644 --- a/src/WebStarter/Program.cs +++ b/src/WebStarter/Program.cs @@ -12,8 +12,8 @@ builder.Services.AddHttpContextAccessor(); // Add BotSharp builder.Services.AddBotSharp(); -// builder.Services.AddLlamaSharp(builder.Configuration); -builder.Services.AddAzureOpenAi(builder.Configuration); +builder.Services.AddLlamaSharp(builder.Configuration); +// builder.Services.AddAzureOpenAi(builder.Configuration); var app = builder.Build(); diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 66ecdcbc..4844661f 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -8,10 +8,14 @@ + + + PreserveNewest + PreserveNewest diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 337ac416..134053e0 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -13,8 +13,10 @@ }, "LlamaSharp": { + "Interactive": true, "ModelPath": "C:\\Users\\haipi\\Downloads\\ggml-vic13b-q5_1.bin", "InstructionFile": "Prompts\\chat-with-bob.txt", + "ChatSampleFile": "Prompts\\chat-samples.txt", "MaxContextLength": 512 }, @@ -22,6 +24,7 @@ "ApiKey": "", "Endpoint": "", "InstructionFile": "Prompts\\chat-with-bob.txt", + "ChatSampleFile": "Prompts\\chat-samples.txt", "DeploymentModel": "" } }