SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Improve IChatCompletionHandler

Browse source
This commit is contained in:
Haiping Chen 2023-05-28 20:06:05 -05:00
parent bae5087642
commit 405a725dc3
13 changed files with 145 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/Infrastructure/BotSharp.Abstraction/IChatCompletionHandler.cs
View file

@ -4,9 +4,9 @@ namespace BotSharp.Abstraction;
public interface IChatCompletionHandler
{
Task GetChatCompletionsAsync(string text,
Func<string> GetInstruction,
Func<List<RoleDialogModel>> GetChatHistory,
Func<string, Task> onChunkReceived,
Func<Task> onChunkCompleted);
string GetInstruction();
List<RoleDialogModel> GetChatSamples();
Task GetChatCompletionsAsync(List<RoleDialogModel> conversations,
Func<string, Task> onChunkReceived);
}

7
src/Infrastructure/BotSharp.Abstraction/IPlatformMidware.cs
View file

@ -4,9 +4,6 @@ namespace BotSharp.Abstraction;
public interface IPlatformMidware
{
Task GetChatCompletionsAsync(string text,
Func<string> GetInstruction,
Func<List<RoleDialogModel>> GetChatHistory,
Func<string, Task> onChunkReceived,
Func<Task> onChunkCompleted);
Task GetChatCompletionsAsync(List<RoleDialogModel> conversations,
Func<string, Task> onChunkReceived);
}

14
src/Infrastructure/BotSharp.Core/Services/PlatformMidware.cs
View file

@ -11,21 +11,15 @@ public class PlatformMidware : IPlatformMidware
_services = services;
}
public async Task GetChatCompletionsAsync(string text,
Func<string> GetInstruction,
Func<List<RoleDialogModel>> GetChatHistory,
Func<string, Task> onChunkReceived,
Func<Task> onChunkCompleted)
public async Task GetChatCompletionsAsync(List<RoleDialogModel> conversations,
Func<string, Task> onChunkReceived)
{
var handlers = _services.GetServices<IChatCompletionHandler>().ToList();
for (int i = 0; i < handlers.Count(); i++)
{
var handler = handlers[i];
await handler.GetChatCompletionsAsync(text,
GetInstruction,
GetChatHistory,
onChunkReceived,
onChunkCompleted);
await handler.GetChatCompletionsAsync(conversations,
onChunkReceived);
}
}
}

9
src/Platforms/BotSharp.Platform.AureAi/AzureAiSettings.cs
View file

@ -2,8 +2,9 @@ namespace BotSharp.Platform.AzureAi;
public class AzureAiSettings
{
public string ApiKey { get; set; }
public string Endpoint { get; set; }
public string DeploymentModel { get; set; }
public string InstructionFile { get; set; }
public string ApiKey { get; set; } = string.Empty;
public string Endpoint { get; set; } = string.Empty;
public string DeploymentModel { get; set; } = string.Empty;
public string InstructionFile { get; set; } = string.Empty;
public string ChatSampleFile { get; set; } = string.Empty;
}

57
src/Platforms/BotSharp.Platform.AureAi/Handlers/ChatCompletionHandler.cs
View file

@ -18,14 +18,11 @@ public class ChatCompletionHandler : IChatCompletionHandler
_settings = settings;
}
public async Task GetChatCompletionsAsync(string text,
Func<string> GetInstruction,
Func<List<RoleDialogModel>> GetChatHistory,
Func<string, Task> onChunkReceived,
Func<Task> onChunkCompleted)
public async Task GetChatCompletionsAsync(List<RoleDialogModel> conversations,
Func<string, Task> onChunkReceived)
{
var client = new OpenAIClient(new Uri(_settings.Endpoint), new AzureKeyCredential(_settings.ApiKey));
var chatCompletionsOptions = PrepareOptions(text, GetInstruction, GetChatHistory);
var chatCompletionsOptions = PrepareOptions(conversations);
var response = await client.GetChatCompletionsStreamingAsync(_settings.DeploymentModel, chatCompletionsOptions);
using StreamingChatCompletions streaming = response.Value;
@ -44,14 +41,42 @@ public class ChatCompletionHandler : IChatCompletionHandler
}
Console.WriteLine();
await onChunkCompleted();
}
private ChatCompletionsOptions PrepareOptions(string text,
Func<string> GetInstruction,
Func<List<RoleDialogModel>> GetChatHistory)
public List<RoleDialogModel> GetChatSamples()
{
var prompt = File.ReadAllText(_settings.InstructionFile);
var samples = new List<RoleDialogModel>();
if (!string.IsNullOrEmpty(_settings.ChatSampleFile))
{
var lines = File.ReadAllLines(_settings.ChatSampleFile);
for (int i = 0; i < lines.Length; i++)
{
var line = lines[i];
var role = line.Substring(0, line.IndexOf(' ') - 1);
var content = line.Substring(line.IndexOf(' ') + 1);
samples.Add(new RoleDialogModel
{
Role = role,
Content = content
});
}
}
return samples;
}
public string GetInstruction()
{
if (!string.IsNullOrEmpty(_settings.InstructionFile))
{
return File.ReadAllText(_settings.InstructionFile);
}
return string.Empty;
}
private ChatCompletionsOptions PrepareOptions(List<RoleDialogModel> conversations)
{
var prompt = GetInstruction();
var chatCompletionsOptions = new ChatCompletionsOptions()
{
Messages =
@ -60,6 +85,16 @@ public class ChatCompletionHandler : IChatCompletionHandler
}
};
foreach (var message in GetChatSamples())
{
chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content));
}
foreach (var message in conversations)
{
chatCompletionsOptions.Messages.Add(new ChatMessage(message.Role, message.Content));
}
return chatCompletionsOptions;
}
}

72
src/Platforms/BotSharp.Platform.LlamaSharp/Handlers/ChatCompletionHandler.cs
View file

@ -5,6 +5,8 @@ using LLama;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime;
using System.Text;
using System.Threading.Tasks;
@ -19,28 +21,24 @@ public class ChatCompletionHandler : IChatCompletionHandler
{
_settings = settings;
_model = new LLamaModel(new LLamaParams(model: _settings.ModelPath,
n_ctx: 512,
interactive: true,
repeat_penalty: 1.0f,
verbose_prompt: false));
n_ctx: _settings.MaxContextLength,
interactive: _settings.Interactive,
repeat_penalty: _settings.RepeatPenalty,
verbose_prompt: _settings.VerbosePrompt,
n_gpu_layers: _settings.NumberOfGpuLayer));
if (!string.IsNullOrEmpty(settings.InstructionFile))
{
var prompt = File.ReadAllText(settings.InstructionFile);
_model.InitChatPrompt(prompt, "UTF-8");
}
_model.InitChatAntiprompt(new string[] { "User:" });
var prompt = GetInstruction();
_model.InitChatPrompt(prompt, "UTF-8");
_model.InitChatAntiprompt(new string[] { "user:" });
}
public async Task GetChatCompletionsAsync(string text,
Func<string> GetInstruction,
Func<List<RoleDialogModel>> GetChatHistory,
Func<string, Task> onChunkReceived,
Func<Task> onChunkCompleted)
public async Task GetChatCompletionsAsync(List<RoleDialogModel> conversations,
Func<string, Task> onChunkReceived)
{
string totalResponse = "";
foreach (var response in _model.Chat(text, "", "UTF-8"))
// var prompt = GetInstruction();
// var content = string.Join("\n", conversations.Select(x => $"\n{x.Content}"));
foreach (var response in _model.Chat(conversations.Last().Content, "", "UTF-8"))
{
Console.Write(response);
totalResponse += response;
@ -48,6 +46,44 @@ public class ChatCompletionHandler : IChatCompletionHandler
}
Console.WriteLine();
await onChunkCompleted();
}
public List<RoleDialogModel> GetChatSamples()
{
var samples = new List<RoleDialogModel>();
if (!string.IsNullOrEmpty(_settings.ChatSampleFile))
{
var lines = File.ReadAllLines(_settings.ChatSampleFile);
for (int i = 0; i < lines.Length; i++)
{
var line = lines[i];
var role = line.Substring(0, line.IndexOf(' ') - 1);
var content = line.Substring(line.IndexOf(' ') + 1);
samples.Add(new RoleDialogModel
{
Role = role,
Content = content
});
}
}
return samples;
}
public string GetInstruction()
{
var instruction = "";
if (!string.IsNullOrEmpty(_settings.InstructionFile))
{
instruction = File.ReadAllText(_settings.InstructionFile);
}
instruction += "\n";
foreach (var message in GetChatSamples())
{
instruction += $"\n{message.Role}: {message.Content}";
}
return instruction;
}
}

9
src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharpSettings.cs
View file

@ -2,7 +2,12 @@ namespace BotSharp.Platform.LlamaSharp;
public class LlamaSharpSettings
{
public string ModelPath { get; set; }
public string InstructionFile { get; set; }
public string ModelPath { get; set; } = string.Empty;
public string InstructionFile { get; set; } = string.Empty;
public string ChatSampleFile { get; set; } = string.Empty;
public int MaxContextLength { get; set; } = 512;
public float RepeatPenalty { get; set; } = 1.0f;
public bool VerbosePrompt { get; set; }
public bool Interactive { get; set; } = true;
public int NumberOfGpuLayer { get; set; }
}

2
src/Platforms/BotSharp.Platform.LlamaSharp/LlamaSharplServiceCollectionExtensions.cs
View file

@ -16,7 +16,7 @@ public static class LlamaSharplServiceCollectionExtensions
config.Bind("LlamaSharp", settings);
return settings;
});
services.AddScoped<IChatCompletionHandler, ChatCompletionHandler>();
services.AddSingleton<IChatCompletionHandler, ChatCompletionHandler>();
return services;
}
}

20
src/UiAdapters/ChatbotUI/Chatbot-UI.md
View file

@ -9,24 +9,8 @@ npm run dev
* Rename `.env.local.example` to `.env.local`
### Azure OpenAI
```shell
# Chatbot UI
OPENAI_API_TYPE=azure
OPENAI_API_VERSION=2023-03-15-preview
AZURE_DEPLOYMENT_ID=
OPENAI_API_KEY=
DEFAULT_MODEL=gpt-35-turbo
OPENAI_API_HOST=
NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=
```
### OpenAI ChatGPT
```shell
# Chatbot UI
DEFAULT_MODEL=gpt-3.5-turbo
NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=
OPENAI_API_KEY=YOUR_KEY
OPENAI_API_HOST=https://api.openai.com
# Change host to BotSharp service
OPENAI_API_HOST=http://localhost:5500
```

26
src/UiAdapters/ChatbotUI/ChatbotUiController.cs
View file

@ -30,7 +30,6 @@ public class ChatbotUiController : ControllerBase, IBotUiAdapter
}
[HttpGet("/v1/models")]
[HttpGet("/openai/deployments")]
public OpenAiModels GetOpenAiModels()
{
return new OpenAiModels
@ -50,8 +49,7 @@ public class ChatbotUiController : ControllerBase, IBotUiAdapter
}
[HttpPost("/v1/chat/completions")]
[HttpPost("/openai/deployments/one/chat/completions")]
public async Task SendMessage([FromBody] OpenAiMessageInput input, [FromQuery(Name = "api-version")] string apiVersion = "2023-03-15-preview")
public async Task SendMessage([FromBody] OpenAiMessageInput input)
{
Response.StatusCode = 200;
Response.Headers.Add(HeaderNames.ContentType, "text/event-stream");
@ -59,23 +57,19 @@ public class ChatbotUiController : ControllerBase, IBotUiAdapter
Response.Headers.Add(HeaderNames.Connection, "keep-alive");
var outputStream = Response.Body;
await _platform.GetChatCompletionsAsync(input.Messages.Last().Content,
delegate
{
return "";
},
delegate
{
return new List<RoleDialogModel>();
},
var conversations = input.Messages.Skip(1).Select(x => new RoleDialogModel
{
Role = x.Role,
Content = x.Content
}).ToList();
await _platform.GetChatCompletionsAsync(conversations,
async content =>
{
await OnChunkReceived(outputStream, content);
},
async () =>
{
await OnEventCompleted(outputStream);
});
await OnEventCompleted(outputStream);
}
private async Task OnChunkReceived(Stream outputStream, string content)

4
src/WebStarter/Program.cs
View file

@ -12,8 +12,8 @@ builder.Services.AddHttpContextAccessor();
// Add BotSharp
builder.Services.AddBotSharp();
// builder.Services.AddLlamaSharp(builder.Configuration);
builder.Services.AddAzureOpenAi(builder.Configuration);
builder.Services.AddLlamaSharp(builder.Configuration);
// builder.Services.AddAzureOpenAi(builder.Configuration);
var app = builder.Build();

4
src/WebStarter/WebStarter.csproj
View file

@ -8,10 +8,14 @@
</PropertyGroup>
<ItemGroup>
<None Remove="Prompts\chat-samples.txt" />
<None Remove="Prompts\chat-with-bob.txt" />
</ItemGroup>
<ItemGroup>
<Content Include="Prompts\chat-samples.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Prompts\chat-with-bob.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

3
src/WebStarter/appsettings.json
View file

@ -13,8 +13,10 @@
},
"LlamaSharp": {
"Interactive": true,
"ModelPath": "C:\\Users\\haipi\\Downloads\\ggml-vic13b-q5_1.bin",
"InstructionFile": "Prompts\\chat-with-bob.txt",
"ChatSampleFile": "Prompts\\chat-samples.txt",
"MaxContextLength": 512
},
@ -22,6 +24,7 @@
"ApiKey": "",
"Endpoint": "",
"InstructionFile": "Prompts\\chat-with-bob.txt",
"ChatSampleFile": "Prompts\\chat-samples.txt",
"DeploymentModel": ""
}
}