Support HuggingFace Inference API.
This commit is contained in:
Haiping Chen
parent
8552357878
commit
ab09f1fcae
|
|
@ -2,7 +2,7 @@
|
|||
<PropertyGroup>
|
||||
<LangVersion>10.0</LangVersion>
|
||||
<OutputPath>..\..\..\packages</OutputPath>
|
||||
<PackageVersion>0.12.1</PackageVersion>
|
||||
<PackageVersion>0.12.3</PackageVersion>
|
||||
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
<mxfile host="Electron" modified="2023-09-11T03:31:43.410Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/21.6.8 Chrome/114.0.5735.289 Electron/25.5.0 Safari/537.36" etag="lRrajK2n_0sEKLZhDFCg" version="21.6.8" type="device">
|
||||
<mxfile host="Electron" modified="2023-09-17T13:34:27.318Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/21.6.8 Chrome/114.0.5735.289 Electron/25.5.0 Safari/537.36" etag="rOwRDk0nfgbjTZKasu5c" version="21.6.8" type="device">
|
||||
<diagram id="6jfvZ688tm8pOZHgqSFM" name="Page-1">
|
||||
<mxGraphModel dx="1418" dy="820" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||
<mxGraphModel dx="1418" dy="820" grid="0" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||
<root>
|
||||
<mxCell id="0" />
|
||||
<mxCell id="1" parent="0" />
|
||||
<mxCell id="kNc4F-3AqUrd7lAFkTx2-3" value="BotSharp<br><span style="font-weight: normal;">Agent<br>Conversation<br>Routing</span>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#ffcc99;strokeColor=#36393d;fontStyle=1;sketch=1;curveFitting=1;jiggle=2;" parent="1" vertex="1">
|
||||
<mxGeometry x="345.71" y="340" width="214.29" height="60" as="geometry" />
|
||||
<mxCell id="kNc4F-3AqUrd7lAFkTx2-3" value="BotSharp<br><span style="font-weight: normal;">Agent<br>Conversation<br>Routing<br>Reasoning<br></span>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#ffcc99;strokeColor=#36393d;fontStyle=1;sketch=1;curveFitting=1;jiggle=2;" parent="1" vertex="1">
|
||||
<mxGeometry x="345.71" y="330.5" width="214.29" height="79" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="9DShFkwYbzccSmdIPQ0x-7" value="<b>Phone&nbsp;Voice</b>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f9f7ed;strokeColor=#36393d;sketch=1;curveFitting=1;jiggle=2;" parent="1" vertex="1">
|
||||
<mxGeometry x="240" y="240" width="90" height="40" as="geometry" />
|
||||
|
|
|
|||
|
|
@ -64,9 +64,9 @@ author = 'Haiping Chen'
|
|||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
version = '0.9'
|
||||
version = '0.12'
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = '0.9.0'
|
||||
release = '0.12.0'
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
|
|
@ -170,7 +170,7 @@ man_pages = [
|
|||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
(master_doc, 'BotSharp', 'BotSharp Documentation',
|
||||
author, 'BotSharp', 'The LLM powered Chatbot framework.',
|
||||
author, 'BotSharp', 'The LLM application framework.',
|
||||
'Miscellaneous'),
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -3,23 +3,23 @@
|
|||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
The Open Source AI Bot Platform Builder
|
||||
The Open Source LLM Application Framework
|
||||
======================================================
|
||||
|
||||
.. image:: https://img.shields.io/discord/1106946823282761851?label=Discord
|
||||
:target: `discord`_
|
||||
|
||||
**Build the AI chatbot platform from scratch with .NET**
|
||||
**Build the AI bot from scratch with .NET**
|
||||
|
||||
> The LLM powered Conversational Service framework
|
||||
> The LLM powered Conversational Service building blocks and best practice
|
||||
|
||||
*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to .NET developers using BotSharp the Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your robots and precisely control every step of the AI processing pipeline."*
|
||||
|
||||
**BotSharp** is an open source bot framework for AI Bot platform builders. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier.
|
||||
**BotSharp** is an open source AI framework for your enterprise-grade LLM applications. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in business oriented systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier.
|
||||
|
||||
It's witten in C# running on .NET which is a full cross-platform framework. C# is an enterprise-grade programming language which is widely used to code business logic in information management related system. More friendly to corporate developers. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be easier when refactoring code in system scope.
|
||||
|
||||
BotSharp is strictly in accordance with the components principle and decouples every part that is needed in the platform builder. So you can choose different UI/UX, or pick up a different NLP Tagger, or select a more advanced algorithm to do NER tasks. They are all modularized based on unified interfaces.
|
||||
BotSharp is strictly in accordance with the components principle and decouples every part that is needed in the platform builder. So you can choose different UI/UX, or pick up a different Vector Storage, or select a more advanced algorithm to do NLU tasks. They are all modularized based on unified interfaces.
|
||||
|
||||
.. image:: static/logos/BotSharp.png
|
||||
:height: 64px
|
||||
|
|
|
|||
BIN
docs/quick-start/assets/overview.png
Normal file
BIN
docs/quick-start/assets/overview.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 118 KiB |
|
|
@ -1,10 +1,10 @@
|
|||
# Overview
|
||||
|
||||
BotSharp is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out of the box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier.
|
||||
BotSharp is an open source application framework to speed up integrate LLMs into your current business system. This project involves natural language understanding and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out of the box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier.
|
||||
|
||||
BotSharp is an high compatible and high scalable platform builder. It is in accordance with components princple strictly, decouples every part that needed in the platform builder. So you can choose different UI/UX, or pick up a different NLP Tagger, or select a more advanced algrithm to do NER task. They are all modulized based on unfied interfaces.
|
||||
|
||||
|
||||
|
||||
From the chart ahead we can see that based on botsharp you can launch your own chatbot platform with 3 components:
|
||||
|
||||
- Storage module: Botsharp supports memory and redis DB 2 methods.
|
||||
|
|
@ -21,7 +21,7 @@ Even with this simple question, you can see conversational experience are hard t
|
|||
|
||||
Your code would have to handle all these different types of requests ro carry out the same logic: looking up some forecast information for a feature. For this reason, a traditional computer interface would tend to force users to input a well-known, standard request at the detriment of the user experience, because it's just easier.
|
||||
|
||||
However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you.When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand.
|
||||
However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you. When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand.
|
||||
|
||||
Features
|
||||
-------------
|
||||
|
|
@ -30,6 +30,6 @@ Features
|
|||
* Integrate with multiple LLMs like ChatGPT and LLaMA.
|
||||
* Using plug-in design, it is easy to expand functions.
|
||||
* Working with multiple Vector Stores for senmatic search.
|
||||
* Supporting different UI providers like [Chatbot UI](https://github.com/mckaywrigley/chatbot-ui) and [HuggingChat UI](https://github.com/huggingface/chat-ui).
|
||||
* Supporting different UI providers like [Chatbot UI](https://github.com/SciSharp/chatbot-ui) and [HuggingChat UI](https://github.com/huggingface/chat-ui).
|
||||
* Integrated with popular social platforms like Facebook Messenger, Slack and Telegram.
|
||||
* Providing REST APIs to work with your own UI.
|
||||
|
|
@ -8,7 +8,7 @@ public interface IConversationStateService
|
|||
ConversationState Load(string conversationId);
|
||||
string GetState(string name, string defaultValue = "");
|
||||
ConversationState GetStates();
|
||||
IConversationStateService SetState(string name, string value);
|
||||
IConversationStateService SetState<T>(string name, T value);
|
||||
void CleanState();
|
||||
void Save();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,11 +8,17 @@ public class IncomingMessageModel
|
|||
|
||||
public virtual string Channel { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Completion Provider
|
||||
/// </summary>
|
||||
[JsonPropertyName("provider")]
|
||||
public virtual string? Provider { get; set; } = null;
|
||||
|
||||
/// <summary>
|
||||
/// Model name
|
||||
/// </summary>
|
||||
[JsonPropertyName("model")]
|
||||
public virtual string? ModelName { get; set; } = null;
|
||||
public virtual string? Model { get; set; } = null;
|
||||
|
||||
/// <summary>
|
||||
/// The sampling temperature to use that controls the apparent creativity of generated completions.
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.MLTasks;
|
|||
|
||||
public interface IChatCompletion
|
||||
{
|
||||
string ModelName { get; }
|
||||
string Provider { get; }
|
||||
Task<bool> GetChatCompletionsAsync(Agent agent,
|
||||
List<RoleDialogModel> conversations,
|
||||
Func<RoleDialogModel, Task> onMessageReceived,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BotSharp.Abstraction.Utilities;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
using BotSharp.Abstraction.Agents.Enums;
|
||||
using BotSharp.Abstraction.Agents.Models;
|
||||
using BotSharp.Abstraction.Templating;
|
||||
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ public partial class ConversationService
|
|||
{
|
||||
if (!string.IsNullOrEmpty(property.Value.ToString()))
|
||||
{
|
||||
stateService.SetState(property.Name, property.Value.ToString());
|
||||
stateService.SetState(property.Name, property.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,13 +29,18 @@ public class ConversationStateService : IConversationStateService, IDisposable
|
|||
_states = new ConversationState();
|
||||
}
|
||||
|
||||
public IConversationStateService SetState(string name, string value)
|
||||
public IConversationStateService SetState<T>(string name, T value)
|
||||
{
|
||||
if (value == null)
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
var currentValue = value.ToString();
|
||||
var hooks = _services.GetServices<IConversationHook>();
|
||||
string preValue = _states.ContainsKey(name) ? _states[name] : "";
|
||||
if (!_states.ContainsKey(name) || _states[name] != value)
|
||||
if (!_states.ContainsKey(name) || _states[name] != currentValue)
|
||||
{
|
||||
var currentValue = value;
|
||||
_states[name] = currentValue;
|
||||
_logger.LogInformation($"Set state: {name} = {value}");
|
||||
foreach (var hook in hooks)
|
||||
|
|
|
|||
|
|
@ -4,16 +4,16 @@ namespace BotSharp.Core.Infrastructures;
|
|||
|
||||
public class CompletionProvider
|
||||
{
|
||||
public static IChatCompletion GetChatCompletion(IServiceProvider services, string? model = null)
|
||||
public static IChatCompletion GetChatCompletion(IServiceProvider services, string? provider = null)
|
||||
{
|
||||
var completions = services.GetServices<IChatCompletion>();
|
||||
|
||||
var state = services.GetRequiredService<IConversationStateService>();
|
||||
if (model == null)
|
||||
if (provider == null)
|
||||
{
|
||||
model = state.GetState("model", "gpt-3.5-turbo");
|
||||
provider = state.GetState("provider", "azure-gpt-3.5");
|
||||
}
|
||||
|
||||
return completions.FirstOrDefault(x => x.ModelName == model);
|
||||
return completions.FirstOrDefault(x => x.Provider == provider);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ public class Simulator
|
|||
{
|
||||
var wholeDialogs = new List<RoleDialogModel>
|
||||
{
|
||||
new RoleDialogModel(AgentRole.User, @"What's the next step, your response must be in JSON format with ""function"" and ""parameters"". ")
|
||||
new RoleDialogModel(AgentRole.User, @"What's the next step? Response in JSON format with ""function"" and ""parameters"".")
|
||||
};
|
||||
|
||||
var chatCompletion = CompletionProvider.GetChatCompletion(_services);
|
||||
|
|
@ -146,7 +146,7 @@ public class Simulator
|
|||
{
|
||||
if (!string.IsNullOrEmpty(property.Value.ToString()))
|
||||
{
|
||||
stateService.SetState(property.Name, property.Value.ToString());
|
||||
stateService.SetState(property.Name, property.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,9 +45,10 @@ public class ConversationController : ControllerBase, IApiAdapter
|
|||
var conv = _services.GetRequiredService<IConversationService>();
|
||||
conv.SetConversationId(conversationId, input.States);
|
||||
conv.States.SetState("channel", input.Channel)
|
||||
.SetState("model", input.ModelName)
|
||||
.SetState("temperature", input.Temperature.ToString())
|
||||
.SetState("sampling_factor", input.SamplingFactor.ToString());
|
||||
.SetState("provider", input.Provider)
|
||||
.SetState("model", input.Model)
|
||||
.SetState("temperature", input.Temperature)
|
||||
.SetState("sampling_factor", input.SamplingFactor);
|
||||
|
||||
var response = new MessageResponseModel();
|
||||
var stackMsg = new List<RoleDialogModel>();
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ public class ChatCompletionProvider : IChatCompletion
|
|||
protected readonly IServiceProvider _services;
|
||||
protected readonly ILogger _logger;
|
||||
|
||||
public virtual string ModelName => "gpt-3.5-turbo";
|
||||
public virtual string Provider => "azure-gpt-3.5";
|
||||
|
||||
public ChatCompletionProvider(AzureOpenAiSettings settings,
|
||||
ILogger<ChatCompletionProvider> logger,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers;
|
|||
|
||||
public class GPT4CompletionProvider : ChatCompletionProvider
|
||||
{
|
||||
public override string ModelName => "gpt-4";
|
||||
public override string Provider => "azure-gpt-4";
|
||||
|
||||
public GPT4CompletionProvider(AzureOpenAiSettings settings,
|
||||
ILogger<GPT4CompletionProvider> logger,
|
||||
|
|
|
|||
|
|
@ -76,10 +76,11 @@ public class ChatbotUiController : ControllerBase, IApiAdapter
|
|||
|
||||
var conv = _services.GetRequiredService<IConversationService>();
|
||||
conv.SetConversationId(input.ConversationId, input.States);
|
||||
conv.States.SetState("model", input.ModelName);
|
||||
conv.States.SetState("channel", "webchat");
|
||||
conv.States.SetState("temperature", input.Temperature.ToString());
|
||||
conv.States.SetState("sampling_factor", input.SamplingFactor.ToString());
|
||||
conv.States.SetState("provider", input.Provider)
|
||||
.SetState("model", input.Model)
|
||||
.SetState("channel", "webchat")
|
||||
.SetState("temperature", input.Temperature)
|
||||
.SetState("sampling_factor", input.SamplingFactor);
|
||||
|
||||
var result = await conv.SendMessage(input.AgentId,
|
||||
message,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
|
||||
<PackageReference Include="Refit.HttpClientFactory" Version="7.0.0" />
|
||||
<PackageReference Include="System.Text.Json" Version="7.0.3" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
namespace BotSharp.Plugin.HuggingFace.DataModels;
|
||||
|
||||
public class FalconLlmResponse
|
||||
{
|
||||
[JsonPropertyName("generated_text")]
|
||||
public string GeneratedText { get; set; }
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
namespace BotSharp.Plugin.HuggingFace.DataModels;
|
||||
|
||||
public class InferenceInput
|
||||
{
|
||||
[JsonPropertyName("inputs")]
|
||||
public string Inputs { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("parameters")]
|
||||
public InferenceInputParameters Parameters { get; set; }
|
||||
= new InferenceInputParameters();
|
||||
|
||||
[JsonPropertyName("options")]
|
||||
public InferenceInputOptions Options { get; set; }
|
||||
= new InferenceInputOptions();
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
namespace BotSharp.Plugin.HuggingFace.DataModels;
|
||||
|
||||
public class InferenceInputOptions
|
||||
{
|
||||
[JsonPropertyName("use_cache")]
|
||||
public bool UseCache { get; set; } = true;
|
||||
|
||||
[JsonPropertyName("wait_for_model")]
|
||||
public bool WaitForModel { get; set; } = false;
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
namespace BotSharp.Plugin.HuggingFace.DataModels;
|
||||
|
||||
public class InferenceInputParameters
|
||||
{
|
||||
[JsonPropertyName("temperature")]
|
||||
public float Temperature { get; set; } = 1.0f;
|
||||
|
||||
[JsonPropertyName("max_new_tokens")]
|
||||
public int MaxNewTokens { get; set; } = 250;
|
||||
|
||||
[JsonPropertyName("return_full_text")]
|
||||
public bool ReturnFullText { get; set; } = false;
|
||||
|
||||
[JsonPropertyName("num_return_sequences")]
|
||||
public int NumReturnSequences { get; set; } = 1;
|
||||
}
|
||||
|
|
@ -3,12 +3,8 @@ using Microsoft.AspNetCore.Mvc;
|
|||
using Microsoft.Net.Http.Headers;
|
||||
using Newtonsoft.Json.Serialization;
|
||||
using Newtonsoft.Json;
|
||||
using System;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using BotSharp.Plugin.HuggingFace.HuggingChat.ViewModels;
|
||||
using BotSharp.Abstraction.TextGeneratives;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BotSharp.Plugin.HuggingFace.HuggingChat;
|
||||
|
||||
|
|
|
|||
29
src/Plugins/BotSharp.Plugin.HuggingFace/HuggingFacePlugin.cs
Normal file
29
src/Plugins/BotSharp.Plugin.HuggingFace/HuggingFacePlugin.cs
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
using BotSharp.Abstraction.Plugins;
|
||||
using BotSharp.Plugin.HuggingFace.Providers;
|
||||
using BotSharp.Plugin.HuggingFace.Services;
|
||||
using BotSharp.Plugin.HuggingFace.Settings;
|
||||
using Refit;
|
||||
|
||||
namespace BotSharp.Plugin.HuggingFace;
|
||||
|
||||
public class HuggingFacePlugin : IBotSharpPlugin
|
||||
{
|
||||
public void RegisterDI(IServiceCollection services, IConfiguration config)
|
||||
{
|
||||
var settings = new HuggingFaceSettings();
|
||||
config.Bind("HuggingFace", settings);
|
||||
services.AddSingleton(x =>
|
||||
{
|
||||
Console.WriteLine($"Loaded HuggingFace settings: {settings.EndPoint} ({settings.Model}) {settings.Token.SubstringMax(4)}");
|
||||
return settings;
|
||||
});
|
||||
|
||||
services
|
||||
.AddRefitClient<IInferenceApi>()
|
||||
.AddHttpMessageHandler<AuthHeaderHandler>()
|
||||
.ConfigureHttpClient(c => c.BaseAddress = new Uri(settings.EndPoint));
|
||||
|
||||
services.AddTransient<AuthHeaderHandler>();
|
||||
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
using BotSharp.Abstraction.Agents.Enums;
|
||||
using BotSharp.Abstraction.Conversations.Settings;
|
||||
using BotSharp.Plugin.HuggingFace.Services;
|
||||
using BotSharp.Plugin.HuggingFace.Settings;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BotSharp.Plugin.HuggingFace.Providers;
|
||||
|
||||
public class ChatCompletionProvider : IChatCompletion
|
||||
{
|
||||
public string Provider => "huggingface";
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly HuggingFaceSettings _settings;
|
||||
private readonly ILogger _logger;
|
||||
|
||||
public ChatCompletionProvider(IServiceProvider services,
|
||||
HuggingFaceSettings settings,
|
||||
ILogger<ChatCompletionProvider> logger)
|
||||
{
|
||||
_services = services;
|
||||
_settings = settings;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<bool> GetChatCompletionsAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived, Func<RoleDialogModel, Task> onFunctionExecuting)
|
||||
{
|
||||
var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim();
|
||||
content += $"\r\n{AgentRole.Assistant}: ";
|
||||
|
||||
var prompt = agent.Instruction + "\r\n" + content;
|
||||
|
||||
var convSetting = _services.GetRequiredService<ConversationSetting>();
|
||||
if (convSetting.ShowVerboseLog)
|
||||
{
|
||||
_logger.LogInformation(prompt);
|
||||
}
|
||||
|
||||
var api = _services.GetRequiredService<IInferenceApi>();
|
||||
|
||||
if (_settings.Model.Contains('/'))
|
||||
{
|
||||
var space = _settings.Model.Split('/')[0];
|
||||
var model = _settings.Model.Split("/")[1];
|
||||
|
||||
var response = await api.Post(space, model, new InferenceInput
|
||||
{
|
||||
Inputs = prompt
|
||||
});
|
||||
|
||||
var falcon = JsonSerializer.Deserialize<List<FalconLlmResponse>>(response);
|
||||
|
||||
var message = falcon[0].GeneratedText.Trim();
|
||||
_logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}");
|
||||
|
||||
var msg = new RoleDialogModel(AgentRole.Assistant, message)
|
||||
{
|
||||
CurrentAgentId = agent.Id
|
||||
};
|
||||
|
||||
// Text response received
|
||||
await onMessageReceived(msg);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
using BotSharp.Plugin.HuggingFace.Settings;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Threading;
|
||||
|
||||
namespace BotSharp.Plugin.HuggingFace.Services;
|
||||
|
||||
public class AuthHeaderHandler : DelegatingHandler
|
||||
{
|
||||
private readonly HuggingFaceSettings _settings;
|
||||
public AuthHeaderHandler(HuggingFaceSettings settings)
|
||||
{
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _settings.Token);
|
||||
|
||||
return await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
using Refit;
|
||||
|
||||
namespace BotSharp.Plugin.HuggingFace.Services;
|
||||
|
||||
/// <summary>
|
||||
/// https://huggingface.co/docs/api-inference/quicktour
|
||||
/// </summary>
|
||||
public interface IInferenceApi
|
||||
{
|
||||
[Post("/models/{space}/{model}")]
|
||||
Task<JsonDocument> Post(string space, string model, [Body] InferenceInput input);
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
namespace BotSharp.Plugin.HuggingFace.Settings;
|
||||
|
||||
public class HuggingFaceSettings
|
||||
{
|
||||
public string EndPoint { get; set; }
|
||||
public string Model { get; set; }
|
||||
public string Token { get; set; }
|
||||
}
|
||||
14
src/Plugins/BotSharp.Plugin.HuggingFace/Using.cs
Normal file
14
src/Plugins/BotSharp.Plugin.HuggingFace/Using.cs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
global using System;
|
||||
global using System.Collections.Generic;
|
||||
global using System.Text;
|
||||
global using System.Threading.Tasks;
|
||||
global using System.Linq;
|
||||
global using System.Text.Json;
|
||||
global using BotSharp.Abstraction.Conversations.Models;
|
||||
global using BotSharp.Abstraction.Agents.Models;
|
||||
global using BotSharp.Abstraction.MLTasks;
|
||||
global using Microsoft.Extensions.Configuration;
|
||||
global using Microsoft.Extensions.DependencyInjection;
|
||||
global using System.Text.Json.Serialization;
|
||||
global using BotSharp.Plugin.HuggingFace.DataModels;
|
||||
global using BotSharp.Abstraction.Utilities;
|
||||
|
|
@ -2,6 +2,7 @@ using BotSharp.Plugin.LLamaSharp.Settings;
|
|||
using LLama;
|
||||
using LLama.Abstractions;
|
||||
using LLama.Common;
|
||||
using System.IO;
|
||||
|
||||
namespace BotSharp.Plugins.LLamaSharp;
|
||||
|
||||
|
|
@ -22,23 +23,23 @@ public class LlamaAiModel
|
|||
public LlamaAiModel(LlamaSharpSettings settings)
|
||||
{
|
||||
_settings = settings;
|
||||
|
||||
_params = new ModelParams(_settings.ModelPath)
|
||||
{
|
||||
ContextSize = _settings.MaxContextLength,
|
||||
Seed = 1337,
|
||||
GpuLayerCount = _settings.NumberOfGpuLayer
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
public void LoadModel()
|
||||
public void LoadModel(string model)
|
||||
{
|
||||
if (_model != null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_params = new ModelParams(Path.Combine(_settings.ModelDir, model))
|
||||
{
|
||||
ContextSize = _settings.MaxContextLength,
|
||||
Seed = 1337,
|
||||
GpuLayerCount = _settings.NumberOfGpuLayer
|
||||
};
|
||||
|
||||
_model = LLamaWeights.LoadFromFile(_params);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
using BotSharp.Abstraction.Agents.Enums;
|
||||
using BotSharp.Abstraction.Agents.Models;
|
||||
using BotSharp.Abstraction.Conversations;
|
||||
using BotSharp.Abstraction.Conversations.Models;
|
||||
using BotSharp.Abstraction.Conversations.Settings;
|
||||
using BotSharp.Abstraction.MLTasks;
|
||||
using BotSharp.Plugin.LLamaSharp.Settings;
|
||||
using BotSharp.Plugins.LLamaSharp;
|
||||
using LLama;
|
||||
using LLama.Common;
|
||||
|
|
@ -19,39 +21,44 @@ public class ChatCompletionProvider : IChatCompletion
|
|||
{
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger _logger;
|
||||
private readonly LlamaSharpSettings _settings;
|
||||
|
||||
public ChatCompletionProvider(IServiceProvider services,
|
||||
ILogger<ChatCompletionProvider> logger)
|
||||
ILogger<ChatCompletionProvider> logger,
|
||||
LlamaSharpSettings settings)
|
||||
{
|
||||
_services = services;
|
||||
_logger = logger;
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
public string ModelName => "llama-2";
|
||||
|
||||
public string Provider => "llama-sharp";
|
||||
|
||||
public async Task<bool> GetChatCompletionsAsync(Agent agent,
|
||||
List<RoleDialogModel> conversations,
|
||||
Func<RoleDialogModel, Task> onMessageReceived,
|
||||
Func<RoleDialogModel, Task> onFunctionExecuting)
|
||||
{
|
||||
var content = string.Join("\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
|
||||
content += $"\n{AgentRole.Assistant}: ";
|
||||
var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
|
||||
content += $"\r\n{AgentRole.Assistant}: ";
|
||||
|
||||
var state = _services.GetRequiredService<IConversationStateService>();
|
||||
var model = state.GetState("model", _settings.DefaultModel);
|
||||
|
||||
var llama = _services.GetRequiredService<LlamaAiModel>();
|
||||
llama.LoadModel();
|
||||
llama.LoadModel(model);
|
||||
var executor = llama.GetStatelessExecutor();
|
||||
|
||||
var inferenceParams = new InferenceParams()
|
||||
{
|
||||
Temperature = 1.0f,
|
||||
AntiPrompts = new List<string> { $"{AgentRole.User}:", "\n", "?" },
|
||||
Temperature = 0.9f,
|
||||
AntiPrompts = new List<string> { $"{AgentRole.User}:" },
|
||||
MaxTokens = 256
|
||||
};
|
||||
|
||||
string totalResponse = "";
|
||||
|
||||
var prompt = agent.Instruction + content;
|
||||
var prompt = agent.Instruction + "\r\n" + content;
|
||||
|
||||
var convSetting = _services.GetRequiredService<ConversationSetting>();
|
||||
if (convSetting.ShowVerboseLog)
|
||||
|
|
@ -70,7 +77,15 @@ public class ChatCompletionProvider : IChatCompletion
|
|||
totalResponse = totalResponse.Replace(anti, "").Trim();
|
||||
}
|
||||
|
||||
await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, totalResponse));
|
||||
_logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {totalResponse}");
|
||||
|
||||
var msg = new RoleDialogModel(AgentRole.Assistant, totalResponse)
|
||||
{
|
||||
CurrentAgentId = agent.Id
|
||||
};
|
||||
|
||||
// Text response received
|
||||
await onMessageReceived(msg);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -78,11 +93,15 @@ public class ChatCompletionProvider : IChatCompletion
|
|||
public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleDialogModel> conversations, Func<RoleDialogModel, Task> onMessageReceived)
|
||||
{
|
||||
string totalResponse = "";
|
||||
var content = string.Join("\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
|
||||
content += $"\n{AgentRole.Assistant}: ";
|
||||
var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
|
||||
content += $"\r\n{AgentRole.Assistant}: ";
|
||||
|
||||
var state = _services.GetRequiredService<IConversationStateService>();
|
||||
var model = state.GetState("model", "llama-2-7b-chat.Q8_0");
|
||||
|
||||
var llama = _services.GetRequiredService<LlamaAiModel>();
|
||||
llama.LoadModel();
|
||||
llama.LoadModel(model);
|
||||
|
||||
var executor = new StatelessExecutor(llama.Model, llama.Params);
|
||||
var inferenceParams = new InferenceParams() { Temperature = 1.0f, AntiPrompts = new List<string> { $"{AgentRole.User}:" }, MaxTokens = 64 };
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
using BotSharp.Abstraction.Conversations;
|
||||
using BotSharp.Abstraction.MLTasks;
|
||||
using BotSharp.Plugin.LLamaSharp.Settings;
|
||||
using BotSharp.Plugins.LLamaSharp;
|
||||
using LLama;
|
||||
using LLama.Common;
|
||||
|
|
@ -11,16 +13,22 @@ namespace BotSharp.Plugin.LLamaSharp.Providers;
|
|||
public class TextCompletionProvider : ITextCompletion
|
||||
{
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly LlamaSharpSettings _settings;
|
||||
|
||||
public TextCompletionProvider(IServiceProvider services)
|
||||
public TextCompletionProvider(IServiceProvider services,
|
||||
LlamaSharpSettings settings)
|
||||
{
|
||||
_services = services;
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
public Task<string> GetCompletion(string text)
|
||||
{
|
||||
var state = _services.GetRequiredService<IConversationStateService>();
|
||||
var model = state.GetState("model", _settings.DefaultModel);
|
||||
|
||||
var llama = _services.GetRequiredService<LlamaAiModel>();
|
||||
llama.LoadModel();
|
||||
llama.LoadModel(model);
|
||||
|
||||
var executor = new InstructExecutor(llama.Model.CreateContext(llama.Params));
|
||||
var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 };
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ using LLama;
|
|||
using LLama.Common;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
namespace BotSharp.Plugin.LLamaSharp.Providers;
|
||||
|
||||
|
|
@ -24,7 +25,8 @@ public class TextEmbeddingProvider : ITextEmbedding
|
|||
{
|
||||
if (_embedder == null)
|
||||
{
|
||||
_embedder = new LLamaEmbedder(new ModelParams(_settings.ModelPath));
|
||||
var path = Path.Combine(_settings.ModelDir, _settings.DefaultModel);
|
||||
_embedder = new LLamaEmbedder(new ModelParams(path));
|
||||
}
|
||||
|
||||
return _embedder.GetEmbeddings(text);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ namespace BotSharp.Plugin.LLamaSharp.Settings;
|
|||
|
||||
public class LlamaSharpSettings
|
||||
{
|
||||
public string ModelPath { get; set; } = string.Empty;
|
||||
public string ModelDir { get; set; } = string.Empty;
|
||||
public string DefaultModel { get; set; } = "llama-2-7b-chat.Q8_0.gguf";
|
||||
public int MaxContextLength { get; set; } = 512;
|
||||
public float RepeatPenalty { get; set; } = 1.0f;
|
||||
public bool VerbosePrompt { get; set; }
|
||||
|
|
|
|||
|
|
@ -31,9 +31,10 @@
|
|||
|
||||
"LlamaSharp": {
|
||||
"Interactive": true,
|
||||
"ModelPath": "C:/Users/haipi/Downloads/llama-2-7b-guanaco-qlora.Q4_K_S.gguf",
|
||||
"ModelDir": "C:/Users/haipi/Downloads",
|
||||
"DefaultModel": "llama-2-7b-chat.Q8_0.gguf",
|
||||
"MaxContextLength": 1024,
|
||||
"NumberOfGpuLayer": 15
|
||||
"NumberOfGpuLayer": 10
|
||||
},
|
||||
|
||||
"AzureOpenAi": {
|
||||
|
|
@ -45,6 +46,12 @@
|
|||
}
|
||||
},
|
||||
|
||||
"HuggingFace": {
|
||||
"Endpoint": "https://api-inference.huggingface.co",
|
||||
"Model": "tiiuae/falcon-180B-chat",
|
||||
"Token": ""
|
||||
},
|
||||
|
||||
"MetaAi": {
|
||||
"fastText": {
|
||||
"ModelPath": "dbpedia.ftz"
|
||||
|
|
@ -88,13 +95,13 @@
|
|||
},
|
||||
|
||||
"KnowledgeBase": {
|
||||
"VectorDb": "MemVectorDatabase",
|
||||
// "VectorDb": "QdrantDb",
|
||||
"TextEmbedding": "fastTextEmbeddingProvider",
|
||||
// "TextEmbedding": "LLamaSharp.TextEmbeddingProvider",
|
||||
"TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider",
|
||||
// "TextCompletion": "LLamaSharp.TextCompletionProvider",
|
||||
"Pdf2TextConverter": "PaddleSharp.Providers.Pdf2TextConverter"
|
||||
"VectorDb": "MemVectorDatabase",
|
||||
// "VectorDb": "QdrantDb",
|
||||
"TextEmbedding": "fastTextEmbeddingProvider",
|
||||
// "TextEmbedding": "LLamaSharp.TextEmbeddingProvider",
|
||||
"TextCompletion": "AzureOpenAI.Providers.TextCompletionProvider",
|
||||
// "TextCompletion": "LLamaSharp.TextCompletionProvider",
|
||||
"Pdf2TextConverter": "PaddleSharp.Providers.Pdf2TextConverter"
|
||||
},
|
||||
|
||||
"PluginLoader": {
|
||||
|
|
@ -103,6 +110,7 @@
|
|||
"BotSharp.Core",
|
||||
"BotSharp.Plugin.AzureOpenAI",
|
||||
"BotSharp.Plugin.MetaAI",
|
||||
"BotSharp.Plugin.HuggingFace",
|
||||
"BotSharp.Plugin.LLamaSharp",
|
||||
"BotSharp.Plugin.KnowledgeBase",
|
||||
"BotSharp.Plugin.Qdrant",
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
{
|
||||
"name": "Pizza Ordering Bot",
|
||||
"description": "Pizza restaurant ordering AI Chatbot.",
|
||||
"createdDateTime": "2023-08-14T18:14:11.6748378Z",
|
||||
"updatedDateTime": "2023-08-14T18:14:11.6756088Z",
|
||||
"id": "91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a"
|
||||
}
|
||||
|
|
@ -1 +0,0 @@
|
|||
You are now a pizza ordering chatbot, and you can help customers order a pizza according to the user's preferences.
|
||||
|
|
@ -1,9 +1,37 @@
|
|||
[
|
||||
{
|
||||
"userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4",
|
||||
"agentId": "91fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",
|
||||
"agentId": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",
|
||||
"updatedTime": "2023-08-14T18:14:11.6833783Z",
|
||||
"createdTime": "2023-08-14T18:14:11.6829767Z",
|
||||
"id": "1273379c-4419-460a-b0a2-5695afd097f5"
|
||||
},
|
||||
{
|
||||
"userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4",
|
||||
"agentId": "ff431273-0c28-4647-88bf-86d82443c579",
|
||||
"updatedTime": "2023-08-14T18:14:11.6833783Z",
|
||||
"createdTime": "2023-08-14T18:14:11.6829767Z",
|
||||
"id": "5ed47062-3dba-4b96-a7d7-5dbab48b71ad"
|
||||
},
|
||||
{
|
||||
"userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4",
|
||||
"agentId": "c2b57a74-ae4e-4c81-b3ad-9ac5bff982bd",
|
||||
"updatedTime": "2023-08-14T18:14:11.6833783Z",
|
||||
"createdTime": "2023-08-14T18:14:11.6829767Z",
|
||||
"id": "91ec91bc-2854-4c21-be7e-4a6806406d56"
|
||||
},
|
||||
{
|
||||
"userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4",
|
||||
"agentId": "b284db86-e9c2-4c25-a59e-4649797dd130",
|
||||
"updatedTime": "2023-08-14T18:14:11.6833783Z",
|
||||
"createdTime": "2023-08-14T18:14:11.6829767Z",
|
||||
"id": "1d6e86ee-1f72-4ac3-93bc-549a65855dc6"
|
||||
},
|
||||
{
|
||||
"userId": "456e35c5-caf0-4d45-9084-b44a8ca717e4",
|
||||
"agentId": "03d3fb55-9ada-423b-a6b4-f9ecddf4b26e",
|
||||
"updatedTime": "2023-08-14T18:14:11.6833783Z",
|
||||
"createdTime": "2023-08-14T18:14:11.6829767Z",
|
||||
"id": "b10c3209-6fd6-41d1-afd9-ea9543be286d"
|
||||
}
|
||||
]
|
||||
|
|
@ -4,7 +4,7 @@ namespace BotSharp.Plugin.PizzaBot.Functions;
|
|||
|
||||
public class GetBakingTimeFn : IFunctionCallback
|
||||
{
|
||||
public string Name => "get_cooking_remaing_time";
|
||||
public string Name => "get_cooking_remaining_time";
|
||||
|
||||
public async Task<bool> Execute(RoleDialogModel message)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue