diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Settings/AgentSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Settings/AgentSettings.cs index cc4be062..e95f72bc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Settings/AgentSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Settings/AgentSettings.cs @@ -4,6 +4,7 @@ public class AgentSettings { public string DataDir { get; set; } = string.Empty; public string TemplateFormat { get; set; } = "liquid"; + public string HostAgentId { get; set; } = string.Empty; /// /// This is the default LLM config for agent diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderSettingService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs similarity index 53% rename from src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderSettingService.cs rename to src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index bfe5d420..ce602c23 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderSettingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -2,7 +2,9 @@ using BotSharp.Abstraction.MLTasks.Settings; namespace BotSharp.Abstraction.MLTasks; -public interface ILlmProviderSettingService +public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); + List GetProviders(); + List GetProviderModels(string provider); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs index 3e53dd0f..b171de08 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextEmbedding.cs @@ -1,9 +1,11 @@ -using System.Threading; - namespace BotSharp.Abstraction.MLTasks; public interface ITextEmbedding { + /// + /// The Embedding provider like Microsoft Azure, OpenAI, ClaudAI + /// + string Provider { get; } int Dimension { get; } Task GetVectorAsync(string text); Task> GetVectorsAsync(List texts); diff --git a/src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs b/src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs index d5e7ce0f..7f7dbc6f 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs @@ -18,7 +18,7 @@ public class AgentPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(provider => diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 2838133b..83070b06 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + netstandard2.1 @@ -45,6 +45,8 @@ + + @@ -55,6 +57,12 @@ + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs index 25ee82f5..1ee38e2e 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs @@ -38,7 +38,7 @@ public class TokenStatistics : ITokenStatistics _promptTokenCount += stats.PromptCount; _completionTokenCount += stats.CompletionCount; - var settingsService = _services.GetRequiredService(); + var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(stats.Provider, _model); _promptCost += stats.PromptCount / 1000f * settings.PromptCost; diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 17a0ebe7..27547bd1 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -26,7 +26,7 @@ public class CompletionProvider model = state.GetState("model", model ?? "gpt-35-turbo-4k"); } - var settingsService = services.GetRequiredService(); + var settingsService = services.GetRequiredService(); var settings = settingsService.GetSetting(provider, model); if (settings.Type == LlmModelType.Text) diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderSettingService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs similarity index 50% rename from src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderSettingService.cs rename to src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index a3f8a8d3..011183fb 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderSettingService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -1,19 +1,43 @@ using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.MLTasks.Settings; +using BotSharp.Abstraction.Settings; namespace BotSharp.Core.Infrastructures; -public class LlmProviderSettingService : ILlmProviderSettingService +public class LlmProviderService : ILlmProviderService { private readonly IServiceProvider _services; private readonly ILogger _logger; - public LlmProviderSettingService(IServiceProvider services, ILogger logger) + public LlmProviderService(IServiceProvider services, ILogger logger) { _services = services; _logger = logger; } + public List GetProviders() + { + var providers = new List(); + var services1 = _services.GetServices(); + providers.AddRange(services1.Select(x => x.Provider)); + + var services2 = _services.GetServices(); + providers.AddRange(services2.Select(x => x.Provider)); + + var services3 = _services.GetServices(); + providers.AddRange(services3.Select(x => x.Provider)); + + return providers.Distinct().ToList(); + } + + public List GetProviderModels(string provider) + { + var settingService = _services.GetRequiredService(); + return settingService.Bind>($"LlmProviders") + .FirstOrDefault(x => x.Provider.Equals(provider)) + ?.Models ?? new List(); + } + public LlmModelSetting? GetSetting(string provider, string model) { var settings = _services.GetRequiredService>(); diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs index 4f4c13e6..fe3d9007 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/SettingService.cs @@ -41,9 +41,9 @@ public class SettingService : ISettingService public static string Mask(string value) { - if (value == null) + if (string.IsNullOrEmpty(value)) { - return null; + return string.Empty; } value = value.Substring(0, value.Length / 2 - 1) + string.Join("", Enumerable.Repeat("*", value.Length / 2)); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json new file mode 100644 index 00000000..b701d67b --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/agent.json @@ -0,0 +1,10 @@ +{ + "name": "Chatbot", + "description": "AI chatbot that can do variaty of tasks", + "createdDateTime": "2024-01-15T10:39:32Z", + "updatedDateTime": "2024-01-15T14:39:32Z", + "id": "01e2fc5c-2c89-4ec7-8470-7688608b496c", + "iconUrl": "/images/users/bot.png", + "disabled": false, + "isPublic": true +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instruction.liquid new file mode 100644 index 00000000..d944594b --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/instruction.liquid @@ -0,0 +1 @@ +You are a AI Assistant. You can answer user's question. diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 36c48476..9259cb54 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Routing.Settings; - namespace BotSharp.OpenAPI.Controllers; [Authorize] diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs new file mode 100644 index 00000000..94de179e --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs @@ -0,0 +1,29 @@ +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.MLTasks.Settings; + +namespace BotSharp.OpenAPI.Controllers; + +[Authorize] +[ApiController] +public class LlmProviderController : ControllerBase +{ + private readonly IServiceProvider _services; + private readonly ILlmProviderService _llmProvider; + public LlmProviderController(IServiceProvider services, ILlmProviderService llmProvider) + { + _services = services; + _llmProvider = llmProvider; + } + + [HttpGet("/llm-providers")] + public IEnumerable GetLlmProviders() + { + return _llmProvider.GetProviders(); + } + + [HttpGet("/llm-provider/{provider}/models")] + public IEnumerable GetLlmProviderModels([FromRoute] string provider) + { + return _llmProvider.GetProviderModels(provider); + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs index a88459f0..dd17be41 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs @@ -12,7 +12,7 @@ public class ProviderHelper { public static OpenAIClient GetClient(string model, IServiceProvider services) { - var settingsService = services.GetRequiredService(); + var settingsService = services.GetRequiredService(); var settings = settingsService.GetSetting("azure-openai", model); var client = new OpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey)); return client; diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputOptions.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputOptions.cs index 960ca8a2..591d2ade 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputOptions.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputOptions.cs @@ -6,5 +6,5 @@ public class InferenceInputOptions public bool UseCache { get; set; } = true; [JsonPropertyName("wait_for_model")] - public bool WaitForModel { get; set; } = false; + public bool WaitForModel { get; set; } = true; } diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputParameters.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputParameters.cs index 13ec492c..04f2f4c5 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputParameters.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/InferenceInputParameters.cs @@ -3,10 +3,10 @@ namespace BotSharp.Plugin.HuggingFace.DataModels; public class InferenceInputParameters { [JsonPropertyName("temperature")] - public float Temperature { get; set; } = 1.0f; + public float Temperature { get; set; } = 0.7f; [JsonPropertyName("max_new_tokens")] - public int MaxNewTokens { get; set; } = 250; + public int MaxNewTokens { get; set; } = 128; [JsonPropertyName("return_full_text")] public bool ReturnFullText { get; set; } = false; diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/FalconLlmResponse.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/TextGenResponse.cs similarity index 82% rename from src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/FalconLlmResponse.cs rename to src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/TextGenResponse.cs index a44a1f76..fe357a88 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/FalconLlmResponse.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/DataModels/TextGenResponse.cs @@ -1,6 +1,6 @@ namespace BotSharp.Plugin.HuggingFace.DataModels; -public class FalconLlmResponse +public class TextGenResponse { [JsonPropertyName("generated_text")] public string GeneratedText { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index 2899a97b..ed0f78db 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -31,34 +31,27 @@ public class ChatCompletionProvider : IChatCompletion var hooks = _services.GetServices().ToList(); // Before chat completion hook - Task.WaitAll(hooks.Select(hook => - hook.BeforeGenerating(agent, conversations)).ToArray()); + foreach (var hook in hooks) + { + await hook.BeforeGenerating(agent, conversations); + } - var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim(); + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; var prompt = agent.Instruction + "\r\n" + content; - var convSetting = _services.GetRequiredService(); - if (convSetting.ShowVerboseLog) - { - _logger.LogInformation(prompt); - } - var api = _services.GetRequiredService(); var space = _model.Split('/')[0]; var model = _model.Split("/")[1]; - var response = await api.Post(space, model, new InferenceInput + var response = await api.TextGenerate(space, model, new InferenceInput { Inputs = prompt }); - var falcon = JsonSerializer.Deserialize>(response); - - var message = falcon[0].GeneratedText.Trim(); - _logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}"); + var message = response[0].GeneratedText.Trim(); var msg = new RoleDialogModel(AgentRole.Assistant, message) { @@ -66,11 +59,15 @@ public class ChatCompletionProvider : IChatCompletion }; // After chat completion hook - Task.WaitAll(hooks.Select(hook => - hook.AfterGenerated(msg, new TokenStatsModel + foreach (var hook in hooks) + { + await hook.AfterGenerated(msg, new TokenStatsModel { + Prompt = prompt, + Provider = Provider, Model = _model - })).ToArray()); + }); + } // Text response received await onMessageReceived(msg); @@ -93,36 +90,37 @@ public class ChatCompletionProvider : IChatCompletion var hooks = _services.GetServices().ToList(); // Before chat completion hook - Task.WaitAll(hooks.Select(hook => - hook.BeforeGenerating(agent, conversations)).ToArray()); + foreach (var hook in hooks) + { + await hook.BeforeGenerating(agent, conversations); + } - var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim(); + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; var agentService = _services.GetRequiredService(); var instruction = agentService.RenderedInstruction(agent); var prompt = instruction + "\r\n" + content; - var convSetting = _services.GetRequiredService(); - if (convSetting.ShowVerboseLog) - { - _logger.LogInformation(prompt); - } - var api = _services.GetRequiredService(); var space = _model.Split('/')[0]; var model = _model.Split("/")[1]; - var response = api.Post(space, model, new InferenceInput + var response = await api.TextGenerate(space, model, new InferenceInput { - Inputs = prompt - }).Result; + Inputs = prompt, + Parameters = new InferenceInputParameters + { + MaxNewTokens = 64, + Temperature = 0.7f + } + }); - var falcon = JsonSerializer.Deserialize>(response); - - var message = falcon[0].GeneratedText.Trim(); - _logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}"); + var message = response[0].GeneratedText + .Split($"{AgentRole.User}:")[0] + .Split($"{AgentRole.Assistant}:")[0] + .Trim(); var msg = new RoleDialogModel(AgentRole.Assistant, message) { @@ -130,11 +128,15 @@ public class ChatCompletionProvider : IChatCompletion }; // After chat completion hook - Task.WaitAll(hooks.Select(hook => - hook.AfterGenerated(msg, new TokenStatsModel + foreach(var hook in hooks) + { + await hook.AfterGenerated(msg, new TokenStatsModel { + Prompt = prompt, + Provider = Provider, Model = _model - })).ToArray()); + }); + } return msg; } diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Services/IInferenceApi.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Services/IInferenceApi.cs index 49b93249..cdc4257d 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Services/IInferenceApi.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Services/IInferenceApi.cs @@ -8,5 +8,5 @@ namespace BotSharp.Plugin.HuggingFace.Services; public interface IInferenceApi { [Post("/models/{space}/{model}")] - Task Post(string space, string model, [Body] InferenceInput input); + Task> TextGenerate(string space, string model, [Body] InferenceInput input); } diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs index 25710428..12006342 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextEmbeddingProvider.cs @@ -9,6 +9,8 @@ public class TextEmbeddingProvider : ITextEmbedding private readonly IServiceProvider _services; public int Dimension => 4096; + public string Provider => "llama-sharp"; + public TextEmbeddingProvider(IServiceProvider services, LlamaSharpSettings settings) { _services = services; diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs index 64aff279..82579eb1 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/fastTextEmbeddingProvider.cs @@ -23,6 +23,8 @@ public class fastTextEmbeddingProvider : ITextEmbedding } } + public string Provider => "meta-ai"; + public fastTextEmbeddingProvider(IServiceProvider services) { _services = services; diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj index 21d06a51..7c33dd7b 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/BotSharp.Plugin.SemanticKernel.csproj @@ -6,7 +6,7 @@ $(LangVersion) $(BotSharpVersion) $(GeneratePackageOnBuild) - True + $(GeneratePackageOnBuild) diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs index 0b2fa22c..ef6efd49 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelTextEmbeddingProvider.cs @@ -1,13 +1,8 @@ using BotSharp.Abstraction.MLTasks; using Microsoft.Extensions.Configuration; -using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.AI.Embeddings; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Plugins.Memory; -using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; namespace BotSharp.Plugin.SemanticKernel @@ -33,6 +28,8 @@ namespace BotSharp.Plugin.SemanticKernel /// public int Dimension { get; set; } + public string Provider => "semantic-kernel"; + /// public async Task GetVectorAsync(string text) {