SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add image generation provider

Browse source
This commit is contained in:
Jicheng Lu 2024-06-24 14:32:52 -05:00
parent 6cc9f14b8c
commit 9d3244e075
19 changed files with 219 additions and 141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs
View file

@ -24,7 +24,4 @@ public interface IChatCompletion
Task<bool> GetChatCompletionsStreamingAsync(Agent agent,
List<RoleDialogModel> conversations,
Func<RoleDialogModel, Task> onMessageReceived);
Task<RoleDialogModel> GetImageGeneration(Agent agent,
List<RoleDialogModel> conversations);
}

17
src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs Normal file
View file

@ -0,0 +1,17 @@
namespace BotSharp.Abstraction.MLTasks;
public interface IImageGeneration
{
/// <summary>
/// The LLM provider like Microsoft Azure, OpenAI, ClaudAI
/// </summary>
string Provider { get; }
/// <summary>
/// Set model name, one provider can consume different model or version(s)
/// </summary>
/// <param name="model">deployment name</param>
void SetModelName(string model);
Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations);
}

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs
View file

@ -6,6 +6,6 @@ public interface ILlmProviderService
{
LlmModelSetting GetSetting(string provider, string model);
List<string> GetProviders();
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null);
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false);
List<LlmModelSetting> GetProviderModels(string provider);
}

5
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -32,6 +32,11 @@ public class LlmModelSetting
/// </summary>
public bool MultiModal { get; set; }
/// <summary>
/// If true, allow generating images
/// </summary>
public bool ImageGeneration { get; set; }
/// <summary>
/// Prompt cost per 1K token
/// </summary>

98
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -55,42 +55,6 @@ public class CompletionProvider
return completer;
}
private static (string, string) GetProviderAndModel(IServiceProvider services,
string? provider = null,
string? model = null,
string? modelId = null,
bool? multiModal = null,
AgentLlmConfig? agentConfig = null)
{
var agentSetting = services.GetRequiredService<AgentSettings>();
var state = services.GetRequiredService<IConversationStateService>();
if (string.IsNullOrEmpty(provider))
{
provider = agentConfig?.Provider ?? agentSetting.LlmConfig?.Provider;
provider = state.GetState("provider", provider ?? "azure-openai");
}
if (string.IsNullOrEmpty(model))
{
model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model;
if (state.ContainsState("model"))
{
model = state.GetState("model", model ?? "gpt-35-turbo-4k");
}
else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId))
{
var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId;
var llmProviderService = services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal: multiModal)?.Name;
}
}
state.SetState("provider", provider);
state.SetState("model", model);
return (provider, model);
}
public static ITextCompletion GetTextCompletion(IServiceProvider services,
string? provider = null,
string? model = null,
@ -111,4 +75,66 @@ public class CompletionProvider
return completer;
}
public static IImageGeneration GetImageGeneration(IServiceProvider services,
string? provider = null,
string? model = null,
string? modelId = null,
bool imageGenerate = false,
AgentLlmConfig? agentConfig = null)
{
var completions = services.GetServices<IImageGeneration>();
(provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId,
imageGenerate: imageGenerate, agentConfig: agentConfig);
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve completion provider by {provider}");
}
completer?.SetModelName(model);
return completer;
}
private static (string, string) GetProviderAndModel(IServiceProvider services,
string? provider = null,
string? model = null,
string? modelId = null,
bool? multiModal = null,
bool imageGenerate = false,
AgentLlmConfig? agentConfig = null)
{
var agentSetting = services.GetRequiredService<AgentSettings>();
var state = services.GetRequiredService<IConversationStateService>();
if (string.IsNullOrEmpty(provider))
{
provider = agentConfig?.Provider ?? agentSetting.LlmConfig?.Provider;
provider = state.GetState("provider", provider ?? "azure-openai");
}
if (string.IsNullOrEmpty(model))
{
model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model;
if (state.ContainsState("model"))
{
model = state.GetState("model", model ?? "dall-e-3");
}
else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId))
{
var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId;
var llmProviderService = services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel(provider, modelIdentity,
multiModal: multiModal, imageGenerate: imageGenerate)?.Name;
}
}
state.SetState("provider", provider);
state.SetState("model", model);
return (provider, model);
}
}

4
src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs
View file

@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService
?.Models ?? new List<LlmModelSetting>();
}
public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null)
public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false)
{
var models = GetProviderModels(provider)
.Where(x => x.Id == id);
@ -54,6 +54,8 @@ public class LlmProviderService : ILlmProviderService
models = models.Where(x => x.MultiModal == multiModal);
}
models = models.Where(x => x.ImageGeneration == imageGenerate);
var random = new Random();
var index = random.Next(0, models.Count());
var modelSetting = models.ElementAt(index);

13
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -99,8 +99,9 @@ public class InstructModeController : ControllerBase
}
catch (Exception ex)
{
_logger.LogError($"Error in analyzing files. {ex.Message}");
return $"Error in analyzing files.";
var error = $"Error in analyzing files. {ex.Message}";
_logger.LogError(error);
return error;
}
}
@ -113,8 +114,8 @@ public class InstructModeController : ControllerBase
try
{
var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai",
modelId: input.ModelId ?? "dall-e");
var completion = CompletionProvider.GetImageGeneration(_services, provider: input.Provider ?? "openai",
modelId: input.ModelId ?? "dall-e", imageGenerate: true);
var message = await completion.GetImageGeneration(new Agent()
{
Id = Guid.Empty.ToString(),
@ -129,8 +130,8 @@ public class InstructModeController : ControllerBase
}
catch (Exception ex)
{
var error = "Error in image generation.";
_logger.LogError($"{error} {ex.Message}");
var error = $"Error in image generation. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}

3
src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs
View file

@ -24,6 +24,7 @@ public class LlmProviderController : ControllerBase
[HttpGet("/llm-provider/{provider}/models")]
public IEnumerable<LlmModelSetting> GetLlmProviderModels([FromRoute] string provider)
{
return _llmProvider.GetProviderModels(provider);
var list = _llmProvider.GetProviderModels(provider);
return list.Where(x => !x.ImageGeneration);
}
}

5
src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs
View file

@ -264,9 +264,4 @@ public class ChatCompletionProvider : IChatCompletion
{
_model = model;
}
public Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
throw new NotImplementedException();
}
}

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs
View file

@ -29,5 +29,7 @@ public class AzureOpenAiPlugin : IBotSharpPlugin
services.AddScoped<ITextCompletion, TextCompletionProvider>();
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<IChatCompletion, OpenAiChatCompletionProvider>();
services.AddScoped<IImageGeneration, ImageGenerationProvider>();
services.AddScoped<IImageGeneration, OpenAiImageGenerationProvider>();
}
}

58
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
View file

@ -446,62 +446,4 @@ public class ChatCompletionProvider : IChatCompletion
functionResultData = "31 celsius";
return new ChatRequestToolMessage(functionResultData.ToString(), toolCall.Id);
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
foreach (var hook in contentHooks)
{
await hook.BeforeGenerating(agent, conversations);
}
var client = ProviderHelper.GetClient(Provider, _model, _services);
var options = BuildImageGenerationOptions(conversations);
var response = await client.GetImageGenerationsAsync(options);
var image = response.Value.Data.First();
var content = string.Empty;
if (!string.IsNullOrEmpty(image.RevisedPrompt))
{
content = image.RevisedPrompt;
}
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
Data = image.Url.AbsoluteUri ?? image.Base64Data
};
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(responseMessage, new TokenStatsModel
{
Prompt = options.Prompt,
Provider = Provider,
Model = _model,
PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(),
CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count()
});
}
return responseMessage;
}
private ImageGenerationOptions BuildImageGenerationOptions(List<RoleDialogModel> conversations)
{
var state = _services.GetRequiredService<IConversationStateService>();
var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024";
var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard";
var options = new ImageGenerationOptions
{
DeploymentName = _model,
Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty,
Size = new ImageSize(sizeValue),
Quality = new ImageGenerationQuality(qualityValue)
};
return options;
}
}

104
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs Normal file
View file

@ -0,0 +1,104 @@
using Azure.AI.OpenAI;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Plugin.AzureOpenAI.Settings;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AzureOpenAI.Providers;
public class ImageGenerationProvider : IImageGeneration
{
protected readonly AzureOpenAiSettings _settings;
protected readonly IServiceProvider _services;
protected readonly ILogger _logger;
protected string _model;
public virtual string Provider => "azure-openai";
public ImageGenerationProvider(
AzureOpenAiSettings settings,
ILogger<ImageGenerationProvider> logger,
IServiceProvider services)
{
_settings = settings;
_services = services;
_logger = logger;
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// Before
foreach (var hook in contentHooks)
{
await hook.BeforeGenerating(agent, conversations);
}
var client = ProviderHelper.GetClient(Provider, _model, _services);
var options = PrepareOptions(conversations);
var response = await client.GetImageGenerationsAsync(options);
var image = response.Value.Data.First();
var content = string.Empty;
if (!string.IsNullOrEmpty(image.RevisedPrompt))
{
content = image.RevisedPrompt;
}
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
Data = image.Url.AbsoluteUri ?? image.Base64Data
};
// After
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(responseMessage, new TokenStatsModel
{
Prompt = options.Prompt,
Provider = Provider,
Model = _model,
PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(),
CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count()
});
}
return responseMessage;
}
private ImageGenerationOptions PrepareOptions(List<RoleDialogModel> conversations)
{
var state = _services.GetRequiredService<IConversationStateService>();
var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024";
var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard";
var options = new ImageGenerationOptions
{
DeploymentName = _model,
Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty,
Size = new ImageSize(sizeValue),
Quality = new ImageGenerationQuality(qualityValue)
};
return options;
}
public void SetModelName(string model)
{
_model = model;
}
}

16
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs Normal file
View file

@ -0,0 +1,16 @@
using BotSharp.Plugin.AzureOpenAI.Settings;
using Microsoft.Extensions.Logging;
using System;
namespace BotSharp.Plugin.AzureOpenAI.Providers;
public class OpenAiImageGenerationProvider : ImageGenerationProvider
{
public override string Provider => "openai";
public OpenAiImageGenerationProvider(AzureOpenAiSettings settings,
ILogger<OpenAiImageGenerationProvider> logger,
IServiceProvider services) : base(settings, logger, services)
{
}
}

5
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs
View file

@ -149,9 +149,4 @@ public class ChatCompletionProvider : IChatCompletion
{
_model = model;
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
throw new NotImplementedException();
}
}

5
src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs
View file

@ -139,9 +139,4 @@ public class ChatCompletionProvider : IChatCompletion
return msg;
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
throw new NotImplementedException();
}
}

5
src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs
View file

@ -191,9 +191,4 @@ public class ChatCompletionProvider : IChatCompletion
{
_model = model;
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
throw new NotImplementedException();
}
}

5
src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs
View file

@ -231,11 +231,6 @@ public class ChatCompletionProvider : IChatCompletion
throw new NotImplementedException();
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
throw new NotImplementedException();
}
public void SetModelName(string model)
{
_model = model;

5
src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs
View file

@ -102,10 +102,5 @@ namespace BotSharp.Plugin.SemanticKernel
{
_model = model;
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
throw new NotImplementedException();
}
}
}

5
src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs
View file

@ -268,9 +268,4 @@ public class ChatCompletionProvider : IChatCompletion
FunctionDef functionDef = new FunctionDef(def.Name, def.Description, fundef.ToArray());
return functionDef;
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
throw new NotImplementedException();
}
}