SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #510 from iceljc/features/add-image-generation

Browse source 
add image generation
This commit is contained in:
C. Oceania 2024-06-26 07:46:59 -05:00 committed by GitHub
parent fecae6236b fa9e0bffe8
commit c544ddd4cc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
37 changed files with 451 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs
View file

@ -16,7 +16,8 @@ public enum AgentField
Template,
Response,
Sample,
LlmConfig
LlmConfig,
Tool
}
public enum AgentTaskField

8
src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentTool.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Abstraction.Agents.Enums;
public class AgentTool
{
public const string FileAnalyzer = "file-analyzer";
public const string ImageGenerator = "image-generator";
public const string HttpHandler = "http-handler";
}

2
src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs
View file

@ -51,4 +51,6 @@ public interface IAgentService
List<Agent> GetAgentsByUser(string userId);
PluginDef GetPlugin(string agentId);
IEnumerable<string> GetAgentTools();
}

13
src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs
View file

@ -90,6 +90,12 @@ public class Agent
public List<string> Profiles { get; set; }
= new List<string>();
/// <summary>
/// Useful tools
/// </summary>
public List<string> Tools { get; set; }
= new List<string>();
/// <summary>
/// Inherit from agent
/// </summary>
@ -121,6 +127,7 @@ public class Agent
Functions = agent.Functions,
Responses = agent.Responses,
Samples = agent.Samples,
Tools = agent.Tools,
Knowledges = agent.Knowledges,
IsPublic = agent.IsPublic,
Disabled = agent.Disabled,
@ -162,6 +169,12 @@ public class Agent
return this;
}
public Agent SetTools(List<string> tools)
{
Tools = tools ?? new List<string>();
return this;
}
public Agent SetResponses(List<AgentResponse> responses)
{
Responses = responses ?? new List<AgentResponse>(); ;

2
src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs
View file

@ -58,4 +58,6 @@ public interface IConversationService
Task<string> GetConversationSummary(IEnumerable<string> conversationId);
Task<Conversation> GetConversationRecordOrCreateNew(string agentId);
bool IsConversationMode();
}

17
src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs Normal file
View file

@ -0,0 +1,17 @@
namespace BotSharp.Abstraction.MLTasks;
public interface IImageGeneration
{
/// <summary>
/// The LLM provider like Microsoft Azure, OpenAI, ClaudAI
/// </summary>
string Provider { get; }
/// <summary>
/// Set model name, one provider can consume different model or version(s)
/// </summary>
/// <param name="model">deployment name</param>
void SetModelName(string model);
Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations);
}

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs
View file

@ -6,6 +6,6 @@ public interface ILlmProviderService
{
LlmModelSetting GetSetting(string provider, string model);
List<string> GetProviders();
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null);
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false);
List<LlmModelSetting> GetProviderModels(string provider);
}

8
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -32,6 +32,11 @@ public class LlmModelSetting
/// </summary>
public bool MultiModal { get; set; }
/// <summary>
/// If true, allow generating images
/// </summary>
public bool ImageGeneration { get; set; }
/// <summary>
/// Prompt cost per 1K token
/// </summary>
@ -51,5 +56,6 @@ public class LlmModelSetting
public enum LlmModelType
{
Text = 1,
Chat = 2
Chat = 2,
Image = 3
}

4
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs
View file

@ -7,7 +7,7 @@ public partial class AgentService
[MemoryCache(10 * 60, perInstanceCache: true)]
public async Task<Agent> LoadAgent(string id)
{
if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString())
if (string.IsNullOrEmpty(id))
{
return null;
}
@ -28,7 +28,7 @@ public partial class AgentService
var agent = await GetAgent(id);
if (agent == null)
{
throw new Exception($"Can't load agent by id: {id}");
return null;
}
if (agent.InheritAgentId != null)

4
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs
View file

@ -1,8 +1,6 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Repositories.Enums;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.Users.Enums;
using Microsoft.EntityFrameworkCore.Metadata;
using System.IO;
namespace BotSharp.Core.Agents.Services;
@ -34,6 +32,7 @@ public partial class AgentService
record.Templates = agent.Templates ?? new List<AgentTemplate>();
record.Responses = agent.Responses ?? new List<AgentResponse>();
record.Samples = agent.Samples ?? new List<string>();
record.Tools = agent.Tools ?? new List<string>();
if (agent.LlmConfig != null && !agent.LlmConfig.IsInherit)
{
record.LlmConfig = agent.LlmConfig;
@ -95,6 +94,7 @@ public partial class AgentService
.SetFunctions(foundAgent.Functions)
.SetResponses(foundAgent.Responses)
.SetSamples(foundAgent.Samples)
.SetTools(foundAgent.Tools)
.SetLlmConfig(foundAgent.LlmConfig);
_db.UpdateAgent(clonedAgent, AgentField.All);

11
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs
View file

@ -1,4 +1,5 @@
using System.IO;
using System.Reflection;
namespace BotSharp.Core.Agents.Services;
@ -53,4 +54,14 @@ public partial class AgentService : IAgentService
var agents = _db.GetAgentsByUser(userId);
return agents;
}
public IEnumerable<string> GetAgentTools()
{
var tools = typeof(AgentTool).GetFields(BindingFlags.Public | BindingFlags.Static)
.Where(f => f.IsLiteral && f.FieldType == typeof(string))
.Select(x => x.GetRawConstantValue()?.ToString())
.ToList();
return tools;
}
}

16
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -46,6 +46,10 @@
</PropertyGroup>
<ItemGroup>
<None Remove="data\agents\00000000-0000-0000-0000-000000000000\agent.json" />
<None Remove="data\agents\00000000-0000-0000-0000-000000000000\instruction.liquid" />
<None Remove="data\agents\00000000-0000-0000-0000-000000000000\functions.json" />
<None Remove="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment_prompt.liquid" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\agent.json" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\functions.json" />
<None Remove="data\agents\01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b\instruction.liquid" />
@ -146,6 +150,18 @@
<Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\conversation.summary.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\00000000-0000-0000-0000-000000000000\agent.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\00000000-0000-0000-0000-000000000000\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\00000000-0000-0000-0000-000000000000\functions.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\00000000-0000-0000-0000-000000000000\templates\load_attachment_prompt.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\plugins\config.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

5
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs
View file

@ -161,4 +161,9 @@ public partial class ConversationService : IConversationService
return converation;
}
public bool IsConversationMode()
{
return !string.IsNullOrWhiteSpace(_conversationId);
}
}

10
src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs
View file

@ -10,9 +10,9 @@ public class LoadAttachmentFn : IFunctionCallback
private readonly IServiceProvider _services;
private readonly ILogger<LoadAttachmentFn> _logger;
private const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a";
private readonly IEnumerable<string> _imageTypes = new List<string> { "image", "images", "png", "jpg", "jpeg" };
private readonly IEnumerable<string> _pdfTypes = new List<string> { "pdf" };
private static string TOOL_ASSISTANT = Guid.Empty.ToString();
public LoadAttachmentFn(
IServiceProvider services,
@ -29,13 +29,13 @@ public class LoadAttachmentFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var wholeDialogs = conv.GetDialogHistory();
var fileTypes = args?.FileTypes?.Split(",")?.ToList() ?? new List<string>();
var fileTypes = args?.FileTypes?.Split(",", StringSplitOptions.RemoveEmptyEntries)?.ToList() ?? new List<string>();
var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes);
var agent = await agentService.LoadAgent(!string.IsNullOrEmpty(message.CurrentAgentId) ? message.CurrentAgentId : AIAssistant);
var agent = await agentService.LoadAgent(TOOL_ASSISTANT);
var fileAgent = new Agent
{
Id = agent.Id,
Name = agent.Name,
Id = agent?.Id ?? Guid.Empty.ToString(),
Name = agent?.Name ?? "Unkown",
Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.",
TemplateDict = new Dictionary<string, object>()
};

62
src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs
View file

@ -1,63 +1,53 @@
using Microsoft.EntityFrameworkCore;
namespace BotSharp.Core.Files.Hooks;
public class AttachmentProcessingHook : AgentHookBase
{
private readonly IServiceProvider _services;
private static string TOOL_ASSISTANT = Guid.Empty.ToString();
public override string SelfId => string.Empty;
public AttachmentProcessingHook(IServiceProvider services, AgentSettings settings)
: base(services, settings)
{
_services = services;
}
public override void OnAgentLoaded(Agent agent)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var conv = _services.GetRequiredService<IConversationService>();
var hasConvFiles = fileService.HasConversationUserFiles(conv.ConversationId);
var isConvMode = conv.IsConversationMode();
var isEnabled = !agent.Tools.IsNullOrEmpty() && agent.Tools.Contains(AgentTool.FileAnalyzer);
if (hasConvFiles)
if (isConvMode && isEnabled)
{
agent.Instruction += "\r\n\r\nPlease call load_attachment if user wants to describe files, such as images, pdf.\r\n\r\n";
if (agent.Functions != null)
var (prompt, loadAttachmentFn) = GetLoadAttachmentFn();
if (loadAttachmentFn != null)
{
var json = JsonSerializer.Serialize(new
if (!string.IsNullOrWhiteSpace(prompt))
{
user_request = new
{
type = "string",
description = "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time."
},
file_types = new
{
type = "string",
description = "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'"
}
});
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
agent.Functions.Add(new FunctionDef
if (agent.Functions == null)
{
Name = "load_attachment",
Description = "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.",
Parameters =
{
Properties = JsonSerializer.Deserialize<JsonDocument>(json),
Required = new List<string>
{
"user_request",
"file_types"
}
}
});
agent.Functions = new List<FunctionDef> { loadAttachmentFn };
}
else
{
agent.Functions.Add(loadAttachmentFn);
}
}
}
base.OnAgentLoaded(agent);
}
private (string, FunctionDef?) GetLoadAttachmentFn()
{
var fnName = "load_attachment";
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(TOOL_ASSISTANT);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fnName}_prompt"))?.Content ?? string.Empty;
var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fnName));
return (prompt, loadAttachmentFn);
}
}

98
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -55,42 +55,6 @@ public class CompletionProvider
return completer;
}
private static (string, string) GetProviderAndModel(IServiceProvider services,
string? provider = null,
string? model = null,
string? modelId = null,
bool? multiModal = null,
AgentLlmConfig? agentConfig = null)
{
var agentSetting = services.GetRequiredService<AgentSettings>();
var state = services.GetRequiredService<IConversationStateService>();
if (string.IsNullOrEmpty(provider))
{
provider = agentConfig?.Provider ?? agentSetting.LlmConfig?.Provider;
provider = state.GetState("provider", provider ?? "azure-openai");
}
if (string.IsNullOrEmpty(model))
{
model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model;
if (state.ContainsState("model"))
{
model = state.GetState("model", model ?? "gpt-35-turbo-4k");
}
else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId))
{
var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId;
var llmProviderService = services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal: multiModal)?.Name;
}
}
state.SetState("provider", provider);
state.SetState("model", model);
return (provider, model);
}
public static ITextCompletion GetTextCompletion(IServiceProvider services,
string? provider = null,
string? model = null,
@ -111,4 +75,66 @@ public class CompletionProvider
return completer;
}
public static IImageGeneration GetImageGeneration(IServiceProvider services,
string? provider = null,
string? model = null,
string? modelId = null,
bool imageGenerate = false,
AgentLlmConfig? agentConfig = null)
{
var completions = services.GetServices<IImageGeneration>();
(provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId,
imageGenerate: imageGenerate, agentConfig: agentConfig);
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve completion provider by {provider}");
}
completer?.SetModelName(model);
return completer;
}
private static (string, string) GetProviderAndModel(IServiceProvider services,
string? provider = null,
string? model = null,
string? modelId = null,
bool? multiModal = null,
bool imageGenerate = false,
AgentLlmConfig? agentConfig = null)
{
var agentSetting = services.GetRequiredService<AgentSettings>();
var state = services.GetRequiredService<IConversationStateService>();
if (string.IsNullOrEmpty(provider))
{
provider = agentConfig?.Provider ?? agentSetting.LlmConfig?.Provider;
provider = state.GetState("provider", provider ?? "azure-openai");
}
if (string.IsNullOrEmpty(model))
{
model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model;
if (state.ContainsState("model"))
{
model = state.GetState("model", model ?? "dall-e-3");
}
else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId))
{
var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId;
var llmProviderService = services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel(provider, modelIdentity,
multiModal: multiModal, imageGenerate: imageGenerate)?.Name;
}
}
state.SetState("provider", provider);
state.SetState("model", model);
return (provider, model);
}
}

4
src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs
View file

@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService
?.Models ?? new List<LlmModelSetting>();
}
public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null)
public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false)
{
var models = GetProviderModels(provider)
.Where(x => x.Id == id);
@ -54,6 +54,8 @@ public class LlmProviderService : ILlmProviderService
models = models.Where(x => x.MultiModal == multiModal);
}
models = models.Where(x => x.ImageGeneration == imageGenerate);
var random = new Random();
var index = random.Next(0, models.Count());
var modelSetting = models.ElementAt(index);

17
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs
View file

@ -53,6 +53,9 @@ namespace BotSharp.Core.Repository
case AgentField.LlmConfig:
UpdateAgentLlmConfig(agent.Id, agent.LlmConfig);
break;
case AgentField.Tool:
UpdateAgentTools(agent.Id, agent.Tools);
break;
case AgentField.All:
UpdateAgentAllFields(agent);
break;
@ -145,6 +148,19 @@ namespace BotSharp.Core.Repository
File.WriteAllText(agentFile, json);
}
private void UpdateAgentTools(string agentId, List<string> tools)
{
if (tools == null) return;
var (agent, agentFile) = GetAgentFromFile(agentId);
if (agent == null) return;
agent.Tools = tools;
agent.UpdatedDateTime = DateTime.UtcNow;
var json = JsonSerializer.Serialize(agent, _options);
File.WriteAllText(agentFile, json);
}
private void UpdateAgentRoutingRules(string agentId, List<RoutingRule> rules)
{
if (rules == null) return;
@ -271,6 +287,7 @@ namespace BotSharp.Core.Repository
agent.Disabled = inputAgent.Disabled;
agent.Type = inputAgent.Type;
agent.Profiles = inputAgent.Profiles;
agent.Tools = inputAgent.Tools;
agent.RoutingRules = inputAgent.RoutingRules;
agent.LlmConfig = inputAgent.LlmConfig;
agent.UpdatedDateTime = DateTime.UtcNow;

13
src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json Normal file
View file

@ -0,0 +1,13 @@
{
"id": "00000000-0000-0000-0000-000000000000",
"name": "Tool Assistant",
"description": "Tool assistant that can be used to complete many different tasks",
"type": "static",
"createdDateTime": "2023-06-24T10:39:32.2349685Z",
"updatedDateTime": "2023-06-24T14:39:32.2349686Z",
"iconUrl": "https://cdn.iconscout.com/icon/premium/png-256-thumb/route-1613278-1368497.png",
"disabled": false,
"isPublic": false,
"profiles": [ "tool" ],
"routingRules": []
}

20
src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json Normal file
View file

@ -0,0 +1,20 @@
[
{
"name": "load_attachment",
"description": "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.",
"parameters": {
"type": "object",
"properties": {
"user_request": {
"type": "string",
"description": "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time."
},
"file_types": {
"type": "string",
"description": "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'."
}
},
"required": [ "user_request", "file_types" ]
}
}
]

1
src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid Normal file
View file

@ -0,0 +1 @@
You are a tool agent.

1
src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid Normal file
View file

@ -0,0 +1 @@
Please call load_attachment if user wants to describe files, such as images, pdf.

6
src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs
View file

@ -140,4 +140,10 @@ public class AgentController : ControllerBase
{
return await _agentService.DeleteAgent(agentId);
}
[HttpGet("/agent/tools")]
public IEnumerable<string> GetAgentTools()
{
return _agentService.GetAgentTools();
}
}

37
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -99,8 +99,41 @@ public class InstructModeController : ControllerBase
}
catch (Exception ex)
{
_logger.LogError($"Error in analyzing files. {ex.Message}");
return $"Error in analyzing files.";
var error = $"Error in analyzing files. {ex.Message}";
_logger.LogError(error);
return error;
}
}
[HttpPost("/instruct/image-generation")]
public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] IncomingMessageModel input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var imageViewModel = new ImageGenerationViewModel();
try
{
var completion = CompletionProvider.GetImageGeneration(_services, provider: input.Provider ?? "openai",
modelId: input.ModelId ?? "dall-e", imageGenerate: true);
var message = await completion.GetImageGeneration(new Agent()
{
Id = Guid.Empty.ToString(),
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, input.Text)
});
imageViewModel.RevisedPrompt = message.Content;
imageViewModel.Data = message.Data;
return imageViewModel;
}
catch (Exception ex)
{
var error = $"Error in image generation. {ex.Message}";
_logger.LogError(error);
imageViewModel.Message = error;
return imageViewModel;
}
}
}

3
src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs
View file

@ -24,6 +24,7 @@ public class LlmProviderController : ControllerBase
[HttpGet("/llm-provider/{provider}/models")]
public IEnumerable<LlmModelSetting> GetLlmProviderModels([FromRoute] string provider)
{
return _llmProvider.GetProviderModels(provider);
var list = _llmProvider.GetProviderModels(provider);
return list.Where(x => !x.ImageGeneration);
}
}

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs
View file

@ -43,6 +43,7 @@ public class AgentCreationModel
/// Combine different Agents together to form a Profile.
/// </summary>
public List<string> Profiles { get; set; } = new List<string>();
public List<string> Tools { get; set; } = new List<string>();
public List<RoutingRuleUpdateModel> RoutingRules { get; set; } = new List<RoutingRuleUpdateModel>();
public AgentLlmConfig? LlmConfig { get; set; }
@ -57,6 +58,7 @@ public class AgentCreationModel
Functions = Functions,
Responses = Responses,
Samples = Samples,
Tools = Tools,
IsPublic = IsPublic,
Type = Type,
Disabled = Disabled,

6
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs
View file

@ -25,6 +25,11 @@ public class AgentUpdateModel
/// </summary>
public List<string>? Samples { get; set; }
/// <summary>
/// Tools
/// </summary>
public List<string>? Tools { get; set; }
/// <summary>
/// Functions
/// </summary>
@ -71,6 +76,7 @@ public class AgentUpdateModel
Templates = Templates ?? new List<AgentTemplate>(),
Functions = Functions ?? new List<FunctionDef>(),
Responses = Responses ?? new List<AgentResponse>(),
Tools = Tools ?? new List<string>(),
LlmConfig = LlmConfig
};

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs
View file

@ -17,6 +17,7 @@ public class AgentViewModel
public List<FunctionDef> Functions { get; set; }
public List<AgentResponse> Responses { get; set; }
public List<string> Samples { get; set; }
public List<string> Tools { get; set; }
[JsonPropertyName("is_public")]
public bool IsPublic { get; set; }
@ -63,6 +64,7 @@ public class AgentViewModel
Functions = agent.Functions,
Responses = agent.Responses,
Samples = agent.Samples,
Tools = agent.Tools,
IsPublic= agent.IsPublic,
Disabled = agent.Disabled,
IconUrl = agent.IconUrl,

18
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs Normal file
View file

@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Instructs;
public class ImageGenerationViewModel
{
[JsonPropertyName("revised_prompt")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? RevisedPrompt { get; set; }
[JsonPropertyName("data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public object? Data { get; set; }
[JsonPropertyName("message")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Message { get; set; }
}

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs
View file

@ -29,5 +29,7 @@ public class AzureOpenAiPlugin : IBotSharpPlugin
services.AddScoped<ITextCompletion, TextCompletionProvider>();
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<IChatCompletion, OpenAiChatCompletionProvider>();
services.AddScoped<IImageGeneration, ImageGenerationProvider>();
services.AddScoped<IImageGeneration, OpenAiImageGenerationProvider>();
}
}

104
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs Normal file
View file

@ -0,0 +1,104 @@
using Azure.AI.OpenAI;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Plugin.AzureOpenAI.Settings;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AzureOpenAI.Providers;
public class ImageGenerationProvider : IImageGeneration
{
protected readonly AzureOpenAiSettings _settings;
protected readonly IServiceProvider _services;
protected readonly ILogger _logger;
protected string _model;
public virtual string Provider => "azure-openai";
public ImageGenerationProvider(
AzureOpenAiSettings settings,
ILogger<ImageGenerationProvider> logger,
IServiceProvider services)
{
_settings = settings;
_services = services;
_logger = logger;
}
public async Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations)
{
var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
// Before
foreach (var hook in contentHooks)
{
await hook.BeforeGenerating(agent, conversations);
}
var client = ProviderHelper.GetClient(Provider, _model, _services);
var options = PrepareOptions(conversations);
var response = await client.GetImageGenerationsAsync(options);
var image = response.Value.Data.First();
var content = string.Empty;
if (!string.IsNullOrEmpty(image.RevisedPrompt))
{
content = image.RevisedPrompt;
}
var responseMessage = new RoleDialogModel(AgentRole.Assistant, content)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
Data = image.Url.AbsoluteUri ?? image.Base64Data
};
// After
foreach (var hook in contentHooks)
{
await hook.AfterGenerated(responseMessage, new TokenStatsModel
{
Prompt = options.Prompt,
Provider = Provider,
Model = _model,
PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(),
CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count()
});
}
return responseMessage;
}
private ImageGenerationOptions PrepareOptions(List<RoleDialogModel> conversations)
{
var state = _services.GetRequiredService<IConversationStateService>();
var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024";
var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard";
var options = new ImageGenerationOptions
{
DeploymentName = _model,
Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty,
Size = new ImageSize(sizeValue),
Quality = new ImageGenerationQuality(qualityValue)
};
return options;
}
public void SetModelName(string model)
{
_model = model;
}
}

16
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs Normal file
View file

@ -0,0 +1,16 @@
using BotSharp.Plugin.AzureOpenAI.Settings;
using Microsoft.Extensions.Logging;
using System;
namespace BotSharp.Plugin.AzureOpenAI.Providers;
public class OpenAiImageGenerationProvider : ImageGenerationProvider
{
public override string Provider => "openai";
public OpenAiImageGenerationProvider(AzureOpenAiSettings settings,
ILogger<OpenAiImageGenerationProvider> logger,
IServiceProvider services) : base(settings, logger, services)
{
}
}

2
src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs
View file

@ -36,7 +36,7 @@ public class WebSocketsMiddleware
{
var regexes = new List<Regex>
{
new Regex(@"/conversation/[a-z0-9-]+/message/[a-z0-9-]+/[a-z]+/file/[a-z0-9-]+/[a-z0-9-]+", RegexOptions.IgnoreCase),
new Regex(@"/conversation/(.*?)/message/(.*?)/(.*?)/file/(.*?)/(.*?)", RegexOptions.IgnoreCase),
new Regex(@"/user/avatar", RegexOptions.IgnoreCase)
};

1
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs
View file

@ -14,6 +14,7 @@ public class AgentDocument : MongoBase
public List<FunctionDefMongoElement> Functions { get; set; }
public List<AgentResponseMongoElement> Responses { get; set; }
public List<string> Samples { get; set; }
public List<string> Tools { get; set; }
public bool IsPublic { get; set; }
public bool Disabled { get; set; }
public List<string> Profiles { get; set; }

18
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs
View file

@ -55,6 +55,9 @@ public partial class MongoRepository
case AgentField.LlmConfig:
UpdateAgentLlmConfig(agent.Id, agent.LlmConfig);
break;
case AgentField.Tool:
UpdateAgentTools(agent.Id, agent.Tools);
break;
case AgentField.All:
UpdateAgentAllFields(agent);
break;
@ -216,6 +219,18 @@ public partial class MongoRepository
_dc.Agents.UpdateOne(filter, update);
}
private void UpdateAgentTools(string agentId, List<string> tools)
{
if (tools == null) return;
var filter = Builders<AgentDocument>.Filter.Eq(x => x.Id, agentId);
var update = Builders<AgentDocument>.Update
.Set(x => x.Tools, tools)
.Set(x => x.UpdatedTime, DateTime.UtcNow);
_dc.Agents.UpdateOne(filter, update);
}
private void UpdateAgentLlmConfig(string agentId, AgentLlmConfig? config)
{
var llmConfig = AgentLlmConfigMongoElement.ToMongoElement(config);
@ -242,6 +257,7 @@ public partial class MongoRepository
.Set(x => x.Functions, agent.Functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList())
.Set(x => x.Responses, agent.Responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList())
.Set(x => x.Samples, agent.Samples)
.Set(x => x.Tools, agent.Tools)
.Set(x => x.LlmConfig, AgentLlmConfigMongoElement.ToMongoElement(agent.LlmConfig))
.Set(x => x.IsPublic, agent.IsPublic)
.Set(x => x.UpdatedTime, DateTime.UtcNow);
@ -367,6 +383,7 @@ public partial class MongoRepository
.Select(r => AgentResponseMongoElement.ToMongoElement(r))?
.ToList() ?? new List<AgentResponseMongoElement>(),
Samples = x.Samples ?? new List<string>(),
Tools = x.Tools ?? new List<string>(),
IsPublic = x.IsPublic,
Type = x.Type,
InheritAgentId = x.InheritAgentId,
@ -456,6 +473,7 @@ public partial class MongoRepository
.Select(r => AgentResponseMongoElement.ToDomainElement(r))
.ToList() : new List<AgentResponse>(),
Samples = agentDoc.Samples ?? new List<string>(),
Tools = agentDoc.Tools ?? new List<string>(),
IsPublic = agentDoc.IsPublic,
Disabled = agentDoc.Disabled,
Type = agentDoc.Type,

2
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs
View file

@ -50,6 +50,7 @@ public partial class MongoRepository
.Select(r => AgentResponseMongoElement.ToMongoElement(r))?
.ToList() ?? new List<AgentResponseMongoElement>(),
Samples = x.Samples ?? new List<string>(),
Tools = x.Tools ?? new List<string>(),
IsPublic = x.IsPublic,
Type = x.Type,
InheritAgentId = x.InheritAgentId,
@ -74,6 +75,7 @@ public partial class MongoRepository
.Set(x => x.Functions, agent.Functions)
.Set(x => x.Responses, agent.Responses)
.Set(x => x.Samples, agent.Samples)
.Set(x => x.Tools, agent.Tools)
.Set(x => x.IsPublic, agent.IsPublic)
.Set(x => x.Type, agent.Type)
.Set(x => x.InheritAgentId, agent.InheritAgentId)

2
src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs
View file

@ -1,8 +1,6 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.Routing;
using Sdcb.SparkDesk.ResponseInternals;
namespace BotSharp.Plugin.SparkDesk.Providers;