SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'SciSharp:master' into master

Browse source
This commit is contained in:
C. Oceania 2024-05-22 09:47:22 -05:00 committed by GitHub
parent 018d889ad3 e75fd5db12
commit 3fd43964b7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 471 additions and 174 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Directory.Build.props
View file

@ -2,8 +2,8 @@
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<LangVersion>10.0</LangVersion>
<BotSharpVersion>1.3.1</BotSharpVersion>
<GeneratePackageOnBuild>false</GeneratePackageOnBuild>
<BotSharpVersion>1.4.0</BotSharpVersion>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<GenerateDocumentationFile>false</GenerateDocumentationFile>
</PropertyGroup>
</Project>

10
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -3,7 +3,8 @@ namespace BotSharp.Abstraction.Files;
public interface IBotSharpFileService
{
string GetDirectory(string conversationId);
IEnumerable<OutputFileModel> GetConversationFiles(string conversationId, string messageId);
IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int offset = 2);
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, bool imageOnly = false);
string? GetMessageFile(string conversationId, string messageId, string fileName);
void SaveMessageFiles(string conversationId, string messageId, List<BotSharpFile> files);
@ -17,4 +18,11 @@ public interface IBotSharpFileService
/// <returns></returns>
bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null);
bool DeleteConversationFiles(IEnumerable<string> conversationIds);
/// <summary>
/// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
(string, byte[]) GetFileInfoFromData(string data);
}

14
src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs
View file

@ -4,14 +4,14 @@ namespace BotSharp.Abstraction.Files.Models;
public class BotSharpFile
{
[JsonPropertyName("file_name")]
public string FileName { get; set; }
public string FileName { get; set; } = string.Empty;
/// <summary>
/// File data, e.g., "data:image/png;base64,aaaaaaaa"
/// </summary>
[JsonPropertyName("file_data")]
public string FileData { get; set; }
public string FileData { get; set; } = string.Empty;
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
[JsonPropertyName("file_size")]
public int FileSize { get; set; }
[JsonPropertyName("file_url")]
public string FileUrl { get; set; } = string.Empty;
}

32
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs Normal file
View file

@ -0,0 +1,32 @@
namespace BotSharp.Abstraction.Files.Models;
public class MessageFileModel
{
[JsonPropertyName("message_id")]
public string MessageId { get; set; }
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
[JsonPropertyName("file_storage_url")]
public string FileStorageUrl { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_type")]
public string FileType { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
public MessageFileModel()
{
}
public override string ToString()
{
return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}";
}
}

13
src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs
View file

@ -1,13 +0,0 @@
namespace BotSharp.Abstraction.Files.Models;
public class OutputFileModel
{
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_type")]
public string FileType { get; set; }
}

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs
View file

@ -6,6 +6,6 @@ public interface ILlmProviderService
{
LlmModelSetting GetSetting(string provider, string model);
List<string> GetProviders();
LlmModelSetting GetProviderModel(string provider, string id);
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null);
List<LlmModelSetting> GetProviderModels(string provider);
}

5
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -27,6 +27,11 @@ public class LlmModelSetting
public string Endpoint { get; set; }
public LlmModelType Type { get; set; } = LlmModelType.Chat;
/// <summary>
/// If true, allow sending images/vidoes to this model
/// </summary>
public bool MultiModal { get; set; }
/// <summary>
/// Prompt cost per 1K token
/// </summary>

10
src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationInput.cs Normal file
View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Translation.Models;
public class TranslationInput
{
[JsonPropertyName("id")]
public int Id { get; set; } = -1;
[JsonPropertyName("text")]
public string Text { get; set; } = null!;
}

2
src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationOutput.cs
View file

@ -9,5 +9,5 @@ public class TranslationOutput
public string OutputLanguage { get; set; } = LanguageType.ENGLISH;
[JsonPropertyName("texts")]
public string[] Texts { get; set; } = Array.Empty<string>();
public TranslationInput[] Texts { get; set; } = Array.Empty<TranslationInput>();
}

2
src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs
View file

@ -6,6 +6,6 @@ public interface IUserService
{
Task<User> GetUser(string id);
Task<User> CreateUser(User user);
Task<Token> GetToken(string authorization);
Task<Token?> GetToken(string authorization);
Task<User> GetMyProfile();
}

1
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -159,6 +159,7 @@
<PackageReference Include="Colorful.Console" Version="1.2.15" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="8.2.1" />
<PackageReference Include="Fluid.Core" Version="2.8.0" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Nanoid" Version="3.0.0" />
<PackageReference Include="RedLock.net" Version="2.3.2" />
</ItemGroup>

173
src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs
View file

@ -1,3 +1,4 @@
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
using System.Threading;
@ -7,16 +8,22 @@ public class BotSharpFileService : IBotSharpFileService
{
private readonly BotSharpDatabaseSettings _dbSettings;
private readonly IServiceProvider _services;
private readonly ILogger<BotSharpFileService> _logger;
private readonly string _baseDir;
private readonly IEnumerable<string> _allowedTypes = new List<string> { "image/png", "image/jpeg" };
private const string CONVERSATION_FOLDER = "conversations";
private const string FILE_FOLDER = "files";
private const int MIN_OFFSET = 1;
private const int MAX_OFFSET = 5;
public BotSharpFileService(
BotSharpDatabaseSettings dbSettings,
ILogger<BotSharpFileService> logger,
IServiceProvider services)
{
_dbSettings = dbSettings;
_logger = logger;
_services = services;
_baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository);
}
@ -31,29 +38,67 @@ public class BotSharpFileService : IBotSharpFileService
return dir;
}
public IEnumerable<OutputFileModel> GetConversationFiles(string conversationId, string messageId)
public IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int offset = 2)
{
var outputFiles = new List<OutputFileModel>();
var dir = GetConversationFileDirectory(conversationId, messageId);
if (string.IsNullOrEmpty(dir))
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty())
{
return outputFiles;
return files;
}
foreach (var file in Directory.GetFiles(dir))
if (offset <= 0)
{
var fileName = Path.GetFileNameWithoutExtension(file);
var extension = Path.GetExtension(file);
var fileType = extension.Substring(1);
var model = new OutputFileModel()
{
FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}",
FileName = fileName,
FileType = fileType
};
outputFiles.Add(model);
offset = MIN_OFFSET;
}
return outputFiles;
else if (offset > MAX_OFFSET)
{
offset = MAX_OFFSET;
}
var messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset).ToList();
files = GetMessageFiles(conversationId, messageIds, imageOnly: true).ToList();
return files;
}
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, bool imageOnly = false)
{
var files = new List<MessageFileModel>();
if (messageIds.IsNullOrEmpty()) return files;
foreach (var messageId in messageIds)
{
var dir = GetConversationFileDirectory(conversationId, messageId);
if (string.IsNullOrEmpty(dir))
{
continue;
}
foreach (var file in Directory.GetFiles(dir))
{
var contentType = GetFileContentType(file);
if (imageOnly && !_allowedTypes.Contains(contentType))
{
continue;
}
var fileName = Path.GetFileNameWithoutExtension(file);
var extension = Path.GetExtension(file);
var fileType = extension.Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}",
FileStorageUrl = file,
FileName = fileName,
FileType = fileType,
ContentType = contentType
};
files.Add(model);
}
}
return files;
}
public string? GetMessageFile(string conversationId, string messageId, string fileName)
@ -75,19 +120,26 @@ public class BotSharpFileService : IBotSharpFileService
var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true);
if (string.IsNullOrEmpty(dir)) return;
for (int i = 0; i < files.Count; i++)
try
{
var file = files[i];
if (string.IsNullOrEmpty(file.FileData))
for (int i = 0; i < files.Count; i++)
{
continue;
}
var file = files[i];
if (string.IsNullOrEmpty(file.FileData))
{
continue;
}
var bytes = GetFileBytes(file.FileData);
var fileType = Path.GetExtension(file.FileName);
var fileName = $"{i + 1}{fileType}";
Thread.Sleep(100);
File.WriteAllBytes(Path.Combine(dir, fileName), bytes);
var (_, bytes) = GetFileInfoFromData(file.FileData);
var fileType = Path.GetExtension(file.FileName);
var fileName = $"{i + 1}{fileType}";
Thread.Sleep(100);
File.WriteAllBytes(Path.Combine(dir, fileName), bytes);
}
}
catch (Exception ex)
{
_logger.LogError($"Error when saving conversation files: {ex.Message}");
}
}
@ -137,6 +189,23 @@ public class BotSharpFileService : IBotSharpFileService
return true;
}
public (string, byte[]) GetFileInfoFromData(string data)
{
if (string.IsNullOrEmpty(data))
{
return (string.Empty, new byte[0]);
}
var typeStartIdx = data.IndexOf(':');
var typeEndIdx = data.IndexOf(';');
var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
var base64startIdx = data.IndexOf(',');
var base64Str = data.Substring(base64startIdx + 1);
return (contentType, Convert.FromBase64String(base64Str));
}
#region Private methods
private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false)
{
@ -170,54 +239,16 @@ public class BotSharpFileService : IBotSharpFileService
return dir;
}
private byte[] GetFileBytes(string data)
private string GetFileContentType(string filePath)
{
if (string.IsNullOrEmpty(data))
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
{
return new byte[0];
contentType = string.Empty;
}
var startIdx = data.IndexOf(',');
var base64Str = data.Substring(startIdx + 1);
return Convert.FromBase64String(base64Str);
}
private string GetFileType(string data)
{
if (string.IsNullOrEmpty(data))
{
return string.Empty;
}
var startIdx = data.IndexOf(':');
var endIdx = data.IndexOf(';');
var fileType = data.Substring(startIdx + 1, endIdx - startIdx - 1);
return fileType;
}
private string ParseFileFormat(string type)
{
var parsed = string.Empty;
switch (type)
{
case "image/png":
parsed = ".png";
break;
case "image/jpeg":
case "image/jpg":
parsed = ".jpeg";
break;
case "application/pdf":
parsed = ".pdf";
break;
case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
parsed = ".xlsx";
break;
case "text/plain":
parsed = ".txt";
break;
}
return parsed;
return contentType;
}
#endregion
}

15
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -35,10 +35,13 @@ public class CompletionProvider
public static IChatCompletion GetChatCompletion(IServiceProvider services,
string? provider = null,
string? model = null,
string? modelId = null,
bool? multiModal = null,
AgentLlmConfig? agentConfig = null)
{
var completions = services.GetServices<IChatCompletion>();
(provider, model) = GetProviderAndModel(services, provider: provider, model: model, agentConfig: agentConfig);
(provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId,
multiModal: multiModal, agentConfig: agentConfig);
var completer = completions.FirstOrDefault(x => x.Provider == provider);
if (completer == null)
@ -47,7 +50,7 @@ public class CompletionProvider
logger.LogError($"Can't resolve completion provider by {provider}");
}
completer.SetModelName(model);
completer?.SetModelName(model);
return completer;
}
@ -55,6 +58,8 @@ public class CompletionProvider
private static (string, string) GetProviderAndModel(IServiceProvider services,
string? provider = null,
string? model = null,
string? modelId = null,
bool? multiModal = null,
AgentLlmConfig? agentConfig = null)
{
var agentSetting = services.GetRequiredService<AgentSettings>();
@ -73,11 +78,11 @@ public class CompletionProvider
{
model = state.GetState("model", model ?? "gpt-35-turbo-4k");
}
else if (state.ContainsState("model_id"))
else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId))
{
var modelId = state.GetState("model_id");
var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId;
var llmProviderService = services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel(provider, modelId)?.Name;
model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal: multiModal)?.Name;
}
}

10
src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs
View file

@ -44,11 +44,15 @@ public class LlmProviderService : ILlmProviderService
?.Models ?? new List<LlmModelSetting>();
}
public LlmModelSetting GetProviderModel(string provider, string id)
public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null)
{
var models = GetProviderModels(provider)
.Where(x => x.Id == id)
.ToList();
.Where(x => x.Id == id);
if (multiModal.HasValue)
{
models = models.Where(x => x.MultiModal == multiModal);
}
var random = new Random();
var index = random.Next(0, models.Count());

8
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs
View file

@ -17,18 +17,18 @@ public partial class RoutingService
return false;
}
var provide = agent.LlmConfig.Provider;
var provider = agent.LlmConfig.Provider;
var model = agent.LlmConfig.Model;
if (provide == null || model == null)
if (provider == null || model == null)
{
var agentSettings = _services.GetRequiredService<AgentSettings>();
provide = agentSettings.LlmConfig.Provider;
provider = agentSettings.LlmConfig.Provider;
model = agentSettings.LlmConfig.Model;
}
var chatCompletion = CompletionProvider.GetChatCompletion(_services,
provider: provide,
provider: provider,
model: model);
var message = dialogs.Last();

2
src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
View file

@ -88,7 +88,7 @@ public partial class RoutingService : IRoutingService
{
var translator = _services.GetRequiredService<ITranslationService>();
var language = states.GetState(StateConst.LANGUAGE, LanguageType.UNKNOWN);
var language = states.GetState(StateConst.LANGUAGE, LanguageType.ENGLISH);
if (language != LanguageType.ENGLISH)
{
message.SecondaryContent = message.Content;

2
src/Infrastructure/BotSharp.Core/Templating/ResponseTemplateService.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.Templating;
using System.Reflection;

2
src/Infrastructure/BotSharp.Core/Templating/TemplateRender.cs
View file

@ -3,6 +3,7 @@ using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.Templating;
using BotSharp.Abstraction.Translation.Models;
using Fluid;
namespace BotSharp.Core.Templating;
@ -30,6 +31,7 @@ public class TemplateRender : ITemplateRender
_options.MemberAccessStrategy.Register<FunctionDef>();
_options.MemberAccessStrategy.Register<FunctionParametersDef>();
_options.MemberAccessStrategy.Register<UserIdentity>();
_options.MemberAccessStrategy.Register<TranslationInput>();
}
public string Render(string template, Dictionary<string, object> dict)

28
src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs
View file

@ -6,6 +6,7 @@ using BotSharp.Abstraction.Templating;
using BotSharp.Abstraction.Translation.Models;
using System.Collections;
using System.Reflection;
using System.Text.Encodings.Web;
namespace BotSharp.Core.Translation;
@ -57,12 +58,23 @@ public class TranslationService : ITranslationService
var keys = unique.ToArray();
var texts = unique.ToArray()
.Select((text, i) => $"{i + 1}. \"{text}\"")
.ToList();
var translatedStringList = await InnerTranslate(texts, language, template);
.Select((text, i) => new TranslationInput
{
Id = i + 1,
Text = text
}).ToList();
try
{
var translatedStringList = await InnerTranslate(texts, language, template);
int retry = 0;
while (translatedStringList.Texts.Length != texts.Count && retry < 3)
{
translatedStringList = await InnerTranslate(texts, language, template);
retry++;
}
// Override language if it's Unknown, it's used to output the corresponding language.
var states = _services.GetRequiredService<IConversationStateService>();
if (!states.ContainsState(StateConst.LANGUAGE))
@ -76,7 +88,7 @@ public class TranslationService : ITranslationService
for (var i = 0; i < texts.Count; i++)
{
map[keys[i]] = translatedTexts[i];
map[keys[i]] = translatedTexts[i].Text;
}
clonedData = Assign(clonedData, map);
@ -297,15 +309,19 @@ public class TranslationService : ITranslationService
/// <param name="list"></param>
/// <param name="language"></param>
/// <returns></returns>
private async Task<TranslationOutput> InnerTranslate(List<string> texts, string language, string template)
private async Task<TranslationOutput> InnerTranslate(List<TranslationInput> texts, string language, string template)
{
var options = new JsonSerializerOptions() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping };
var jsonString = JsonSerializer.Serialize(texts, options);
var translator = new Agent
{
Id = Guid.Empty.ToString(),
Name = "Translator",
Instruction = "You are a translation expert.",
TemplateDict = new Dictionary<string, object>
{
{ "text_list", texts },
{ "text_list", jsonString },
{ "text_list_size", texts.Count },
{ StateConst.LANGUAGE, language }
}
};

7
src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
View file

@ -65,7 +65,7 @@ public class UserService : IUserService
return record;
}
public async Task<Token> GetToken(string authorization)
public async Task<Token?> GetToken(string authorization)
{
var base64 = Encoding.UTF8.GetString(Convert.FromBase64String(authorization));
var (id, password) = base64.SplitAsTuple(":");
@ -77,13 +77,14 @@ public class UserService : IUserService
record = db.GetUserByUserName(id);
}
User? user = null;
var hooks = _services.GetServices<IAuthenticationHook>();
if (record == null || record.Source != "internal")
{
// check 3rd party user
foreach (var hook in hooks)
{
var user = await hook.Authenticate(id, password);
user = await hook.Authenticate(id, password);
if (user == null)
{
continue;
@ -114,7 +115,7 @@ public class UserService : IUserService
}
}
if (record == null)
if ((!hooks.IsNullOrEmpty() && user == null) || record == null)
{
return default;
}

9
src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/translation_prompt.liquid
View file

@ -1,7 +1,6 @@
{% for text in text_list %}
{{ text }}
{% endfor %}
{{ text_list }}
=====
Translate the above sentences in the list into {{ language }}.
Output the translated text in JSON {"input_lang":"", "output_lang":"{{ language }}", "texts":[]}, input_lang is based on the original sentences.
Translate all the above sentences into {{ language }}.
Output the translated text in JSON {"input_lang":"original text language", "output_count": {{ text_list_size }}, "output_lang":"{{ language }}", "texts":[{"id": 1, "text":""},{"id": 2, "text":""}]}.
The "output_count" must equal the length of the "texts" array in the output.

64
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -1,8 +1,4 @@
using BotSharp.Abstraction.Routing;
using Newtonsoft.Json.Serialization;
using Newtonsoft.Json;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files;
namespace BotSharp.OpenAPI.Controllers;
@ -138,6 +134,35 @@ public class ConversationController : ControllerBase
return result;
}
[HttpGet("/conversation/{conversationId}/user")]
public async Task<UserViewModel> GetConversationUser([FromRoute] string conversationId)
{
var service = _services.GetRequiredService<IConversationService>();
var conversations = await service.GetConversations(new ConversationFilter
{
Id = conversationId
});
var userService = _services.GetRequiredService<IUserService>();
var conversation = conversations?.Items?.FirstOrDefault();
var userId = conversation == null ? _user.Id : conversation.UserId;
var user = await userService.GetUser(userId);
if (user == null)
{
return new UserViewModel
{
Id = _user.Id,
UserName = _user.UserName,
FirstName = _user.FirstName,
LastName = _user.LastName,
Email = _user.Email,
Source = "Unknown"
};
}
return UserViewModel.FromUser(user);
}
[HttpDelete("/conversation/{conversationId}")]
public async Task<bool> DeleteConversation([FromRoute] string conversationId)
{
@ -232,7 +257,11 @@ public class ConversationController : ControllerBase
conv.SetConversationId(conversationId, input.States);
SetStates(conv, input);
var response = new ChatResponseModel();
var response = new ChatResponseModel
{
ConversationId = conversationId,
MessageId = inputMsg.MessageId,
};
Response.StatusCode = 200;
Response.Headers.Append(Microsoft.Net.Http.Headers.HeaderNames.ContentType, "text/event-stream");
@ -241,6 +270,7 @@ public class ConversationController : ControllerBase
await conv.SendMessage(agentId, inputMsg,
replyMessage: input.Postback,
// responsed generated
async msg =>
{
response.Text = !string.IsNullOrEmpty(msg.SecondaryContent) ? msg.SecondaryContent : msg.Content;
@ -249,18 +279,21 @@ public class ConversationController : ControllerBase
response.Instruction = msg.Instruction;
response.Data = msg.Data;
await OnChunkReceived(Response, msg);
await OnChunkReceived(Response, response);
},
// executing
async msg =>
{
var message = new RoleDialogModel(AgentRole.Function, msg.Content)
var indicator = new ChatResponseModel
{
FunctionArgs = msg.FunctionArgs,
FunctionName = msg.FunctionName,
Indication = msg.Indication
ConversationId = conversationId,
MessageId = msg.MessageId,
Text = msg.Indication,
Function = "indicating",
};
await OnChunkReceived(Response, message);
await OnChunkReceived(Response, indicator);
},
// executed
async msg =>
{
@ -274,14 +307,9 @@ public class ConversationController : ControllerBase
// await OnEventCompleted(Response);
}
private async Task OnChunkReceived(HttpResponse response, RoleDialogModel message)
private async Task OnChunkReceived(HttpResponse response, ChatResponseModel message)
{
var json = JsonConvert.SerializeObject(message, new JsonSerializerSettings
{
Formatting = Formatting.None,
ContractResolver = new CamelCasePropertyNamesContractResolver(),
NullValueHandling = NullValueHandling.Ignore,
});
var json = JsonSerializer.Serialize(message);
var buffer = Encoding.UTF8.GetBytes($"data:{json}\n");
await response.Body.WriteAsync(buffer, 0, buffer.Length);

5
src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs
View file

@ -38,10 +38,11 @@ public class FileController : ControllerBase
}
[HttpGet("/conversation/{conversationId}/files/{messageId}")]
public IEnumerable<OutputFileModel> GetConversationFiles([FromRoute] string conversationId, [FromRoute] string messageId)
public IEnumerable<MessageFileViewModel> GetMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
return fileService.GetConversationFiles(conversationId, messageId);
var files = fileService.GetMessageFiles(conversationId, new List<string> { messageId });
return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List<MessageFileViewModel>();
}
[HttpGet("/conversation/{conversationId}/message/{messageId}/file/{fileName}")]

33
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -11,10 +11,12 @@ namespace BotSharp.OpenAPI.Controllers;
public class InstructModeController : ControllerBase
{
private readonly IServiceProvider _services;
private readonly ILogger<InstructModeController> _logger;
public InstructModeController(IServiceProvider services)
public InstructModeController(IServiceProvider services, ILogger<InstructModeController> logger)
{
_services = services;
_logger = logger;
}
[HttpPost("/instruct/{agentId}")]
@ -72,4 +74,33 @@ public class InstructModeController : ControllerBase
});
return message.Content;
}
[HttpPost("/instruct/multi-modal")]
public async Task<string> MultiModalCompletion([FromBody] IncomingMessageModel input)
{
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
try
{
var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai",
modelId: input.ModelId ?? "gpt-4", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = Guid.Empty.ToString(),
}, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, input.Text)
{
Files = input.Files
}
});
return message.Content;
}
catch (Exception ex)
{
_logger.LogError($"Error in analyzing files. {ex.Message}");
return $"Error in analyzing files.";
}
}
}

3
src/Infrastructure/BotSharp.OpenAPI/Using.cs
View file

@ -28,4 +28,5 @@ global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.OpenAPI.ViewModels.Conversations;
global using BotSharp.OpenAPI.ViewModels.Users;
global using BotSharp.OpenAPI.ViewModels.Agents;
global using BotSharp.OpenAPI.ViewModels.Agents;
global using BotSharp.OpenAPI.ViewModels.Files;

34
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs Normal file
View file

@ -0,0 +1,34 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Files;
public class MessageFileViewModel
{
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_type")]
public string FileType { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
public MessageFileViewModel()
{
}
public static MessageFileViewModel Transform(MessageFileModel model)
{
return new MessageFileViewModel
{
FileUrl = model.FileUrl,
FileName = model.FileName,
FileType = model.FileType,
ContentType = model.ContentType
};
}
}

10
src/Plugins/BotSharp.Plugin.AnthropicAI/BotSharp.Plugin.AnthropicAI.csproj
View file

@ -1,9 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<TargetFramework>netstandard2.1</TargetFramework>
<Nullable>enable</Nullable>
<LangVersion>$(LangVersion)</LangVersion>
<VersionPrefix>$(BotSharpVersion)</VersionPrefix>
<GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
<GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
<OutputPath>$(SolutionDir)packages</OutputPath>
</PropertyGroup>
<ItemGroup>

6
src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs
View file

@ -1,3 +1,9 @@
global using System;
global using System.Collections.Generic;
global using System.Text;
global using System.Threading.Tasks;
global using System.Linq;
global using System.Text.Json;
global using Anthropic.SDK;
global using Anthropic.SDK.Constants;
global using Anthropic.SDK.Messaging;

89
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
View file

@ -4,14 +4,19 @@ using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Utilities;
using BotSharp.Plugin.AzureOpenAI.Settings;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices.ComTypes;
using System.Threading.Tasks;
namespace BotSharp.Plugin.AzureOpenAI.Providers;
@ -218,6 +223,17 @@ public class ChatCompletionProvider : IChatCompletion
protected (string, ChatCompletionsOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var allowMultiModal = settings != null && settings.MultiModal;
var chatFiles = new List<MessageFileModel>();
if (allowMultiModal)
{
chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList();
}
var chatCompletionsOptions = new ChatCompletionsOptions();
@ -279,17 +295,63 @@ public class ChatCompletionProvider : IChatCompletion
else if (message.Role == ChatRole.User)
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
var userMessage = new ChatRequestUserMessage(text)
{
// To display Planner name in log
Name = message.FunctionName,
};
if (!string.IsNullOrEmpty(message.ImageUrl))
ChatRequestUserMessage userMessage = null;
if (allowMultiModal)
{
var uri = new Uri(message.ImageUrl);
userMessage.MultimodalContentItems.Add(
new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
var chatItems = new List<ChatMessageContentItem>()
{
new ChatMessageTextContentItem(text)
};
var files = chatFiles.Where(x => x.MessageId == message.MessageId).ToList();
if (!files.IsNullOrEmpty())
{
foreach (var file in files)
{
using var stream = File.OpenRead(file.FileStorageUrl);
chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low));
}
}
if (!message.Files.IsNullOrEmpty())
{
foreach (var file in message.Files)
{
if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
chatItems.Add(new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
}
else if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
using var stream = new MemoryStream(bytes, 0, bytes.Length);
chatItems.Add(new ChatMessageImageContentItem(stream, contentType, ChatMessageImageDetailLevel.Low));
}
}
}
//if (!string.IsNullOrEmpty(message.ImageUrl))
//{
// var uri = new Uri(message.ImageUrl);
// userMessage.MultimodalContentItems.Add(
// new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
//}
userMessage = new ChatRequestUserMessage(chatItems)
{
// To display Planner name in log
Name = message.FunctionName,
};
}
else
{
userMessage = new ChatRequestUserMessage(text)
{
// To display Planner name in log
Name = message.FunctionName,
};
}
chatCompletionsOptions.Messages.Add(userMessage);
@ -301,7 +363,7 @@ public class ChatCompletionProvider : IChatCompletion
}
// https://community.openai.com/t/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api-a-few-tips-and-tricks-on-controlling-the-creativity-deterministic-output-of-prompt-responses/172683
var state = _services.GetRequiredService<IConversationStateService>();
//var state = _services.GetRequiredService<IConversationStateService>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
var samplingFactor = float.Parse(state.GetState("sampling_factor", "0.0"));
chatCompletionsOptions.Temperature = temperature;
@ -347,9 +409,12 @@ public class ChatCompletionProvider : IChatCompletion
else if (x.Role == ChatRole.User)
{
var m = x as ChatRequestUserMessage;
var content = m.Content ?? string.Join(", ", m.MultimodalContentItems
.Where(m => m is ChatMessageTextContentItem)
.Select(m => (m as ChatMessageTextContentItem)?.Text));
return !string.IsNullOrEmpty(m.Name) && m.Name != "route_to_agent" ?
$"{m.Name}: {m.Content}" :
$"{m.Role}: {m.Content}";
$"{m.Name}: {content}" :
$"{m.Role}: {content}";
}
else if (x.Role == ChatRole.Assistant)
{

50
src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.Loggers.Enums;
@ -49,6 +50,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public override async Task OnMessageReceived(RoleDialogModel message)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var log = $"{GetMessageContent(message)}";
var input = new ContentLogInputModel(conversationId, message)
@ -63,6 +66,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public override async Task OnPostbackMessageReceived(RoleDialogModel message, PostbackMessageModel replyMsg)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var log = $"{GetMessageContent(message)}";
var replyContent = JsonSerializer.Serialize(replyMsg, _options.JsonSerializerOptions);
log += $"\r\n```json\r\n{replyContent}\r\n```";
@ -81,6 +86,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
if (!_convSettings.ShowVerboseLog) return;
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var log = $"{agent.Name} is using template {name}";
var message = new RoleDialogModel(AgentRole.System, log)
@ -104,12 +110,11 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public override async Task OnFunctionExecuting(RoleDialogModel message)
{
if (message.FunctionName == "route_to_agent")
{
return;
}
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
if (message.FunctionName == "route_to_agent") return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
message.FunctionArgs = message.FunctionArgs ?? "{}";
var args = JsonSerializer.Serialize(JsonDocument.Parse(message.FunctionArgs), _options.JsonSerializerOptions);
@ -127,12 +132,11 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public override async Task OnFunctionExecuted(RoleDialogModel message)
{
if (message.FunctionName == "route_to_agent")
{
return;
}
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
if (message.FunctionName == "route_to_agent") return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
message.FunctionArgs = message.FunctionArgs ?? "{}";
// var args = JsonSerializer.Serialize(JsonDocument.Parse(message.FunctionArgs), _options.JsonSerializerOptions);
@ -159,6 +163,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
if (!_convSettings.ShowVerboseLog) return;
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var log = tokenStats.Prompt;
@ -180,8 +186,10 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
/// <returns></returns>
public override async Task OnResponseGenerated(RoleDialogModel message)
{
var conv = _services.GetRequiredService<IConversationService>();
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var conv = _services.GetRequiredService<IConversationService>();
await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStateLogGenerated", BuildStateLog(conv.ConversationId, _state.GetStates(), message));
if (message.Role == AgentRole.Assistant)
@ -208,6 +216,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public override async Task OnTaskCompleted(RoleDialogModel message)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var log = $"{GetMessageContent(message)}";
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
@ -223,6 +233,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public override async Task OnConversationEnding(RoleDialogModel message)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var log = $"Conversation ended";
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
@ -237,6 +249,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public override async Task OnBreakpointUpdated(string conversationId, bool resetStates)
{
if (string.IsNullOrEmpty(conversationId)) return;
var log = $"Conversation breakpoint is updated";
if (resetStates)
{
@ -263,6 +277,9 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public override async Task OnStateChanged(StateChangeModel stateChange)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
if (stateChange == null) return;
await _chatHub.Clients.User(_user.Id).SendAsync("OnStateChangeGenerated", BuildStateChangeLog(stateChange));
@ -273,6 +290,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public async Task OnAgentEnqueued(string agentId, string preAgentId, string? reason = null)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(agentId);
// Agent queue log
@ -298,6 +317,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public async Task OnAgentDequeued(string agentId, string currentAgentId, string? reason = null)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(agentId);
var currentAgent = await _agentService.LoadAgent(currentAgentId);
@ -324,6 +345,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public async Task OnAgentReplaced(string fromAgentId, string toAgentId, string? reason = null)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var fromAgent = await _agentService.LoadAgent(fromAgentId);
var toAgent = await _agentService.LoadAgent(toAgentId);
@ -350,6 +373,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public async Task OnAgentQueueEmptied(string agentId, string? reason = null)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
// Agent queue log
var log = $"Agent queue is empty";
@ -374,6 +398,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public async Task OnRoutingInstructionReceived(FunctionCallFromLlm instruct, RoleDialogModel message)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var log = JsonSerializer.Serialize(instruct, _options.JsonSerializerOptions);
log = $"```json\r\n{log}\r\n```";
@ -391,6 +417,8 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR
public async Task OnRoutingInstructionRevised(FunctionCallFromLlm instruct, RoleDialogModel message)
{
var conversationId = _state.GetConversationId();
if (string.IsNullOrEmpty(conversationId)) return;
var agent = await _agentService.LoadAgent(message.CurrentAgentId);
var log = $"Revised user goal agent to {instruct.OriginalAgent}";