SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add chat files

Browse source
This commit is contained in:
Jicheng Lu 2024-05-13 16:53:41 -05:00
parent d12954824b
commit f9e63097a9
12 changed files with 193 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs
View file

@ -3,7 +3,8 @@ namespace BotSharp.Abstraction.Files;
public interface IBotSharpFileService
{
string GetDirectory(string conversationId);
IEnumerable<OutputFileModel> GetConversationFiles(string conversationId, string messageId);
IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int offset = 2);
IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, bool imageOnly = false);
string? GetMessageFile(string conversationId, string messageId, string fileName);
void SaveMessageFiles(string conversationId, string messageId, List<BotSharpFile> files);

32
src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs Normal file
View file

@ -0,0 +1,32 @@
namespace BotSharp.Abstraction.Files.Models;
public class MessageFileModel
{
[JsonPropertyName("message_id")]
public string MessageId { get; set; }
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
[JsonPropertyName("file_storage_url")]
public string FileStorageUrl { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_type")]
public string FileType { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
public MessageFileModel()
{
}
public override string ToString()
{
return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}";
}
}

13
src/Infrastructure/BotSharp.Abstraction/Files/Models/OutputFileModel.cs
View file

@ -1,13 +0,0 @@
namespace BotSharp.Abstraction.Files.Models;
public class OutputFileModel
{
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_type")]
public string FileType { get; set; }
}

5
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -27,6 +27,11 @@ public class LlmModelSetting
public string Endpoint { get; set; }
public LlmModelType Type { get; set; } = LlmModelType.Chat;
/// <summary>
/// If true, allow sending images/vidoes to this model
/// </summary>
public bool MultiModal { get; set; }
/// <summary>
/// Prompt cost per 1K token
/// </summary>

1
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -159,6 +159,7 @@
<PackageReference Include="Colorful.Console" Version="1.2.15" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="8.2.1" />
<PackageReference Include="Fluid.Core" Version="2.8.0" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Nanoid" Version="3.0.0" />
<PackageReference Include="RedLock.net" Version="2.3.2" />
</ItemGroup>

7
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs
View file

@ -151,6 +151,13 @@ public partial class ConversationService
Message = new TextMessage(response.SecondaryContent ?? response.Content)
};
response.RichContent = new RichContent<IRichMessage>
{
Recipient = new Recipient { Id = state.GetConversationId() },
Editor = "file",
Message = new TextMessage(response.SecondaryContent ?? response.Content)
};
// Patch return function name
if (response.PostbackFunctionName != null)
{

114
src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs
View file

@ -1,3 +1,4 @@
using Microsoft.AspNetCore.StaticFiles;
using System.IO;
using System.Threading;
@ -8,9 +9,12 @@ public class BotSharpFileService : IBotSharpFileService
private readonly BotSharpDatabaseSettings _dbSettings;
private readonly IServiceProvider _services;
private readonly string _baseDir;
private readonly IEnumerable<string> _allowedTypes = new List<string> { "image/png", "image/jpeg" };
private const string CONVERSATION_FOLDER = "conversations";
private const string FILE_FOLDER = "files";
private const int MIN_OFFSET = 1;
private const int MAX_OFFSET = 5;
public BotSharpFileService(
BotSharpDatabaseSettings dbSettings,
@ -31,29 +35,67 @@ public class BotSharpFileService : IBotSharpFileService
return dir;
}
public IEnumerable<OutputFileModel> GetConversationFiles(string conversationId, string messageId)
public IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int offset = 2)
{
var outputFiles = new List<OutputFileModel>();
var dir = GetConversationFileDirectory(conversationId, messageId);
if (string.IsNullOrEmpty(dir))
var files = new List<MessageFileModel>();
if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty())
{
return outputFiles;
return files;
}
foreach (var file in Directory.GetFiles(dir))
if (offset <= 0)
{
var fileName = Path.GetFileNameWithoutExtension(file);
var extension = Path.GetExtension(file);
var fileType = extension.Substring(1);
var model = new OutputFileModel()
{
FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}",
FileName = fileName,
FileType = fileType
};
outputFiles.Add(model);
offset = MIN_OFFSET;
}
return outputFiles;
else if (offset > MAX_OFFSET)
{
offset = MAX_OFFSET;
}
var messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset).ToList();
files = GetMessageFiles(conversationId, messageIds, imageOnly: true).ToList();
return files;
}
public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, bool imageOnly = false)
{
var files = new List<MessageFileModel>();
if (messageIds.IsNullOrEmpty()) return files;
foreach (var messageId in messageIds)
{
var dir = GetConversationFileDirectory(conversationId, messageId);
if (string.IsNullOrEmpty(dir))
{
continue;
}
foreach (var file in Directory.GetFiles(dir))
{
var contentType = GetFileContentType(file);
if (imageOnly && !_allowedTypes.Contains(contentType))
{
continue;
}
var fileName = Path.GetFileNameWithoutExtension(file);
var extension = Path.GetExtension(file);
var fileType = extension.Substring(1);
var model = new MessageFileModel()
{
MessageId = messageId,
FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}",
FileStorageUrl = file,
FileName = fileName,
FileType = fileType,
ContentType = contentType
};
files.Add(model);
}
}
return files;
}
public string? GetMessageFile(string conversationId, string messageId, string fileName)
@ -182,42 +224,16 @@ public class BotSharpFileService : IBotSharpFileService
return Convert.FromBase64String(base64Str);
}
private string GetFileType(string data)
private string GetFileContentType(string filePath)
{
if (string.IsNullOrEmpty(data))
string contentType;
var provider = new FileExtensionContentTypeProvider();
if (!provider.TryGetContentType(filePath, out contentType))
{
return string.Empty;
contentType = string.Empty;
}
var startIdx = data.IndexOf(':');
var endIdx = data.IndexOf(';');
var fileType = data.Substring(startIdx + 1, endIdx - startIdx - 1);
return fileType;
}
private string ParseFileFormat(string type)
{
var parsed = string.Empty;
switch (type)
{
case "image/png":
parsed = ".png";
break;
case "image/jpeg":
case "image/jpg":
parsed = ".jpeg";
break;
case "application/pdf":
parsed = ".pdf";
break;
case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
parsed = ".xlsx";
break;
case "text/plain":
parsed = ".txt";
break;
}
return parsed;
return contentType;
}
#endregion
}

8
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs
View file

@ -17,18 +17,18 @@ public partial class RoutingService
return false;
}
var provide = agent.LlmConfig.Provider;
var provider = agent.LlmConfig.Provider;
var model = agent.LlmConfig.Model;
if (provide == null || model == null)
if (provider == null || model == null)
{
var agentSettings = _services.GetRequiredService<AgentSettings>();
provide = agentSettings.LlmConfig.Provider;
provider = agentSettings.LlmConfig.Provider;
model = agentSettings.LlmConfig.Model;
}
var chatCompletion = CompletionProvider.GetChatCompletion(_services,
provider: provide,
provider: provider,
model: model);
var message = dialogs.Last();

5
src/Infrastructure/BotSharp.OpenAPI/Controllers/FileController.cs
View file

@ -38,10 +38,11 @@ public class FileController : ControllerBase
}
[HttpGet("/conversation/{conversationId}/files/{messageId}")]
public IEnumerable<OutputFileModel> GetConversationFiles([FromRoute] string conversationId, [FromRoute] string messageId)
public IEnumerable<MessageFileViewModel> GetMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId)
{
var fileService = _services.GetRequiredService<IBotSharpFileService>();
return fileService.GetConversationFiles(conversationId, messageId);
var files = fileService.GetMessageFiles(conversationId, new List<string> { messageId });
return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List<MessageFileViewModel>();
}
[HttpGet("/conversation/{conversationId}/message/{messageId}/file/{fileName}")]

3
src/Infrastructure/BotSharp.OpenAPI/Using.cs
View file

@ -28,4 +28,5 @@ global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files;
global using BotSharp.OpenAPI.ViewModels.Conversations;
global using BotSharp.OpenAPI.ViewModels.Users;
global using BotSharp.OpenAPI.ViewModels.Agents;
global using BotSharp.OpenAPI.ViewModels.Agents;
global using BotSharp.OpenAPI.ViewModels.Files;

34
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs Normal file
View file

@ -0,0 +1,34 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Files;
public class MessageFileViewModel
{
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
[JsonPropertyName("file_type")]
public string FileType { get; set; }
[JsonPropertyName("content_type")]
public string ContentType { get; set; }
public MessageFileViewModel()
{
}
public static MessageFileViewModel Transform(MessageFileModel model)
{
return new MessageFileViewModel
{
FileUrl = model.FileUrl,
FileName = model.FileName,
FileType = model.FileType,
ContentType = model.ContentType
};
}
}

47
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
View file

@ -4,13 +4,17 @@ using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Utilities;
using BotSharp.Plugin.AzureOpenAI.Settings;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
@ -218,6 +222,16 @@ public class ChatCompletionProvider : IChatCompletion
protected (string, ChatCompletionsOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
{
var agentService = _services.GetRequiredService<IAgentService>();
var fileService = _services.GetRequiredService<IBotSharpFileService>();
var state = _services.GetRequiredService<IConversationStateService>();
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(Provider, _model);
var chatFiles = new List<MessageFileModel>();
if (settings != null && settings.MultiModal)
{
chatFiles = fileService.GetChatImages(state.GetConversationId(), conversations, offset: 2).ToList();
}
var chatCompletionsOptions = new ChatCompletionsOptions();
@ -279,19 +293,34 @@ public class ChatCompletionProvider : IChatCompletion
else if (message.Role == ChatRole.User)
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
var userMessage = new ChatRequestUserMessage(text)
var chatItems = new List<ChatMessageContentItem>()
{
new ChatMessageTextContentItem(text)
};
var files = chatFiles.Where(x => x.MessageId == message.MessageId).ToList();
if (!files.IsNullOrEmpty())
{
foreach (var file in files)
{
using var stream = File.OpenRead(file.FileStorageUrl);
chatItems.Add(new ChatMessageImageContentItem(stream, file.ContentType, ChatMessageImageDetailLevel.Low));
}
}
//if (!string.IsNullOrEmpty(message.ImageUrl))
//{
// var uri = new Uri(message.ImageUrl);
// userMessage.MultimodalContentItems.Add(
// new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
//}
var userMessage = new ChatRequestUserMessage(chatItems)
{
// To display Planner name in log
Name = message.FunctionName,
};
if (!string.IsNullOrEmpty(message.ImageUrl))
{
var uri = new Uri(message.ImageUrl);
userMessage.MultimodalContentItems.Add(
new ChatMessageImageContentItem(uri, ChatMessageImageDetailLevel.Low));
}
chatCompletionsOptions.Messages.Add(userMessage);
}
else if (message.Role == ChatRole.Assistant)
@ -301,7 +330,7 @@ public class ChatCompletionProvider : IChatCompletion
}
// https://community.openai.com/t/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api-a-few-tips-and-tricks-on-controlling-the-creativity-deterministic-output-of-prompt-responses/172683
var state = _services.GetRequiredService<IConversationStateService>();
//var state = _services.GetRequiredService<IConversationStateService>();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
var samplingFactor = float.Parse(state.GetState("sampling_factor", "0.0"));
chatCompletionsOptions.Temperature = temperature;