use role content

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs

@@ -140,6 +140,25 @@ public class RoleDialogModel : ITrackableMessage
     [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
     public bool IsFromAssistant => Role == AgentRole.Assistant || Role == AgentRole.Model;
 
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    public string RoleContent
+    {
+        get
+        {
+            var text = Content;
+            if (Role == AgentRole.User)
+            {
+                text = !string.IsNullOrWhiteSpace(Payload) ? Payload : Content;
+            }
+            else
+            {
+                text = !string.IsNullOrWhiteSpace(RichContent?.Message?.Text) ? RichContent.Message.Text : Content;
+            }
+
+            return text;
+        }
+    }
+
     public RoleDialogModel()
     {
     }

src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs

@@ -136,17 +136,17 @@ public class ChatCompletionProvider : IChatCompletion
             filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
         }
 
-        foreach (var conv in filteredMessages)
+        foreach (var message in filteredMessages)
         {
-            if (conv.Role == AgentRole.User)
+            if (message.Role == AgentRole.User)
             {
-                messages.Add(new Message(RoleType.User, conv.Payload ?? conv.Content));
+                messages.Add(new Message(RoleType.User, message.RoleContent));
             }
-            else if (conv.Role == AgentRole.Assistant)
+            else if (message.Role == AgentRole.Assistant)
             {
-                messages.Add(new Message(RoleType.Assistant, conv.Content));
+                messages.Add(new Message(RoleType.Assistant, message.RoleContent));
             }
-            else if (conv.Role == AgentRole.Function)
+            else if (message.Role == AgentRole.Function)
             {
                 messages.Add(new Message
                 {
@@ -155,9 +155,9 @@ public class ChatCompletionProvider : IChatCompletion
                     {
                         new ToolUseContent()
                         {
-                            Id = conv.ToolCallId,
+                            Id = message.ToolCallId,
-                            Name = conv.FunctionName,
+                            Name = message.FunctionName,
-                            Input = JsonNode.Parse(conv.FunctionArgs ?? "{}")
+                            Input = JsonNode.Parse(message.FunctionArgs ?? "{}")
                         }
                     }
                 });
@@ -169,8 +169,8 @@ public class ChatCompletionProvider : IChatCompletion
                     {
                         new ToolResultContent()
                         {
-                            ToolUseId = conv.ToolCallId,
+                            ToolUseId = message.ToolCallId,
-                            Content = [new TextContent() { Text = conv.Content }]
+                            Content = [new TextContent() { Text = message.RoleContent }]
                         }
                     }
                 });

src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs

@@ -432,11 +432,11 @@ public class ChatCompletionProvider : IChatCompletion
                     ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
                 }));
 
-                messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content));
+                messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent));
             }
             else if (message.Role == AgentRole.User)
             {
-                var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
+                var text = message.RoleContent;
                 var textPart = ChatMessageContentPart.CreateTextPart(text);
                 var contentParts = new List<ChatMessageContentPart> { textPart };
 
@@ -448,7 +448,7 @@ public class ChatCompletionProvider : IChatCompletion
             }
             else if (message.Role == AgentRole.Assistant)
             {
-                var text = message.Content;
+                var text = message.RoleContent;
                 var textPart = ChatMessageContentPart.CreateTextPart(text);
                 var contentParts = new List<ChatMessageContentPart> { textPart };
 

src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs

@@ -399,11 +399,11 @@ public class ChatCompletionProvider : IChatCompletion
                     ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
                 }));
 
-                messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content));
+                messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent));
             }
             else if (message.Role == AgentRole.User)
             {
-                var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
+                var text = message.RoleContent;
                 var textPart = ChatMessageContentPart.CreateTextPart(text);
                 var contentParts = new List<ChatMessageContentPart> { textPart };
 
@@ -415,7 +415,7 @@ public class ChatCompletionProvider : IChatCompletion
             }
             else if (message.Role == AgentRole.Assistant)
             {
-                var text = message.Content;
+                var text = message.RoleContent;
                 var textPart = ChatMessageContentPart.CreateTextPart(text);
                 var contentParts = new List<ChatMessageContentPart> { textPart };
 

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs

@@ -263,17 +263,17 @@ public class GeminiChatCompletionProvider : IChatCompletion
                             Name = message.FunctionName,
                             Response = new JsonObject()
                             {
-                                ["result"] = message.Content ?? string.Empty
+                                ["result"] = message.RoleContent ?? string.Empty
                             }
                         }
                     }
                 ], AgentRole.Function));
 
-                convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
+                convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.RoleContent}");
             }
             else if (message.Role == AgentRole.User)
             {
-                var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
+                var text = message.RoleContent;
                 var contentParts = new List<Part> { new() { Text = text } };
 
                 if (allowMultiModal && !message.Files.IsNullOrEmpty())
@@ -285,7 +285,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
             }
             else if (message.Role == AgentRole.Assistant)
             {
-                var text = message.Content;
+                var text = message.RoleContent;
                 var contentParts = new List<Part> { new() { Text = text } };
 
                 if (allowMultiModal && !message.Files.IsNullOrEmpty())
@@ -294,7 +294,7 @@ public class GeminiChatCompletionProvider : IChatCompletion
                 }
 
                 contents.Add(new Content(contentParts, AgentRole.Model));
-                convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
+                convPrompts.Add($"{AgentRole.Assistant}: {text}");
             }
         }
 

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs

@@ -108,7 +108,7 @@ public class PalmChatCompletionProvider : IChatCompletion
             renderedInstructions.Add(prompt);
         }
 
-        var messages = conversations.Select(c => new PalmChatMessage(c.Content, c.Role == AgentRole.User ? "user" : "AI"))
+        var messages = conversations.Select(c => new PalmChatMessage(c.RoleContent, c.Role == AgentRole.User ? "user" : "AI"))
             .ToList();
 
         if (!functions.IsNullOrEmpty())
@@ -124,8 +124,8 @@ public class PalmChatCompletionProvider : IChatCompletion
             foreach (var dialog in conversations)
             {
                 prompt += dialog.Role == AgentRole.Function ?
-                    $"{dialog.Role}: {dialog.FunctionName} => {dialog.Content}\r\n" :
+                    $"{dialog.Role}: {dialog.FunctionName} => {dialog.RoleContent}\r\n" :
-                    $"{dialog.Role}: {dialog.Content}\r\n";
+                    $"{dialog.Role}: {dialog.RoleContent}\r\n";
             }
 
             prompt += "\r\n\r\n" + router.Templates.FirstOrDefault(x => x.Name == "response_with_function").Content;

src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -482,7 +482,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
     }
 
 
-    private (string, GenerateContentRequest) PrepareOptions(Agent agent,
+    private (string, GenerateContentRequest) PrepareOptions(
+        Agent agent,
         List<RoleDialogModel> conversations)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
@@ -564,25 +565,25 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
                             Name = message.FunctionName ?? string.Empty,
                             Response = new JsonObject()
                             {
-                                ["result"] = message.Content ?? string.Empty
+                                ["result"] = message.RoleContent ?? string.Empty
                             }
                         }
                     }
                 ], AgentRole.Function));
 
                 convPrompts.Add(
-                    $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.Content}");
+                    $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.RoleContent}");
             }
             else if (message.Role == AgentRole.User)
             {
-                var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
+                var text = message.RoleContent;
                 contents.Add(new Content(text, AgentRole.User));
                 convPrompts.Add($"{AgentRole.User}: {text}");
             }
             else if (message.Role == AgentRole.Assistant)
             {
-                contents.Add(new Content(message.Content, AgentRole.Model));
+                contents.Add(new Content(message.RoleContent, AgentRole.Model));
-                convPrompts.Add($"{AgentRole.Assistant}: {message.Content}");
+                convPrompts.Add($"{AgentRole.Assistant}: {message.RoleContent}");
             }
         }
 

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs

@@ -42,7 +42,7 @@ public class ChatCompletionProvider : IChatCompletion
             await hook.BeforeGenerating(agent, conversations);
         }
 
-        var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
+        var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.RoleContent}")).Trim();
         content += $"\r\n{AgentRole.Assistant}: ";
 
         var llama = _services.GetRequiredService<LlamaAiModel>();
@@ -118,7 +118,7 @@ public class ChatCompletionProvider : IChatCompletion
         Func<RoleDialogModel, Task> onMessageReceived,
         Func<RoleDialogModel, Task> onFunctionExecuting)
     {
-        var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
+        var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.RoleContent}")).Trim();
         content += $"\r\n{AgentRole.Assistant}: ";
 
         var state = _services.GetRequiredService<IConversationStateService>();

src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs

@@ -41,7 +41,7 @@ namespace BotSharp.Plugin.VertexAI.Providers
             var client = new VertexAIProvider(_config);
             var model = new VertexAIChatModel(client, _model);
             var messages = conversations
-                .Select(c => new Message(c.Content, c.Role == AgentRole.User ? MessageRole.Human : MessageRole.Ai)).ToList();
+                .Select(c => new Message(c.RoleContent, c.Role == AgentRole.User ? MessageRole.Human : MessageRole.Ai)).ToList();
 
             var response = await model.GenerateAsync(new ChatRequest { Messages = messages }, _settings);
 

src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs

@@ -111,8 +111,8 @@ public class ChatCompletionProvider : IChatCompletion
         foreach (var message in samples)
         {
             messages.Add(message.Role == AgentRole.User ?
-                new MessageItem("user", message.Content) :
+                new MessageItem("user", message.RoleContent) :
-                new MessageItem("assistant", message.Content));
+                new MessageItem("assistant", message.RoleContent));
         }
 
         foreach (var function in functions)
@@ -129,13 +129,13 @@ public class ChatCompletionProvider : IChatCompletion
             }
             else if (message.Role == "user")
             {
-                var userMessage = new MessageItem("user",message.Content);
+                var userMessage = new MessageItem("user",message.RoleContent);
 
                 messages.Add(userMessage);
             }
             else if (message.Role == "assistant")
             {
-                messages.Add(new MessageItem("assistant", message.Content));
+                messages.Add(new MessageItem("assistant", message.RoleContent));
             }
         }
 

src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs

@@ -109,16 +109,16 @@ public sealed class MicrosoftExtensionsAIChatCompletionProvider : IChatCompletio
                 messages.Add(new(ChatRole.Assistant,
                 [
                     new FunctionCallContent(x.FunctionName, x.FunctionName, JsonSerializer.Deserialize<Dictionary<string, object?>>(x.FunctionArgs ?? "{}")),
-                    new FunctionResultContent(x.FunctionName, x.Content)
+                    new FunctionResultContent(x.FunctionName, x.RoleContent)
                 ]));
             }
             else if (x.Role == AgentRole.System || x.Role == AgentRole.Assistant)
             {
-                messages.Add(new(x.Role == AgentRole.System ? ChatRole.System : ChatRole.Assistant, x.Content));
+                messages.Add(new(x.Role == AgentRole.System ? ChatRole.System : ChatRole.Assistant, x.RoleContent));
             }
             else if (x.Role == AgentRole.User)
             {
-                List<AIContent> contents = [new TextContent(!string.IsNullOrWhiteSpace(x.Payload) ? x.Payload : x.Content)];
+                List<AIContent> contents = [new TextContent(x.RoleContent)];
                 if (allowMultiModal)
                 {
                     foreach (var file in x.Files)

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs

@@ -402,11 +402,11 @@ public class ChatCompletionProvider : IChatCompletion
                     ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
                 }));
 
-                messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content));
+                messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent));
             }
             else if (message.Role == AgentRole.User)
             {
-                var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
+                var text = message.RoleContent;
                 var textPart = ChatMessageContentPart.CreateTextPart(text);
                 var contentParts = new List<ChatMessageContentPart> { textPart };
 
@@ -418,7 +418,7 @@ public class ChatCompletionProvider : IChatCompletion
             }
             else if (message.Role == AgentRole.Assistant)
             {
-                var text = message.Content;
+                var text = message.RoleContent;
                 var textPart = ChatMessageContentPart.CreateTextPart(text);
                 var contentParts = new List<ChatMessageContentPart> { textPart };
 

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -555,10 +555,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var state = _services.GetRequiredService<IConversationStateService>();
-        var fileStorage = _services.GetRequiredService<IFileStorageService>();
         var settingsService = _services.GetRequiredService<ILlmProviderService>();
         var settings = settingsService.GetSetting(Provider, _model);
-        var allowMultiModal = settings != null && settings.MultiModal;
 
         var messages = new List<ChatMessage>();
 
@@ -623,44 +621,15 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
                     ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
                 }));
 
-                messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.Content));
+                messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent));
             }
             else if (message.Role == AgentRole.User)
             {
-                var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
+                messages.Add(new UserChatMessage(message.RoleContent));
-                var textPart = ChatMessageContentPart.CreateTextPart(text);
-                var contentParts = new List<ChatMessageContentPart> { textPart };
-
-                if (allowMultiModal && !message.Files.IsNullOrEmpty())
-                {
-                    foreach (var file in message.Files)
-                    {
-                        if (!string.IsNullOrEmpty(file.FileData))
-                        {
-                            var (contentType, binary) = FileUtility.GetFileInfoFromData(file.FileData);
-                            var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto);
-                            contentParts.Add(contentPart);
-                        }
-                        else if (!string.IsNullOrEmpty(file.FileStorageUrl))
-                        {
-                            var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
-                            var binary = fileStorage.GetFileBytes(file.FileStorageUrl);
-                            var contentPart = ChatMessageContentPart.CreateImagePart(binary, contentType, ChatImageDetailLevel.Auto);
-                            contentParts.Add(contentPart);
-                        }
-                        else if (!string.IsNullOrEmpty(file.FileUrl))
-                        {
-                            var uri = new Uri(file.FileUrl);
-                            var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto);
-                            contentParts.Add(contentPart);
-                        }
-                    }
-                }
-                messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
             }
             else if (message.Role == AgentRole.Assistant)
             {
-                messages.Add(new AssistantChatMessage(message.Content));
+                messages.Add(new AssistantChatMessage(message.RoleContent));
             }
         }
 

src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs

@@ -99,7 +99,7 @@ public class PyProgrammerFn : IFunctionCallback
                         Language = "python"
                     }
                 };
-                message.StopCompletion = true;
+                //message.StopCompletion = true;
 
                 // Restore the original stdout/stderr
                 sys.stdout = sys.__stdout__;

src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs

@@ -269,13 +269,13 @@ public class ChatCompletionProvider : IChatCompletion
             }
             else if (message.Role == "user")
             {
-                var userMessage = ChatMessage.FromUser(message.Content);
+                var userMessage = ChatMessage.FromUser(message.RoleContent);
 
                 messages.Add(userMessage);
             }
             else if (message.Role == "assistant")
             {
-                messages.Add(ChatMessage.FromAssistant(message.Content));
+                messages.Add(ChatMessage.FromAssistant(message.RoleContent));
             }
         }