1. refine chart handler instruction

2. add mechanism to allow sending multiple messages in a single round
This commit is contained in:
Jicheng Lu 2025-09-05 01:17:18 -05:00
parent 27ada7c5a8
commit 8778a5f8ed
11 changed files with 181 additions and 126 deletions

View file

@ -42,6 +42,9 @@ public class ChatResponseDto : InstructResult
[JsonPropertyName("is_streaming")] [JsonPropertyName("is_streaming")]
public bool IsStreaming { get; set; } public bool IsStreaming { get; set; }
[JsonPropertyName("is_append")]
public bool IsAppend { get; set; }
[JsonPropertyName("created_at")] [JsonPropertyName("created_at")]
public DateTime CreatedAt { get; set; } = DateTime.UtcNow; public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
} }

View file

@ -127,6 +127,13 @@ public class RoleDialogModel : ITrackableMessage
[JsonIgnore(Condition = JsonIgnoreCondition.Always)] [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
public bool IsStreaming { get; set; } public bool IsStreaming { get; set; }
/// <summary>
/// Additional messages that can be sent sequentially and save to db
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.Always)]
public ChatMessageWrapper? AdditionalMessageWrapper { get; set; }
public RoleDialogModel() public RoleDialogModel()
{ {
} }
@ -171,7 +178,15 @@ public class RoleDialogModel : ITrackableMessage
Instruction = source.Instruction, Instruction = source.Instruction,
Data = source.Data, Data = source.Data,
IsStreaming = source.IsStreaming, IsStreaming = source.IsStreaming,
Annotations = source.Annotations Annotations = source.Annotations,
AdditionalMessageWrapper = source.AdditionalMessageWrapper
}; };
} }
} }
public class ChatMessageWrapper
{
public int IntervalMilliSeconds { get; set; } = 1000;
public bool SaveToDb { get; set; }
public List<RoleDialogModel>? Messages { get; set; }
}

View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Hooks;
using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging;
using BotSharp.Abstraction.Messaging.Models.RichContent; using BotSharp.Abstraction.Messaging.Models.RichContent;

View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging;
using BotSharp.Abstraction.Messaging.Models.RichContent; using BotSharp.Abstraction.Messaging.Models.RichContent;
using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Options;
@ -31,63 +32,21 @@ public class ConversationStorage : IConversationStorage
foreach ( var dialog in dialogs) foreach ( var dialog in dialogs)
{ {
if (dialog.Role == AgentRole.Function) var innerList = new List<RoleDialogModel> { dialog };
if (dialog.AdditionalMessageWrapper != null
&& dialog.AdditionalMessageWrapper.SaveToDb
&& dialog.AdditionalMessageWrapper.Messages?.Count > 0)
{ {
var meta = new DialogMetaData innerList.AddRange(dialog.AdditionalMessageWrapper.Messages);
{
Role = dialog.Role,
AgentId = dialog.CurrentAgentId,
MessageId = dialog.MessageId,
MessageType = dialog.MessageType,
FunctionName = dialog.FunctionName,
FunctionArgs = dialog.FunctionArgs,
ToolCallId = dialog.ToolCallId,
CreatedTime = dialog.CreatedAt
};
var content = dialog.Content.RemoveNewLine();
if (string.IsNullOrEmpty(content))
{
continue;
}
dialogElements.Add(new DialogElement
{
MetaData = meta,
Content = dialog.Content,
SecondaryContent = dialog.SecondaryContent,
Payload = dialog.Payload
});
} }
else
foreach (var item in innerList)
{ {
var meta = new DialogMetaData var element = BuildDialogElement(item);
if (element != null)
{ {
Role = dialog.Role, dialogElements.Add(element);
AgentId = dialog.CurrentAgentId,
MessageId = dialog.MessageId,
MessageType = dialog.MessageType,
SenderId = dialog.SenderId,
FunctionName = dialog.FunctionName,
CreatedTime = dialog.CreatedAt
};
var content = dialog.Content.RemoveNewLine();
if (string.IsNullOrEmpty(content))
{
continue;
} }
var richContent = dialog.RichContent != null ? JsonSerializer.Serialize(dialog.RichContent, _options.JsonSerializerOptions) : null;
var secondaryRichContent = dialog.SecondaryRichContent != null ? JsonSerializer.Serialize(dialog.SecondaryRichContent, _options.JsonSerializerOptions) : null;
dialogElements.Add(new DialogElement
{
MetaData = meta,
Content = dialog.Content,
SecondaryContent = dialog.SecondaryContent,
RichContent = richContent,
SecondaryRichContent = secondaryRichContent,
Payload = dialog.Payload
});
} }
} }
@ -148,4 +107,67 @@ public class ConversationStorage : IConversationStorage
return results; return results;
} }
private DialogElement? BuildDialogElement(RoleDialogModel dialog)
{
DialogElement? element = null;
if (dialog.Role == AgentRole.Function)
{
var meta = new DialogMetaData
{
Role = dialog.Role,
AgentId = dialog.CurrentAgentId,
MessageId = dialog.MessageId,
MessageType = dialog.MessageType,
FunctionName = dialog.FunctionName,
FunctionArgs = dialog.FunctionArgs,
ToolCallId = dialog.ToolCallId,
CreatedTime = dialog.CreatedAt
};
var content = dialog.Content.RemoveNewLine();
if (!string.IsNullOrEmpty(content))
{
element = new DialogElement
{
MetaData = meta,
Content = dialog.Content,
SecondaryContent = dialog.SecondaryContent,
Payload = dialog.Payload
};
}
}
else
{
var meta = new DialogMetaData
{
Role = dialog.Role,
AgentId = dialog.CurrentAgentId,
MessageId = dialog.MessageId,
MessageType = dialog.MessageType,
SenderId = dialog.SenderId,
FunctionName = dialog.FunctionName,
CreatedTime = dialog.CreatedAt
};
var content = dialog.Content.RemoveNewLine();
if (!string.IsNullOrEmpty(content))
{
var richContent = dialog.RichContent != null ? JsonSerializer.Serialize(dialog.RichContent, _options.JsonSerializerOptions) : null;
var secondaryRichContent = dialog.SecondaryRichContent != null ? JsonSerializer.Serialize(dialog.SecondaryRichContent, _options.JsonSerializerOptions) : null;
element = new DialogElement
{
MetaData = meta,
Content = dialog.Content,
SecondaryContent = dialog.SecondaryContent,
RichContent = richContent,
SecondaryRichContent = secondaryRichContent,
Payload = dialog.Payload
};
}
}
return element;
}
} }

View file

@ -65,6 +65,7 @@ public partial class RoutingService
message.StopCompletion = clonedMessage.StopCompletion; message.StopCompletion = clonedMessage.StopCompletion;
message.RichContent = clonedMessage.RichContent; message.RichContent = clonedMessage.RichContent;
message.Data = clonedMessage.Data; message.Data = clonedMessage.Data;
message.AdditionalMessageWrapper = clonedMessage.AdditionalMessageWrapper;
} }
catch (JsonException ex) catch (JsonException ex)
{ {

View file

@ -30,6 +30,5 @@
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" /> <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
<ProjectReference Include="..\BotSharp.Plugin.ChatHub\BotSharp.Plugin.ChatHub.csproj" />
</ItemGroup> </ItemGroup>
</Project> </Project>

View file

@ -1,9 +1,4 @@
using BotSharp.Abstraction.Conversations.Dtos;
using BotSharp.Abstraction.Conversations.Enums;
using BotSharp.Abstraction.Messaging.Models.RichContent.Template; using BotSharp.Abstraction.Messaging.Models.RichContent.Template;
using BotSharp.Abstraction.Users;
using BotSharp.Plugin.ChatHub.Helpers;
using System.Runtime.CompilerServices;
namespace BotSharp.Plugin.ChartHandler.Functions; namespace BotSharp.Plugin.ChartHandler.Functions;
@ -11,6 +6,7 @@ public class PlotChartFn : IFunctionCallback
{ {
private readonly IServiceProvider _services; private readonly IServiceProvider _services;
private readonly ILogger<PlotChartFn> _logger; private readonly ILogger<PlotChartFn> _logger;
private readonly ChartHandlerSettings _settings;
private readonly BotSharpOptions _options; private readonly BotSharpOptions _options;
public string Name => "util-chart-plot_chart"; public string Name => "util-chart-plot_chart";
@ -20,10 +16,12 @@ public class PlotChartFn : IFunctionCallback
public PlotChartFn( public PlotChartFn(
IServiceProvider services, IServiceProvider services,
ILogger<PlotChartFn> logger, ILogger<PlotChartFn> logger,
ChartHandlerSettings settings,
BotSharpOptions options) BotSharpOptions options)
{ {
_services = services; _services = services;
_logger = logger; _logger = logger;
_settings = settings;
_options = options; _options = options;
} }
@ -43,7 +41,7 @@ public class PlotChartFn : IFunctionCallback
Instruction = inst, Instruction = inst,
LlmConfig = new AgentLlmConfig LlmConfig = new AgentLlmConfig
{ {
MaxOutputTokens = 8192 MaxOutputTokens = _settings?.ChartPlot?.MaxOutputTokens ?? 8192
}, },
TemplateDict = new Dictionary<string, object> TemplateDict = new Dictionary<string, object>
{ {
@ -52,7 +50,8 @@ public class PlotChartFn : IFunctionCallback
} }
}; };
var response = await GetChatCompletion(innerAgent, [ var response = await GetChatCompletion(innerAgent,
[
new RoleDialogModel(AgentRole.User, "Please follow the instruction to generate the javascript code.") new RoleDialogModel(AgentRole.User, "Please follow the instruction to generate the javascript code.")
{ {
CurrentAgentId = message.CurrentAgentId, CurrentAgentId = message.CurrentAgentId,
@ -72,59 +71,32 @@ public class PlotChartFn : IFunctionCallback
} }
}; };
// Send report summary after 1.5 seconds if exists
if (!string.IsNullOrEmpty(obj?.ReportSummary)) if (!string.IsNullOrEmpty(obj?.ReportSummary))
{ {
_ = Task.Run(async () => message.AdditionalMessageWrapper = new()
{ {
var services = _services.CreateScope().ServiceProvider; IntervalMilliSeconds = 1500,
await Task.Delay(1500); SaveToDb = true,
await SendDelayedMessage(services, obj.ReportSummary, convService.ConversationId, agent.Id, agent.Name); Messages = new List<RoleDialogModel>
}); {
new()
{
Role = AgentRole.Assistant,
MessageId = message.MessageId,
CurrentAgentId = message.CurrentAgentId,
Content = obj.ReportSummary,
FunctionName = message.FunctionName,
FunctionArgs = message.FunctionArgs,
CreatedAt = DateTime.UtcNow
}
}
};
} }
message.StopCompletion = true; message.StopCompletion = true;
return true; return true;
} }
private async Task SendDelayedMessage(IServiceProvider services, string text, string conversationId, string agentId, string agentName)
{
try
{
var messageId = Guid.NewGuid().ToString();
var messageData = new ChatResponseDto
{
ConversationId = conversationId,
MessageId = messageId,
Text = text,
Sender = new() { FirstName = agentName, LastName = "", Role = AgentRole.Assistant }
};
var dialogModel = new RoleDialogModel(AgentRole.Assistant, text)
{
MessageId = messageId,
CurrentAgentId = agentId,
CreatedAt = DateTime.UtcNow
};
var storage = services.GetService<IConversationStorage>();
storage?.Append(conversationId, dialogModel);
await SendEvent(services, ChatEvent.OnMessageReceivedFromAssistant, conversationId, messageData);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to send delayed message");
}
}
private async Task SendEvent<T>(IServiceProvider services, string @event, string conversationId, T data, [CallerMemberName] string callerName = "")
{
var user = services.GetService<IUserIdentity>();
var json = JsonSerializer.Serialize(data, _options.JsonSerializerOptions);
await EventEmitter.SendChatEvent(services, _logger, @event, conversationId, user?.Id, json, nameof(PlotChartFn), callerName);
}
private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel> dialogs) private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel> dialogs)
{ {
try try
@ -169,12 +141,11 @@ public class PlotChartFn : IFunctionCallback
var model = "gpt-5"; var model = "gpt-5";
var state = _services.GetRequiredService<IConversationStateService>(); var state = _services.GetRequiredService<IConversationStateService>();
var settings = _services.GetRequiredService<ChartHandlerSettings>();
provider = state.GetState("chart_plot_llm_provider") provider = state.GetState("chart_plot_llm_provider")
.IfNullOrEmptyAs(settings.ChartPlot?.LlmProvider) .IfNullOrEmptyAs(_settings.ChartPlot?.LlmProvider)
.IfNullOrEmptyAs(provider); .IfNullOrEmptyAs(provider);
model = state.GetState("chart_plot_llm_model") model = state.GetState("chart_plot_llm_model")
.IfNullOrEmptyAs(settings.ChartPlot?.LlmModel) .IfNullOrEmptyAs(_settings.ChartPlot?.LlmModel)
.IfNullOrEmptyAs(model); .IfNullOrEmptyAs(model);
return (provider, model); return (provider, model);

View file

@ -7,6 +7,7 @@ public class ChartHandlerSettings
public class ChartPlotSetting public class ChartPlotSetting
{ {
public string LlmProvider { get; set; } public string? LlmProvider { get; set; }
public string LlmModel { get; set; } public string? LlmModel { get; set; }
public int? MaxOutputTokens { get; set; }
} }

View file

@ -1,29 +1,39 @@
Please take a look at "Plotting Requirement" and generate a javascript code that can be used to render the charts on an html element. Please take a look at "Plotting Requirement" and generate a javascript code that can be used to render the charts on an html element.
You must strictly follow the "Hard Requirements", "Render Requirements", "Code Requirements" and "Response Format" below.
=== Plotting Requirement === === Plotting Requirement ===
{{ plotting_requirement }} {{ plotting_requirement }}
***** Hard Requirements ***** ***** Hard Requirements *****
** Your output javascript code must be wrapped in one or multiple <script>...</script> blocks with everything needed inside. ** Your output javascript code must be wrapped in one or multiple <script>...</script> blocks with everything needed inside.
** You need to import ECharts.js to plot the charts. The script source is "https://cdnjs.cloudflare.com/ajax/libs/echarts/6.0.0/echarts.min.js". ** You need to import ECharts.js exactly once to plot the charts. The script source is "https://cdnjs.cloudflare.com/ajax/libs/echarts/5.5.1/echarts.min.js".
** You need to add the MODE bar for each chart you plot. ** You must add an ECharts Toolbox bar at the top left corner of the chart.
** You must render the charts under the div html element with id {{ chart_element_id }}. ** ALWAYS add a "Full screen" button right next to the Toolbox bar.
** Add a custom mode bar button named "Fullscreen" that toggles the chart container in and out of fullscreen using the Fullscreen API. Requirements for this button: ** The "Full screen" button can toggle the chart container in and out of fullscreen using the Fullscreen API. Requirements for this button:
* Always call the Fullscreen API on the chart container div itself (document.getElementById("{{ chart_element_id }}")), not on the document or on Plotlys SVG. * Always call the Fullscreen API on the chart container div itself (document.getElementById("{{ chart_element_id }}")), not on the document.
* Use el.requestFullscreen() with fallbacks to el.webkitRequestFullscreen || el.msRequestFullscreen. * Use el.requestFullscreen() with fallbacks to el.webkitRequestFullscreen || el.msRequestFullscreen.
* Exit fullscreen with document.exitFullscreen() and vendor fallbacks. * Exit fullscreen with document.exitFullscreen() and vendor fallbacks.
* Listen for fullscreenchange, webkitfullscreenchange, and msfullscreenchange to keep the button working across repeated clicks and ESC exits. * Listen for fullscreenchange, webkitfullscreenchange, and msfullscreenchange to keep the button working across repeated clicks and ESC exits.
* Ensure the chart fully expands and scales to the entire screen when fullscreen is active. * Ensure the chart fully expands and scales to the entire screen when fullscreen is active.
* Provide a simple inline SVG path icon for the button (no external assets). * fullscreenBtn must be a fully-formed object {show: true, name, title, icon: 'path://M3 3 H9 V5 H5 V9 H3 Z M15 3 H21 V9 H19 V5 H15 Z M3 15 H5 V19 H9 V21 H3 Z M19 15 H21 V21 H15 V19 H19 Z', onclick}.
* Use Plotly.newPlot(container, data, layout, {displayModeBar:true, modeBarButtonsToAdd:[fullscreenBtn]}); * When using "chart.setOption" to define the fullscreen button, DO NOT use "graphic". Include the fullscreenBtn object in toolbox.feature with name 'myFullscreen'.
* fullscreenBtn must be a fully-formed object {name, title, icon, click}.
***** Render Requirements *****
** You must render the charts under the div html element with id {{ chart_element_id }}.
** You must not create any new html element. ** You must not create any new html element.
** You must ensure the generated charts have visible height (at least 500px) and width (at least 800px). DO NOT generate charts with zero height or zero width.
** You must not apply any styles on any html element. ** You must not apply any styles on any html element.
** Keep code compact (few tokens), but fix all errors before returning.
** Do not generate charts with zero height and zero width.
***** Code Requirements *****
** You must strictly follow the valid javascript and ECharts.js syntax.
** You must ensure no syntax/runtime error before returning the response.
** Please ensure the code can be executed and the charts are rendered correctly. ** Please ensure the code can be executed and the charts are rendered correctly.
*** Response Format ***
***** Response Format *****
You must output the response in the following JSON format: You must output the response in the following JSON format:
{ {
"greeting_message": "A short polite message that informs user that the charts have been generated.", "greeting_message": "A short polite message that informs user that the charts have been generated.",

View file

@ -3,9 +3,9 @@ using System.Runtime.CompilerServices;
namespace BotSharp.Plugin.ChatHub.Helpers; namespace BotSharp.Plugin.ChatHub.Helpers;
public class EventEmitter internal class EventEmitter
{ {
public static async Task SendChatEvent<T>( internal static async Task SendChatEvent<T>(
IServiceProvider services, IServiceProvider services,
ILogger logger, ILogger logger,
string @event, string @event,

View file

@ -5,6 +5,7 @@ using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.SideCar; using BotSharp.Abstraction.SideCar;
using BotSharp.Abstraction.Users.Dtos; using BotSharp.Abstraction.Users.Dtos;
using Microsoft.AspNetCore.SignalR; using Microsoft.AspNetCore.SignalR;
using System;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
namespace BotSharp.Plugin.ChatHub.Hooks; namespace BotSharp.Plugin.ChatHub.Hooks;
@ -112,15 +113,48 @@ public class ChatHubConversationHook : ConversationHookBase
} }
}; };
// Send typing-off to client // Send type-off to client
var action = new ConversationSenderActionModel var action = new ConversationSenderActionModel
{ {
ConversationId = conv.ConversationId, ConversationId = conv.ConversationId,
SenderAction = SenderActionEnum.TypingOff SenderAction = SenderActionEnum.TypingOff
}; };
await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action); await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action);
await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data); await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data);
if (message.AdditionalMessageWrapper?.Messages?.Count > 0)
{
action.SenderAction = SenderActionEnum.TypingOn;
await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action);
foreach (var item in message.AdditionalMessageWrapper.Messages)
{
await Task.Delay(message.AdditionalMessageWrapper.IntervalMilliSeconds);
data = new ChatResponseDto
{
ConversationId = conv.ConversationId,
MessageId = item.MessageId,
Text = !string.IsNullOrEmpty(item.SecondaryContent) ? item.SecondaryContent : item.Content,
Function = item.FunctionName,
RichContent = item.SecondaryRichContent ?? item.RichContent,
Data = item.Data,
States = state.GetStates(),
IsAppend = true,
Sender = new()
{
FirstName = "AI",
LastName = "Assistant",
Role = AgentRole.Assistant
}
};
await SendEvent(ChatEvent.OnMessageReceivedFromAssistant, conv.ConversationId, data);
}
action.SenderAction = SenderActionEnum.TypingOff;
await SendEvent(ChatEvent.OnSenderActionGenerated, conv.ConversationId, action);
}
await base.OnResponseGenerated(message); await base.OnResponseGenerated(message);
} }