SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #432 from hchen2020/server-sent-events

Browse source 
Server-sent Events.
This commit is contained in:
C. Oceania 2024-04-30 16:29:36 -05:00 committed by GitHub
parent 0fa8e5cf7b 0ae2c4ba99
commit ed4f567e7d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 142 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs
View file

@ -28,6 +28,11 @@ public class RoleDialogModel : ITrackableMessage
public string? SecondaryContent { get; set; }
/// <summary>
/// Indicator message used to provide UI feedback for function execution
/// </summary>
public string? Indication { get; set; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string CurrentAgentId { get; set; }

6
src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs
View file

@ -3,5 +3,11 @@ namespace BotSharp.Abstraction.Functions;
public interface IFunctionCallback
{
string Name { get; }
/// <summary>
/// Indicator message used to provide UI feedback for function execution
/// </summary>
string Indication => string.Empty;
Task<bool> Execute(RoleDialogModel message);
}

2
src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHandler.cs
View file

@ -16,5 +16,5 @@ public interface IRoutingHandler
void SetDialogs(List<RoleDialogModel> dialogs);
Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message);
Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message, Func<RoleDialogModel, Task> onFunctionExecuting);
}

8
src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Routing.Models;
namespace BotSharp.Abstraction.Routing;
public interface IRoutingService
@ -30,9 +28,9 @@ public interface IRoutingService
List<RoutingHandlerDef> GetHandlers(Agent router);
void ResetRecursiveCounter();
Task<bool> InvokeAgent(string agentId, List<RoleDialogModel> dialogs);
Task<bool> InvokeFunction(string name, RoleDialogModel message);
Task<RoleDialogModel> InstructLoop(RoleDialogModel message, List<RoleDialogModel> dialogs);
Task<bool> InvokeAgent(string agentId, List<RoleDialogModel> dialogs, Func<RoleDialogModel, Task> onFunctionExecuting);
Task<bool> InvokeFunction(string name, RoleDialogModel messages, Func<RoleDialogModel, Task>? onFunctionExecuting = null);
Task<RoleDialogModel> InstructLoop(RoleDialogModel message, List<RoleDialogModel> dialogs, Func<RoleDialogModel, Task> onFunctionExecuting);
/// <summary>
/// Talk to a specific Agent directly, bypassing the Router

3
src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IExecutor.cs
View file

@ -7,5 +7,6 @@ public interface IExecutor
Task<RoleDialogModel> Execute(IRoutingService routing,
FunctionCallFromLlm inst,
RoleDialogModel message,
List<RoleDialogModel> dialogs);
List<RoleDialogModel> dialogs,
Func<RoleDialogModel, Task> onFunctionExecuting);
}

2
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs
View file

@ -77,7 +77,7 @@ public partial class ConversationService
var settings = _services.GetRequiredService<RoutingSettings>();
response = agent.Type == AgentType.Routing ?
await routing.InstructLoop(message, dialogs) :
await routing.InstructLoop(message, dialogs, onFunctionExecuting) :
await routing.InstructDirect(agent, message);
routing.ResetRecursiveCounter();

2
src/Infrastructure/BotSharp.Core/Routing/Functions/FallbackToRouterFn.cs
View file

@ -34,7 +34,7 @@ public class FallbackToRouterFn : IFunctionCallback
routing.Context.Replace(targetAgent.Id);
message.CurrentAgentId = targetAgent.Id;
var response = await routing.InstructLoop(message, dialogs);
var response = await routing.InstructLoop(message, dialogs, null);
message.Content = response.Content;
message.StopCompletion = true;

2
src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs
View file

@ -30,7 +30,7 @@ public class ResponseToUserRoutingHandler : RoutingHandlerBase, IRoutingHandler
{
}
public async Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message)
public async Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message, Func<RoleDialogModel, Task> onFunctionExecuting)
{
var response = new RoleDialogModel(AgentRole.Assistant, inst.Response)
{

4
src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs
View file

@ -34,7 +34,7 @@ public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase, IRoutingH
{
}
public async Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message)
public async Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message, Func<RoleDialogModel, Task> onFunctionExecuting)
{
var context = _services.GetRequiredService<IRoutingContext>();
var agentId = context.GetCurrentAgentId();
@ -47,7 +47,7 @@ public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase, IRoutingH
}
};
var ret = await routing.InvokeAgent(agentId, dialogs);
var ret = await routing.InvokeAgent(agentId, dialogs, onFunctionExecuting);
var response = dialogs.Last();
inst.Response = response.Content;

4
src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs
View file

@ -39,7 +39,7 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler
{
}
public async Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message)
public async Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst, RoleDialogModel message, Func<RoleDialogModel, Task> onFunctionExecuting)
{
var states = _services.GetRequiredService<IConversationStateService>();
var goalAgent = states.GetState(StateConst.EXPECTED_GOAL_AGENT);
@ -85,7 +85,7 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler
}
else
{
ret = await routing.InvokeAgent(agentId, _dialogs);
ret = await routing.InvokeAgent(agentId, _dialogs, onFunctionExecuting);
}
var response = _dialogs.Last();

7
src/Infrastructure/BotSharp.Core/Routing/Planning/InstructExecutor.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Routing;
using BotSharp.Abstraction.Routing.Planning;
namespace BotSharp.Core.Routing.Planning;
@ -18,7 +16,8 @@ public class InstructExecutor : IExecutor
public async Task<RoleDialogModel> Execute(IRoutingService routing,
FunctionCallFromLlm inst,
RoleDialogModel message,
List<RoleDialogModel> dialogs)
List<RoleDialogModel> dialogs,
Func<RoleDialogModel, Task> onFunctionExecuting)
{
message.Instruction = inst;
@ -26,7 +25,7 @@ public class InstructExecutor : IExecutor
var handler = handlers.FirstOrDefault(x => x.Name == inst.Function);
handler.SetDialogs(dialogs);
var handled = await handler.Handle(routing, inst, message);
var handled = await handler.Handle(routing, inst, message, onFunctionExecuting);
// For client display purpose
var response = dialogs.Last();

11
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs
View file

@ -5,7 +5,7 @@ namespace BotSharp.Core.Routing;
public partial class RoutingService
{
private int _currentRecursionDepth = 0;
public async Task<bool> InvokeAgent(string agentId, List<RoleDialogModel> dialogs)
public async Task<bool> InvokeAgent(string agentId, List<RoleDialogModel> dialogs, Func<RoleDialogModel, Task> onFunctionExecuting)
{
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(agentId);
@ -34,7 +34,8 @@ public partial class RoutingService
message.FunctionName = response.FunctionName;
message.FunctionArgs = response.FunctionArgs;
message.CurrentAgentId = agent.Id;
await InvokeFunction(message, dialogs);
await InvokeFunction(message, dialogs, onFunctionExecuting);
}
else
{
@ -54,7 +55,7 @@ public partial class RoutingService
return true;
}
private async Task<bool> InvokeFunction(RoleDialogModel message, List<RoleDialogModel> dialogs)
private async Task<bool> InvokeFunction(RoleDialogModel message, List<RoleDialogModel> dialogs, Func<RoleDialogModel, Task>? onFunctionExecuting = null)
{
// execute function
// Save states
@ -63,7 +64,7 @@ public partial class RoutingService
var routing = _services.GetRequiredService<IRoutingService>();
// Call functions
await routing.InvokeFunction(message.FunctionName, message);
await routing.InvokeFunction(message.FunctionName, message, onFunctionExecuting);
// Pass execution result to LLM to get response
if (!message.StopCompletion)
@ -88,7 +89,7 @@ public partial class RoutingService
// Send to Next LLM
var agentId = routing.Context.GetCurrentAgentId();
await InvokeAgent(agentId, dialogs);
await InvokeAgent(agentId, dialogs, onFunctionExecuting);
}
}
else

8
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Core.Routing;
public partial class RoutingService
{
public async Task<bool> InvokeFunction(string name, RoleDialogModel message)
public async Task<bool> InvokeFunction(string name, RoleDialogModel message, Func<RoleDialogModel, Task>? onFunctionExecuting = null)
{
var function = _services.GetServices<IFunctionCallback>().FirstOrDefault(x => x.Name == name);
if (function == null)
@ -24,6 +24,12 @@ public partial class RoutingService
.ToList();
// Before executing functions
clonedMessage.Indication = function.Indication;
if (onFunctionExecuting != null)
{
await onFunctionExecuting(clonedMessage);
}
foreach (var hook in hooks)
{
await hook.OnFunctionExecuting(clonedMessage);

19
src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
View file

@ -1,19 +1,8 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.Routing.Planning;
using BotSharp.Abstraction.Routing.Settings;
using BotSharp.Abstraction.Templating;
using BotSharp.Core.Routing.Planning;
using Fluid.Ast;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System.Diagnostics.Metrics;
using System.Drawing;
using System.Runtime.InteropServices;
using System.Security.Cryptography.X509Certificates;
using ThirdParty.Json.LitJson;
using static System.Net.Mime.MediaTypeNames;
namespace BotSharp.Core.Routing;
@ -67,7 +56,7 @@ public partial class RoutingService : IRoutingService
ExecutingDirectly = true
};
var result = await handler.Handle(this, inst, message);
var result = await handler.Handle(this, inst, message, null);
var response = dialogs.Last();
response.MessageId = message.MessageId;
@ -76,7 +65,7 @@ public partial class RoutingService : IRoutingService
return response;
}
public async Task<RoleDialogModel> InstructLoop(RoleDialogModel message, List<RoleDialogModel> dialogs)
public async Task<RoleDialogModel> InstructLoop(RoleDialogModel message, List<RoleDialogModel> dialogs, Func<RoleDialogModel, Task> onFunctionExecuting)
{
RoleDialogModel response = default;
@ -134,12 +123,12 @@ public partial class RoutingService : IRoutingService
if (inst.HandleDialogsByPlanner)
{
var dialogWithoutContext = planner.BeforeHandleContext(inst, message, dialogs);
response = await executor.Execute(this, inst, message, dialogWithoutContext);
response = await executor.Execute(this, inst, message, dialogWithoutContext, onFunctionExecuting);
planner.AfterHandleContext(dialogs, dialogWithoutContext);
}
else
{
response = await executor.Execute(this, inst, message, dialogs);
response = await executor.Execute(this, inst, message, dialogs, onFunctionExecuting);
}
await planner.AgentExecuted(_router, inst, response, dialogs);

103
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Routing;
using Newtonsoft.Json.Serialization;
using Newtonsoft.Json;
namespace BotSharp.OpenAPI.Controllers;
@ -149,6 +151,15 @@ public class ConversationController : ControllerBase
return response;
}
private void SetStates(IConversationService conv, NewMessageModel input)
{
conv.States.SetState("channel", input.Channel, source: StateSource.External)
.SetState("provider", input.Provider, source: StateSource.External)
.SetState("model", input.Model, source: StateSource.External)
.SetState("temperature", input.Temperature, source: StateSource.External)
.SetState("sampling_factor", input.SamplingFactor, source: StateSource.External);
}
[HttpPost("/conversation/{agentId}/{conversationId}")]
public async Task<ChatResponseModel> SendMessage([FromRoute] string agentId,
[FromRoute] string conversationId,
@ -165,11 +176,7 @@ public class ConversationController : ControllerBase
routing.Context.SetMessageId(conversationId, inputMsg.MessageId);
conv.SetConversationId(conversationId, input.States);
conv.States.SetState("channel", input.Channel, source: StateSource.External)
.SetState("provider", input.Provider, source: StateSource.External)
.SetState("model", input.Model, source: StateSource.External)
.SetState("temperature", input.Temperature, source: StateSource.External)
.SetState("sampling_factor", input.SamplingFactor, source: StateSource.External);
SetStates(conv, input);
var response = new ChatResponseModel();
@ -194,6 +201,92 @@ public class ConversationController : ControllerBase
return response;
}
[HttpPost("/conversation/{agentId}/{conversationId}/sse")]
public async Task SendMessageSse([FromRoute] string agentId,
[FromRoute] string conversationId,
[FromBody] NewMessageModel input)
{
var conv = _services.GetRequiredService<IConversationService>();
if (!string.IsNullOrEmpty(input.TruncateMessageId))
{
await conv.TruncateConversation(conversationId, input.TruncateMessageId);
}
var inputMsg = new RoleDialogModel(AgentRole.User, input.Text);
var routing = _services.GetRequiredService<IRoutingService>();
routing.Context.SetMessageId(conversationId, inputMsg.MessageId);
conv.SetConversationId(conversationId, input.States);
SetStates(conv, input);
var response = new ChatResponseModel();
Response.StatusCode = 200;
Response.Headers.Append(Microsoft.Net.Http.Headers.HeaderNames.ContentType, "text/event-stream");
Response.Headers.Append(Microsoft.Net.Http.Headers.HeaderNames.CacheControl, "no-cache");
Response.Headers.Append(Microsoft.Net.Http.Headers.HeaderNames.Connection, "keep-alive");
await conv.SendMessage(agentId, inputMsg,
replyMessage: input.Postback,
async msg =>
{
response.Text = !string.IsNullOrEmpty(msg.SecondaryContent) ? msg.SecondaryContent : msg.Content;
response.Function = msg.FunctionName;
response.RichContent = msg.SecondaryRichContent ?? msg.RichContent;
response.Instruction = msg.Instruction;
response.Data = msg.Data;
await OnChunkReceived(Response, msg);
},
async msg =>
{
var message = new RoleDialogModel(AgentRole.Function, msg.Content)
{
FunctionArgs = msg.FunctionArgs,
FunctionName = msg.FunctionName,
Indication = msg.Indication
};
await OnChunkReceived(Response, message);
},
async msg =>
{
});
var state = _services.GetRequiredService<IConversationStateService>();
response.States = state.GetStates();
response.MessageId = inputMsg.MessageId;
response.ConversationId = conversationId;
// await OnEventCompleted(Response);
}
private async Task OnChunkReceived(HttpResponse response, RoleDialogModel message)
{
var json = JsonConvert.SerializeObject(message, new JsonSerializerSettings
{
Formatting = Formatting.None,
ContractResolver = new CamelCasePropertyNamesContractResolver(),
NullValueHandling = NullValueHandling.Ignore,
});
var buffer = Encoding.UTF8.GetBytes($"data:{json}\n");
await response.Body.WriteAsync(buffer, 0, buffer.Length);
await Task.Delay(10);
buffer = Encoding.UTF8.GetBytes("\n");
await response.Body.WriteAsync(buffer, 0, buffer.Length);
}
private async Task OnEventCompleted(HttpResponse response)
{
var buffer = Encoding.UTF8.GetBytes("data:[DONE]\n");
await response.Body.WriteAsync(buffer, 0, buffer.Length);
buffer = Encoding.UTF8.GetBytes("\n");
await response.Body.WriteAsync(buffer, 0, buffer.Length);
}
[HttpPost("/conversation/{conversationId}/attachments")]
public IActionResult UploadAttachments([FromRoute] string conversationId,
IFormFile[] files)