SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add GetChatCompletions and clean routing code.

Browse source
This commit is contained in:
Haiping 2023-09-27 07:51:15 -05:00
parent 38b84f7c37
commit 3740a8cc7a
28 changed files with 569 additions and 473 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/Infrastructure/BotSharp.Abstraction/Agents/IAgentRouting.cs
View file

@ -1,10 +0,0 @@
using BotSharp.Abstraction.Routing.Models;
namespace BotSharp.Abstraction.Agents;
public interface IAgentRouting
{
string AgentId { get; }
Task<Agent> LoadRouter();
RoutingRule[] GetRulesByName(string name);
}

3
src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs
View file

@ -13,6 +13,9 @@ public interface IChatCompletion
/// <param name="model"></param>
void SetModelName(string model);
RoleDialogModel GetChatCompletions(Agent agent,
List<RoleDialogModel> conversations);
Task<bool> GetChatCompletionsAsync(Agent agent,
List<RoleDialogModel> conversations,
Func<RoleDialogModel, Task> onMessageReceived,

13
src/Infrastructure/BotSharp.Abstraction/Routing/IRouterInstance.cs Normal file
View file

@ -0,0 +1,13 @@
using BotSharp.Abstraction.Routing.Models;
namespace BotSharp.Abstraction.Routing;
public interface IRouterInstance
{
string AgentId { get; }
Agent Router { get; }
List<RoutingHandlerDef> GetHandlers();
IRouterInstance Load();
IRouterInstance WithDialogs(List<RoleDialogModel> dialogs);
RoutingRule[] GetRulesByName(string name);
}

8
src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHandler.cs
View file

@ -13,11 +13,7 @@ public interface IRoutingHandler
void SetRouter(Agent router) { }
void SetDialogs(List<RoleDialogModel> dialogs) { }
void SetDialogs(List<RoleDialogModel> dialogs) { }
Task<FunctionCallFromLlm> GetNextInstructionFromReasoner(string prompt)
=> throw new NotImplementedException("");
Task<RoleDialogModel> Handle(FunctionCallFromLlm inst)
=> throw new NotImplementedException("");
Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst);
}

5
src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs
View file

@ -1,9 +1,12 @@
using BotSharp.Abstraction.Functions.Models;
namespace BotSharp.Abstraction.Routing;
public interface IRoutingService
{
Agent LoadRouter();
List<RoleDialogModel> Dialogs { get; }
Task<FunctionCallFromLlm> GetNextInstruction(string prompt);
Task<RoleDialogModel> InvokeAgent(string agentId);
Task<RoleDialogModel> InstructLoop();
Task<RoleDialogModel> ExecuteOnce(Agent agent);
}

32
src/Infrastructure/BotSharp.Abstraction/Routing/RoutingHandlerBase.cs Normal file
View file

@ -0,0 +1,32 @@
using BotSharp.Abstraction.Routing.Settings;
using Microsoft.Extensions.Logging;
namespace BotSharp.Abstraction.Routing;
public abstract class RoutingHandlerBase
{
protected Agent _router;
protected readonly IServiceProvider _services;
protected readonly ILogger _logger;
protected RoutingSettings _settings;
protected List<RoleDialogModel> _dialogs;
public RoutingHandlerBase(IServiceProvider services,
ILogger logger,
RoutingSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public void SetRouter(Agent router)
{
_router = router;
}
public void SetDialogs(List<RoleDialogModel> dialogs)
{
_dialogs = dialogs;
}
}

4
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs
View file

@ -23,10 +23,10 @@ public partial class AgentService
public async Task<Agent> GetAgent(string id)
{
var settings = _services.GetRequiredService<RoutingSettings>();
var routingService = _services.GetRequiredService<IRoutingService>();
var routerInstance = _services.GetRequiredService<IRouterInstance>();
if (settings.RouterId == id)
{
return routingService.LoadRouter();
return routerInstance.Load().Router;
}
var profile = _db.GetAgent(id);

2
src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs
View file

@ -52,7 +52,7 @@ public static class BotSharpServiceCollectionExtensions
config.Bind("Router", routingSettings);
services.AddSingleton((IServiceProvider x) => routingSettings);
services.AddScoped<IAgentRouting, Router>();
services.AddScoped<IRouterInstance, RouterInstance>();
services.AddScoped<IRoutingService, RoutingService>();
if (myDatabaseSettings.Default == "FileRepository")

3
src/Infrastructure/BotSharp.Core/Routing/Handlers/ContinueExecuteTaskRoutingHandler.cs
View file

@ -26,9 +26,8 @@ public class ContinueExecuteTaskRoutingHandler : RoutingHandlerBase, IRoutingHan
{
}
public async Task<RoleDialogModel> Handle(FunctionCallFromLlm inst)
public async Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
{
var routing = _services.GetRequiredService<IAgentRouting>();
var db = _services.GetRequiredService<IBotSharpRepository>();
var record = db.Agents.First(x => x.Name.ToLower() == inst.AgentName.ToLower());

2
src/Infrastructure/BotSharp.Core/Routing/Handlers/ConversationEndRoutingHandler.cs
View file

@ -23,7 +23,7 @@ public class ConversationEndRoutingHandler : RoutingHandlerBase, IRoutingHandler
{
}
public Task<RoleDialogModel> Handle(FunctionCallFromLlm inst)
public Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
{
throw new NotImplementedException();
}

24
src/Infrastructure/BotSharp.Core/Routing/Handlers/GetNextInstructionRoutingHandler.cs
View file

@ -1,24 +0,0 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Routing;
using BotSharp.Abstraction.Routing.Settings;
namespace BotSharp.Core.Routing.Handlers;
public class GetNextInstructionRoutingHandler : RoutingHandlerBase, IRoutingHandler
{
public string Name => "get_next_instruction";
public string Description => "";
public bool IsReasoning => false;
public GetNextInstructionRoutingHandler(IServiceProvider services, ILogger<GetNextInstructionRoutingHandler> logger, RoutingSettings settings)
: base(services, logger, settings)
{
}
public async Task<RoleDialogModel> Handle(FunctionCallFromLlm inst)
{
throw new NotImplementedException();
}
}

2
src/Infrastructure/BotSharp.Core/Routing/Handlers/InterruptTaskExecutionRoutingHandler.cs
View file

@ -24,7 +24,7 @@ public class InterruptTaskExecutionRoutingHandler : RoutingHandlerBase, IRouting
{
}
public async Task<RoleDialogModel> Handle(FunctionCallFromLlm inst)
public async Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
{
var result = new RoleDialogModel(AgentRole.User, inst.Reason)
{

2
src/Infrastructure/BotSharp.Core/Routing/Handlers/ResponseToUserRoutingHandler.cs
View file

@ -24,7 +24,7 @@ public class ResponseToUserRoutingHandler : RoutingHandlerBase, IRoutingHandler
{
}
public async Task<RoleDialogModel> Handle(FunctionCallFromLlm inst)
public async Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
{
var result = new RoleDialogModel(AgentRole.Assistant, inst.Answer)
{

11
src/Infrastructure/BotSharp.Core/Routing/Handlers/RetrieveDataFromAgentRoutingHandler.cs
View file

@ -27,17 +27,12 @@ public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase, IRoutingH
{
}
public async Task<RoleDialogModel> Handle(FunctionCallFromLlm inst)
public async Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
{
if (string.IsNullOrEmpty(inst.AgentName))
{
inst = await GetNextInstructionFromReasoner($"What's the next step? your response must have agent name.");
}
// Retrieve information from specific agent
var db = _services.GetRequiredService<IBotSharpRepository>();
var record = db.Agents.First(x => x.Name.ToLower() == inst.AgentName.ToLower());
var response = await InvokeAgent(record.Id);
var response = await routing.InvokeAgent(record.Id);
inst.Answer = response.Content;
@ -60,7 +55,7 @@ public class RetrieveDataFromAgentRoutingHandler : RoutingHandlerBase, IRoutingH
_router.Instruction += $"\r\n{AgentRole.Function}: {response.Content}";
// Got the response from agent, then send to reasoner again to make the decision
inst = await GetNextInstructionFromReasoner($"What's the next step based on user's original goal and function result?");
// inst = await GetNextInstructionFromReasoner($"What's the next step based on user's original goal and function result?");
return null;
}

4
src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs
View file

@ -27,7 +27,7 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler
{
}
public async Task<RoleDialogModel> Handle(FunctionCallFromLlm inst)
public async Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
{
var function = _services.GetServices<IFunctionCallback>().FirstOrDefault(x => x.Name == inst.Function);
var message = new RoleDialogModel(AgentRole.Function, inst.Question)
@ -41,7 +41,7 @@ public class RouteToAgentRoutingHandler : RoutingHandlerBase, IRoutingHandler
var ret = await function.Execute(message);
var result = await InvokeAgent(message.CurrentAgentId);
var result = await routing.InvokeAgent(message.CurrentAgentId);
result.ExecutionData = result.ExecutionData ?? message.ExecutionData;
return result;

215
src/Infrastructure/BotSharp.Core/Routing/Handlers/RoutingHandlerBase.cs
View file

@ -1,215 +0,0 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.Routing.Settings;
using BotSharp.Abstraction.Templating;
using System.Drawing;
using System.Text.RegularExpressions;
namespace BotSharp.Core.Routing.Handlers;
public abstract class RoutingHandlerBase
{
protected Agent _router;
protected readonly IServiceProvider _services;
protected readonly ILogger _logger;
protected RoutingSettings _settings;
protected List<RoleDialogModel> _dialogs;
public virtual bool RequireAgent => true;
public RoutingHandlerBase(IServiceProvider services,
ILogger logger,
RoutingSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public void SetRouter(Agent router)
{
_router = router;
}
public void SetDialogs(List<RoleDialogModel> dialogs)
{
_dialogs = dialogs;
}
public async Task<FunctionCallFromLlm> GetNextInstructionFromReasoner(string prompt)
{
var responseFormat = _settings.EnableReasoning ?
JsonSerializer.Serialize(new FunctionCallFromLlm()) :
JsonSerializer.Serialize(new RoutingArgs
{
Function = "route_to_agent"
});
var content = $"{prompt} Response must be in JSON format {responseFormat}";
var chatCompletion = CompletionProvider.GetChatCompletion(_services,
provider: _settings.Provider,
model: _settings.Model);
RoleDialogModel response = null;
await chatCompletion.GetChatCompletionsAsync(_router, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, content)
}, async msg
=> response = msg, fn
=> Task.CompletedTask);
var args = new FunctionCallFromLlm();
try
{
#if DEBUG
Console.WriteLine(response.Content, Color.Gray);
#else
_logger.LogInformation(response.Content);
#endif
var pattern = @"\{(?:[^{}]|(?<open>\{)|(?<-open>\}))+(?(open)(?!))\}";
response.Content = Regex.Match(response.Content, pattern).Value;
args = JsonSerializer.Deserialize<FunctionCallFromLlm>(response.Content);
// Sometimes it populate malformed Function in Agent name
if (!string.IsNullOrEmpty(args.Function) && args.Function == args.AgentName)
{
args.Function = "route_to_agent";
_logger.LogWarning($"Captured LLM malformed response");
}
// Another case of malformed response
var agentService = _services.GetRequiredService<IAgentService>();
var agents = await agentService.GetAgents();
if (string.IsNullOrEmpty(args.AgentName) && agents.Select(x => x.Name).Contains(args.Function))
{
args.AgentName = args.Function;
args.Function = "route_to_agent";
_logger.LogWarning($"Captured LLM malformed response");
}
}
catch (Exception ex)
{
_logger.LogError($"{ex.Message}: {response.Content}");
args.Function = "response_to_user";
args.Answer = ex.Message;
args.AgentName = _settings.RouterName;
}
if (args.Arguments != null)
{
SaveStateByArgs(args.Arguments);
}
args.Function = args.Function.Split('.').Last();
#if DEBUG
Console.WriteLine($"*** Next Instruction *** {args}", Color.Green);
#else
_logger.LogInformation($"*** Next Instruction *** {args}");
#endif
return args;
}
public async Task<RoleDialogModel> GetResponseFromReasoner()
{
var wholeDialogs = new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, $"How to response to user?")
};
var chatCompletion = CompletionProvider.GetChatCompletion(_services,
provider: _settings.Provider,
model: _settings.Model);
RoleDialogModel response = null;
await chatCompletion.GetChatCompletionsAsync(_router, wholeDialogs, async msg
=> response = msg, fn
=> Task.CompletedTask);
return response;
}
const int MAXIMUM_RECURSION_DEPTH = 2;
int CurrentRecursionDepth = 0;
protected async Task<RoleDialogModel> InvokeAgent(string agentId)
{
CurrentRecursionDepth++;
if (CurrentRecursionDepth > MAXIMUM_RECURSION_DEPTH)
{
return _dialogs.Last();
}
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(agentId);
var chatCompletion = CompletionProvider.GetChatCompletion(_services);
RoleDialogModel response = null;
await chatCompletion.GetChatCompletionsAsync(agent, _dialogs,
async msg =>
{
response = msg;
}, async fn =>
{
// execute function
// Save states
SaveStateByArgs(JsonSerializer.Deserialize<JsonDocument>(fn.FunctionArgs));
var conversationService = _services.GetRequiredService<IConversationService>();
// Call functions
await conversationService.CallFunctions(fn);
if (string.IsNullOrEmpty(fn.Content))
{
fn.Content = fn.ExecutionResult;
}
_dialogs.Add(fn);
if (!fn.StopCompletion)
{
// Find response template
var templateService = _services.GetRequiredService<IResponseTemplateService>();
var quickResponse = await templateService.RenderFunctionResponse(agent.Id, fn);
if (!string.IsNullOrEmpty(quickResponse))
{
response = new RoleDialogModel(AgentRole.Assistant, quickResponse)
{
CurrentAgentId = agent.Id
};
}
else
{
response = await InvokeAgent(fn.CurrentAgentId);
}
}
else
{
response = fn;
}
});
return response;
}
protected void SaveStateByArgs(JsonDocument args)
{
if (args == null)
{
return;
}
var stateService = _services.GetRequiredService<IConversationStateService>();
if (args.RootElement is JsonElement root)
{
foreach (JsonProperty property in root.EnumerateObject())
{
if (!string.IsNullOrEmpty(property.Value.ToString()))
{
stateService.SetState(property.Name, property.Value);
}
}
}
}
}

2
src/Infrastructure/BotSharp.Core/Routing/Handlers/TaskEndRoutingHandler.cs
View file

@ -23,7 +23,7 @@ public class TaskEndRoutingHandler : RoutingHandlerBase, IRoutingHandler
{
}
public Task<RoleDialogModel> Handle(FunctionCallFromLlm inst)
public Task<RoleDialogModel> Handle(IRoutingService routing, FunctionCallFromLlm inst)
{
throw new NotImplementedException();
}

3
src/Infrastructure/BotSharp.Core/Routing/RouteToAgentFn.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Routing;
using BotSharp.Abstraction.Routing.Models;
using System.Drawing;
@ -57,7 +58,7 @@ public class RouteToAgentFn : IFunctionCallback
private bool HasMissingRequiredField(RoleDialogModel message, out string agentId)
{
var args = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs);
var router = _services.GetRequiredService<IAgentRouting>();
var router = _services.GetRequiredService<IRouterInstance>();
var routingRules = router.GetRulesByName(args.AgentName);

74
src/Infrastructure/BotSharp.Core/Routing/Router.cs
View file

@ -1,74 +0,0 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.Routing.Settings;
namespace BotSharp.Core.Routing;
public class Router : IAgentRouting
{
protected readonly IServiceProvider _services;
protected readonly ILogger _logger;
protected readonly RoutingSettings _settings;
public virtual string AgentId => _settings.RouterId;
public Router(IServiceProvider services,
ILogger<Router> logger,
RoutingSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public virtual async Task<Agent> LoadRouter()
{
var agentService = _services.GetRequiredService<IAgentService>();
return await agentService.LoadAgent(AgentId);
}
#if !DEBUG
[MemoryCache(10 * 60)]
#endif
protected RoutingRule[] GetRoutingRecords()
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray();
var records = agents.SelectMany(x =>
{
x.RoutingRules.ForEach(r =>
{
r.AgentId = x.Id;
r.AgentName = x.Name;
});
return x.RoutingRules;
}).ToArray();
// Filter agents by profile
var state = _services.GetRequiredService<IConversationStateService>();
var name = state.GetState("channel");
var specifiedProfile = agents.FirstOrDefault(x => x.Profiles.Contains(name));
if (specifiedProfile != null)
{
records = records.Where(x => specifiedProfile.Profiles.Contains(name)).ToArray();
}
return records;
}
public RoutingRule[] GetRulesByName(string name)
{
return GetRoutingRecords()
.Where(x => x.AgentName.ToLower() == name.ToLower())
.ToArray();
}
public RoutingRule[] GetRulesByAgentId(string id)
{
return GetRoutingRecords()
.Where(x => x.AgentId == id)
.ToArray();
}
}

173
src/Infrastructure/BotSharp.Core/Routing/RouterInstance.cs Normal file
View file

@ -0,0 +1,173 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Routing;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.Routing.Settings;
namespace BotSharp.Core.Routing;
public class RouterInstance : IRouterInstance
{
protected readonly IServiceProvider _services;
protected readonly ILogger _logger;
protected readonly RoutingSettings _settings;
private Agent _router;
public Agent Router => _router;
public virtual string AgentId => _router.Id;
public RouterInstance(IServiceProvider services,
ILogger<RouterInstance> logger,
RoutingSettings settings)
{
_services = services;
_logger = logger;
_settings = settings;
}
public IRouterInstance Load()
{
var db = _services.GetRequiredService<IBotSharpRepository>();
_router = new Agent()
{
Id = _settings.RouterId,
Name = _settings.RouterName,
Description = _settings.Description
};
var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray();
// Assemble prompt
var prompt = @$"You're {_settings.RouterName} ({_settings.Description}). Follow these steps to handle user's request:
1. Read the CONVERSATION context.
2. Select a appropriate function from FUNCTIONS.
3. Determine which agent is suitable according to conversation context.
4. Re-think about selected function is from FUNCTIONS to handle the request.
5. Make sure agent is not in args.";
// Append function
prompt += "\r\n";
prompt += "\r\nFUNCTIONS";
GetHandlers().Select((handler, i) =>
{
prompt += "\r\n";
prompt += $"\r\n{i + 1}. {handler.Name}";
prompt += $"\r\n{handler.Description}";
// Append parameters
if (handler.Parameters.Any())
{
prompt += "\r\nParameters:";
handler.Parameters.Select((p, i) =>
{
prompt += $"\r\n - {p.Name}: {p.Description}";
return p;
}).ToList();
}
return handler;
}).ToList();
prompt += "\r\n";
prompt += "\r\nAGENTS";
agents.Select(x => new RoutingItem
{
AgentId = x.Id,
Description = x.Description,
Name = x.Name,
RequiredFields = x.RoutingRules.Where(x => x.Required)
.Select(x => new NameDesc(x.Field, x.Description))
.ToList()
}).Select((agent, i) =>
{
prompt += "\r\n";
prompt += $"\r\n{i + 1}. {agent.Name}";
prompt += $"\r\n{agent.Description}";
// Append parameters
if (agent.RequiredFields.Any())
{
prompt += $"\r\nRequired:";
agent.RequiredFields.Select((field, i) =>
{
prompt += $"\r\n - {field.Name}: {field.Description}";
return field;
}).ToList();
}
return agent;
}).ToList();
prompt += "\r\n";
prompt += "\r\nCONVERSATION";
_router.Instruction = prompt;
return this;
}
public IRouterInstance WithDialogs(List<RoleDialogModel> dialogs)
{
foreach (var dialog in dialogs.TakeLast(20))
{
_router.Instruction += $"\r\n{dialog.Role}: {dialog.Content}";
}
return this;
}
public List<RoutingHandlerDef> GetHandlers()
{
return _services.GetServices<IRoutingHandler>()
.Where(x => x.IsReasoning == _settings.EnableReasoning)
.Where(x => !string.IsNullOrEmpty(x.Description))
.Select(x => new RoutingHandlerDef
{
Name = x.Name,
Description = x.Description,
Parameters = x.Parameters
}).ToList();
}
#if !DEBUG
[MemoryCache(10 * 60)]
#endif
protected RoutingRule[] GetRoutingRecords()
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray();
var records = agents.SelectMany(x =>
{
x.RoutingRules.ForEach(r =>
{
r.AgentId = x.Id;
r.AgentName = x.Name;
});
return x.RoutingRules;
}).ToArray();
// Filter agents by profile
var state = _services.GetRequiredService<IConversationStateService>();
var name = state.GetState("channel");
var specifiedProfile = agents.FirstOrDefault(x => x.Profiles.Contains(name));
if (specifiedProfile != null)
{
records = records.Where(x => specifiedProfile.Profiles.Contains(name)).ToArray();
}
return records;
}
public RoutingRule[] GetRulesByName(string name)
{
return GetRoutingRecords()
.Where(x => x.AgentName.ToLower() == name.ToLower())
.ToArray();
}
public RoutingRule[] GetRulesByAgentId(string id)
{
return GetRoutingRecords()
.Where(x => x.AgentId == id)
.ToArray();
}
}

82
src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs Normal file
View file

@ -0,0 +1,82 @@
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Routing;
using BotSharp.Abstraction.Routing.Models;
using System.Drawing;
using System.Text.RegularExpressions;
namespace BotSharp.Core.Routing;
public partial class RoutingService
{
public async Task<FunctionCallFromLlm> GetNextInstruction(string prompt)
{
var responseFormat = _settings.EnableReasoning ?
JsonSerializer.Serialize(new FunctionCallFromLlm()) :
JsonSerializer.Serialize(new RoutingArgs
{
Function = "route_to_agent"
});
var content = $"{prompt} Response must be in JSON format {responseFormat}";
var chatCompletion = CompletionProvider.GetChatCompletion(_services,
provider: _settings.Provider,
model: _settings.Model);
var response = chatCompletion.GetChatCompletions(_routerInstance.Router, new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, content)
});
var args = new FunctionCallFromLlm();
try
{
#if DEBUG
Console.WriteLine(response.Content, Color.Gray);
#else
_logger.LogInformation(response.Content);
#endif
var pattern = @"\{(?:[^{}]|(?<open>\{)|(?<-open>\}))+(?(open)(?!))\}";
response.Content = Regex.Match(response.Content, pattern).Value;
args = JsonSerializer.Deserialize<FunctionCallFromLlm>(response.Content);
// Sometimes it populate malformed Function in Agent name
if (!string.IsNullOrEmpty(args.Function) && args.Function == args.AgentName)
{
args.Function = "route_to_agent";
_logger.LogWarning($"Captured LLM malformed response");
}
// Another case of malformed response
var agentService = _services.GetRequiredService<IAgentService>();
var agents = await agentService.GetAgents();
if (string.IsNullOrEmpty(args.AgentName) && agents.Select(x => x.Name).Contains(args.Function))
{
args.AgentName = args.Function;
args.Function = "route_to_agent";
_logger.LogWarning($"Captured LLM malformed response");
}
}
catch (Exception ex)
{
_logger.LogError($"{ex.Message}: {response.Content}");
args.Function = "response_to_user";
args.Answer = ex.Message;
args.AgentName = _settings.RouterName;
}
if (args.Arguments != null)
{
SaveStateByArgs(args.Arguments);
}
args.Function = args.Function.Split('.').Last();
#if DEBUG
Console.WriteLine($"*** Next Instruction *** {args}", Color.Green);
#else
_logger.LogInformation($"*** Next Instruction *** {args}");
#endif
return args;
}
}

69
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs Normal file
View file

@ -0,0 +1,69 @@
using BotSharp.Abstraction.Templating;
namespace BotSharp.Core.Routing;
public partial class RoutingService
{
const int MAXIMUM_RECURSION_DEPTH = 2;
int CurrentRecursionDepth = 0;
public async Task<RoleDialogModel> InvokeAgent(string agentId)
{
CurrentRecursionDepth++;
if (CurrentRecursionDepth > MAXIMUM_RECURSION_DEPTH)
{
return Dialogs.Last();
}
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(agentId);
var chatCompletion = CompletionProvider.GetChatCompletion(_services);
RoleDialogModel response = null;
await chatCompletion.GetChatCompletionsAsync(agent, Dialogs,
async msg =>
{
response = msg;
}, async fn =>
{
// execute function
// Save states
SaveStateByArgs(JsonSerializer.Deserialize<JsonDocument>(fn.FunctionArgs));
var conversationService = _services.GetRequiredService<IConversationService>();
// Call functions
await conversationService.CallFunctions(fn);
if (string.IsNullOrEmpty(fn.Content))
{
fn.Content = fn.ExecutionResult;
}
Dialogs.Add(fn);
if (!fn.StopCompletion)
{
// Find response template
var templateService = _services.GetRequiredService<IResponseTemplateService>();
var quickResponse = await templateService.RenderFunctionResponse(agent.Id, fn);
if (!string.IsNullOrEmpty(quickResponse))
{
response = new RoleDialogModel(AgentRole.Assistant, quickResponse)
{
CurrentAgentId = agent.Id
};
}
else
{
response = await InvokeAgent(fn.CurrentAgentId);
}
}
else
{
response = fn;
}
});
return response;
}
}

128
src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
View file

@ -1,16 +1,15 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Abstraction.Routing;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Abstraction.Routing.Settings;
namespace BotSharp.Core.Routing;
public class RoutingService : IRoutingService
public partial class RoutingService : IRoutingService
{
private readonly IServiceProvider _services;
private readonly RoutingSettings _settings;
private readonly IRouterInstance _routerInstance;
private readonly ILogger _logger;
private List<RoleDialogModel> _dialogs;
public List<RoleDialogModel> Dialogs {
@ -28,11 +27,13 @@ public class RoutingService : IRoutingService
public RoutingService(IServiceProvider services,
RoutingSettings settings,
ILogger<RoutingService> logger)
ILogger<RoutingService> logger,
IRouterInstance routerInstance)
{
_services = services;
_settings = settings;
_logger = logger;
_routerInstance = routerInstance;
}
@ -44,7 +45,7 @@ public class RoutingService : IRoutingService
var handler = handlers.FirstOrDefault(x => x.Name == "route_to_agent");
handler.SetDialogs(Dialogs);
var result = await handler.Handle(new FunctionCallFromLlm
var result = await handler.Handle(this, new FunctionCallFromLlm
{
Function = "route_to_agent",
Question = message,
@ -57,24 +58,18 @@ public class RoutingService : IRoutingService
public async Task<RoleDialogModel> InstructLoop()
{
var router = LoadRouter();
_routerInstance.Load().WithDialogs(Dialogs);
var router = _routerInstance.Router;
var result = new RoleDialogModel(AgentRole.Assistant, "Can you repeat your request again?")
{
CurrentAgentId = router.Id
};
var message = Dialogs.Last().Content;
foreach (var dialog in Dialogs.TakeLast(20))
{
router.Instruction += $"\r\n{dialog.Role}: {dialog.Content}";
}
var handlers = _services.GetServices<IRoutingHandler>();
var handler = handlers.FirstOrDefault(x => x.Name == "get_next_instruction");
handler.SetRouter(router);
handler.SetDialogs(Dialogs);
int loopCount = 0;
var stop = false;
while (!stop && loopCount < 5)
@ -82,20 +77,21 @@ public class RoutingService : IRoutingService
loopCount++;
var prompt = _settings.EnableReasoning ? "Tell me the next step?" : "Which agent is suitable to handle user's request?";
var inst = await handler.GetNextInstructionFromReasoner(prompt);
prompt += " Or you can handle without asking specific agent.";
var inst = await GetNextInstruction(prompt);
inst.Question = inst.Question ?? message;
handler = handlers.FirstOrDefault(x => x.Name == inst.Function);
var handler = handlers.FirstOrDefault(x => x.Name == inst.Function);
if (handler == null)
{
handler = handlers.FirstOrDefault(x => x.Name == "get_next_instruction");
router.Instruction += $"\r\n{AgentRole.System}: the function must be one of {string.Join(",", GetHandlers().Select(x => x.Name))}.";
router.Instruction += $"\r\n{AgentRole.System}: the function must be one of {string.Join(",", _routerInstance.GetHandlers().Select(x => x.Name))}.";
continue;
}
handler.SetRouter(router);
handler.SetDialogs(Dialogs);
result = await handler.Handle(inst);
result = await handler.Handle(this, inst);
message = result.Content.Replace("\r\n", " ");
router.Instruction += $"\r\n{result.Role}: {message}";
@ -106,95 +102,23 @@ public class RoutingService : IRoutingService
return result;
}
public Agent LoadRouter()
protected void SaveStateByArgs(JsonDocument args)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var router = new Agent()
if (args == null)
{
Id = _settings.RouterId,
Name = _settings.RouterName,
Description = _settings.Description
};
var agents = db.Agents.Where(x => !x.Disabled && x.AllowRouting).ToArray();
return;
}
// Assemble prompt
var prompt = @$"You're {_settings.RouterName} ({_settings.Description}). Follow these steps to handle user's request:
1. Read the CONVERSATION context.
2. Select a appropriate function from FUNCTIONS.
3. Determine which agent is suitable according to conversation context.
4. Re-think about selected function is from FUNCTIONS to handle the request.
5. Make sure agent is not in args.";
// Append function
prompt += "\r\n";
prompt += "\r\nFUNCTIONS";
GetHandlers().Select((handler, i) =>
var stateService = _services.GetRequiredService<IConversationStateService>();
if (args.RootElement is JsonElement root)
{
prompt += "\r\n";
prompt += $"\r\n{i + 1}. {handler.Name}";
prompt += $"\r\n{handler.Description}";
// Append parameters
if (handler.Parameters.Any())
foreach (JsonProperty property in root.EnumerateObject())
{
prompt += "\r\nParameters:";
handler.Parameters.Select((p, i) =>
if (!string.IsNullOrEmpty(property.Value.ToString()))
{
prompt += $"\r\n - {p.Name}: {p.Description}";
return p;
}).ToList();
stateService.SetState(property.Name, property.Value);
}
}
return handler;
}).ToList();
prompt += "\r\n";
prompt += "\r\nAGENTS";
agents.Select(x => new RoutingItem
{
AgentId = x.Id,
Description = x.Description,
Name = x.Name,
RequiredFields = x.RoutingRules.Where(x => x.Required)
.Select(x => new NameDesc(x.Field, x.Description))
.ToList()
}).Select((agent, i) =>
{
prompt += "\r\n";
prompt += $"\r\n{i + 1}. {agent.Name}";
prompt += $"\r\n{agent.Description}";
// Append parameters
if (agent.RequiredFields.Any())
{
prompt += $"\r\nRequired:";
agent.RequiredFields.Select((field, i) =>
{
prompt += $"\r\n - {field.Name}: {field.Description}";
return field;
}).ToList();
}
return agent;
}).ToList();
prompt += "\r\n";
prompt += "\r\nCONVERSATION";
router.Instruction = prompt;
return router;
}
private List<RoutingHandlerDef> GetHandlers()
{
return _services.GetServices<IRoutingHandler>()
.Where(x => x.IsReasoning == _settings.EnableReasoning)
.Where(x => !string.IsNullOrEmpty(x.Description))
.Select(x => new RoutingHandlerDef
{
Name = x.Name,
Description = x.Description,
Parameters = x.Parameters
}).ToList();
}
}
}

4
src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs
View file

@ -26,8 +26,8 @@ public class AgentController : ControllerBase, IApiAdapter
var agents = await _agentService.GetAgents();
// Add the router as agent
var routing = _services.GetRequiredService<IRoutingService>();
agents.Insert(0, routing.LoadRouter());
var routing = _services.GetRequiredService<IRouterInstance>();
agents.Insert(0, routing.Load().Router);
return agents.Select(x => AgentViewModel.FromAgent(x)).ToList();
}

48
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs
View file

@ -95,6 +95,54 @@ public class ChatCompletionProvider : IChatCompletion
return functions;
}
public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
{
var (client, deploymentModel) = GetClient();
var chatCompletionsOptions = PrepareOptions(agent, conversations);
var response = client.GetChatCompletions(deploymentModel, chatCompletionsOptions);
var choice = response.Value.Choices[0];
var message = choice.Message;
_tokenStatistics.AddToken(new TokenStatsModel
{
Model = _model,
PromptCount = response.Value.Usage.PromptTokens,
CompletionCount = response.Value.Usage.CompletionTokens,
PromptCost = 0.0015f,
CompletionCost = 0.002f
});
if (choice.FinishReason == CompletionsFinishReason.FunctionCall)
{
_logger.LogInformation($"[{agent.Name}]: {message.FunctionCall.Name} => {message.FunctionCall.Arguments}");
var funcContextIn = new RoleDialogModel(AgentRole.Function, message.Content)
{
CurrentAgentId = agent.Id,
FunctionName = message.FunctionCall.Name,
FunctionArgs = message.FunctionCall.Arguments
};
// Somethings LLM will generate a function name with agent name.
if (!string.IsNullOrEmpty(funcContextIn.FunctionName))
{
funcContextIn.FunctionName = funcContextIn.FunctionName.Split('.').Last();
}
return funcContextIn;
}
else
{
var msg = new RoleDialogModel(AgentRole.Assistant, message.Content)
{
CurrentAgentId = agent.Id
};
return msg;
}
}
public async Task<bool> GetChatCompletionsAsync(Agent agent,
List<RoleDialogModel> conversations,
Func<RoleDialogModel, Task> onMessageReceived,

69
src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs
View file

@ -39,29 +39,26 @@ public class ChatCompletionProvider : IChatCompletion
var api = _services.GetRequiredService<IInferenceApi>();
if (_model.Contains('/'))
var space = _model.Split('/')[0];
var model = _model.Split("/")[1];
var response = await api.Post(space, model, new InferenceInput
{
var space = _model.Split('/')[0];
var model = _model.Split("/")[1];
Inputs = prompt
});
var response = await api.Post(space, model, new InferenceInput
{
Inputs = prompt
});
var falcon = JsonSerializer.Deserialize<List<FalconLlmResponse>>(response);
var falcon = JsonSerializer.Deserialize<List<FalconLlmResponse>>(response);
var message = falcon[0].GeneratedText.Trim();
_logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}");
var message = falcon[0].GeneratedText.Trim();
_logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}");
var msg = new RoleDialogModel(AgentRole.Assistant, message)
{
CurrentAgentId = agent.Id
};
var msg = new RoleDialogModel(AgentRole.Assistant, message)
{
CurrentAgentId = agent.Id
};
// Text response received
await onMessageReceived(msg);
}
// Text response received
await onMessageReceived(msg);
return true;
}
@ -75,4 +72,40 @@ public class ChatCompletionProvider : IChatCompletion
{
_model = model;
}
public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
{
var content = string.Join("\r\n", conversations.Select(x => $"{AgentRole.System}: {x.Content}")).Trim();
content += $"\r\n{AgentRole.Assistant}: ";
var prompt = agent.Instruction + "\r\n" + content;
var convSetting = _services.GetRequiredService<ConversationSetting>();
if (convSetting.ShowVerboseLog)
{
_logger.LogInformation(prompt);
}
var api = _services.GetRequiredService<IInferenceApi>();
var space = _model.Split('/')[0];
var model = _model.Split("/")[1];
var response = api.Post(space, model, new InferenceInput
{
Inputs = prompt
}).Result;
var falcon = JsonSerializer.Deserialize<List<FalconLlmResponse>>(response);
var message = falcon[0].GeneratedText.Trim();
_logger.LogInformation($"[{agent.Name}] {AgentRole.Assistant}: {message}");
var msg = new RoleDialogModel(AgentRole.Assistant, message)
{
CurrentAgentId = agent.Id
};
return msg;
}
}

48
src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs
View file

@ -36,6 +36,54 @@ public class ChatCompletionProvider : IChatCompletion
public string Provider => "llama-sharp";
public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> conversations)
{
var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
content += $"\r\n{AgentRole.Assistant}: ";
var state = _services.GetRequiredService<IConversationStateService>();
var model = state.GetState("model", _settings.DefaultModel);
var llama = _services.GetRequiredService<LlamaAiModel>();
llama.LoadModel(model);
var executor = llama.GetStatelessExecutor();
var inferenceParams = new InferenceParams()
{
Temperature = 0.1f,
AntiPrompts = new List<string> { $"{AgentRole.User}:", "[/INST]" },
MaxTokens = 64
};
string totalResponse = "";
var prompt = agent.Instruction + "\r\n" + content;
var convSetting = _services.GetRequiredService<ConversationSetting>();
if (convSetting.ShowVerboseLog)
{
_logger.LogInformation(prompt);
}
foreach (var response in executor.Infer(prompt, inferenceParams))
{
Console.Write(response);
totalResponse += response;
}
foreach (var anti in inferenceParams.AntiPrompts)
{
totalResponse = totalResponse.Replace(anti, "").Trim();
}
var msg = new RoleDialogModel(AgentRole.Assistant, totalResponse)
{
CurrentAgentId = agent.Id
};
return msg;
}
public async Task<bool> GetChatCompletionsAsync(Agent agent,
List<RoleDialogModel> conversations,
Func<RoleDialogModel, Task> onMessageReceived,

2
src/WebStarter/appsettings.json
View file

@ -19,7 +19,7 @@
"Description": "Pizza restaurant AI Bot",
"EnableReasoning": false,
"Provider": "azure-openai",
"Model": "gpt-3.5"
"Model": "gpt-3.5-turbo"
},
"Agent": {