Merge pull request #1060 from iceljc/test/google-realtime
add session reconnect
This commit is contained in:
commit
3dfb921126
|
|
@ -8,7 +8,7 @@ public enum AgentField
|
|||
IsPublic,
|
||||
Disabled,
|
||||
Type,
|
||||
Mode,
|
||||
RoutingMode,
|
||||
InheritAgentId,
|
||||
Profile,
|
||||
Label,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ public class Agent
|
|||
public string Id { get; set; } = string.Empty;
|
||||
public string Name { get; set; } = string.Empty;
|
||||
public string Description { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Agent Type
|
||||
/// </summary>
|
||||
|
|
@ -17,7 +18,8 @@ public class Agent
|
|||
/// <summary>
|
||||
/// Routing Mode: lazy or eager
|
||||
/// </summary>
|
||||
public string Mode { get; set; } = "eager";
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Mode { get; set; }
|
||||
|
||||
public DateTime CreatedDateTime { get; set; }
|
||||
public DateTime UpdatedDateTime { get; set; }
|
||||
|
|
@ -277,18 +279,18 @@ public class Agent
|
|||
return this;
|
||||
}
|
||||
|
||||
public Agent SetAgentType(string type)
|
||||
public Agent SetType(string type)
|
||||
{
|
||||
Type = type;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Set agent mode: lazy or eager
|
||||
/// Set agent routing mode: lazy or eager
|
||||
/// </summary>
|
||||
/// <param name="mode"></param>
|
||||
/// <returns></returns>
|
||||
public Agent SetAgentMode(string mode)
|
||||
public Agent SetRoutingMode(string? mode)
|
||||
{
|
||||
Mode = mode;
|
||||
return this;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
namespace BotSharp.Abstraction.Crontab.Settings;
|
||||
|
||||
public class CrontabSettings
|
||||
{
|
||||
public CrontabBaseSetting EventSubscriber { get; set; } = new();
|
||||
public CrontabBaseSetting Watcher { get; set; } = new();
|
||||
}
|
||||
|
||||
public class CrontabBaseSetting
|
||||
{
|
||||
public bool Enabled { get; set; } = true;
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
using BotSharp.Abstraction.Users;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.AspNetCore.Mvc.Filters;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace BotSharp.Abstraction.Infrastructures.Attributes;
|
||||
|
||||
/// <summary>
|
||||
/// BotSharp authorization: check whether the request user is admin or root role.
|
||||
/// </summary>
|
||||
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)]
|
||||
public class BotSharpAuthAttribute : Attribute, IAsyncAuthorizationFilter
|
||||
{
|
||||
public BotSharpAuthAttribute()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
public async Task OnAuthorizationAsync(AuthorizationFilterContext context)
|
||||
{
|
||||
var services = context.HttpContext.RequestServices;
|
||||
|
||||
var userIdentity = services.GetRequiredService<IUserIdentity>();
|
||||
var userService = services.GetRequiredService<IUserService>();
|
||||
|
||||
var (isAdmin, user) = await userService.IsAdminUser(userIdentity.Id);
|
||||
if (!isAdmin || user == null)
|
||||
{
|
||||
context.Result = new UnauthorizedResult();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -19,6 +19,8 @@ public interface IRealTimeCompletion
|
|||
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
|
||||
Func<Task> onInterruptionDetected);
|
||||
|
||||
Task Reconnect(RealtimeHubConnection conn);
|
||||
|
||||
Task AppenAudioBuffer(string message);
|
||||
Task AppenAudioBuffer(ArraySegment<byte> data, int length);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
using BotSharp.Abstraction.Hooks;
|
||||
using BotSharp.Abstraction.MLTasks;
|
||||
using BotSharp.Abstraction.Realtime.Models;
|
||||
|
||||
namespace BotSharp.Abstraction.Realtime;
|
||||
|
||||
|
|
@ -8,4 +9,5 @@ public interface IRealtimeHook : IHookBase
|
|||
Task OnModelReady(Agent agent, IRealTimeCompletion completer);
|
||||
string[] OnModelTranscriptPrompt(Agent agent);
|
||||
Task OnTranscribeCompleted(RoleDialogModel message, TranscriptionData data);
|
||||
Task<bool> ShouldReconnect(RealtimeHubConnection conn) => Task.FromResult(false);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
using Microsoft.Extensions.Logging;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace BotSharp.Abstraction.Realtime.Models.Session;
|
||||
|
||||
public class ChatSessionOptions
|
||||
{
|
||||
public string Provider { get; set; }
|
||||
public int? BufferSize { get; set; }
|
||||
public JsonSerializerOptions? JsonOptions { get; set; }
|
||||
public ILogger? Logger { get; set; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
namespace BotSharp.Abstraction.Repositories;
|
||||
namespace BotSharp.Abstraction.Repositories.Settings;
|
||||
|
||||
public class BotSharpDatabaseSettings : DatabaseBasicSettings
|
||||
{
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
namespace BotSharp.Abstraction.Routing.Enums;
|
||||
|
||||
public class RoutingMode
|
||||
{
|
||||
public const string Eager = "eager";
|
||||
public const string Lazy = "lazy";
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ namespace BotSharp.Abstraction.Utilities;
|
|||
|
||||
public static class StringExtensions
|
||||
{
|
||||
public static string IfNullOrEmptyAs(this string str, string defaultValue)
|
||||
public static string IfNullOrEmptyAs(this string? str, string defaultValue)
|
||||
=> string.IsNullOrEmpty(str) ? defaultValue : str;
|
||||
|
||||
public static string SubstringMax(this string str, int maxLength)
|
||||
|
|
|
|||
|
|
@ -32,11 +32,22 @@ public class CrontabPlugin : IBotSharpPlugin
|
|||
|
||||
public void RegisterDI(IServiceCollection services, IConfiguration config)
|
||||
{
|
||||
var settings = new CrontabSettings();
|
||||
config.Bind("Crontab", settings);
|
||||
services.AddSingleton(settings);
|
||||
|
||||
services.AddScoped<IAgentUtilityHook, CrontabUtilityHook>();
|
||||
services.AddScoped<ICrontabService, CrontabService>();
|
||||
services.AddScoped<ITaskFeeder, CrontabService>();
|
||||
|
||||
services.AddHostedService<CrontabWatcher>();
|
||||
services.AddHostedService<CrontabEventSubscription>();
|
||||
if (settings.Watcher?.Enabled == true)
|
||||
{
|
||||
services.AddHostedService<CrontabWatcher>();
|
||||
}
|
||||
|
||||
if (settings.EventSubscriber?.Enabled == true)
|
||||
{
|
||||
services.AddHostedService<CrontabEventSubscription>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ global using Microsoft.Extensions.DependencyInjection;
|
|||
global using BotSharp.Abstraction.Agents.Enums;
|
||||
global using BotSharp.Abstraction.Crontab;
|
||||
global using BotSharp.Abstraction.Crontab.Models;
|
||||
global using BotSharp.Abstraction.Crontab.Settings;
|
||||
global using BotSharp.Abstraction.Agents;
|
||||
global using BotSharp.Abstraction.Plugins;
|
||||
global using BotSharp.Abstraction.Conversations.Models;
|
||||
|
|
|
|||
|
|
@ -98,8 +98,6 @@ public class RealtimeHub : IRealtimeHub
|
|||
}
|
||||
|
||||
await routing.InvokeFunction(message.FunctionName, message);
|
||||
dialogs.Add(message);
|
||||
storage.Append(_conn.ConversationId, message);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -107,8 +105,8 @@ public class RealtimeHub : IRealtimeHub
|
|||
dialogs.Add(message);
|
||||
storage.Append(_conn.ConversationId, message);
|
||||
|
||||
var hooks = _services.GetHooksOrderByPriority<IConversationHook>(_conn.CurrentAgentId);
|
||||
foreach (var hook in hooks)
|
||||
var convHooks = _services.GetHooksOrderByPriority<IConversationHook>(_conn.CurrentAgentId);
|
||||
foreach (var hook in convHooks)
|
||||
{
|
||||
hook.SetAgent(agent)
|
||||
.SetConversation(conversation);
|
||||
|
|
@ -117,6 +115,19 @@ public class RealtimeHub : IRealtimeHub
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
var isReconnect = false;
|
||||
var realtimeHooks = _services.GetHooks<IRealtimeHook>(_conn.CurrentAgentId);
|
||||
foreach (var hook in realtimeHooks)
|
||||
{
|
||||
isReconnect = await hook.ShouldReconnect(_conn);
|
||||
if (isReconnect) break;
|
||||
}
|
||||
|
||||
if (isReconnect)
|
||||
{
|
||||
await _completer.Reconnect(_conn);
|
||||
}
|
||||
},
|
||||
onConversationItemCreated: async response =>
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
using BotSharp.Abstraction.Tasks.Models;
|
||||
using BotSharp.Abstraction.Users.Enums;
|
||||
using System.IO;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using BotSharp.Abstraction.Repositories.Enums;
|
||||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
using System.IO;
|
||||
|
||||
namespace BotSharp.Core.Agents.Services;
|
||||
|
|
@ -16,14 +17,6 @@ public partial class AgentService
|
|||
return refreshResult;
|
||||
}
|
||||
|
||||
var userIdentity = _services.GetRequiredService<IUserIdentity>();
|
||||
var userService = _services.GetRequiredService<IUserService>();
|
||||
var (isValid, _) = await userService.IsAdminUser(userIdentity.Id);
|
||||
if (!isValid)
|
||||
{
|
||||
return "Unauthorized user.";
|
||||
}
|
||||
|
||||
var agentDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
|
||||
dbSettings.FileRepository,
|
||||
_agentSettings.DataDir);
|
||||
|
|
@ -74,7 +67,7 @@ public partial class AgentService
|
|||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError($"Failed to migrate agent in file directory: {dir}\r\nError: {ex.Message}");
|
||||
_logger.LogError(ex, $"Failed to migrate agent in file directory: {dir}\r\nError: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using BotSharp.Abstraction.Repositories.Enums;
|
||||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
using BotSharp.Abstraction.Users.Enums;
|
||||
using BotSharp.Abstraction.Users.Models;
|
||||
using System.IO;
|
||||
|
|
@ -93,12 +94,12 @@ public partial class AgentService
|
|||
{
|
||||
clonedAgent.SetId(foundAgent.Id)
|
||||
.SetName(foundAgent.Name)
|
||||
.SetDescription(foundAgent.Description)
|
||||
.SetType(foundAgent.Type)
|
||||
.SetRoutingMode(foundAgent.Mode)
|
||||
.SetIsPublic(foundAgent.IsPublic)
|
||||
.SetDisabled(foundAgent.Disabled)
|
||||
.SetDescription(foundAgent.Description)
|
||||
.SetMergeUtility(foundAgent.MergeUtility)
|
||||
.SetAgentType(foundAgent.Type)
|
||||
.SetAgentMode(foundAgent.Mode)
|
||||
.SetProfiles(foundAgent.Profiles)
|
||||
.SetLabels(foundAgent.Labels)
|
||||
.SetRoutingRules(foundAgent.RoutingRules)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ using BotSharp.Abstraction.Templating;
|
|||
using BotSharp.Core.Templating;
|
||||
using BotSharp.Abstraction.Infrastructures.Enums;
|
||||
using BotSharp.Abstraction.Realtime;
|
||||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
|
||||
namespace BotSharp.Core;
|
||||
|
||||
|
|
@ -71,17 +72,6 @@ public static class BotSharpCoreExtensions
|
|||
return services;
|
||||
}
|
||||
|
||||
//public static IServiceCollection UsingFileRepository(this IServiceCollection services, IConfiguration config)
|
||||
//{
|
||||
// services.AddScoped<IBotSharpRepository>(sp =>
|
||||
// {
|
||||
// var myDatabaseSettings = sp.GetRequiredService<BotSharpDatabaseSettings>();
|
||||
// return new FileRepository(myDatabaseSettings, sp);
|
||||
// });
|
||||
|
||||
// return services;
|
||||
//}
|
||||
|
||||
public static IApplicationBuilder UseBotSharp(this IApplicationBuilder app)
|
||||
{
|
||||
if (app == null)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ using BotSharp.Abstraction.Hooks;
|
|||
using BotSharp.Abstraction.Infrastructures.Enums;
|
||||
using BotSharp.Abstraction.Messaging;
|
||||
using BotSharp.Abstraction.Messaging.Models.RichContent;
|
||||
using BotSharp.Abstraction.Routing.Enums;
|
||||
using BotSharp.Abstraction.Routing.Settings;
|
||||
|
||||
namespace BotSharp.Core.Conversations.Services;
|
||||
|
|
@ -83,10 +84,10 @@ public partial class ConversationService
|
|||
{
|
||||
// Check the routing mode
|
||||
var states = _services.GetRequiredService<IConversationStateService>();
|
||||
var routingMode = states.GetState(StateConst.ROUTING_MODE, "eager");
|
||||
var routingMode = states.GetState(StateConst.ROUTING_MODE, RoutingMode.Eager);
|
||||
routing.Context.Push(agent.Id, reason: "request started", updateLazyRouting: false);
|
||||
|
||||
if (routingMode == "lazy")
|
||||
if (routingMode == RoutingMode.Lazy)
|
||||
{
|
||||
message.CurrentAgentId = states.GetState(StateConst.LAZY_ROUTING_AGENT_ID, message.CurrentAgentId);
|
||||
routing.Context.Push(message.CurrentAgentId, reason: "lazy routing", updateLazyRouting: false);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
using System.IO;
|
||||
|
||||
namespace BotSharp.Core.Files.Services;
|
||||
|
|
|
|||
|
|
@ -24,7 +24,9 @@ public class DistributedLocker : IDistributedLocker
|
|||
var redis = _services.GetService<IConnectionMultiplexer>();
|
||||
if (redis == null)
|
||||
{
|
||||
#if !DEBUG
|
||||
_logger.LogInformation($"The Redis server is experiencing issues and is not functioning as expected.");
|
||||
#endif
|
||||
await action();
|
||||
return true;
|
||||
}
|
||||
|
|
@ -50,7 +52,9 @@ public class DistributedLocker : IDistributedLocker
|
|||
var redis = _services.GetRequiredService<IConnectionMultiplexer>();
|
||||
if (redis == null)
|
||||
{
|
||||
#if !DEBUG
|
||||
_logger.LogWarning($"The Redis server is experiencing issues and is not functioning as expected.");
|
||||
#endif
|
||||
action();
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
using BotSharp.Abstraction.Realtime.Models.Session;
|
||||
using System.ClientModel;
|
||||
using System.Net.WebSockets;
|
||||
|
||||
|
|
@ -7,16 +6,16 @@ namespace BotSharp.Core.Infrastructures.Websocket;
|
|||
internal class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<ClientResult>
|
||||
{
|
||||
private readonly WebSocket _webSocket;
|
||||
private readonly ChatSessionOptions? _sessionOptions;
|
||||
private readonly ChatSessionOptions? _options;
|
||||
private readonly CancellationToken _cancellationToken;
|
||||
|
||||
public AsyncWebsocketDataCollectionResult(
|
||||
WebSocket webSocket,
|
||||
ChatSessionOptions? sessionOptions,
|
||||
ChatSessionOptions? options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
_webSocket = webSocket;
|
||||
_sessionOptions = sessionOptions;
|
||||
_options = options;
|
||||
_cancellationToken = cancellationToken;
|
||||
}
|
||||
|
||||
|
|
@ -27,7 +26,7 @@ internal class AsyncWebsocketDataCollectionResult : AsyncCollectionResult<Client
|
|||
|
||||
public override async IAsyncEnumerable<ClientResult> GetRawPagesAsync()
|
||||
{
|
||||
await using var enumerator = new AsyncWebsocketDataResultEnumerator(_webSocket, _sessionOptions, _cancellationToken);
|
||||
await using var enumerator = new AsyncWebsocketDataResultEnumerator(_webSocket, _options, _cancellationToken);
|
||||
while (await enumerator.MoveNextAsync().ConfigureAwait(false))
|
||||
{
|
||||
yield return enumerator.Current;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using Microsoft.AspNetCore.Builder;
|
||||
using System.Buffers;
|
||||
using System.ClientModel;
|
||||
using System.Net.WebSockets;
|
||||
|
|
@ -7,7 +8,7 @@ namespace BotSharp.Core.Infrastructures.Websocket;
|
|||
internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResult>
|
||||
{
|
||||
private readonly WebSocket _webSocket;
|
||||
private readonly ChatSessionOptions? _sessionOptions;
|
||||
private readonly ChatSessionOptions? _options;
|
||||
private readonly CancellationToken _cancellationToken;
|
||||
private readonly byte[] _buffer;
|
||||
|
||||
|
|
@ -15,13 +16,13 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResul
|
|||
|
||||
public AsyncWebsocketDataResultEnumerator(
|
||||
WebSocket webSocket,
|
||||
ChatSessionOptions? sessionOptions,
|
||||
ChatSessionOptions? options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
_webSocket = webSocket;
|
||||
_sessionOptions = sessionOptions;
|
||||
_options = options;
|
||||
_cancellationToken = cancellationToken;
|
||||
var bufferSize = sessionOptions?.BufferSize > 0 ? sessionOptions.BufferSize.Value : DEFAULT_BUFFER_SIZE;
|
||||
var bufferSize = options?.BufferSize > 0 ? options.BufferSize.Value : DEFAULT_BUFFER_SIZE;
|
||||
_buffer = ArrayPool<byte>.Shared.Rent(bufferSize);
|
||||
}
|
||||
|
||||
|
|
@ -44,7 +45,10 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator<ClientResul
|
|||
if (receivedResult.CloseStatus.HasValue)
|
||||
{
|
||||
#if DEBUG
|
||||
Console.WriteLine($"Websocket close: {receivedResult.CloseStatus} {receivedResult.CloseStatusDescription}");
|
||||
if (_options?.Logger != null)
|
||||
{
|
||||
_options.Logger.LogWarning($"{_options?.Provider} Websocket close: ({receivedResult.CloseStatus}) {receivedResult.CloseStatusDescription}");
|
||||
}
|
||||
#endif
|
||||
Current = null;
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
using Microsoft.Data.SqlClient;
|
||||
using System.Data.Common;
|
||||
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ namespace BotSharp.Core.Repository
|
|||
case AgentField.Type:
|
||||
UpdateAgentType(agent.Id, agent.Type);
|
||||
break;
|
||||
case AgentField.Mode:
|
||||
UpdateAgentMode(agent.Id, agent.Mode);
|
||||
case AgentField.RoutingMode:
|
||||
UpdateAgentRoutingMode(agent.Id, agent.Mode);
|
||||
break;
|
||||
case AgentField.InheritAgentId:
|
||||
UpdateAgentInheritAgentId(agent.Id, agent.InheritAgentId);
|
||||
|
|
@ -145,7 +145,7 @@ namespace BotSharp.Core.Repository
|
|||
File.WriteAllText(agentFile, json);
|
||||
}
|
||||
|
||||
private void UpdateAgentMode(string agentId, string mode)
|
||||
private void UpdateAgentRoutingMode(string agentId, string? mode)
|
||||
{
|
||||
var (agent, agentFile) = GetAgentFromFile(agentId);
|
||||
if (agent == null) return;
|
||||
|
|
@ -383,11 +383,12 @@ namespace BotSharp.Core.Repository
|
|||
if (agent == null) return;
|
||||
|
||||
agent.Name = inputAgent.Name;
|
||||
agent.Description = inputAgent.Description;
|
||||
agent.Type = inputAgent.Type;
|
||||
agent.Mode = inputAgent.Mode;
|
||||
agent.IsPublic = inputAgent.IsPublic;
|
||||
agent.Disabled = inputAgent.Disabled;
|
||||
agent.Description = inputAgent.Description;
|
||||
agent.MergeUtility = inputAgent.MergeUtility;
|
||||
agent.Type = inputAgent.Type;
|
||||
agent.Profiles = inputAgent.Profiles;
|
||||
agent.Labels = inputAgent.Labels;
|
||||
agent.Utilities = inputAgent.Utilities;
|
||||
|
|
|
|||
|
|
@ -263,8 +263,15 @@ public partial class FileRepository
|
|||
}
|
||||
|
||||
var userFile = Path.Combine(dir, USER_FILE);
|
||||
user.UpdatedTime = DateTime.UtcNow;
|
||||
File.WriteAllText(userFile, JsonSerializer.Serialize(user, _options));
|
||||
var userJson = File.ReadAllText(userFile);
|
||||
var curUser = JsonSerializer.Deserialize<User>(userJson, _options);
|
||||
if (curUser == null) return false;
|
||||
|
||||
curUser.Type = user.Type;
|
||||
curUser.Role = user.Role;
|
||||
curUser.Permissions = user.Permissions;
|
||||
curUser.UpdatedTime = DateTime.UtcNow;
|
||||
File.WriteAllText(userFile, JsonSerializer.Serialize(curUser, _options));
|
||||
|
||||
if (updateUserAgents)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ using FunctionDef = BotSharp.Abstraction.Functions.Models.FunctionDef;
|
|||
using BotSharp.Abstraction.Users.Models;
|
||||
using BotSharp.Abstraction.Plugins.Models;
|
||||
using BotSharp.Abstraction.Tasks.Models;
|
||||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
|
||||
namespace BotSharp.Core.Repository;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using BotSharp.Abstraction.Repositories.Enums;
|
||||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
using BotSharp.Abstraction.Settings;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using BotSharp.Abstraction.Infrastructures.Enums;
|
||||
using BotSharp.Abstraction.Routing.Enums;
|
||||
using BotSharp.Abstraction.Routing.Settings;
|
||||
|
||||
namespace BotSharp.Core.Routing;
|
||||
|
|
@ -290,8 +291,8 @@ public class RoutingContext : IRoutingContext
|
|||
|
||||
// Set next handling agent for lazy routing mode
|
||||
var states = _services.GetRequiredService<IConversationStateService>();
|
||||
var routingMode = states.GetState(StateConst.ROUTING_MODE, "eager");
|
||||
if (routingMode == "lazy")
|
||||
var routingMode = states.GetState(StateConst.ROUTING_MODE, RoutingMode.Eager);
|
||||
if (routingMode == RoutingMode.Lazy)
|
||||
{
|
||||
var agentId = GetCurrentAgentId();
|
||||
if (agentId != BuiltInAgentId.Fallback)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ public class BotSharpRealtimeSession : IDisposable
|
|||
private readonly ChatSessionOptions? _sessionOptions;
|
||||
private readonly object _singleReceiveLock = new();
|
||||
private AsyncWebsocketDataCollectionResult _receivedCollectionResult;
|
||||
private bool _disposed = false;
|
||||
|
||||
public BotSharpRealtimeSession(
|
||||
IServiceProvider services,
|
||||
|
|
@ -57,23 +58,30 @@ public class BotSharpRealtimeSession : IDisposable
|
|||
|
||||
public async Task SendEventAsync(string message)
|
||||
{
|
||||
if (_websocket.State == WebSocketState.Open)
|
||||
if (_disposed || _websocket.State != WebSocketState.Open)
|
||||
{
|
||||
var buffer = Encoding.UTF8.GetBytes(message);
|
||||
await _websocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
|
||||
return;
|
||||
}
|
||||
|
||||
var buffer = Encoding.UTF8.GetBytes(message);
|
||||
await _websocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, CancellationToken.None);
|
||||
}
|
||||
|
||||
public async Task DisconnectAsync()
|
||||
{
|
||||
if (_websocket.State == WebSocketState.Open)
|
||||
if (_disposed || _websocket.State != WebSocketState.Open)
|
||||
{
|
||||
await _websocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
|
||||
return;
|
||||
}
|
||||
|
||||
await _websocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) return;
|
||||
|
||||
_disposed = true;
|
||||
_websocket.Dispose();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ public class LlmRealtimeSession : IDisposable
|
|||
private readonly object _singleReceiveLock = new();
|
||||
private readonly SemaphoreSlim _clientEventSemaphore = new(initialCount: 1, maxCount: 1);
|
||||
private AsyncWebsocketDataCollectionResult _receivedCollectionResult;
|
||||
private bool _disposed = false;
|
||||
|
||||
public LlmRealtimeSession(
|
||||
IServiceProvider services,
|
||||
|
|
@ -24,6 +25,7 @@ public class LlmRealtimeSession : IDisposable
|
|||
|
||||
public async Task ConnectAsync(Uri uri, Dictionary<string, string>? headers = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_disposed = false;
|
||||
_webSocket?.Dispose();
|
||||
_webSocket = new ClientWebSocket();
|
||||
|
||||
|
|
@ -73,15 +75,20 @@ public class LlmRealtimeSession : IDisposable
|
|||
|
||||
public async Task SendEventToModelAsync(object message)
|
||||
{
|
||||
if (_webSocket.State != WebSocketState.Open)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _clientEventSemaphore.WaitAsync();
|
||||
|
||||
try
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _clientEventSemaphore.WaitAsync();
|
||||
|
||||
if (_webSocket.State != WebSocketState.Open)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (message is not string data)
|
||||
{
|
||||
data = JsonSerializer.Serialize(message, _sessionOptions?.JsonOptions);
|
||||
|
|
@ -92,12 +99,17 @@ public class LlmRealtimeSession : IDisposable
|
|||
}
|
||||
finally
|
||||
{
|
||||
_clientEventSemaphore.Release();
|
||||
if (!_disposed)
|
||||
{
|
||||
_clientEventSemaphore.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DisconnectAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
|
||||
if (_webSocket.State == WebSocketState.Open)
|
||||
{
|
||||
await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None);
|
||||
|
|
@ -106,6 +118,9 @@ public class LlmRealtimeSession : IDisposable
|
|||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) return;
|
||||
|
||||
_disposed = true;
|
||||
_clientEventSemaphore?.Dispose();
|
||||
_webSocket?.Dispose();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using BotSharp.Abstraction.Agents.Models;
|
||||
using BotSharp.Abstraction.Infrastructures.Attributes;
|
||||
|
||||
namespace BotSharp.OpenAPI.Controllers;
|
||||
|
||||
|
|
@ -108,6 +109,7 @@ public class AgentController : ControllerBase
|
|||
return AgentViewModel.FromAgent(createdAgent);
|
||||
}
|
||||
|
||||
[BotSharpAuth]
|
||||
[HttpPost("/refresh-agents")]
|
||||
public async Task<string> RefreshAgents()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using BotSharp.Abstraction.Infrastructures.Attributes;
|
||||
using BotSharp.Abstraction.Plugins.Models;
|
||||
using BotSharp.Abstraction.Users.Enums;
|
||||
using BotSharp.Core.Plugins;
|
||||
|
|
@ -10,15 +11,10 @@ public class PluginController(IServiceProvider services, IUserIdentity user, Plu
|
|||
{
|
||||
private readonly IUserIdentity _user = user;
|
||||
|
||||
[BotSharpAuth]
|
||||
[HttpGet("/plugins")]
|
||||
public async Task<PagedItems<PluginDef>> GetPlugins([FromQuery] PluginFilter filter)
|
||||
{
|
||||
var isValid = await IsValidUser();
|
||||
if (!isValid)
|
||||
{
|
||||
return new PagedItems<PluginDef>();
|
||||
}
|
||||
|
||||
var loader = services.GetRequiredService<PluginLoader>();
|
||||
return loader.GetPagedPlugins(services, filter);
|
||||
}
|
||||
|
|
@ -72,6 +68,7 @@ public class PluginController(IServiceProvider services, IUserIdentity user, Plu
|
|||
return menu;
|
||||
}
|
||||
|
||||
[BotSharpAuth]
|
||||
[HttpPost("/plugin/{id}/install")]
|
||||
public PluginDef InstallPlugin([FromRoute] string id)
|
||||
{
|
||||
|
|
@ -79,17 +76,11 @@ public class PluginController(IServiceProvider services, IUserIdentity user, Plu
|
|||
return loader.UpdatePluginStatus(services, id, true);
|
||||
}
|
||||
|
||||
[BotSharpAuth]
|
||||
[HttpPost("/plugin/{id}/remove")]
|
||||
public PluginDef RemovePluginStats([FromRoute] string id)
|
||||
{
|
||||
var loader = services.GetRequiredService<PluginLoader>();
|
||||
return loader.UpdatePluginStatus(services, id, false);
|
||||
}
|
||||
|
||||
private async Task<bool> IsValidUser()
|
||||
{
|
||||
var userService = services.GetRequiredService<IUserService>();
|
||||
var (isAdmin, _) = await userService.IsAdminUser(_user.Id);
|
||||
return isAdmin;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using BotSharp.Abstraction.Infrastructures.Attributes;
|
||||
using BotSharp.Abstraction.Roles;
|
||||
|
||||
namespace BotSharp.OpenAPI.Controllers;
|
||||
|
|
@ -20,15 +21,10 @@ public class RoleController : ControllerBase
|
|||
_user = user;
|
||||
}
|
||||
|
||||
[BotSharpAuth]
|
||||
[HttpPost("/role/refresh")]
|
||||
public async Task<bool> RefreshRoles()
|
||||
{
|
||||
var isValid = await IsValidUser();
|
||||
if (!isValid)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return await _roleService.RefreshRoles();
|
||||
}
|
||||
|
||||
|
|
@ -39,6 +35,7 @@ public class RoleController : ControllerBase
|
|||
return await _roleService.GetRoleOptions();
|
||||
}
|
||||
|
||||
[BotSharpAuth]
|
||||
[HttpPost("/roles")]
|
||||
public async Task<IEnumerable<RoleViewModel>> GetRoles([FromBody] RoleFilter? filter = null)
|
||||
{
|
||||
|
|
@ -47,12 +44,6 @@ public class RoleController : ControllerBase
|
|||
filter = RoleFilter.Empty();
|
||||
}
|
||||
|
||||
var isValid = await IsValidUser();
|
||||
if (!isValid)
|
||||
{
|
||||
return Enumerable.Empty<RoleViewModel>();
|
||||
}
|
||||
|
||||
var roles = await _roleService.GetRoles(filter);
|
||||
return roles.Select(x => RoleViewModel.FromRole(x)).ToList();
|
||||
}
|
||||
|
|
@ -64,25 +55,13 @@ public class RoleController : ControllerBase
|
|||
return RoleViewModel.FromRole(role);
|
||||
}
|
||||
|
||||
[BotSharpAuth]
|
||||
[HttpPut("/role")]
|
||||
public async Task<bool> UpdateRole([FromBody] RoleUpdateModel model)
|
||||
{
|
||||
if (model == null) return false;
|
||||
|
||||
var isValid = await IsValidUser();
|
||||
if (!isValid)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var role = RoleUpdateModel.ToRole(model);
|
||||
return await _roleService.UpdateRole(role, isUpdateRoleAgents: true);
|
||||
}
|
||||
|
||||
private async Task<bool> IsValidUser()
|
||||
{
|
||||
var userService = _services.GetRequiredService<IUserService>();
|
||||
var (isAdmin, _) = await userService.IsAdminUser(_user.Id);
|
||||
return isAdmin;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using BotSharp.Abstraction.Infrastructures.Attributes;
|
||||
using BotSharp.Abstraction.Users.Settings;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.AspNetCore.Authentication.Cookies;
|
||||
|
|
@ -181,12 +182,6 @@ public class UserController : ControllerBase
|
|||
public async Task<PagedItems<UserViewModel>> GetUsers([FromBody] UserFilter filter)
|
||||
{
|
||||
var userService = _services.GetRequiredService<IUserService>();
|
||||
var isValid = await IsValidUser();
|
||||
if (!isValid)
|
||||
{
|
||||
return new PagedItems<UserViewModel>();
|
||||
}
|
||||
|
||||
var users = await userService.GetUsers(filter);
|
||||
var views = users.Items.Select(x => UserViewModel.FromUser(x)).ToList();
|
||||
|
||||
|
|
@ -197,6 +192,7 @@ public class UserController : ControllerBase
|
|||
};
|
||||
}
|
||||
|
||||
[BotSharpAuth]
|
||||
[HttpGet("/user/{id}/details")]
|
||||
public async Task<UserViewModel> GetUserDetails(string id)
|
||||
{
|
||||
|
|
@ -205,17 +201,12 @@ public class UserController : ControllerBase
|
|||
return UserViewModel.FromUser(user);
|
||||
}
|
||||
|
||||
[BotSharpAuth]
|
||||
[HttpPut("/user")]
|
||||
public async Task<bool> UpdateUser([FromBody] UserUpdateModel model)
|
||||
{
|
||||
if (model == null) return false;
|
||||
|
||||
var isValid = await IsValidUser();
|
||||
if (!isValid)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var userService = _services.GetRequiredService<IUserService>();
|
||||
var updated = await userService.UpdateUser(UserUpdateModel.ToUser(model), isUpdateUserAgents: true);
|
||||
return updated;
|
||||
|
|
@ -251,13 +242,6 @@ public class UserController : ControllerBase
|
|||
|
||||
|
||||
#region Private methods
|
||||
private async Task<bool> IsValidUser()
|
||||
{
|
||||
var userService = _services.GetRequiredService<IUserService>();
|
||||
var (isAdmin, _) = await userService.IsAdminUser(_user.Id);
|
||||
return isAdmin;
|
||||
}
|
||||
|
||||
private FileContentResult BuildFileResult(string file)
|
||||
{
|
||||
var fileStorage = _services.GetRequiredService<IFileStorageService>();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
using BotSharp.Abstraction.Agents.Models;
|
||||
using BotSharp.Abstraction.Functions.Models;
|
||||
using BotSharp.Abstraction.Routing.Enums;
|
||||
|
||||
namespace BotSharp.OpenAPI.ViewModels.Agents;
|
||||
|
||||
|
|
@ -9,6 +10,11 @@ public class AgentCreationModel
|
|||
public string Description { get; set; }
|
||||
public string Type { get; set; } = AgentType.Task;
|
||||
|
||||
/// <summary>
|
||||
/// Agent routing mode
|
||||
/// </summary>
|
||||
public string? Mode { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// LLM default system instructions
|
||||
/// </summary>
|
||||
|
|
@ -66,6 +72,10 @@ public class AgentCreationModel
|
|||
return new Agent
|
||||
{
|
||||
Name = Name,
|
||||
Type = Type,
|
||||
Mode = Mode,
|
||||
Disabled = Disabled,
|
||||
IsPublic = IsPublic,
|
||||
Description = Description,
|
||||
Instruction = Instruction,
|
||||
ChannelInstructions = ChannelInstructions,
|
||||
|
|
@ -75,9 +85,6 @@ public class AgentCreationModel
|
|||
Samples = Samples,
|
||||
Utilities = Utilities,
|
||||
McpTools = McpTools,
|
||||
IsPublic = IsPublic,
|
||||
Type = Type,
|
||||
Disabled = Disabled,
|
||||
MergeUtility = MergeUtility,
|
||||
MaxMessageCount = MaxMessageCount,
|
||||
Profiles = Profiles,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,12 @@ public class AgentUpdateModel
|
|||
public string Name { get; set; } = string.Empty;
|
||||
public string Description { get; set; } = string.Empty;
|
||||
public string Type { get; set; } = AgentType.Task;
|
||||
|
||||
/// <summary>
|
||||
/// Agent routing mode
|
||||
/// </summary>
|
||||
public string? Mode { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Instruction
|
||||
/// </summary>
|
||||
|
|
@ -93,12 +99,13 @@ public class AgentUpdateModel
|
|||
var agent = new Agent()
|
||||
{
|
||||
Name = Name ?? string.Empty,
|
||||
Description = Description ?? string.Empty,
|
||||
Type = Type,
|
||||
Mode = Mode,
|
||||
IsPublic = IsPublic,
|
||||
Disabled = Disabled,
|
||||
Description = Description ?? string.Empty,
|
||||
MergeUtility = MergeUtility,
|
||||
MaxMessageCount = MaxMessageCount,
|
||||
Type = Type,
|
||||
Profiles = Profiles ?? [],
|
||||
Labels = Labels ?? [],
|
||||
RoutingRules = RoutingRules?.Select(x => RoutingRuleUpdateModel.ToDomainElement(x))?.ToList() ?? [],
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@ public class AgentViewModel
|
|||
public string Name { get; set; }
|
||||
public string Description { get; set; }
|
||||
public string Type { get; set; } = AgentType.Task;
|
||||
public string Mode { get; set; } = null!;
|
||||
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Mode { get; set; }
|
||||
public string Instruction { get; set; }
|
||||
|
||||
[JsonPropertyName("channel_instructions")]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
using Whisper.net;
|
||||
using Whisper.net.Ggml;
|
||||
|
||||
|
|
|
|||
|
|
@ -305,7 +305,7 @@ public class ChatCompletionProvider : IChatCompletion
|
|||
{
|
||||
messages.Add(new AssistantChatMessage(new List<ChatToolCall>
|
||||
{
|
||||
ChatToolCall.CreateFunctionToolCall(message.ToolCallId ?? message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
|
||||
ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
|
||||
}));
|
||||
|
||||
messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content));
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ public class ChatStreamMiddleware
|
|||
_session?.Dispose();
|
||||
_session = new BotSharpRealtimeSession(services, webSocket, new ChatSessionOptions
|
||||
{
|
||||
Provider = "BotSharp Chat Stream",
|
||||
BufferSize = 1024 * 16,
|
||||
JsonOptions = BotSharpOptions.defaultJsonOptions
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ internal class RealtimeTranscriptionResponse : IDisposable
|
|||
|
||||
}
|
||||
|
||||
private bool _disposed = false;
|
||||
|
||||
private MemoryStream _contentStream = new();
|
||||
public Stream? ContentStream
|
||||
{
|
||||
|
|
@ -20,6 +22,8 @@ internal class RealtimeTranscriptionResponse : IDisposable
|
|||
|
||||
public void Collect(string text)
|
||||
{
|
||||
if (_disposed) return;
|
||||
|
||||
var binary = BinaryData.FromString(text);
|
||||
var bytes = binary.ToArray();
|
||||
|
||||
|
|
@ -30,7 +34,7 @@ internal class RealtimeTranscriptionResponse : IDisposable
|
|||
|
||||
public string GetText()
|
||||
{
|
||||
if (_contentStream.Length == 0)
|
||||
if (_disposed || _contentStream.Length == 0)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
|
@ -42,12 +46,21 @@ internal class RealtimeTranscriptionResponse : IDisposable
|
|||
|
||||
public void Clear()
|
||||
{
|
||||
_contentStream.Position = 0;
|
||||
_contentStream.SetLength(0);
|
||||
try
|
||||
{
|
||||
if (_disposed) return;
|
||||
|
||||
_contentStream.Position = 0;
|
||||
_contentStream.SetLength(0);
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) return;
|
||||
|
||||
_disposed = true;
|
||||
_contentStream?.Dispose();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
public string Provider => "google-ai";
|
||||
public string Model => _model;
|
||||
|
||||
private string _model = GoogleAIModels.Gemini2FlashExp;
|
||||
private string _model = GoogleAIModels.Gemini2FlashLive001;
|
||||
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly ILogger _logger;
|
||||
|
|
@ -35,7 +35,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
|
||||
private RealtimeTranscriptionResponse _inputStream = new();
|
||||
private RealtimeTranscriptionResponse _outputStream = new();
|
||||
|
||||
private bool _isBlocking = false;
|
||||
|
||||
private RealtimeHubConnection _conn;
|
||||
private Func<Task> _onModelReady;
|
||||
|
|
@ -73,6 +73,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
|
||||
Func<Task> onInterruptionDetected)
|
||||
{
|
||||
_logger.LogInformation($"Connecting {Provider} realtime server...");
|
||||
|
||||
_conn = conn;
|
||||
_onModelReady = onModelReady;
|
||||
_onModelAudioDeltaReceived = onModelAudioDeltaReceived;
|
||||
|
|
@ -90,11 +92,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
var modelSettings = settingsService.GetSetting(Provider, _model);
|
||||
|
||||
Reset();
|
||||
_isBlocking = true;
|
||||
_inputStream = new();
|
||||
_outputStream = new();
|
||||
_session = new LlmRealtimeSession(_services, new ChatSessionOptions
|
||||
{
|
||||
JsonOptions = _jsonOptions
|
||||
Provider = Provider,
|
||||
JsonOptions = _jsonOptions,
|
||||
Logger = _logger
|
||||
});
|
||||
|
||||
var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta");
|
||||
|
|
@ -124,6 +129,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
if (response.SetupComplete != null)
|
||||
{
|
||||
_logger.LogInformation($"Session setup completed.");
|
||||
_isBlocking = false;
|
||||
}
|
||||
else if (response.SessionResumptionUpdate != null)
|
||||
{
|
||||
|
|
@ -155,13 +161,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
|
||||
if (response.ServerContent.ModelTurn != null)
|
||||
{
|
||||
_logger.LogInformation($"Model audio delta received.");
|
||||
|
||||
// Handle input transcription
|
||||
var inputTranscription = _inputStream.GetText();
|
||||
if (!string.IsNullOrEmpty(inputTranscription))
|
||||
if (!string.IsNullOrWhiteSpace(inputTranscription))
|
||||
{
|
||||
var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription);
|
||||
var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription ?? string.Empty);
|
||||
await _onInputAudioTranscriptionDone(message);
|
||||
}
|
||||
_inputStream.Clear();
|
||||
|
|
@ -188,9 +192,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
|
||||
// Handle output transcription
|
||||
var outputTranscription = _outputStream.GetText();
|
||||
if (!string.IsNullOrEmpty(outputTranscription))
|
||||
if (!string.IsNullOrWhiteSpace(outputTranscription))
|
||||
{
|
||||
var messages = await OnResponseDone(_conn, outputTranscription, response.UsageMetaData);
|
||||
var messages = await OnResponseDone(_conn, outputTranscription ?? string.Empty, response.UsageMetaData);
|
||||
await _onModelResponseDone(messages);
|
||||
}
|
||||
_inputStream.Clear();
|
||||
|
|
@ -200,7 +204,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, $"Error when deserializing server response. {ex.Message}");
|
||||
_logger.LogError(ex, $"Error when handling server response. {ex.Message}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -211,8 +215,30 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
}
|
||||
|
||||
|
||||
public async Task Reconnect(RealtimeHubConnection conn)
|
||||
{
|
||||
_logger.LogInformation($"Reconnecting {Provider} realtime server...");
|
||||
|
||||
_isBlocking = true;
|
||||
_conn = conn;
|
||||
await Disconnect();
|
||||
await Task.Delay(500);
|
||||
await Connect(
|
||||
_conn,
|
||||
_onModelReady,
|
||||
_onModelAudioDeltaReceived,
|
||||
_onModelAudioResponseDone,
|
||||
_onModelAudioTranscriptDone,
|
||||
_onModelResponseDone,
|
||||
_onConversationItemCreated,
|
||||
_onInputAudioTranscriptionDone,
|
||||
_onInterruptionDetected);
|
||||
}
|
||||
|
||||
public async Task Disconnect()
|
||||
{
|
||||
_logger.LogInformation($"Disconnecting {Provider} realtime server...");
|
||||
|
||||
if (_session != null)
|
||||
{
|
||||
_inputStream?.Dispose();
|
||||
|
|
@ -224,6 +250,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
|
||||
public async Task AppenAudioBuffer(string message)
|
||||
{
|
||||
if (_isBlocking) return;
|
||||
|
||||
await SendEventToModel(new RealtimeClientPayload
|
||||
{
|
||||
RealtimeInput = new()
|
||||
|
|
@ -235,6 +263,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
|
||||
public async Task AppenAudioBuffer(ArraySegment<byte> data, int length)
|
||||
{
|
||||
if (_isBlocking) return;
|
||||
|
||||
var buffer = data.AsSpan(0, length).ToArray();
|
||||
await SendEventToModel(new RealtimeClientPayload
|
||||
{
|
||||
|
|
@ -313,7 +343,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
}).ToArray();
|
||||
|
||||
await HookEmitter.Emit<IContentGeneratingHook>(_services,
|
||||
async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit); }, agent.Id);
|
||||
async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit: false); }, agent.Id);
|
||||
|
||||
if (_settings.Gemini.UseGoogleSearch)
|
||||
{
|
||||
|
|
@ -332,8 +362,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
|
|||
Model = Model.ToModelId(),
|
||||
SystemInstruction = request.SystemInstruction,
|
||||
Tools = request.Tools?.ToArray(),
|
||||
InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
|
||||
OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null
|
||||
InputAudioTranscription = new(),
|
||||
OutputAudioTranscription = new()
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ public sealed class MicrosoftExtensionsAITextEmbeddingProvider : ITextEmbedding
|
|||
|
||||
/// <inheritdoc/>
|
||||
public async Task<float[]> GetVectorAsync(string text) =>
|
||||
(await _generator.GenerateEmbeddingVectorAsync(text, CreateOptions())).ToArray();
|
||||
(await _generator.GenerateVectorAsync(text, CreateOptions())).ToArray();
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<List<float[]>> GetVectorsAsync(List<string> texts)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ public class AgentDocument : MongoBase
|
|||
public string Name { get; set; } = default!;
|
||||
public string Description { get; set; } = default!;
|
||||
public string Type { get; set; } = default!;
|
||||
public string Mode { get; set; } = default!;
|
||||
public string? Mode { get; set; }
|
||||
public string? InheritAgentId { get; set; }
|
||||
public string? IconUrl { get; set; }
|
||||
public string Instruction { get; set; } = default!;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
|
||||
namespace BotSharp.Plugin.MongoStorage;
|
||||
|
||||
public class MongoDbContext
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using BotSharp.Abstraction.Repositories.Enums;
|
||||
using BotSharp.Abstraction.Repositories.Settings;
|
||||
using BotSharp.Plugin.MongoStorage.Repository;
|
||||
|
||||
namespace BotSharp.Plugin.MongoStorage;
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ public partial class MongoRepository
|
|||
case AgentField.Type:
|
||||
UpdateAgentType(agent.Id, agent.Type);
|
||||
break;
|
||||
case AgentField.Mode:
|
||||
UpdateAgentMode(agent.Id, agent.Mode);
|
||||
case AgentField.RoutingMode:
|
||||
UpdateAgentRoutingMode(agent.Id, agent.Mode);
|
||||
break;
|
||||
case AgentField.InheritAgentId:
|
||||
UpdateAgentInheritAgentId(agent.Id, agent.InheritAgentId);
|
||||
|
|
@ -139,7 +139,7 @@ public partial class MongoRepository
|
|||
_dc.Agents.UpdateOne(filter, update);
|
||||
}
|
||||
|
||||
private void UpdateAgentMode(string agentId, string mode)
|
||||
private void UpdateAgentRoutingMode(string agentId, string? mode)
|
||||
{
|
||||
var filter = Builders<AgentDocument>.Filter.Eq(x => x.Id, agentId);
|
||||
var update = Builders<AgentDocument>.Update
|
||||
|
|
|
|||
|
|
@ -272,7 +272,7 @@ public class ChatCompletionProvider : IChatCompletion
|
|||
{
|
||||
messages.Add(new AssistantChatMessage(new List<ChatToolCall>
|
||||
{
|
||||
ChatToolCall.CreateFunctionToolCall(message.ToolCallId ?? message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
|
||||
ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}"))
|
||||
}));
|
||||
|
||||
messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content));
|
||||
|
|
|
|||
|
|
@ -16,8 +16,19 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
private readonly ILogger<RealTimeCompletionProvider> _logger;
|
||||
private readonly BotSharpOptions _botsharpOptions;
|
||||
|
||||
protected string _model = "gpt-4o-mini-realtime-preview";
|
||||
private string _model = "gpt-4o-mini-realtime-preview";
|
||||
private LlmRealtimeSession _session;
|
||||
private bool _isBlocking = false;
|
||||
|
||||
private RealtimeHubConnection _conn;
|
||||
private Func<Task> _onModelReady;
|
||||
private Func<string, string, Task> _onModelAudioDeltaReceived;
|
||||
private Func<Task> _onModelAudioResponseDone;
|
||||
private Func<string, Task> _onModelAudioTranscriptDone;
|
||||
private Func<List<RoleDialogModel>, Task> _onModelResponseDone;
|
||||
private Func<string, Task> _onConversationItemCreated;
|
||||
private Func<RoleDialogModel, Task> _onInputAudioTranscriptionDone;
|
||||
private Func<Task> _onInterruptionDetected;
|
||||
|
||||
public RealTimeCompletionProvider(
|
||||
IServiceProvider services,
|
||||
|
|
@ -40,6 +51,18 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
|
||||
Func<Task> onInterruptionDetected)
|
||||
{
|
||||
_logger.LogInformation($"Connecting {Provider} realtime server...");
|
||||
|
||||
_conn = conn;
|
||||
_onModelReady = onModelReady;
|
||||
_onModelAudioDeltaReceived = onModelAudioDeltaReceived;
|
||||
_onModelAudioResponseDone = onModelAudioResponseDone;
|
||||
_onModelAudioTranscriptDone = onModelAudioTranscriptDone;
|
||||
_onModelResponseDone = onModelResponseDone;
|
||||
_onConversationItemCreated = onConversationItemCreated;
|
||||
_onInputAudioTranscriptionDone = onInputAudioTranscriptionDone;
|
||||
_onInterruptionDetected = onInterruptionDetected;
|
||||
|
||||
var settingsService = _services.GetRequiredService<ILlmProviderService>();
|
||||
var realtimeSettings = _services.GetRequiredService<RealtimeModelSettings>();
|
||||
|
||||
|
|
@ -49,7 +72,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
_session?.Dispose();
|
||||
_session = new LlmRealtimeSession(_services, new ChatSessionOptions
|
||||
{
|
||||
JsonOptions = _botsharpOptions.JsonSerializerOptions
|
||||
Provider = Provider,
|
||||
JsonOptions = _botsharpOptions.JsonSerializerOptions,
|
||||
Logger = _logger
|
||||
});
|
||||
|
||||
await _session.ConnectAsync(
|
||||
|
|
@ -61,30 +86,10 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
},
|
||||
cancellationToken: CancellationToken.None);
|
||||
|
||||
_ = ReceiveMessage(
|
||||
realtimeSettings,
|
||||
conn,
|
||||
onModelReady,
|
||||
onModelAudioDeltaReceived,
|
||||
onModelAudioResponseDone,
|
||||
onModelAudioTranscriptDone,
|
||||
onModelResponseDone,
|
||||
onConversationItemCreated,
|
||||
onInputAudioTranscriptionDone,
|
||||
onInterruptionDetected);
|
||||
_ = ReceiveMessage(realtimeSettings);
|
||||
}
|
||||
|
||||
private async Task ReceiveMessage(
|
||||
RealtimeModelSettings realtimeSettings,
|
||||
RealtimeHubConnection conn,
|
||||
Func<Task> onModelReady,
|
||||
Func<string, string, Task> onModelAudioDeltaReceived,
|
||||
Func<Task> onModelAudioResponseDone,
|
||||
Func<string, Task> onModelAudioTranscriptDone,
|
||||
Func<List<RoleDialogModel>, Task> onModelResponseDone,
|
||||
Func<string, Task> onConversationItemCreated,
|
||||
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
|
||||
Func<Task> onInterruptionDetected)
|
||||
private async Task ReceiveMessage(RealtimeModelSettings realtimeSettings)
|
||||
{
|
||||
DateTime? startTime = null;
|
||||
|
||||
|
|
@ -121,7 +126,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
else if (response.Type == "session.created")
|
||||
{
|
||||
_logger.LogInformation($"{response.Type}: {receivedText}");
|
||||
await onModelReady();
|
||||
_isBlocking = false;
|
||||
await _onModelReady();
|
||||
}
|
||||
else if (response.Type == "session.updated")
|
||||
{
|
||||
|
|
@ -135,7 +141,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
{
|
||||
_logger.LogInformation($"{response.Type}: {receivedText}");
|
||||
var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
|
||||
await onModelAudioTranscriptDone(data.Transcript);
|
||||
await _onModelAudioTranscriptDone(data.Transcript);
|
||||
}
|
||||
else if (response.Type == "response.audio.delta")
|
||||
{
|
||||
|
|
@ -143,13 +149,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
if (audio?.Delta != null)
|
||||
{
|
||||
_logger.LogDebug($"{response.Type}: {receivedText}");
|
||||
await onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
|
||||
await _onModelAudioDeltaReceived(audio.Delta, audio.ItemId);
|
||||
}
|
||||
}
|
||||
else if (response.Type == "response.audio.done")
|
||||
{
|
||||
_logger.LogInformation($"{response.Type}: {receivedText}");
|
||||
await onModelAudioResponseDone();
|
||||
await _onModelAudioResponseDone();
|
||||
}
|
||||
else if (response.Type == "response.done")
|
||||
{
|
||||
|
|
@ -159,14 +165,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
{
|
||||
if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens")
|
||||
{
|
||||
await onInterruptionDetected();
|
||||
await _onInterruptionDetected();
|
||||
await TriggerModelInference("Response user concisely");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var messages = await OnResponsedDone(conn, receivedText);
|
||||
await onModelResponseDone(messages);
|
||||
var messages = await OnResponsedDone(_conn, receivedText);
|
||||
await _onModelResponseDone(messages);
|
||||
}
|
||||
}
|
||||
else if (response.Type == "conversation.item.created")
|
||||
|
|
@ -179,23 +185,23 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
startTime = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
await onConversationItemCreated(receivedText);
|
||||
await _onConversationItemCreated(receivedText);
|
||||
}
|
||||
else if (response.Type == "conversation.item.input_audio_transcription.completed")
|
||||
{
|
||||
_logger.LogInformation($"{response.Type}: {receivedText}");
|
||||
|
||||
var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
|
||||
var message = await OnUserAudioTranscriptionCompleted(_conn, receivedText);
|
||||
if (!string.IsNullOrEmpty(message.Content))
|
||||
{
|
||||
await onInputAudioTranscriptionDone(message);
|
||||
await _onInputAudioTranscriptionDone(message);
|
||||
}
|
||||
}
|
||||
else if (response.Type == "input_audio_buffer.speech_started")
|
||||
{
|
||||
_logger.LogInformation($"{response.Type}: {receivedText}");
|
||||
// Handle user interuption
|
||||
await onInterruptionDetected();
|
||||
await _onInterruptionDetected();
|
||||
}
|
||||
else if (response.Type == "input_audio_buffer.speech_stopped")
|
||||
{
|
||||
|
|
@ -210,8 +216,31 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
_session.Dispose();
|
||||
}
|
||||
|
||||
|
||||
public async Task Reconnect(RealtimeHubConnection conn)
|
||||
{
|
||||
_logger.LogInformation($"Reconnecting {Provider} realtime server...");
|
||||
|
||||
_isBlocking = true;
|
||||
_conn = conn;
|
||||
await Disconnect();
|
||||
await Task.Delay(500);
|
||||
await Connect(
|
||||
_conn,
|
||||
_onModelReady,
|
||||
_onModelAudioDeltaReceived,
|
||||
_onModelAudioResponseDone,
|
||||
_onModelAudioTranscriptDone,
|
||||
_onModelResponseDone,
|
||||
_onConversationItemCreated,
|
||||
_onInputAudioTranscriptionDone,
|
||||
_onInterruptionDetected);
|
||||
}
|
||||
|
||||
public async Task Disconnect()
|
||||
{
|
||||
_logger.LogInformation($"Disconnecting {Provider} realtime server...");
|
||||
|
||||
if (_session != null)
|
||||
{
|
||||
await _session.DisconnectAsync();
|
||||
|
|
@ -221,6 +250,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
|
||||
public async Task AppenAudioBuffer(string message)
|
||||
{
|
||||
if (_isBlocking) return;
|
||||
|
||||
var audioAppend = new
|
||||
{
|
||||
type = "input_audio_buffer.append",
|
||||
|
|
@ -232,6 +263,8 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
|
||||
public async Task AppenAudioBuffer(ArraySegment<byte> data, int length)
|
||||
{
|
||||
if (_isBlocking) return;
|
||||
|
||||
var message = Convert.ToBase64String(data.AsSpan(0, length).ToArray());
|
||||
await AppenAudioBuffer(message);
|
||||
}
|
||||
|
|
@ -348,7 +381,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
|
|||
|
||||
await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
|
||||
{
|
||||
await hook.OnSessionUpdated(agent, instruction, functions, isInit);
|
||||
await hook.OnSessionUpdated(agent, instruction, functions, isInit: false);
|
||||
}, agent.Id);
|
||||
|
||||
await SendEventToModel(sessionUpdate);
|
||||
|
|
|
|||
|
|
@ -9,7 +9,12 @@ using StackExchange.Redis;
|
|||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
builder.Host.UseSerilog();
|
||||
Log.Logger = new LoggerConfiguration()
|
||||
.MinimumLevel.Information()
|
||||
.WriteTo.Console()
|
||||
.CreateLogger();
|
||||
|
||||
builder.Host.UseSerilog(Log.Logger);
|
||||
|
||||
string[] allowedOrigins = builder.Configuration.GetSection("AllowedOrigins").Get<string[]>() ?? new[]
|
||||
{
|
||||
|
|
@ -18,8 +23,6 @@ string[] allowedOrigins = builder.Configuration.GetSection("AllowedOrigins").Get
|
|||
"https://chat.scisharpstack.org"
|
||||
};
|
||||
|
||||
|
||||
|
||||
// Add BotSharp
|
||||
builder.Services.AddBotSharpCore(builder.Configuration, options =>
|
||||
{
|
||||
|
|
|
|||
|
|
@ -325,6 +325,15 @@
|
|||
"Enabled": false
|
||||
},
|
||||
|
||||
"Crontab": {
|
||||
"Watcher": {
|
||||
"Enabled": false
|
||||
},
|
||||
"EventSubscriber": {
|
||||
"Enabled": false
|
||||
}
|
||||
},
|
||||
|
||||
"Instruction": {
|
||||
"Logging": {
|
||||
"Enabled": true,
|
||||
|
|
@ -427,6 +436,7 @@
|
|||
"BucketName": "",
|
||||
"Region": ""
|
||||
},
|
||||
|
||||
"Qdrant": {
|
||||
"Url": "",
|
||||
"ApiKey": ""
|
||||
|
|
@ -468,6 +478,7 @@
|
|||
"ApiSecret": "",
|
||||
"ModelVersion": "V3_5"
|
||||
},
|
||||
|
||||
"MetaGLM": {
|
||||
"ApiKey": "6b6c8b3fca3e5da21d633e350980744d.938gruOqrK4BDqW8",
|
||||
"BaseAddress": "http://localhost:8100/v1/",
|
||||
|
|
|
|||
Loading…
Reference in a new issue