diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..ed4cf565 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,44 @@ +name: build + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + build: + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v1 + - name: Setup .NET Core + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '8.0.x' + - name: Set env + run: | + echo "DOTNET_CLI_TELEMETRY_OPTOUT=1" >> $GITHUB_ENV + echo "DOTNET_hostBuilder:reloadConfigOnChange=false" >> $GITHUB_ENV + - name: Install required workloads + run: | + dotnet workload install aspire --source https://aka.ms/dotnet8/nuget/index.json --source https://api.nuget.org/v3/index.json + - name: Clean + run: | + dotnet clean ./BotSharp.sln --configuration Release + dotnet nuget locals all --clear + - name: Build + run: dotnet build ./BotSharp.sln -c Release + - name: Test + run: | + cd ./tests/UnitTest + dotnet test --logger "console;verbosity=detailed" + cd ../BotSharp.Plugin.SemanticKernel.UnitTests + dotnet test --logger "console;verbosity=detailed" \ No newline at end of file diff --git a/README.md b/README.md index f8f76eb4..9924c4d1 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ The core module is mainly composed of abstraction and framework function impleme ### Plugins -BotSharp uses component design, the kernel is kept to a minimum, and business functions are implemented by external components. The modular design also allows contributors to better participate. Below are the bulit-in plugins: +BotSharp uses component design, the kernel is kept to a minimum, and business functions are implemented by external components. The modular design also allows contributors to better participate. Below are the built-in plugins: #### Data Storages - BotSharp.Core.Repository diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 4475e1b4..f189d323 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -25,6 +25,7 @@ + diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs index c4e73d69..c13f26a8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs @@ -4,4 +4,6 @@ public static class MessageTypeName { public const string Plain = "plain"; public const string Notification = "notification"; + public const string FunctionCall = "function"; + public const string Audio = "audio"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index ddf985b4..01aba146 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -63,4 +63,13 @@ public interface IConversationService bool IsConversationMode(); void SaveStates(); + + /// + /// Get conversation keys for searching + /// + /// search query + /// conversation limit + /// if pre-loading, then keys are not filter by the search query + /// + Task> GetConversationStateSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs index c452b7ab..466893d3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs @@ -10,4 +10,6 @@ public class StateConst public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason"; public const string LANGUAGE = "language"; + + public const string SUB_CONVERSATION_ID = "sub_conversation_id"; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 5eebb5b3..8e1d11d5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -5,8 +5,28 @@ namespace BotSharp.Abstraction.MLTasks; public interface IRealTimeCompletion { string Provider { get; } + string Model { get; } void SetModelName(string model); + Task Connect(RealtimeHubConnection conn, + Action onModelReady, + Action onModelAudioDeltaReceived, + Action onModelAudioResponseDone, + Action onAudioTranscriptDone, + Action> onModelResponseDone, + Action onConversationItemCreated, + Action onInputAudioTranscriptionCompleted, + Action onUserInterrupted); + Task AppenAudioBuffer(string message); + + Task SendEventToModel(object message); + Task Disconnect(); + Task CreateSession(Agent agent, List conversations); + Task UpdateInitialSession(RealtimeHubConnection conn); + Task InsertConversationItem(RoleDialogModel message); + Task TriggerModelInference(string? instructions = null); + Task> OnResponsedDone(RealtimeHubConnection conn, string response); + Task OnConversationItemCreated(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs new file mode 100644 index 00000000..67d0f18c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs @@ -0,0 +1,12 @@ +using BotSharp.Abstraction.Realtime.Models; +using System.Net.WebSockets; + +namespace BotSharp.Abstraction.Realtime; + +/// +/// Realtime hub interface. Manage the WebSocket connection include User, Agent and Model. +/// +public interface IRealtimeHub +{ + Task Listen(WebSocket userWebSocket, Func onUserMessageReceived); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs new file mode 100644 index 00000000..60fec1dc --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Abstraction.Realtime.Models; + +public class RealtimeHubConnection +{ + public string Event { get; set; } = null!; + public string StreamId { get; set; } = null!; + public string EntryAgentId { get; set; } = null!; + public string ConversationId { get; set; } = null!; + public string Data { get; set; } = string.Empty; + public string Model { get; set; } = null!; + public Func OnModelMessageReceived { get; set; } = null!; + public Func OnModelAudioResponseDone { get; set; } = null!; + public Func OnModelUserInterrupted { get; set; } = null!; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 67a1f901..a58600e5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -146,7 +146,9 @@ public interface IBotSharpRepository : IHaveServiceProvider => throw new NotImplementedException(); List GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds) => throw new NotImplementedException(); - IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + => throw new NotImplementedException(); + List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperlimit = 100) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs index d801e943..742b39e9 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs @@ -15,6 +15,8 @@ using BotSharp.Core.Roles.Services; using BotSharp.Abstraction.Templating; using BotSharp.Core.Templating; using BotSharp.Abstraction.Infrastructures.Enums; +using BotSharp.Abstraction.Realtime; +using BotSharp.Core.Realtime; namespace BotSharp.Core; @@ -171,5 +173,7 @@ public static class BotSharpCoreExtensions }); services.AddSingleton(loader); + + services.AddScoped(); } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 0c02918e..63a257e5 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -107,7 +107,7 @@ public partial class ConversationService : IConversationService record.Id = sess.Id.IfNullOrEmptyAs(Guid.NewGuid().ToString()); record.UserId = sess.UserId.IfNullOrEmptyAs(foundUserId); record.Tags = sess.Tags; - record.Title = "New Conversation"; + record.Title = string.IsNullOrEmpty(record.Title) ? "New Conversation" : record.Title; db.CreateNewConversation(record); @@ -221,4 +221,18 @@ public partial class ConversationService : IConversationService { _state.Save(); } + + public async Task> GetConversationStateSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false) + { + var keys = new List(); + if (!preLoad && string.IsNullOrWhiteSpace(query)) + { + return keys; + } + + var db = _services.GetRequiredService(); + keys = db.GetConversationStateSearchKeys(convUpperlimit: convlimit); + keys = preLoad ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList(); + return keys.OrderBy(x => x).Take(keyLimit).ToList(); + } } diff --git a/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs new file mode 100644 index 00000000..c44f74d0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs @@ -0,0 +1,170 @@ +using BotSharp.Abstraction.Realtime; +using System.Net.WebSockets; +using System; +using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Agents.Models; + +namespace BotSharp.Core.Realtime; + +public class RealtimeHub : IRealtimeHub +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + public RealtimeHub(IServiceProvider services, ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Listen(WebSocket userWebSocket, + Func onUserMessageReceived) + { + var buffer = new byte[1024 * 4]; + WebSocketReceiveResult result; + + var llmProviderService = _services.GetRequiredService(); + var model = llmProviderService.GetProviderModel("openai", "gpt-4", + realTime: true).Name; + + var completer = _services.GetServices().First(x => x.Provider == "openai"); + completer.SetModelName(model); + + do + { + result = await userWebSocket.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); + string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + _logger.LogDebug($"Received from user: {receivedText}"); + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + + var conn = onUserMessageReceived(receivedText); + conn.Model = model; + + if (conn.Event == "user_connected") + { + await ConnectToModel(completer, userWebSocket, conn); + } + else if (conn.Event == "user_data_received") + { + await completer.AppenAudioBuffer(conn.Data); + } + else if (conn.Event == "user_disconnected") + { + await completer.Disconnect(); + } + } while (!result.CloseStatus.HasValue); + + await userWebSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userWebSocket, RealtimeHubConnection conn) + { + var hookProvider = _services.GetRequiredService(); + var storage = _services.GetRequiredService(); + + var convService = _services.GetRequiredService(); + convService.SetConversationId(conn.ConversationId, []); + var conversation = await convService.GetConversation(conn.ConversationId); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(conversation.AgentId); + conn.EntryAgentId = agent.Id; + + var routing = _services.GetRequiredService(); + var dialogs = convService.GetDialogHistory(); + routing.Context.SetDialogs(dialogs); + + await completer.Connect(conn, + onModelReady: async () => + { + // Control initial session + await completer.UpdateInitialSession(conn); + + + // Add dialog history + foreach (var item in dialogs) + { + await completer.InsertConversationItem(item); + } + + if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) + { + // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); + } + else + { + await completer.TriggerModelInference("Reply based on the conversation context."); + } + }, + onModelAudioDeltaReceived: async audioDeltaData => + { + var data = conn.OnModelMessageReceived(audioDeltaData); + await SendEventToUser(userWebSocket, data); + }, + onModelAudioResponseDone: async () => + { + var data = conn.OnModelAudioResponseDone(); + await SendEventToUser(userWebSocket, data); + }, + onAudioTranscriptDone: async transcript => + { + + }, + onModelResponseDone: async messages => + { + foreach (var message in messages) + { + // Invoke function + if (message.MessageType == "function_call") + { + await routing.InvokeFunction(message.FunctionName, message); + message.Role = AgentRole.Function; + await completer.InsertConversationItem(message); + await completer.TriggerModelInference("Reply based on the function's output."); + } + else + { + // append transcript to conversation + storage.Append(conn.ConversationId, message); + dialogs.Add(message); + + foreach (var hook in hookProvider.HooksOrderByPriority) + { + hook.SetAgent(agent) + .SetConversation(conversation); + + if (!string.IsNullOrEmpty(message.Content)) + { + await hook.OnMessageReceived(message); + } + } + } + } + }, + onConversationItemCreated: async response => + { + + }, + onInputAudioTranscriptionCompleted: async message => + { + // append transcript to conversation + storage.Append(conn.ConversationId, message); + dialogs.Add(message); + }, + onUserInterrupted: async () => + { + var data = conn.OnModelUserInterrupted(); + await SendEventToUser(userWebSocket, data); + }); + } + + private async Task SendEventToUser(WebSocket webSocket, object message) + { + var data = JsonSerializer.Serialize(message); + var buffer = Encoding.UTF8.GetBytes(data); + await webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 7d96357d..100bbf16 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -131,7 +131,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository public void UpdateConversationStatus(string conversationId, string status) => throw new NotImplementedException(); - public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) => throw new NotImplementedException(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index c1cdd69f..25682662 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -1,276 +1,235 @@ using BotSharp.Abstraction.Loggers.Models; using System.IO; -namespace BotSharp.Core.Repository +namespace BotSharp.Core.Repository; + +public partial class FileRepository { - public partial class FileRepository + public void CreateNewConversation(Conversation conversation) { - public void CreateNewConversation(Conversation conversation) + var utcNow = DateTime.UtcNow; + conversation.CreatedTime = utcNow; + conversation.UpdatedTime = utcNow; + conversation.Tags ??= new(); + + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversation.Id); + if (!Directory.Exists(dir)) { - var utcNow = DateTime.UtcNow; - conversation.CreatedTime = utcNow; - conversation.UpdatedTime = utcNow; - conversation.Tags ??= new(); - - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversation.Id); - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - - var convFile = Path.Combine(dir, CONVERSATION_FILE); - if (!File.Exists(convFile)) - { - File.WriteAllText(convFile, JsonSerializer.Serialize(conversation, _options)); - } - - var dialogFile = Path.Combine(dir, DIALOG_FILE); - if (!File.Exists(dialogFile)) - { - File.WriteAllText(dialogFile, "[]"); - } - - var stateFile = Path.Combine(dir, STATE_FILE); - if (!File.Exists(stateFile)) - { - File.WriteAllText(stateFile, JsonSerializer.Serialize(new List(), _options)); - } - - var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE); - if (!File.Exists(breakpointFile)) - { - File.WriteAllText(breakpointFile, JsonSerializer.Serialize(new List(), _options)); - } + Directory.CreateDirectory(dir); } - public bool DeleteConversations(IEnumerable conversationIds) + var convFile = Path.Combine(dir, CONVERSATION_FILE); + if (!File.Exists(convFile)) { - if (conversationIds.IsNullOrEmpty()) return false; - - foreach (var conversationId in conversationIds) - { - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) continue; - - Directory.Delete(convDir, true); - } - - return true; + File.WriteAllText(convFile, JsonSerializer.Serialize(conversation, _options)); } - [SideCar] - public List GetConversationDialogs(string conversationId) + var dialogFile = Path.Combine(dir, DIALOG_FILE); + if (!File.Exists(dialogFile)) { - var dialogs = new List(); - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var dialogDir = Path.Combine(convDir, DIALOG_FILE); - var texts = File.ReadAllText(dialogDir); - try - { - dialogs = JsonSerializer.Deserialize>(texts, _options) ?? new List(); - } - catch - { - dialogs = new List(); - } - } - - return dialogs; + File.WriteAllText(dialogFile, "[]"); } - [SideCar] - public void AppendConversationDialogs(string conversationId, List dialogs) + var stateFile = Path.Combine(dir, STATE_FILE); + if (!File.Exists(stateFile)) + { + File.WriteAllText(stateFile, JsonSerializer.Serialize(new List(), _options)); + } + + var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE); + if (!File.Exists(breakpointFile)) + { + File.WriteAllText(breakpointFile, JsonSerializer.Serialize(new List(), _options)); + } + } + + public bool DeleteConversations(IEnumerable conversationIds) + { + if (conversationIds.IsNullOrEmpty()) return false; + + foreach (var conversationId in conversationIds) { var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) + if (string.IsNullOrEmpty(convDir)) continue; + + Directory.Delete(convDir, true); + } + + return true; + } + + [SideCar] + public List GetConversationDialogs(string conversationId) + { + var dialogs = new List(); + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var dialogDir = Path.Combine(convDir, DIALOG_FILE); + var texts = File.ReadAllText(dialogDir); + try { - var dialogFile = Path.Combine(convDir, DIALOG_FILE); - if (File.Exists(dialogFile)) - { - var prevDialogs = File.ReadAllText(dialogFile); - var elements = JsonSerializer.Deserialize>(prevDialogs, _options); - if (elements != null) - { - elements.AddRange(dialogs); - } - else - { - elements = elements ?? new List(); - } - - File.WriteAllText(dialogFile, JsonSerializer.Serialize(elements, _options)); - } - - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - if (File.Exists(convFile)) - { - var json = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(json, _options); - if (conv != null) - { - conv.DialogCount += dialogs.Count(); - conv.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - } - } + dialogs = JsonSerializer.Deserialize>(texts, _options) ?? new List(); + } + catch + { + dialogs = new List(); } } - public void UpdateConversationTitle(string conversationId, string title) + return dialogs; + } + + [SideCar] + public void AppendConversationDialogs(string conversationId, List dialogs) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var content = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(content, _options); - if (record != null) - { - record.Title = title; - record.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); - } - } - } - public void UpdateConversationTitleAlias(string conversationId, string titleAlias) - { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var content = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(content, _options); - if (record != null) - { - record.TitleAlias = titleAlias; - record.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); - } - } - } - - public bool UpdateConversationTags(string conversationId, List tags) - { - if (string.IsNullOrEmpty(conversationId)) return false; - - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return false; - - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - if (!File.Exists(convFile)) return false; - - var json = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(json, _options); - conv.Tags = tags ?? new(); - conv.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - return true; - } - - public bool AppendConversationTags(string conversationId, List tags) - { - if (string.IsNullOrEmpty(conversationId) || tags.IsNullOrEmpty()) return false; - - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return false; - - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - if (!File.Exists(convFile)) return false; - - var json = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(json, _options); - - var curTags = conv.Tags ?? new(); - var newTags = curTags.Concat(tags).Distinct(StringComparer.InvariantCultureIgnoreCase).ToList(); - conv.Tags = newTags; - conv.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - return true; - } - - public bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request) - { - if (string.IsNullOrEmpty(conversationId)) return false; - - var dialogs = GetConversationDialogs(conversationId); - var candidates = dialogs.Where(x => x.MetaData.MessageId == request.Message.MetaData.MessageId - && x.MetaData.Role == request.Message.MetaData.Role).ToList(); - - var found = candidates.Where((_, idx) => idx == request.InnderIndex).FirstOrDefault(); - if (found == null) return false; - - found.Content = request.Message.Content; - found.RichContent = request.Message.RichContent; - - if (!string.IsNullOrEmpty(found.SecondaryContent)) - { - found.SecondaryContent = request.Message.Content; - } - - if (!string.IsNullOrEmpty(found.SecondaryRichContent)) - { - found.SecondaryRichContent = request.Message.RichContent; - } - - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return false; - var dialogFile = Path.Combine(convDir, DIALOG_FILE); - File.WriteAllText(dialogFile, JsonSerializer.Serialize(dialogs, _options)); - return true; - } - - [SideCar] - public void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint) - { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) + if (File.Exists(dialogFile)) { - var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE); - - if (!File.Exists(breakpointFile)) + var prevDialogs = File.ReadAllText(dialogFile); + var elements = JsonSerializer.Deserialize>(prevDialogs, _options); + if (elements != null) { - File.Create(breakpointFile); - } - - var content = File.ReadAllText(breakpointFile); - var records = JsonSerializer.Deserialize>(content, _options); - var newBreakpoint = new List() - { - new ConversationBreakpoint - { - MessageId = breakpoint.MessageId, - Breakpoint = breakpoint.Breakpoint, - Reason = breakpoint.Reason, - CreatedTime = DateTime.UtcNow, - } - }; - - if (records != null && !records.IsNullOrEmpty()) - { - records = records.Concat(newBreakpoint).ToList(); + elements.AddRange(dialogs); } else { - records = newBreakpoint; + elements = elements ?? new List(); } - File.WriteAllText(breakpointFile, JsonSerializer.Serialize(records, _options)); + File.WriteAllText(dialogFile, JsonSerializer.Serialize(elements, _options)); + } + + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + if (File.Exists(convFile)) + { + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + if (conv != null) + { + conv.DialogCount += dialogs.Count(); + conv.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + } } } + } - [SideCar] - public ConversationBreakpoint? GetConversationBreakpoint(string conversationId) + public void UpdateConversationTitle(string conversationId, string title) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) { - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var content = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(content, _options); + if (record != null) { - return null; + record.Title = title; + record.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); } + } + } + public void UpdateConversationTitleAlias(string conversationId, string titleAlias) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var content = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(content, _options); + if (record != null) + { + record.TitleAlias = titleAlias; + record.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options)); + } + } + } + public bool UpdateConversationTags(string conversationId, List tags) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return false; + + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + if (!File.Exists(convFile)) return false; + + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + conv.Tags = tags ?? new(); + conv.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + return true; + } + + public bool AppendConversationTags(string conversationId, List tags) + { + if (string.IsNullOrEmpty(conversationId) || tags.IsNullOrEmpty()) return false; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return false; + + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + if (!File.Exists(convFile)) return false; + + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + + var curTags = conv.Tags ?? new(); + var newTags = curTags.Concat(tags).Distinct(StringComparer.InvariantCultureIgnoreCase).ToList(); + conv.Tags = newTags; + conv.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + return true; + } + + public bool UpdateConversationMessage(string conversationId, UpdateMessageRequest request) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var dialogs = GetConversationDialogs(conversationId); + var candidates = dialogs.Where(x => x.MetaData.MessageId == request.Message.MetaData.MessageId + && x.MetaData.Role == request.Message.MetaData.Role).ToList(); + + var found = candidates.Where((_, idx) => idx == request.InnderIndex).FirstOrDefault(); + if (found == null) return false; + + found.Content = request.Message.Content; + found.RichContent = request.Message.RichContent; + + if (!string.IsNullOrEmpty(found.SecondaryContent)) + { + found.SecondaryContent = request.Message.Content; + } + + if (!string.IsNullOrEmpty(found.SecondaryRichContent)) + { + found.SecondaryRichContent = request.Message.RichContent; + } + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return false; + + var dialogFile = Path.Combine(convDir, DIALOG_FILE); + File.WriteAllText(dialogFile, JsonSerializer.Serialize(dialogs, _options)); + return true; + } + + [SideCar] + public void UpdateConversationBreakpoint(string conversationId, ConversationBreakpoint breakpoint) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE); + if (!File.Exists(breakpointFile)) { File.Create(breakpointFile); @@ -278,517 +237,599 @@ namespace BotSharp.Core.Repository var content = File.ReadAllText(breakpointFile); var records = JsonSerializer.Deserialize>(content, _options); - - return records?.LastOrDefault(); - } - - public ConversationState GetConversationStates(string conversationId) - { - var states = new List(); - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) + var newBreakpoint = new List() { - var stateFile = Path.Combine(convDir, STATE_FILE); - states = CollectConversationStates(stateFile); - } - - return new ConversationState(states); - } - - public void UpdateConversationStates(string conversationId, List states) - { - if (states.IsNullOrEmpty()) return; - - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var stateFile = Path.Combine(convDir, STATE_FILE); - if (File.Exists(stateFile)) + new ConversationBreakpoint { - var stateStr = JsonSerializer.Serialize(states, _options); - File.WriteAllText(stateFile, stateStr); + MessageId = breakpoint.MessageId, + Breakpoint = breakpoint.Breakpoint, + Reason = breakpoint.Reason, + CreatedTime = DateTime.UtcNow, } - } - } - - public void UpdateConversationStatus(string conversationId, string status) - { - var convDir = FindConversationDirectory(conversationId); - if (!string.IsNullOrEmpty(convDir)) - { - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - if (File.Exists(convFile)) - { - var json = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(json, _options); - conv.Status = status; - conv.UpdatedTime = DateTime.UtcNow; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - } - } - } - - public Conversation GetConversation(string conversationId) - { - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) return null; - - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var content = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(content, _options); - - var dialogFile = Path.Combine(convDir, DIALOG_FILE); - if (record != null) - { - record.Dialogs = CollectDialogElements(dialogFile); - } - - var stateFile = Path.Combine(convDir, STATE_FILE); - if (record != null) - { - var states = CollectConversationStates(stateFile); - var curStates = new Dictionary(); - states.ForEach(x => - { - curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; - }); - record.States = curStates; - } - - return record; - } - - public PagedItems GetConversations(ConversationFilter filter) - { - if (filter == null) - { - filter = ConversationFilter.Empty(); - } - - var records = new List(); - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - var pager = filter?.Pager ?? new Pagination(); - - if (!Directory.Exists(dir)) - { - Directory.CreateDirectory(dir); - } - - var totalDirs = Directory.GetDirectories(dir); - foreach (var d in totalDirs) - { - var convFile = Path.Combine(d, CONVERSATION_FILE); - if (!File.Exists(convFile)) continue; - - var json = File.ReadAllText(convFile); - var record = JsonSerializer.Deserialize(json, _options); - if (record == null) continue; - - var matched = true; - if (filter?.Id != null) - { - matched = matched && record.Id == filter.Id; - } - if (filter?.Title != null) - { - matched = matched && record.Title.Contains(filter.Title); - } - if (filter?.TitleAlias != null) - { - matched = matched && record.TitleAlias.Contains(filter.TitleAlias); - } - if (filter?.AgentId != null) - { - matched = matched && record.AgentId == filter.AgentId; - } - if (filter?.Status != null) - { - matched = matched && record.Status == filter.Status; - } - if (filter?.Channel != null) - { - matched = matched && record.Channel == filter.Channel; - } - if (filter?.UserId != null) - { - matched = matched && record.UserId == filter.UserId; - } - if (filter?.TaskId != null) - { - matched = matched && record.TaskId == filter.TaskId; - } - if (filter?.StartTime != null) - { - matched = matched && record.CreatedTime >= filter.StartTime.Value; - } - if (filter?.Tags != null && filter.Tags.Any()) - { - matched = matched && !record.Tags.IsNullOrEmpty() && record.Tags.Exists(t => filter.Tags.Contains(t)); - } - - // Check states - if (filter != null && !filter.States.IsNullOrEmpty()) - { - var stateFile = Path.Combine(d, STATE_FILE); - var convStates = CollectConversationStates(stateFile); - foreach (var pair in filter.States) - { - if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; - - var foundState = convStates.FirstOrDefault(x => x.Key.IsEqualTo(pair.Key)); - if (foundState == null) - { - matched = false; - break; - } - - if (!string.IsNullOrWhiteSpace(pair.Value)) - { - var curValue = foundState.Values.LastOrDefault()?.Data; - matched = matched && pair.Value.IsEqualTo(curValue); - } - } - } - - if (!matched) continue; - - records.Add(record); - } - - return new PagedItems - { - Items = records.OrderByDescending(x => x.CreatedTime).Skip(pager.Offset).Take(pager.Size), - Count = records.Count(), }; + + if (records != null && !records.IsNullOrEmpty()) + { + records = records.Concat(newBreakpoint).ToList(); + } + else + { + records = newBreakpoint; + } + + File.WriteAllText(breakpointFile, JsonSerializer.Serialize(records, _options)); + } + } + + [SideCar] + public ConversationBreakpoint? GetConversationBreakpoint(string conversationId) + { + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) + { + return null; } - public List GetLastConversations() + var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE); + if (!File.Exists(breakpointFile)) { - var records = new List(); - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - - foreach (var d in Directory.GetDirectories(dir)) - { - var path = Path.Combine(d, CONVERSATION_FILE); - if (!File.Exists(path)) continue; - - var json = File.ReadAllText(path); - var record = JsonSerializer.Deserialize(json, _options); - if (record == null) continue; - - records.Add(record); - } - return records.GroupBy(r => r.UserId) - .Select(g => g.OrderByDescending(x => x.CreatedTime).First()) - .ToList(); + File.Create(breakpointFile); } - public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds) + var content = File.ReadAllText(breakpointFile); + var records = JsonSerializer.Deserialize>(content, _options); + + return records?.LastOrDefault(); + } + + public ConversationState GetConversationStates(string conversationId) + { + var states = new List(); + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) { - var ids = new List(); - var batchLimit = 100; - var utcNow = DateTime.UtcNow; - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + var stateFile = Path.Combine(convDir, STATE_FILE); + states = CollectConversationStates(stateFile); + } - if (!Directory.Exists(dir)) + return new ConversationState(states); + } + + public void UpdateConversationStates(string conversationId, List states) + { + if (states.IsNullOrEmpty()) return; + + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var stateFile = Path.Combine(convDir, STATE_FILE); + if (File.Exists(stateFile)) { - Directory.CreateDirectory(dir); + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateFile, stateStr); } + } + } - if (batchSize <= 0 || batchSize > batchLimit) + public void UpdateConversationStatus(string conversationId, string status) + { + var convDir = FindConversationDirectory(conversationId); + if (!string.IsNullOrEmpty(convDir)) + { + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + if (File.Exists(convFile)) { - batchSize = batchLimit; - } - - if (bufferHours <= 0) - { - bufferHours = 12; - } - - if (messageLimit <= 0) - { - messageLimit = 2; - } - - foreach (var d in Directory.GetDirectories(dir)) - { - var convFile = Path.Combine(d, CONVERSATION_FILE); - if (!File.Exists(convFile)) - { - Directory.Delete(d, true); - continue; - } - var json = File.ReadAllText(convFile); var conv = JsonSerializer.Deserialize(json, _options); + conv.Status = status; + conv.UpdatedTime = DateTime.UtcNow; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + } + } + } - if (conv == null) - { - Directory.Delete(d, true); - continue; - } + public Conversation GetConversation(string conversationId) + { + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return null; - if (conv.UpdatedTime > utcNow.AddHours(-bufferHours)) - { - continue; - } + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var content = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(content, _options); - if ((excludeAgentIds.Contains(conv.AgentId) && conv.DialogCount == 0) - || (!excludeAgentIds.Contains(conv.AgentId) && conv.DialogCount <= messageLimit)) + var dialogFile = Path.Combine(convDir, DIALOG_FILE); + if (record != null) + { + record.Dialogs = CollectDialogElements(dialogFile); + } + + var stateFile = Path.Combine(convDir, STATE_FILE); + if (record != null) + { + var states = CollectConversationStates(stateFile); + var curStates = new Dictionary(); + states.ForEach(x => + { + curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty; + }); + record.States = curStates; + } + + return record; + } + + public PagedItems GetConversations(ConversationFilter filter) + { + if (filter == null) + { + filter = ConversationFilter.Empty(); + } + + var records = new List(); + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + var pager = filter?.Pager ?? new Pagination(); + + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + var totalDirs = Directory.GetDirectories(dir); + foreach (var d in totalDirs) + { + var convFile = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(convFile)) continue; + + var json = File.ReadAllText(convFile); + var record = JsonSerializer.Deserialize(json, _options); + if (record == null) continue; + + var matched = true; + if (filter?.Id != null) + { + matched = matched && record.Id == filter.Id; + } + if (filter?.Title != null) + { + matched = matched && record.Title.Contains(filter.Title); + } + if (filter?.TitleAlias != null) + { + matched = matched && record.TitleAlias.Contains(filter.TitleAlias); + } + if (filter?.AgentId != null) + { + matched = matched && record.AgentId == filter.AgentId; + } + if (filter?.Status != null) + { + matched = matched && record.Status == filter.Status; + } + if (filter?.Channel != null) + { + matched = matched && record.Channel == filter.Channel; + } + if (filter?.UserId != null) + { + matched = matched && record.UserId == filter.UserId; + } + if (filter?.TaskId != null) + { + matched = matched && record.TaskId == filter.TaskId; + } + if (filter?.StartTime != null) + { + matched = matched && record.CreatedTime >= filter.StartTime.Value; + } + if (filter?.Tags != null && filter.Tags.Any()) + { + matched = matched && !record.Tags.IsNullOrEmpty() && record.Tags.Exists(t => filter.Tags.Contains(t)); + } + + // Check states + if (filter != null && !filter.States.IsNullOrEmpty()) + { + var stateFile = Path.Combine(d, STATE_FILE); + var convStates = CollectConversationStates(stateFile); + foreach (var pair in filter.States) { - ids.Add(conv.Id); - if (ids.Count >= batchSize) + if (pair == null || string.IsNullOrWhiteSpace(pair.Key)) continue; + + var foundState = convStates.FirstOrDefault(x => x.Key.IsEqualTo(pair.Key)); + if (foundState == null) { - return ids; + matched = false; + break; + } + + if (!string.IsNullOrWhiteSpace(pair.Value)) + { + var curValue = foundState.Values.LastOrDefault()?.Data; + matched = matched && pair.Value.IsEqualTo(curValue); } } } - return ids; + + if (!matched) continue; + + records.Add(record); } - - public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + return new PagedItems { - var deletedMessageIds = new List(); - if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) + Items = records.OrderByDescending(x => x.CreatedTime).Skip(pager.Offset).Take(pager.Size), + Count = records.Count(), + }; + } + + public List GetLastConversations() + { + var records = new List(); + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + + foreach (var d in Directory.GetDirectories(dir)) + { + var path = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(path)) continue; + + var json = File.ReadAllText(path); + var record = JsonSerializer.Deserialize(json, _options); + if (record == null) continue; + + records.Add(record); + } + return records.GroupBy(r => r.UserId) + .Select(g => g.OrderByDescending(x => x.CreatedTime).First()) + .ToList(); + } + + public List GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds) + { + var ids = new List(); + var batchLimit = 100; + var utcNow = DateTime.UtcNow; + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + if (batchSize <= 0 || batchSize > batchLimit) + { + batchSize = batchLimit; + } + + if (bufferHours <= 0) + { + bufferHours = 12; + } + + if (messageLimit <= 0) + { + messageLimit = 2; + } + + foreach (var d in Directory.GetDirectories(dir)) + { + var convFile = Path.Combine(d, CONVERSATION_FILE); + if (!File.Exists(convFile)) { - return deletedMessageIds; + Directory.Delete(d, true); + continue; } - var dialogs = new List(); - - var convDir = FindConversationDirectory(conversationId); - if (string.IsNullOrEmpty(convDir)) + var json = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(json, _options); + + if (conv == null) { - return deletedMessageIds; + Directory.Delete(d, true); + continue; } - var dialogDir = Path.Combine(convDir, DIALOG_FILE); - dialogs = CollectDialogElements(dialogDir); - if (dialogs.IsNullOrEmpty()) + if (conv.UpdatedTime > utcNow.AddHours(-bufferHours)) { - return deletedMessageIds; + continue; } - var foundIdx = dialogs.FindIndex(x => x.MetaData?.MessageId == messageId); - if (foundIdx < 0) + if ((excludeAgentIds.Contains(conv.AgentId) && conv.DialogCount == 0) + || (!excludeAgentIds.Contains(conv.AgentId) && conv.DialogCount <= messageLimit)) { - return deletedMessageIds; + ids.Add(conv.Id); + if (ids.Count >= batchSize) + { + return ids; + } } + } + return ids; + } - deletedMessageIds = dialogs.Where((x, idx) => idx >= foundIdx && !string.IsNullOrEmpty(x.MetaData?.MessageId)) - .Select(x => x.MetaData.MessageId).Distinct().ToList(); - - // Handle truncated dialogs - var isSaved = HandleTruncatedDialogs(convDir, dialogDir, dialogs, foundIdx); - - // Handle truncated states - var refTime = dialogs.ElementAt(foundIdx).MetaData.CreateTime; - var stateDir = Path.Combine(convDir, STATE_FILE); - var states = CollectConversationStates(stateDir); - isSaved = HandleTruncatedStates(stateDir, states, messageId, refTime); - - // Handle truncated breakpoints - var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE); - var breakpoints = CollectConversationBreakpoints(breakpointDir); - isSaved = HandleTruncatedBreakpoints(breakpointDir, breakpoints, refTime); - - // Remove logs - if (cleanLog) - { - HandleTruncatedLogs(convDir, refTime); - } + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + { + var deletedMessageIds = new List(); + if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) + { return deletedMessageIds; } - - #region Private methods - private string? FindConversationDirectory(string conversationId) + var dialogs = new List(); + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) { - if (string.IsNullOrEmpty(conversationId)) return null; - - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId); - if (!Directory.Exists(dir)) return null; - - return dir; + return deletedMessageIds; } - private List CollectDialogElements(string dialogDir) + var dialogDir = Path.Combine(convDir, DIALOG_FILE); + dialogs = CollectDialogElements(dialogDir); + if (dialogs.IsNullOrEmpty()) { - var dialogs = new List(); - - if (!File.Exists(dialogDir)) return dialogs; - - var texts = File.ReadAllText(dialogDir); - dialogs = JsonSerializer.Deserialize>(texts) ?? new List(); - return dialogs; + return deletedMessageIds; } - private string ParseDialogElements(List dialogs) + var foundIdx = dialogs.FindIndex(x => x.MetaData?.MessageId == messageId); + if (foundIdx < 0) { - if (dialogs.IsNullOrEmpty()) return "[]"; - - return JsonSerializer.Serialize(dialogs, _options) ?? "[]"; + return deletedMessageIds; } - private List CollectConversationStates(string stateFile) + deletedMessageIds = dialogs.Where((x, idx) => idx >= foundIdx && !string.IsNullOrEmpty(x.MetaData?.MessageId)) + .Select(x => x.MetaData.MessageId).Distinct().ToList(); + + // Handle truncated dialogs + var isSaved = HandleTruncatedDialogs(convDir, dialogDir, dialogs, foundIdx); + + // Handle truncated states + var refTime = dialogs.ElementAt(foundIdx).MetaData.CreateTime; + var stateDir = Path.Combine(convDir, STATE_FILE); + var states = CollectConversationStates(stateDir); + isSaved = HandleTruncatedStates(stateDir, states, messageId, refTime); + + // Handle truncated breakpoints + var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE); + var breakpoints = CollectConversationBreakpoints(breakpointDir); + isSaved = HandleTruncatedBreakpoints(breakpointDir, breakpoints, refTime); + + // Remove logs + if (cleanLog) { - var states = new List(); - if (!File.Exists(stateFile)) return states; - - var stateStr = File.ReadAllText(stateFile); - if (string.IsNullOrEmpty(stateStr)) return states; - - states = JsonSerializer.Deserialize>(stateStr, _options); - return states ?? new List(); + HandleTruncatedLogs(convDir, refTime); } - private List CollectConversationBreakpoints(string breakpointFile) - { - var breakpoints = new List(); - if (!File.Exists(breakpointFile)) return breakpoints; - - var content = File.ReadAllText(breakpointFile); - if (string.IsNullOrEmpty(content)) return breakpoints; - - breakpoints = JsonSerializer.Deserialize>(content, _options); - return breakpoints ?? new List(); - } - - private bool HandleTruncatedDialogs(string convDir, string dialogDir, List dialogs, int foundIdx) - { - var truncatedDialogs = dialogs.Where((x, idx) => idx < foundIdx).ToList(); - var isSaved = SaveTruncatedDialogs(dialogDir, truncatedDialogs); - var convFile = Path.Combine(convDir, CONVERSATION_FILE); - var convJson = File.ReadAllText(convFile); - var conv = JsonSerializer.Deserialize(convJson, _options); - if (conv != null) - { - conv.DialogCount = truncatedDialogs.Count; - File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); - } - return isSaved; - } - - private bool HandleTruncatedStates(string stateDir, List states, string refMsgId, DateTime refTime) - { - var truncatedStates = new List(); - foreach (var state in states) - { - if (!state.Versioning) - { - truncatedStates.Add(state); - continue; - } - - var values = state.Values.Where(x => x.MessageId != refMsgId) - .Where(x => x.UpdateTime < refTime) - .ToList(); - if (values.Count == 0) continue; - - state.Values = values; - truncatedStates.Add(state); - } - - var isSaved = SaveTruncatedStates(stateDir, truncatedStates); - return isSaved; - } - - private bool HandleTruncatedBreakpoints(string breakpointDir, List breakpoints, DateTime refTime) - { - var truncatedBreakpoints = breakpoints?.Where(x => x.CreatedTime < refTime)? - .ToList() ?? new List(); - - var isSaved = SaveTruncatedBreakpoints(breakpointDir, truncatedBreakpoints); - return isSaved; - } - - private bool HandleTruncatedLogs(string convDir, DateTime refTime) - { - var contentLogDir = Path.Combine(convDir, "content_log"); - var stateLogDir = Path.Combine(convDir, "state_log"); - - if (Directory.Exists(contentLogDir)) - { - foreach (var file in Directory.GetFiles(contentLogDir)) - { - var text = File.ReadAllText(file); - var log = JsonSerializer.Deserialize(text); - if (log == null) continue; - - if (log.CreateTime >= refTime) - { - File.Delete(file); - } - } - } - - if (Directory.Exists(stateLogDir)) - { - foreach (var file in Directory.GetFiles(stateLogDir)) - { - var text = File.ReadAllText(file); - var log = JsonSerializer.Deserialize(text); - if (log == null) continue; - - if (log.CreateTime >= refTime) - { - File.Delete(file); - } - } - } - - return true; - } - - private bool SaveTruncatedDialogs(string dialogDir, List dialogs) - { - if (string.IsNullOrEmpty(dialogDir) || dialogs == null) return false; - if (!File.Exists(dialogDir)) File.Create(dialogDir); - - var texts = ParseDialogElements(dialogs); - File.WriteAllText(dialogDir, texts); - return true; - } - - private bool SaveTruncatedStates(string stateDir, List states) - { - if (string.IsNullOrEmpty(stateDir) || states == null) return false; - if (!File.Exists(stateDir)) File.Create(stateDir); - - var stateStr = JsonSerializer.Serialize(states, _options); - File.WriteAllText(stateDir, stateStr); - return true; - } - - private bool SaveTruncatedBreakpoints(string breakpointDir, List breakpoints) - { - if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false; - if (!File.Exists(breakpointDir)) File.Create(breakpointDir); - - var breakpointStr = JsonSerializer.Serialize(breakpoints, _options); - File.WriteAllText(breakpointDir, breakpointStr); - return true; - } - - private string? EncodeText(string? text) - { - if (string.IsNullOrEmpty(text)) return text; - - var bytes = Encoding.UTF8.GetBytes(text); - var encoded = Convert.ToBase64String(bytes); - return encoded; - } - - private string? DecodeText(string? text) - { - if (string.IsNullOrEmpty(text)) return text; - - var decoded = Convert.FromBase64String(text); - var origin = Encoding.UTF8.GetString(decoded); - return origin; - } - #endregion + return deletedMessageIds; } + +#if !DEBUG + [SharpCache(10)] +#endif + public List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperlimit = 100) + { + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + if (!Directory.Exists(dir)) return []; + + var count = 0; + var keys = new List(); + + foreach (var d in Directory.GetDirectories(dir)) + { + var convFile = Path.Combine(d, CONVERSATION_FILE); + var stateFile = Path.Combine(d, STATE_FILE); + if (!File.Exists(convFile) || !File.Exists(stateFile)) + { + continue; + } + + var convJson = File.ReadAllText(convFile); + var stateJson = File.ReadAllText(stateFile); + var conv = JsonSerializer.Deserialize(convJson, _options); + var states = JsonSerializer.Deserialize>(stateJson, _options); + if (conv == null || conv.DialogCount < messageLowerLimit) + { + continue; + } + + var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? []; + keys.AddRange(stateKeys); + count++; + + if (count >= convUpperlimit) + { + break; + } + } + + return keys.Distinct().ToList(); + } + + + #region Private methods + private string? FindConversationDirectory(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return null; + + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId); + if (!Directory.Exists(dir)) return null; + + return dir; + } + + private List CollectDialogElements(string dialogDir) + { + var dialogs = new List(); + + if (!File.Exists(dialogDir)) return dialogs; + + var texts = File.ReadAllText(dialogDir); + dialogs = JsonSerializer.Deserialize>(texts) ?? new List(); + return dialogs; + } + + private string ParseDialogElements(List dialogs) + { + if (dialogs.IsNullOrEmpty()) return "[]"; + + return JsonSerializer.Serialize(dialogs, _options) ?? "[]"; + } + + private List CollectConversationStates(string stateFile) + { + var states = new List(); + if (!File.Exists(stateFile)) return states; + + var stateStr = File.ReadAllText(stateFile); + if (string.IsNullOrEmpty(stateStr)) return states; + + states = JsonSerializer.Deserialize>(stateStr, _options); + return states ?? new List(); + } + + private List CollectConversationBreakpoints(string breakpointFile) + { + var breakpoints = new List(); + if (!File.Exists(breakpointFile)) return breakpoints; + + var content = File.ReadAllText(breakpointFile); + if (string.IsNullOrEmpty(content)) return breakpoints; + + breakpoints = JsonSerializer.Deserialize>(content, _options); + return breakpoints ?? new List(); + } + + private bool HandleTruncatedDialogs(string convDir, string dialogDir, List dialogs, int foundIdx) + { + var truncatedDialogs = dialogs.Where((x, idx) => idx < foundIdx).ToList(); + var isSaved = SaveTruncatedDialogs(dialogDir, truncatedDialogs); + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var convJson = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(convJson, _options); + if (conv != null) + { + conv.DialogCount = truncatedDialogs.Count; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + } + return isSaved; + } + + private bool HandleTruncatedStates(string stateDir, List states, string refMsgId, DateTime refTime) + { + var truncatedStates = new List(); + foreach (var state in states) + { + if (!state.Versioning) + { + truncatedStates.Add(state); + continue; + } + + var values = state.Values.Where(x => x.MessageId != refMsgId) + .Where(x => x.UpdateTime < refTime) + .ToList(); + if (values.Count == 0) continue; + + state.Values = values; + truncatedStates.Add(state); + } + + var isSaved = SaveTruncatedStates(stateDir, truncatedStates); + return isSaved; + } + + private bool HandleTruncatedBreakpoints(string breakpointDir, List breakpoints, DateTime refTime) + { + var truncatedBreakpoints = breakpoints?.Where(x => x.CreatedTime < refTime)? + .ToList() ?? new List(); + + var isSaved = SaveTruncatedBreakpoints(breakpointDir, truncatedBreakpoints); + return isSaved; + } + + private bool HandleTruncatedLogs(string convDir, DateTime refTime) + { + var contentLogDir = Path.Combine(convDir, "content_log"); + var stateLogDir = Path.Combine(convDir, "state_log"); + + if (Directory.Exists(contentLogDir)) + { + foreach (var file in Directory.GetFiles(contentLogDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + if (log.CreateTime >= refTime) + { + File.Delete(file); + } + } + } + + if (Directory.Exists(stateLogDir)) + { + foreach (var file in Directory.GetFiles(stateLogDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + if (log.CreateTime >= refTime) + { + File.Delete(file); + } + } + } + + return true; + } + + private bool SaveTruncatedDialogs(string dialogDir, List dialogs) + { + if (string.IsNullOrEmpty(dialogDir) || dialogs == null) return false; + if (!File.Exists(dialogDir)) File.Create(dialogDir); + + var texts = ParseDialogElements(dialogs); + File.WriteAllText(dialogDir, texts); + return true; + } + + private bool SaveTruncatedStates(string stateDir, List states) + { + if (string.IsNullOrEmpty(stateDir) || states == null) return false; + if (!File.Exists(stateDir)) File.Create(stateDir); + + var stateStr = JsonSerializer.Serialize(states, _options); + File.WriteAllText(stateDir, stateStr); + return true; + } + + private bool SaveTruncatedBreakpoints(string breakpointDir, List breakpoints) + { + if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false; + if (!File.Exists(breakpointDir)) File.Create(breakpointDir); + + var breakpointStr = JsonSerializer.Serialize(breakpoints, _options); + File.WriteAllText(breakpointDir, breakpointStr); + return true; + } + + private string? EncodeText(string? text) + { + if (string.IsNullOrEmpty(text)) return text; + + var bytes = Encoding.UTF8.GetBytes(text); + var encoded = Convert.ToBase64String(bytes); + return encoded; + } + + private string? DecodeText(string? text) + { + if (string.IsNullOrEmpty(text)) return text; + + var decoded = Convert.FromBase64String(text); + var origin = Encoding.UTF8.GetString(decoded); + return origin; + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index b2defb2e..7461518d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -553,6 +553,16 @@ public class ConversationController : ControllerBase } #endregion + #region Search state keys + [HttpGet("/conversation/state/keys")] + public async Task> GetConversationStateKeys([FromQuery] string query, [FromQuery] int keyLimit = 10, [FromQuery] bool preLoad = false) + { + var convService = _services.GetRequiredService(); + var keys = await convService.GetConversationStateSearhKeys(query, keyLimit: keyLimit, preLoad: preLoad); + return keys; + } + #endregion + #region Private methods private void SetStates(IConversationService conv, NewMessageModel input) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs index 12442cdc..a0e66e98 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs @@ -4,5 +4,6 @@ public class ConversationDialogDocument : MongoBase { public string ConversationId { get; set; } public string AgentId { get; set; } + public DateTime UpdatedTime { get; set; } public List Dialogs { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs index 7b515616..d83945a8 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs @@ -4,6 +4,7 @@ public class ConversationStateDocument : MongoBase { public string ConversationId { get; set; } public string AgentId { get; set; } + public DateTime UpdatedTime { get; set; } public List States { get; set; } = new List(); public List Breakpoints { get; set; } = new List(); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentKnowledgeBaseMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentKnowledgeBaseMongoElement.cs index 3e6b3500..b82c6c47 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentKnowledgeBaseMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentKnowledgeBaseMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class AgentKnowledgeBaseMongoElement { public string Name { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs index 32989ad1..b626d7da 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class AgentLlmConfigMongoElement { public string? Provider { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentResponseMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentResponseMongoElement.cs index 26ddebbc..99cad6a5 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentResponseMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentResponseMongoElement.cs @@ -2,7 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; -[BsonIgnoreExtraElements] +[BsonIgnoreExtraElements(Inherited = true)] public class AgentResponseMongoElement { public string Prefix { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentRuleMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentRuleMongoElement.cs index 744fab0c..718ffe17 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentRuleMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentRuleMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class AgentRuleMongoElement { public string TriggerName { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentTemplateMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentTemplateMongoElement.cs index 847ec5c9..f7aa1275 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentTemplateMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentTemplateMongoElement.cs @@ -2,7 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; -[BsonIgnoreExtraElements] +[BsonIgnoreExtraElements(Inherited = true)] public class AgentTemplateMongoElement { public string Name { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs index c05f07ef..f398b37c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentUtilityMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class AgentUtilityMongoElement { public string Name { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs index db17d353..6dd05bf2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs @@ -1,5 +1,6 @@ namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class BreakpointMongoElement { public string? MessageId { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs index 884c638c..91ad7c3a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/ChannelInstructionMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Agents.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class ChannelInstructionMongoElement { public string Channel { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs index 099286b8..06120581 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/CronTaskMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Crontab.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class CronTaskMongoElement { public string Topic { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs index 1179d3cd..126dfee8 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class DialogMongoElement { public DialogMetaDataMongoElement MetaData { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs index 6f72c517..a0078397 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs @@ -3,7 +3,7 @@ using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Models; -[BsonIgnoreExtraElements] +[BsonIgnoreExtraElements(Inherited = true)] public class FunctionDefMongoElement { public string Name { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs index 31049e5d..de6638ee 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeEmbeddingConfigMongoModel.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class KnowledgeEmbeddingConfigMongoModel { public string Provider { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs index bc213dea..9d09e53c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Knowledges.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class KnowledgeFileMetaRefMongoModel { public string Id { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs index 07ade285..3cf23314 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeVectorStoreConfigMongoModel.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.VectorStorage.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class KnowledgeVectorStoreConfigMongoModel { public string Provider { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs index ced9030d..94723817 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs @@ -1,5 +1,6 @@ namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class PromptLogMongoElement { public string MessageId { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/RoutingRuleMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/RoutingRuleMongoElement.cs index 293ffc47..3b8e495c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/RoutingRuleMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/RoutingRuleMongoElement.cs @@ -2,7 +2,7 @@ using BotSharp.Abstraction.Routing.Models; namespace BotSharp.Plugin.MongoStorage.Models; -[BsonIgnoreExtraElements] +[BsonIgnoreExtraElements(Inherited = true)] public class RoutingRuleMongoElement { public string Field { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs index a939f59c..276b17aa 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class StateMongoElement { public string Key { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs index 0d090b78..e5a7ede5 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs @@ -1,5 +1,6 @@ namespace BotSharp.Plugin.MongoStorage.Models; +[BsonIgnoreExtraElements(Inherited = true)] public class TranslationMemoryMongoElement { public string TranslatedText { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index c83ccf17..cabaecd2 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -30,7 +30,8 @@ public partial class MongoRepository Id = Guid.NewGuid().ToString(), ConversationId = convDoc.Id, AgentId = conversation.AgentId, - Dialogs = new List() + Dialogs = [], + UpdatedTime = utcNow }; var stateDoc = new ConversationStateDocument @@ -38,8 +39,9 @@ public partial class MongoRepository Id = Guid.NewGuid().ToString(), ConversationId = convDoc.Id, AgentId = conversation.AgentId, - States = new List(), - Breakpoints = new List() + States = [], + Breakpoints = [], + UpdatedTime = utcNow }; _dc.Conversations.InsertOne(convDoc); @@ -97,7 +99,8 @@ public partial class MongoRepository var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList(); - var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements); + var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements) + .Set(x => x.UpdatedTime, DateTime.UtcNow); var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow) .Inc(x => x.DialogCount, dialogs.Count); @@ -190,7 +193,8 @@ public partial class MongoRepository found.SecondaryRichContent = request.Message.RichContent; } - var update = Builders.Update.Set(x => x.Dialogs, dialogs); + var update = Builders.Update.Set(x => x.Dialogs, dialogs) + .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationDialogs.UpdateOne(filter, update); return true; } @@ -208,7 +212,8 @@ public partial class MongoRepository Reason = breakpoint.Reason }; var filterState = Builders.Filter.Eq(x => x.ConversationId, conversationId); - var updateState = Builders.Update.Push(x => x.Breakpoints, newBreakpoint); + var updateState = Builders.Update.Push(x => x.Breakpoints, newBreakpoint) + .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterState, updateState); } @@ -258,7 +263,8 @@ public partial class MongoRepository var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); - var updateStates = Builders.Update.Set(x => x.States, saveStates); + var updateStates = Builders.Update.Set(x => x.States, saveStates) + .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterStates, updateStates); } @@ -500,7 +506,7 @@ public partial class MongoRepository return conversationIds.Take(batchSize).ToList(); } - public IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false) + public List TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { var deletedMessageIds = new List(); if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) @@ -566,11 +572,13 @@ public partial class MongoRepository } // Update + foundStates.UpdatedTime = DateTime.UtcNow; _dc.ConversationStates.ReplaceOne(stateFilter, foundStates); } // Save dialogs foundDialog.Dialogs = truncatedDialogs; + foundDialog.UpdatedTime = DateTime.UtcNow; _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog); // Update conversation @@ -603,6 +611,29 @@ public partial class MongoRepository return deletedMessageIds; } +#if !DEBUG + [SharpCache(10)] +#endif + public List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperlimit = 100) + { + var convFilter = Builders.Filter.Gte(x => x.DialogCount, messageLowerLimit); + var conversations = _dc.Conversations.Find(convFilter) + .SortByDescending(x => x.UpdatedTime) + .Limit(convUpperlimit) + .ToList(); + + if (conversations.IsNullOrEmpty()) return []; + + var convIds = conversations.Select(x => x.Id).ToList(); + var stateFilter = Builders.Filter.In(x => x.ConversationId, convIds); + + var states = _dc.ConversationStates.Find(stateFilter).ToList(); + var keys = states.SelectMany(x => x.States.Select(x => x.Key)).Distinct().ToList(); + return keys; + } + + + private string ConvertSnakeCaseToPascalCase(string snakeCase) { string[] words = snakeCase.Split('_'); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 3b6dd05f..9c88e8bd 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -429,6 +429,11 @@ public partial class MongoRepository return true; } + public Dashboard? GetDashboard(string userId = null) + { + return null; + } + public void AddDashboardConversation(string userId, string conversationId) { var user = _dc.Users.AsQueryable() diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs index 13b2739b..b39be957 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs @@ -9,6 +9,7 @@ global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Translation.Models; global using BotSharp.Abstraction.SideCar.Attributes; +global using BotSharp.Core.Infrastructures; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using MongoDB.Bson; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs new file mode 100644 index 00000000..b46d8cc7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ConversationItemCreated.cs @@ -0,0 +1,33 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ConversationItemCreated : ServerEventResponse +{ + [JsonPropertyName("item")] + public ConversationItemBody Item { get; set; } = new(); +} + +public class ConversationItemBody +{ + [JsonPropertyName("id")] + public string Id { get; set; } = null!; + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("role")] + public string Role { get; set;} = null!; + + [JsonPropertyName("content")] + public ConversationItemContent[] Content { get; set; } = []; +} + +public class ConversationItemContent +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("transcript")] + public string Transcript { get; set; } = null!; + + [JsonPropertyName("audio")] + public string Audio { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs similarity index 52% rename from src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs rename to src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index ce249e70..1aca181b 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -1,22 +1,42 @@ using BotSharp.Abstraction.Functions.Models; -using System.Text.Json.Serialization; -namespace BotSharp.Plugin.OpenAI.Models; +namespace BotSharp.Plugin.OpenAI.Models.Realtime; -public class RealtimeSessionRequest +public class RealtimeSessionBody { + [JsonPropertyName("id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Id { get; set; } = null!; + + [JsonPropertyName("object")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Object { get; set; } = null!; + [JsonPropertyName("model")] - public string Model { get; set; } = "gpt-4o-mini-realtime-preview-2024-12-17"; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Model { get; set; } = null!; [JsonPropertyName("temperature")] - public float temperature { get; set; } = 0.8f; + public float Temperature { get; set; } = 0.8f; [JsonPropertyName("modalities")] public string[] Modalities { get; set; } = ["audio", "text"]; + [JsonPropertyName("input_audio_format")] + public string InputAudioFormat { get; set; } = "pcm16"; + + [JsonPropertyName("output_audio_format")] + public string OutputAudioFormat { get; set; } = "pcm16"; + + [JsonPropertyName("input_audio_transcription")] + public InputAudioTranscription InputAudioTranscription { get; set; } = new(); + [JsonPropertyName("instructions")] public string Instructions { get; set; } = "You are a friendly assistant."; + [JsonPropertyName("voice")] + public string Voice { get; set; } = "sage"; + [JsonPropertyName("max_response_output_tokens")] public int MaxResponseOutputTokens { get; set; } = 512; @@ -46,4 +66,10 @@ public class RealtimeSessionTurnDetection [JsonPropertyName("type")] public string Type { get; set; } = "server_vad"; +} + +public class InputAudioTranscription +{ + [JsonPropertyName("model")] + public string Model { get; set; } = null!; } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs new file mode 100644 index 00000000..b2125e99 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionRequest.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class RealtimeSessionCreationRequest : RealtimeSessionBody +{ + +} + +/// +/// https://platform.openai.com/docs/api-reference/realtime-client-events/session/update +/// +public class RealtimeSessionUpdateRequest : RealtimeSessionBody +{ + +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs similarity index 76% rename from src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs rename to src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs index 0d0afa1a..a89928d8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionUpdate.cs @@ -1,4 +1,6 @@ -namespace BotSharp.Abstraction.Realtime.Models; +using BotSharp.Abstraction.Realtime.Models; + +namespace BotSharp.Plugin.OpenAI.Models.Realtime; public class RealtimeSessionUpdate { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs new file mode 100644 index 00000000..dbf299b2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioDelta.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ResponseAudioDelta : ServerEventResponse +{ + [JsonPropertyName("response_id")] + public string ResponseId { get; set; } = null!; + + [JsonPropertyName("item_id")] + public string ItemId { get; set; } = null!; + + [JsonPropertyName("output_index")] + public int OutputIndex { get; set; } + + [JsonPropertyName("content_index")] + public int ContentIndex { get; set; } + + [JsonPropertyName("delta")] + public string? Delta { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioTranscript.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioTranscript.cs new file mode 100644 index 00000000..3f83ef1b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseAudioTranscript.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ResponseAudioTranscript : ServerEventResponse +{ + [JsonPropertyName("response_id")] + public string ResponseId { get; set; } = null!; + + [JsonPropertyName("item_id")] + public string ItemId { get; set; } = null!; + + [JsonPropertyName("output_index")] + public int OutputIndex { get; set; } + + [JsonPropertyName("content_index")] + public int ContentIndex { get; set; } + + [JsonPropertyName("transcript")] + public string? Transcript { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs new file mode 100644 index 00000000..ae3db58d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ResponseDone.cs @@ -0,0 +1,102 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ResponseDone : ServerEventResponse +{ + [JsonPropertyName("response")] + public ResponseDoneBody Body { get; set; } = new(); +} + +public class ResponseDoneBody +{ + [JsonPropertyName("id")] + public string Id { get; set; } = null!; + + [JsonPropertyName("object")] + public string Object { get; set; } = null!; + + [JsonPropertyName("status")] + public string Status { get; set; } = null!; + + [JsonPropertyName("status_details")] + public ResponseDoneStatusDetail StatusDetails { get; set; } = new(); + + [JsonPropertyName("conversation_id")] + public string ConversationId { get; set; } = null!; + + [JsonPropertyName("usage")] + public ModelTokenUsage Usage { get; set; } = new(); + + [JsonPropertyName("modalities")] + public string[] Modalities { get; set; } = []; + + [JsonPropertyName("temperature")] + public float Temperature { get; set; } + + [JsonPropertyName("output_audio_format")] + public string OutputAudioFormat { get; set; } = null!; + + [JsonPropertyName("voice")] + public string Voice { get; set; } = null!; + + [JsonPropertyName("output")] + public ModelResponseDoneOutput[] Outputs { get; set; } = []; +} + +public class ModelTokenUsage +{ + [JsonPropertyName("total_tokens")] + public int TotalTokens { get; set; } + + [JsonPropertyName("input_tokens")] + public int InputTokens { get; set; } + + [JsonPropertyName("output_tokens")] + public int OutputTokens { get; set; } +} + +public class ModelResponseDoneOutput +{ + [JsonPropertyName("id")] + public string Id { get; set; } = null!; + [JsonPropertyName("object")] + public string Object { get; set; } = null!; + + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("status")] + public string Status { get; set; } = null!; + + [JsonPropertyName("role")] + public string Role { get; set; } = null!; + + [JsonPropertyName("name")] + public string Name { get; set; } = null!; + + [JsonPropertyName("call_id")] + public string CallId { get; set; } = null!; + + [JsonPropertyName("arguments")] + public string Arguments { get; set; } = null!; + + [JsonPropertyName("content")] + public ResponseDoneOutputContent[] Content { get; set; } = []; +} + +public class ResponseDoneStatusDetail +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("reason")] + public string Reason { get; set; } = null!; +} + +public class ResponseDoneOutputContent +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("transcript")] + public string Transcript { get; set; } = null!; +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventErrorResponse.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventErrorResponse.cs new file mode 100644 index 00000000..f14b5437 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventErrorResponse.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ServerEventErrorResponse : ServerEventResponse +{ + [JsonPropertyName("error")] + public ServerEventErrorBody Body { get; set; } = new(); +} + +public class ServerEventErrorBody +{ + [JsonPropertyName("type")] + public string Type { get; set; } = null!; + + [JsonPropertyName("code")] + public string Code { get; set; } = null!; + + [JsonPropertyName("message")] + public string? Message { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs new file mode 100644 index 00000000..921c4c56 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/ServerEventResponse.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class ServerEventResponse +{ + [JsonPropertyName("event_id")] + public string EventId { get; set; } = null!; + + [JsonPropertyName("type")] + public string Type { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs new file mode 100644 index 00000000..fb1d08f6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/SessionServerEventResponse.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Plugin.OpenAI.Models.Realtime; + +public class SessionServerEventResponse : ServerEventResponse +{ + [JsonPropertyName("session")] + public RealtimeSessionBody Session { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs index c26ce46d..68382dff 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Plugin.OpenAI.Models.Realtime; using Refit; namespace BotSharp.Plugin.OpenAI.Providers.Realtime; @@ -6,5 +7,5 @@ namespace BotSharp.Plugin.OpenAI.Providers.Realtime; public interface IOpenAiRealtimeApi { [Post("/v1/realtime/sessions")] - Task GetSessionAsync(RealtimeSessionRequest model, [Authorize("Bearer")] string token); + Task GetSessionAsync(RealtimeSessionCreationRequest model, [Authorize("Bearer")] string token); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index eb678c5d..9b981cb1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,20 +1,29 @@ using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; +using System.Net.WebSockets; +using System.Text; using System.Text.Json; +using System.Threading; namespace BotSharp.Plugin.OpenAI.Providers.Realtime; +/// +/// Reference to https://platform.openai.com/docs/api-reference/realtime-server-events +/// public class RealTimeCompletionProvider : IRealTimeCompletion { public string Provider => "openai"; + public string Model => _model; protected readonly OpenAiSettings _settings; protected readonly IServiceProvider _services; protected readonly ILogger _logger; protected string _model = "gpt-4o-mini-realtime-preview-2024-12-17"; + private ClientWebSocket _webSocket; public RealTimeCompletionProvider( OpenAiSettings settings, @@ -26,6 +35,184 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _services = services; } + public async Task Connect(RealtimeHubConnection conn, + Action onModelReady, + Action onModelAudioDeltaReceived, + Action onModelAudioResponseDone, + Action onAudioTranscriptDone, + Action> onModelResponseDone, + Action onConversationItemCreated, + Action onInputAudioTranscriptionCompleted, + Action onUserInterrupted) + { + var settingsService = _services.GetRequiredService(); + var settings = settingsService.GetSetting(provider: "openai", conn.Model); + + _webSocket = new ClientWebSocket(); + _webSocket.Options.SetRequestHeader("Authorization", $"Bearer {settings.ApiKey}"); + _webSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1"); + + await _webSocket.ConnectAsync(new Uri($"wss://api.openai.com/v1/realtime?model={conn.Model}"), CancellationToken.None); + + if (_webSocket.State == WebSocketState.Open) + { + onModelReady(); + + // Receive a message + _ = ReceiveMessage(conn, + onModelAudioDeltaReceived, + onModelAudioResponseDone, + onAudioTranscriptDone, + onModelResponseDone, + onConversationItemCreated, + onInputAudioTranscriptionCompleted, + onUserInterrupted); + } + } + + public async Task Disconnect() + { + await _webSocket.CloseAsync(WebSocketCloseStatus.Empty, null, CancellationToken.None); + } + + public async Task AppenAudioBuffer(string message) + { + var audioAppend = new + { + type = "input_audio_buffer.append", + audio = message + }; + + await SendEventToModel(audioAppend); + } + + public async Task TriggerModelInference(string? instructions = null) + { + // Triggering model inference + await SendEventToModel(new + { + type = "response.create", + response = new + { + instructions + } + }); + } + + private async Task ReceiveMessage(RealtimeHubConnection conn, + Action onModelAudioDeltaReceived, + Action onModelAudioResponseDone, + Action onAudioTranscriptDone, + Action> onModelResponseDone, + Action onConversationItemCreated, + Action onInputAudioTranscriptionCompleted, + Action onUserInterrupted) + { + var buffer = new byte[1024 * 1024 * 1]; + WebSocketReceiveResult result; + string lastAssistantItem = ""; + do + { + result = await _webSocket.ReceiveAsync( + new ArraySegment(buffer), CancellationToken.None); + + // Convert received data to text/audio (Twilio sends Base64-encoded audio) + string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count); + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + _logger.LogDebug($"{nameof(RealTimeCompletionProvider)} received: {receivedText}"); + var response = JsonSerializer.Deserialize(receivedText); + + if (response.Type == "error") + { + var error = JsonSerializer.Deserialize(receivedText); + _logger.LogError($"Error: {error.Body.Message}"); + } + else if (response.Type == "session.created") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + } + else if (response.Type == "session.updated") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + } + else if (response.Type == "response.audio_transcript.delta") + { + + } + else if (response.Type == "response.audio_transcript.done") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + var data = JsonSerializer.Deserialize(receivedText); + onAudioTranscriptDone(data.Transcript); + } + else if (response.Type == "response.audio.delta") + { + var audio = JsonSerializer.Deserialize(receivedText); + lastAssistantItem = audio?.ItemId ?? ""; + + if (audio != null && audio.Delta != null) + { + onModelAudioDeltaReceived(audio.Delta); + } + } + else if (response.Type == "response.audio.done") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + onModelAudioResponseDone(); + } + else if (response.Type == "response.done") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + await Task.Delay(1000); + var messages = await OnResponsedDone(conn, receivedText); + onModelResponseDone(messages); + } + else if (response.Type == "conversation.item.created") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + onConversationItemCreated(receivedText); + } + else if (response.Type == "conversation.item.input_audio_transcription.completed") + { + _logger.LogInformation($"{response.Type}: {receivedText}"); + var message = await OnInputAudioTranscriptionCompleted(conn, receivedText); + onInputAudioTranscriptionCompleted(message); + } + else if (response.Type == "input_audio_buffer.speech_started") + { + // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio; + // handle use interuption + var truncateEvent = new + { + type = "conversation.item.truncate", + item_id = lastAssistantItem, + content_index = 0, + audio_end_ms = 100 + }; + + await SendEventToModel(truncateEvent); + onUserInterrupted(); + } + + } while (!result.CloseStatus.HasValue); + + await _webSocket.CloseAsync(result.CloseStatus.Value, result.CloseStatusDescription, CancellationToken.None); + } + + public async Task SendEventToModel(object message) + { + if (message is not string data) + { + data = JsonSerializer.Serialize(message); + } + + var buffer = Encoding.UTF8.GetBytes(data); + await _webSocket.SendAsync(new ArraySegment(buffer), WebSocketMessageType.Text, true, CancellationToken.None); + } + public async Task CreateSession(Agent agent, List conversations) { var contentHooks = _services.GetServices().ToList(); @@ -34,9 +221,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var chatClient = client.GetChatClient(_model); var (prompt, messages, options) = PrepareOptions(agent, conversations); - var args = new RealtimeSessionRequest + var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description; + + var args = new RealtimeSessionCreationRequest { - Instructions = prompt, + Instructions = instruction, ToolChoice = "auto", Tools = options.Tools.Select(x => { @@ -58,6 +247,119 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return session; } + public async Task UpdateInitialSession(RealtimeHubConnection conn) + { + var convService = _services.GetRequiredService(); + var conv = await convService.GetConversation(conn.ConversationId); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(conv.AgentId); + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var chatClient = client.GetChatClient(_model); + var (prompt, messages, options) = PrepareOptions(agent, []); + + var instruction = messages.FirstOrDefault()?.Content.FirstOrDefault()?.Text ?? agent.Description; + + var sessionUpdate = new + { + type = "session.update", + session = new RealtimeSessionUpdateRequest + { + InputAudioFormat = "g711_ulaw", + OutputAudioFormat = "g711_ulaw", + InputAudioTranscription = new InputAudioTranscription + { + Model = "whisper-1", + }, + Voice = "alloy", + Instructions = instruction, + ToolChoice = "auto", + Tools = options.Tools.Select(x => + { + var fn = new FunctionDef + { + Name = x.FunctionName, + Description = x.FunctionDescription + }; + fn.Parameters = JsonSerializer.Deserialize(x.FunctionParameters); + return fn; + }).ToArray(), + Modalities = [ "text", "audio" ], + Temperature = Math.Max(options.Temperature ?? 0f, 0.6f) + } + }; + + await SendEventToModel(sessionUpdate); + } + + public async Task InsertConversationItem(RoleDialogModel message) + { + if (message.Role == AgentRole.Function) + { + var functionConversationItem = new + { + type = "conversation.item.create", + item = new + { + call_id = message.ToolCallId, + type = "function_call_output", + output = message.Content + } + }; + + await SendEventToModel(functionConversationItem); + } + else if (message.Role == AgentRole.Assistant) + { + var conversationItem = new + { + type = "conversation.item.create", + item = new + { + type = "message", + role = message.Role, + content = new object[] + { + new + { + type = "text", + text = message.Content + } + } + } + }; + + await SendEventToModel(conversationItem); + } + else if (message.Role == AgentRole.User) + { + var conversationItem = new + { + type = "conversation.item.create", + item = new + { + type = "message", + role = message.Role, + content = new object[] + { + new + { + type = "input_text", + text = message.Content + } + } + } + }; + + await SendEventToModel(conversationItem); + } + else + { + throw new NotImplementedException(""); + } + } + protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); @@ -171,7 +473,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return (prompt, messages, options); } - private string GetPrompt(IEnumerable messages, ChatCompletionOptions options) { var prompt = string.Empty; @@ -243,4 +544,52 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { _model = model; } + + public async Task> OnResponsedDone(RealtimeHubConnection conn, string response) + { + var outputs = new List(); + + var data = JsonSerializer.Deserialize(response).Body; + foreach (var output in data.Outputs) + { + if (output.Type == "function_call") + { + outputs.Add(new RoleDialogModel(output.Role, output.Arguments) + { + CurrentAgentId = conn.EntryAgentId, + FunctionName = output.Name, + FunctionArgs = output.Arguments, + ToolCallId = output.CallId + }); + } + else if (output.Type == "message") + { + var content = output.Content.FirstOrDefault(); + + outputs.Add(new RoleDialogModel(output.Role, content.Transcript) + { + CurrentAgentId = conn.EntryAgentId + }); + } + } + + return outputs; + } + + public async Task OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + { + var data = JsonSerializer.Deserialize(response); + return new RoleDialogModel(AgentRole.User, data.Transcript) + { + CurrentAgentId = conn.EntryAgentId + }; + } + + public async Task OnConversationItemCreated(RealtimeHubConnection conn, string response) + { + var item = JsonSerializer.Deserialize(response).Item; + var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript); + + return message; + } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs index ac1022a2..e8e458c2 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs @@ -3,8 +3,11 @@ global using System.Collections.Generic; global using System.Linq; global using System.IO; global using System.Threading.Tasks; +global using System.Text.Json.Serialization; + global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; + global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Constants; global using BotSharp.Abstraction.Agents.Models; @@ -17,4 +20,4 @@ global using BotSharp.Abstraction.Files; global using BotSharp.Abstraction.Files.Models; global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.OpenAI.Models; -global using BotSharp.Plugin.OpenAI.Settings; \ No newline at end of file +global using BotSharp.Plugin.OpenAI.Settings; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index aff07e81..91c81e90 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -25,8 +25,8 @@ - - + + diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs new file mode 100644 index 00000000..e6de9fc5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -0,0 +1,112 @@ +using BotSharp.Abstraction.Infrastructures; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Twilio.TwiML.Voice; +using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioStreamController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly IHttpContextAccessor _context; + private readonly ILogger _logger; + + public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _context = context; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/stream")] + public async Task InitiateStreamConversation(ConversationalVoiceRequest request) + { + var text = JsonSerializer.Serialize(request); + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + + VoiceResponse response = null; + var instruction = new ConversationalVoiceResponse + { + SpeechPaths = [], + ActionOnEmptyResult = true + }; + + if (_context.HttpContext.Request.Query.ContainsKey("init_audio_file")) + { + instruction.SpeechPaths.Add(_context.HttpContext.Request.Query["init_audio_file"]); + } + + if (_context.HttpContext.Request.Query.ContainsKey("conversation_id")) + { + request.ConversationId = _context.HttpContext.Request.Query["conversation_id"]; + } + else + { + request.ConversationId = request.CallSid; + } + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreating(request, instruction); + }, new HookEmitOption + { + OnlyOnce = true + }); + + await InitConversation(request); + + var twilio = _services.GetRequiredService(); + + response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction); + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreated(request); + }, new HookEmitOption + { + OnlyOnce = true + }); + + return TwiML(response); + } + + private async Task InitConversation(ConversationalVoiceRequest request) + { + var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(request.ConversationId); + if (conversation == null) + { + var conv = new Conversation + { + Id = request.CallSid, + AgentId = _settings.AgentId, + Channel = ConversationChannel.Phone, + Title = $"Phone call from {request.From}", + Tags = [], + }; + + conversation = await convService.NewConversation(conv); + } + + var states = new List + { + new("channel", ConversationChannel.Phone), + new("calling_phone", request.From) + }; + + convService.SetConversationId(conversation.Id, states); + convService.SaveStates(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 5ef088e3..19144673 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -38,6 +38,13 @@ public class TwilioVoiceController : TwilioController [HttpPost("twilio/voice/welcome")] public async Task InitiateConversation(ConversationalVoiceRequest request) { + foreach(var header in Request.Headers) + { + _logger.LogWarning($"{header.Key}: {header.Value}"); + } + + _logger.LogWarning($"{Request.Path}{Request.QueryString}"); + var text = JsonSerializer.Serialize(request); if (request?.CallSid == null) { @@ -101,7 +108,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - [ValidateRequest] + // [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")] public async Task ReceiveCallerMessage(ConversationalVoiceRequest request) { @@ -195,7 +202,7 @@ public class TwilioVoiceController : TwilioController /// /// /// - [ValidateRequest] + // [ValidateRequest] [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")] public async Task ReplyCallerMessage(ConversationalVoiceRequest request) { @@ -360,7 +367,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - [ValidateRequest] + // [ValidateRequest] [HttpPost("twilio/voice/init-call")] public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQuery] string conversationId) { @@ -381,7 +388,7 @@ public class TwilioVoiceController : TwilioController return TwiML(response); } - [ValidateRequest] + // [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs index 775d4b76..29d4bc7c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Realtime.Models; using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; @@ -23,6 +24,9 @@ public interface ITwilioSessionHook Task OnSessionCreated(ConversationalVoiceRequest request) => Task.CompletedTask; + Task OnStreamingStarted(RealtimeHubConnection conn) + => Task.CompletedTask; + /// /// On received user message /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs new file mode 100644 index 00000000..5ad291b7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventMediaResponse.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventMediaResponse : StreamEventResponse +{ + [JsonPropertyName("media")] + public StreamEventMediaBody Body { get; set; } +} + +public class StreamEventMediaBody +{ + [JsonPropertyName("track")] + public string Track { get; set; } + + [JsonPropertyName("chunk")] + public string Chunk { get; set; } + + [JsonPropertyName("timestamp")] + public string Timestamp { get; set; } + + [JsonPropertyName("payload")] + public string Payload { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs new file mode 100644 index 00000000..5be7aa60 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventResponse.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventResponse +{ + /// + /// connected, start, media, stop + /// + [JsonPropertyName("event")] + public string Event { get; set; } + + [JsonPropertyName("sequenceNumber")] + public string SequenceNumber { get; set; } + + [JsonPropertyName("streamSid")] + public string StreamSid { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs new file mode 100644 index 00000000..28b818d2 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStartResponse.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventStartResponse : StreamEventResponse +{ + [JsonPropertyName("start")] + public StreamEventStartBody Body { get; set; } +} + +public class StreamEventStartBody +{ + [JsonPropertyName("accountSid")] + public string AccountSid { get; set; } + + [JsonPropertyName("callSid")] + public string CallSid { get; set; } + + [JsonPropertyName("tracks")] + public string[] Tracks { get; set; } + + [JsonPropertyName("customParameters")] + public JsonDocument CustomParameters { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs new file mode 100644 index 00000000..7ff6f69a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/StreamEventStopResponse.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class StreamEventStopResponse : StreamEventResponse +{ + [JsonPropertyName("sequenceNumber")] + public string SequenceNumber { get; set; } + + [JsonPropertyName("streamSid")] + public string StreamSid { get; set; } + + [JsonPropertyName("stop")] + public StreamEventStopBody Body { get; set; } +} + +public class StreamEventStopBody +{ + [JsonPropertyName("accountSid")] + public string AccountSid { get; set; } + + [JsonPropertyName("callSid")] + public string CallSid { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs new file mode 100644 index 00000000..11e9c774 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/Stream/TwilioHubCallerContext.cs @@ -0,0 +1,37 @@ +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.SignalR; +using System.Security.Claims; +using System.Threading; + +namespace BotSharp.Plugin.Twilio.Models.Stream; + +public class TwilioHubCallerContext : HubCallerContext +{ + private readonly HubConnectionContext _connection; + + public TwilioHubCallerContext(HubConnectionContext connection) + { + _connection = connection; + } + + /// + public override string ConnectionId => _connection.ConnectionId; + + /// + public override string? UserIdentifier => _connection.UserIdentifier; + + /// + public override ClaimsPrincipal? User => _connection.User; + + /// + public override IDictionary Items => _connection.Items; + + /// + public override IFeatureCollection Features => _connection.Features; + + /// + public override CancellationToken ConnectionAborted => _connection.ConnectionAborted; + + /// + public override void Abort() => _connection.Abort(); +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs index b032d8ed..12b827af 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HandleOutboundPhoneCallFn.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Routing; using BotSharp.Core.Infrastructures; @@ -55,6 +56,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions var routing = _services.GetRequiredService(); var fileStorage = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); + var states = _services.GetRequiredService(); // Fork conversation var entryAgentId = routing.EntryAgentId; @@ -66,7 +68,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions var conversationId = newConv.Id; convStorage.Append(conversationId, new List { - new RoleDialogModel(AgentRole.User, "Hi, I'm calling to check my work order quote status, please help me locate my work order number and let me know what to do next.") + new RoleDialogModel(AgentRole.User, "Hi") { CurrentAgentId = entryAgentId }, @@ -75,6 +77,7 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions CurrentAgentId = entryAgentId } }); + states.SetState(StateConst.SUB_CONVERSATION_ID, conversationId); // Generate audio var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); @@ -83,20 +86,19 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions fileStorage.SaveSpeechFile(conversationId, fileName, data); // Call phone number - await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage + /*await sessionManager.SetAssistantReplyAsync(conversationId, 0, new AssistantMessage { Content = args.InitialMessage, SpeechFileName = fileName - }); + });*/ var call = await CallResource.CreateAsync( - url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), + // url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/init-call?conversationId={conversationId}"), + url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={conversationId}&init_audio_file={fileName}"), to: new PhoneNumber(args.PhoneNumber), - from: new PhoneNumber(_twilioSetting.PhoneNumber), - asyncAmd: "true", - machineDetection: "DetectMessageEnd"); + from: new PhoneNumber(_twilioSetting.PhoneNumber)); - message.Content = $"The generated phone message: {args.InitialMessage}. \r\n[Conversation ID: {conversationId}]" ?? message.Content; + message.Content = $"The generated phone message: {args.InitialMessage}." ?? message.Content; message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs new file mode 100644 index 00000000..492b5dcf --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs @@ -0,0 +1,114 @@ +using BotSharp.Abstraction.Realtime; +using BotSharp.Abstraction.Realtime.Models; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models.Stream; +using Microsoft.AspNetCore.Http; +using System.Net.WebSockets; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Services.Stream; + +/// +/// Refrence to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js +/// +public class TwilioStreamMiddleware +{ + private readonly RequestDelegate _next; + + public TwilioStreamMiddleware(RequestDelegate next) + { + _next = next; + } + + public async Task Invoke(HttpContext httpContext) + { + var request = httpContext.Request; + + if (request.Path.StartsWithSegments("/twilio/stream")) + { + if (httpContext.WebSockets.IsWebSocketRequest) + { + var services = httpContext.RequestServices; + var conversationId = request.Path.Value.Split("/").Last(); + using WebSocket webSocket = await httpContext.WebSockets.AcceptWebSocketAsync(); + await HandleWebSocket(services, conversationId, webSocket); + return; + } + } + + await _next(httpContext); + } + + private async Task HandleWebSocket(IServiceProvider services, string conversationId, WebSocket webSocket) + { + var hub = services.GetRequiredService(); + + var conn = new RealtimeHubConnection + { + ConversationId = conversationId + }; + + // load conversation and state + var convService = services.GetRequiredService(); + convService.SetConversationId(conversationId, []); + var hooks = services.GetServices(); + foreach (var hook in hooks) + { + await hook.OnStreamingStarted(conn); + } + convService.States.Save(); + + await hub.Listen(webSocket, (receivedText) => + { + var response = JsonSerializer.Deserialize(receivedText); + conn.StreamId = response.StreamSid; + conn.Event = response.Event switch + { + "start" => "user_connected", + "media" => "user_data_received", + "stop" => "user_disconnected", + _ => response.Event + }; + + if (string.IsNullOrEmpty(conn.Event)) + { + return conn; + } + + conn.OnModelMessageReceived = message => + new + { + @event = "media", + streamSid = response.StreamSid, + media = new { payload = message } + }; + conn.OnModelAudioResponseDone = () => + new + { + @event = "mark", + streamSid = response.StreamSid, + mark = new { name = "responsePart" } + }; + conn.OnModelUserInterrupted = () => + new + { + @event = "clear", + streamSid = response.StreamSid + }; + + if (response.Event == "start") + { + var startResponse = JsonSerializer.Deserialize(receivedText); + conn.Data = JsonSerializer.Serialize(startResponse.Body.CustomParameters); + } + else if (response.Event == "media") + { + var mediaResponse = JsonSerializer.Deserialize(receivedText); + conn.Data = mediaResponse.Body.Payload; + } + + return conn; + }); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 5d407b23..1ba7120c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Utilities; using BotSharp.Plugin.Twilio.Models; using Twilio.Jwt.AccessToken; +using Twilio.TwiML.Messaging; using Token = Twilio.Jwt.AccessToken.Token; namespace BotSharp.Plugin.Twilio.Services; @@ -175,4 +176,27 @@ public class TwilioService response.Append(gather); return response; } + + /// + /// Bidirectional Media Streams + /// + /// + /// + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse) + { + var response = new VoiceResponse(); + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) + { + foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) + { + response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}")); + } + } + var connect = new Connect(); + var host = _settings.CallbackHost.Split("://").Last(); + connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}"); + response.Append(connect); + + return response; + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index d78489ad..a31dbb02 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -1,7 +1,9 @@ +using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Settings; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; using BotSharp.Plugin.Twilio.Services; +using BotSharp.Plugin.Twilio.Services.Stream; using StackExchange.Redis; using Twilio;