SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

temp save

Browse source
This commit is contained in:
Jicheng Lu 2025-05-15 15:46:42 -05:00
parent 4c4dce852d
commit b64f22ef8e
8 changed files with 105 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs
View file

@ -47,15 +47,19 @@ public class DialogElement
public string Content { get; set; } = default!;
[JsonPropertyName("secondary_content")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? SecondaryContent { get; set; }
[JsonPropertyName("rich_content")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? RichContent { get; set; }
[JsonPropertyName("secondary_rich_content")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? SecondaryRichContent { get; set; }
[JsonPropertyName("payload")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Payload { get; set; }
public DialogElement()
@ -95,8 +99,17 @@ public class DialogMetaData
public string MessageType { get; set; } = default!;
[JsonPropertyName("function_name")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FunctionName { get; set; }
[JsonPropertyName("function_args")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? FunctionArgs { get; set; }
[JsonPropertyName("tool_call_id")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? ToolCallId { get; set; }
[JsonPropertyName("sender_id")]
public string? SenderId { get; set; }

1
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -10,6 +10,7 @@ public class RealtimeHubConnection
public string KeypadInputBuffer { get; set; } = string.Empty;
public string CurrentAgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public string? PrevSessionId { get; set; }
public Func<string> OnModelReady { get; set; } = () => string.Empty;
public Func<string, string> OnModelMessageReceived { get; set; } = null!;
public Func<string> OnModelAudioResponseDone { get; set; } = null!;

6
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
View file

@ -65,9 +65,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
else
{
// Update session for changed states
// TO DO
//var instruction = await hub.Completer.UpdateSession(hub.HubConn);
var instruction = await hub.Completer.UpdateSession(hub.HubConn);
await hub.Completer.InsertConversationItem(message);
if (string.IsNullOrEmpty(message.Content))
@ -81,7 +79,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
//}
//else
//{
// await hub.Completer.TriggerModelInference();
// await hub.Completer.TriggerModelInference(instruction);
//}
}
}

2
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -96,6 +96,8 @@ public class RealtimeHub : IRealtimeHub
}
await routing.InvokeFunction(message.FunctionName, message);
dialogs.Add(message);
storage.Append(_conn.ConversationId, message);
}
else
{

7
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs
View file

@ -40,6 +40,8 @@ public class ConversationStorage : IConversationStorage
MessageId = dialog.MessageId,
MessageType = dialog.MessageType,
FunctionName = dialog.FunctionName,
FunctionArgs = dialog.FunctionArgs,
ToolCallId = dialog.ToolCallId,
CreatedTime = dialog.CreatedAt
};
@ -109,7 +111,6 @@ public class ConversationStorage : IConversationStorage
var currentAgentId = meta.AgentId;
var messageId = meta.MessageId;
var messageType = meta.MessageType;
var function = meta.FunctionName;
var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId;
var createdAt = meta.CreatedTime;
var richContent = !string.IsNullOrEmpty(dialog.RichContent) ?
@ -124,7 +125,9 @@ public class ConversationStorage : IConversationStorage
MessageType = messageType,
CreatedAt = createdAt,
SenderId = senderId,
FunctionName = function,
FunctionName = meta.FunctionName,
FunctionArgs = meta.FunctionArgs,
ToolCallId = meta.ToolCallId,
RichContent = richContent,
SecondaryContent = secondaryContent,
SecondaryRichContent = secondaryRichContent,

121
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -32,6 +32,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement
};
private RealtimeTranscriptionResponse _inputStream = new();
private RealtimeTranscriptionResponse _outputStream = new();
private RealtimeHubConnection _conn;
private Func<Task> _onModelReady;
private Func<string, string, Task> _onModelAudioDeltaReceived;
private Func<Task> _onModelAudioResponseDone;
private Func<string, Task> _onModelAudioTranscriptDone;
private Func<List<RoleDialogModel>, Task> _onModelResponseDone;
private Func<string, Task> _onConversationItemCreated;
private Func<RoleDialogModel, Task> _onInputAudioTranscriptionDone;
private Func<Task> _onInterruptionDetected;
public GoogleRealTimeProvider(
IServiceProvider services,
GoogleAiSettings settings,
@ -58,17 +72,26 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
{
_conn = conn;
_onModelReady = onModelReady;
_onModelAudioDeltaReceived = onModelAudioDeltaReceived;
_onModelAudioResponseDone = onModelAudioResponseDone;
_onModelAudioTranscriptDone = onModelAudioTranscriptDone;
_onModelResponseDone = onModelResponseDone;
_onConversationItemCreated = onConversationItemCreated;
_onInputAudioTranscriptionDone = onInputAudioTranscriptionDone;
_onInterruptionDetected = onInterruptionDetected;
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
_model = realtimeModelSettings.Model;
var modelSettings = settingsService.GetSetting(Provider, _model);
if (_session != null)
{
_session.Dispose();
}
Reset();
_inputStream = new();
_outputStream = new();
_session = new LlmRealtimeSession(_services, new ChatSessionOptions
{
JsonOptions = _jsonOptions
@ -79,32 +102,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
await onModelReady();
_ = ReceiveMessage(
conn,
onModelReady,
onModelAudioDeltaReceived,
onModelAudioResponseDone,
onModelAudioTranscriptDone,
onModelResponseDone,
onConversationItemCreated,
onInputAudioTranscriptionDone,
onInterruptionDetected);
_ = ReceiveMessage();
}
private async Task ReceiveMessage(
RealtimeHubConnection conn,
Func<Task> onModelReady,
Func<string, string, Task> onModelAudioDeltaReceived,
Func<Task> onModelAudioResponseDone,
Func<string, Task> onModelAudioTranscriptDone,
Func<List<RoleDialogModel>, Task> onModelResponseDone,
Func<string, Task> onConversationItemCreated,
Func<RoleDialogModel, Task> onInputAudioTranscriptionDone,
Func<Task> onInterruptionDetected)
private async Task ReceiveMessage()
{
using var inputStream = new RealtimeTranscriptionResponse();
using var outputStream = new RealtimeTranscriptionResponse();
await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None))
{
var receivedText = update?.RawResponse;
@ -128,6 +130,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
else if (response.SessionResumptionUpdate != null)
{
_logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}");
_conn.PrevSessionId = response.SessionResumptionUpdate?.NewHandle;
}
else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty())
{
@ -137,20 +140,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
if (functionCall != null)
{
var messages = OnFunctionCall(conn, functionCall);
await onModelResponseDone(messages);
var messages = OnFunctionCall(_conn, functionCall);
await _onModelResponseDone(messages);
}
}
else if (response.ServerContent != null)
{
if (response.ServerContent.InputTranscription?.Text != null)
{
inputStream.Collect(response.ServerContent.InputTranscription.Text);
_inputStream.Collect(response.ServerContent.InputTranscription.Text);
}
if (response.ServerContent.OutputTranscription?.Text != null)
{
outputStream.Collect(response.ServerContent.OutputTranscription.Text);
_outputStream.Collect(response.ServerContent.OutputTranscription.Text);
}
if (response.ServerContent.ModelTurn != null)
@ -158,13 +161,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_logger.LogInformation($"Model audio delta received.");
// Handle input transcription
var inputTranscription = inputStream.GetText();
var inputTranscription = _inputStream.GetText();
if (!string.IsNullOrEmpty(inputTranscription))
{
var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription);
await onInputAudioTranscriptionDone(message);
var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription);
await _onInputAudioTranscriptionDone(message);
}
inputStream.Clear();
_inputStream.Clear();
var parts = response.ServerContent.ModelTurn.Parts;
if (!parts.IsNullOrEmpty())
@ -173,7 +176,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
if (!string.IsNullOrEmpty(part.InlineData?.Data))
{
await onModelAudioDeltaReceived(part.InlineData.Data, string.Empty);
await _onModelAudioDeltaReceived(part.InlineData.Data, string.Empty);
}
}
}
@ -187,14 +190,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
_logger.LogInformation($"Model turn completed.");
// Handle output transcription
var outputTranscription = outputStream.GetText();
var outputTranscription = _outputStream.GetText();
if (!string.IsNullOrEmpty(outputTranscription))
{
var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData);
await onModelResponseDone(messages);
var messages = await OnResponseDone(_conn, outputTranscription, response.UsageMetaData);
await _onModelResponseDone(messages);
}
inputStream.Clear();
outputStream.Clear();
_inputStream.Clear();
_outputStream.Clear();
}
}
}
@ -205,6 +208,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}
}
_inputStream.Dispose();
_outputStream.Dispose();
_session.Dispose();
}
@ -213,7 +218,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
if (_session != null)
{
_inputStream?.Dispose();
_outputStream?.Dispose();
await _session.DisconnectAsync();
_session.Dispose();
}
}
@ -242,8 +250,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task TriggerModelInference(string? instructions = null)
{
var content = new Content(instructions ?? "Please respond to user.", AgentRole.User);
if (string.IsNullOrWhiteSpace(instructions)) return;
var content = new Content(instructions, AgentRole.User);
await SendEventToModel(new BidiClientPayload
{
ClientContent = new()
@ -273,13 +282,18 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task<string> UpdateSession(RealtimeHubConnection conn, bool isInit = false)
{
if (!isInit)
{
return null;
}
var agentService = _services.GetRequiredService<IAgentService>();
var realtimeSetting = _services.GetRequiredService<RealtimeModelSettings>();
var agent = await agentService.LoadAgent(conn.CurrentAgentId);
var (prompt, request) = PrepareOptions(agent, []);
var config = request.GenerationConfig;
var config = request.GenerationConfig ?? new();
if (config != null)
{
//Output Modality can either be text or audio
@ -317,7 +331,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
});
}
await SendEventToModel(new RealtimeClientPayload
var payload = new RealtimeClientPayload
{
Setup = new RealtimeGenerateContentSetup()
{
@ -328,8 +343,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
SessionResumption = new()
{
Handle = _conn.PrevSessionId
}
}
});
};
Console.WriteLine($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}");
await SendEventToModel(payload);
return prompt;
}
@ -596,5 +617,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}");
}
private void Reset()
{
_inputStream?.Clear();
_outputStream?.Clear();
_inputStream?.Dispose();
_outputStream?.Dispose();
_session?.Dispose();
}
#endregion
}

6
src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs
View file

@ -47,6 +47,8 @@ public class DialogMetaDataMongoElement
public string MessageId { get; set; } = default!;
public string MessageType { get; set; } = default!;
public string? FunctionName { get; set; }
public string? FunctionArgs { get; set; }
public string? ToolCallId { get; set; }
public string? SenderId { get; set; }
public DateTime CreateTime { get; set; }
@ -59,6 +61,8 @@ public class DialogMetaDataMongoElement
MessageId = meta.MessageId,
MessageType = meta.MessageType,
FunctionName = meta.FunctionName,
FunctionArgs = meta.FunctionArgs,
ToolCallId = meta.ToolCallId,
SenderId = meta.SenderId,
CreatedTime = meta.CreateTime,
};
@ -73,6 +77,8 @@ public class DialogMetaDataMongoElement
MessageId = meta.MessageId,
MessageType = meta.MessageType,
FunctionName = meta.FunctionName,
FunctionArgs = meta.FunctionArgs,
ToolCallId = meta.ToolCallId,
SenderId = meta.SenderId,
CreateTime = meta.CreatedTime,
};

1
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Repositories.Filters;
using MongoDB.Driver;
using System.Text.Json;
namespace BotSharp.Plugin.MongoStorage.Repository;