diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 89619484..de2c8909 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -96,6 +96,8 @@ + + @@ -204,6 +206,14 @@ PreserveNewest + + + + PreserveNewest + + + PreserveNewest + diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs b/src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs new file mode 100644 index 00000000..cabfbdbb --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Options; + +namespace BotSharp.Core.Functions; + +public class GetLocationFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + + public GetLocationFn(IServiceProvider services) + { + _services = services; + } + + public string Name => "get_location"; + public string Indication => "Finding location"; + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + + message.Content = $"There are a lot of fun events here in {args.City}"; + return true; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs new file mode 100644 index 00000000..759bc68c --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs @@ -0,0 +1,56 @@ +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Models; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.SideCar; +using System.Text.Json.Serialization; + +namespace BotSharp.Core.Functions; + +public class GetWeatherFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + + public GetWeatherFn(IServiceProvider services) + { + _services = services; + } + + public string Name => "get_weather"; + public string Indication => "Querying weather"; + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + + var sidecar = _services.GetService(); + var states = GetSideCarStates(); + + var userMessage = $"Please find the information at location {args.City}, {args.State}"; + var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states); + message.Content = $"It is a sunny day {response.Content}."; + return true; + } + + private List GetSideCarStates() + { + var sideCarStates = new List() + { + new("channel", "email") + }; + return sideCarStates; + } +} + +class Location +{ + [JsonPropertyName("city")] + public string? City { get; set; } + + [JsonPropertyName("state")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? State { get; set; } + + [JsonPropertyName("county")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? County { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs index 692e7220..af2abf8f 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs @@ -44,6 +44,7 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator headers, CancellationToken cancellationToken = default) + public async Task ConnectAsync(Uri uri, Dictionary? headers = null, CancellationToken cancellationToken = default) { _webSocket?.Dispose(); _webSocket = new ClientWebSocket(); - foreach (var header in headers) + if (!headers.IsNullOrEmpty()) { - _webSocket.Options.SetRequestHeader(header.Key, header.Value); + foreach (var header in headers) + { + _webSocket.Options.SetRequestHeader(header.Key, header.Value); + } } await _webSocket.ConnectAsync(uri, cancellationToken); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json new file mode 100644 index 00000000..b0716218 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json @@ -0,0 +1,20 @@ +{ + "name": "get_location", + "description": "Get location information for user.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "visibility_expression": "{% if states.channel == 'email' %}visible{% endif %}", + "description": "The location city that user wants to know about." + }, + "county": { + "type": "string", + "visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}", + "description": "The location county that user wants to know about." + } + }, + "required": [ "city", "county" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json new file mode 100644 index 00000000..bdb679a2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json @@ -0,0 +1,19 @@ +{ + "name": "get_weather", + "description": "Get weather information for user.", + "visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city where the user wants to get weather information." + }, + "state": { + "type": "string", + "description": "The state where the user wants to get weather information." + } + }, + "required": [ "city", "state" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 98896ac3..d901f981 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,8 +1,15 @@ +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Realtime.Models.Session; +using BotSharp.Core.Session; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Live; using GenerativeAI.Live.Extensions; using GenerativeAI.Types; +using GenerativeAI.Types.Converters; +using Google.Ai.Generativelanguage.V1Beta2; +using Google.Api; +using System.Threading; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -18,14 +25,18 @@ public class GoogleRealTimeProvider : IRealTimeCompletion private readonly ILogger _logger; private List renderedInstructions = []; + private LlmRealtimeSession _session; + private readonly BotSharpOptions _botsharpOptions; private readonly GoogleAiSettings _settings; public GoogleRealTimeProvider( IServiceProvider services, GoogleAiSettings settings, + BotSharpOptions botSharpOptions, ILogger logger) { _settings = settings; + _botsharpOptions = botSharpOptions; _services = services; _logger = logger; } @@ -66,8 +77,48 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; _onUserInterrupted = onUserInterrupted; + var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; + var modelSettings = settingsService.GetSetting(Provider, _model); + + //if (_session != null) + //{ + // _session.Dispose(); + //} + + //_session = new LlmRealtimeSession(_services, new ChatSessionOptions + //{ + // JsonOptions = new JsonSerializerOptions + // { + // PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + // PropertyNameCaseInsensitive = true, + // Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() }, + // DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + // TypeInfoResolver = TypesSerializerContext.Default, + // UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement, + + // } + //}); + + //await _session.ConnectAsync( + // uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"), + // cancellationToken: CancellationToken.None); + + ////await UpdateSession(conn, true); + + //_ = ReceiveMessage( + // conn, + // onModelReady, + // onModelAudioDeltaReceived, + // onModelAudioResponseDone, + // onModelAudioTranscriptDone, + // onModelResponseDone, + // onConversationItemCreated, + // onInputAudioTranscriptionCompleted, + // onUserInterrupted); + var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); _chatClient = client.CreateGenerativeModel(_model); @@ -75,7 +126,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion config: new GenerationConfig { ResponseModalities = [Modality.AUDIO], - }, + }, systemInstruction: "You are a helpful assistant.", logger: _logger); @@ -84,37 +135,107 @@ public class GoogleRealTimeProvider : IRealTimeCompletion await _client.ConnectAsync(false); } + + private async Task ReceiveMessage( + RealtimeHubConnection conn, + Action onModelReady, + Action onModelAudioDeltaReceived, + Action onModelAudioResponseDone, + Action onModelAudioTranscriptDone, + Action> onModelResponseDone, + Action onConversationItemCreated, + Action onUserAudioTranscriptionCompleted, + Action onInterruptionDetected) + { + await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) + { + var receivedText = update?.RawResponse; + Console.WriteLine($"Received text: {receivedText}"); + + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + + + } + + _session.Dispose(); + } + + public async Task Disconnect() { + //if (_session != null) + //{ + // await _session.Disconnect(); + //} + if (_client != null) + { await _client.DisconnectAsync(); + } } public async Task AppenAudioBuffer(string message) { await _client.SendAudioAsync(Convert.FromBase64String(message)); + + //await SendEventToModel(new BidiClientPayload + //{ + // RealtimeInput = new() + // { + // MediaChunks = [ new() { Data = message, MimeType = "audio/pcm; rate=16000" } ] + // } + //}); } public async Task AppenAudioBuffer(ArraySegment data, int length) { var buffer = data.AsSpan(0, length).ToArray(); - await _client.SendAudioAsync(buffer,"audio/pcm;rate=16000"); + await _client.SendAudioAsync(buffer, "audio/pcm; rate=16000"); + + //await SendEventToModel(new BidiClientPayload + //{ + // RealtimeInput = new() + // { + // MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = "audio/pcm; rate=16000" }] + // } + //}); } public async Task TriggerModelInference(string? instructions = null) { + var content = !string.IsNullOrWhiteSpace(instructions) + ? new Content(instructions, AgentRole.User) + : null; + await _client.SendClientContentAsync(new BidiGenerateContentClientContent() { + Turns = content != null ? [content] : null, TurnComplete = true, }); + + + + //await SendEventToModel(new BidiClientPayload + //{ + // ClientContent = new() + // { + // Turns = content != null ? [content] : null, + // TurnComplete = true + // } + //}); } public async Task CancelModelResponse() { + } public async Task RemoveConversationItem(string itemId) { + } private Task AttachEvents(MultiModalLiveClient client) @@ -236,6 +357,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task SendEventToModel(object message) { //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete + + //if (_session == null) return; + + //await _session.SendEventToModel(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) @@ -246,13 +371,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(conn.CurrentAgentId); - var (prompt, request) = PrepareOptions(_chatClient, agent, new List()); + var (prompt, request) = PrepareOptions(agent, []); var config = request.GenerationConfig; //Output Modality can either be text or audio if (config != null) { - config.ResponseModalities = new List([Modality.AUDIO]); + config.ResponseModalities = [Modality.AUDIO]; var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); @@ -270,10 +395,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { Name = x.Name ?? string.Empty, Description = x.Description ?? string.Empty, + Parameters = x.Parameters != null + ? JsonSerializer.Deserialize(JsonSerializer.Serialize(x.Parameters)) + : null }; - fn.Parameters = x.Parameters != null - ? JsonSerializer.Deserialize(JsonSerializer.Serialize(x.Parameters)) - : null; return fn; }).ToArray(); @@ -282,33 +407,39 @@ public class GoogleRealTimeProvider : IRealTimeCompletion if (_settings.Gemini.UseGoogleSearch) { - if (request.Tools == null) - request.Tools = new List(); + request.Tools ??= []; request.Tools.Add(new Tool() { GoogleSearch = new GoogleSearchTool() }); } - // if(request.Tools.Count == 0) - // request.Tools = null; - // config.MaxOutputTokens = null; - await _client.SendSetupAsync(new BidiGenerateContentSetup() { GenerationConfig = config, Model = Model.ToModelId(), SystemInstruction = request.SystemInstruction, - Tools = request.Tools?.ToArray(), + //Tools = request.Tools?.ToArray(), }); + //await SendEventToModel(new BidiClientPayload + //{ + // Setup = new BidiGenerateContentSetup() + // { + // GenerationConfig = config, + // Model = $"models/{_model}", + // SystemInstruction = new Content(agent.Instruction, AgentRole.System), + // //Tools = request.Tools?.ToArray(), + // } + //}); + return prompt; } public async Task InsertConversationItem(RoleDialogModel message) { - if (_client == null) - throw new Exception("Client is not initialized"); + //if (_client == null) + // throw new Exception("Client is not initialized"); if (message.Role == AgentRole.Function) { var function = new FunctionResponse() @@ -321,13 +452,38 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { FunctionResponses = [function] }); + + //await SendEventToModel(new BidiClientPayload + //{ + // ToolResponse = new() + // { + // FunctionResponses = [function] + // } + //}); } else if (message.Role == AgentRole.Assistant) { + //await SendEventToModel(new BidiClientPayload + //{ + // ClientContent = new() + // { + // Turns = [new Content(message.Content, AgentRole.Model)], + // TurnComplete = true + // } + //}); } else if (message.Role == AgentRole.User) { await _client.SentTextAsync(message.Content); + + //await SendEventToModel(new BidiClientPayload + //{ + // ClientContent = new() + // { + // Turns = [new Content(message.Content, AgentRole.User)], + // TurnComplete = true + // } + //}); } else { @@ -335,33 +491,24 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } } - public Task> OnResponsedDone(RealtimeHubConnection conn, string response) + public async Task> OnResponsedDone(RealtimeHubConnection conn, string response) { - throw new NotImplementedException(""); + return []; } - public Task OnConversationItemCreated(RealtimeHubConnection conn, string response) + public async Task OnConversationItemCreated(RealtimeHubConnection conn, string response) { - return Task.FromResult(new RoleDialogModel(AgentRole.User, response)); + return await Task.FromResult(new RoleDialogModel(AgentRole.User, response)); } - private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent, + private (string, GenerateContentRequest) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); var googleSettings = _settings; renderedInstructions = []; - // Add settings - aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch; - aiModel.UseGrounding = googleSettings.Gemini.UseGrounding; - - aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour() - { - AutoCallFunction = false - }; - // Assembly messages var contents = new List(); var tools = new List(); @@ -458,6 +605,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) ? tokens : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + var request = new GenerateContentRequest { SystemInstruction = !systemPrompts.IsNullOrEmpty() diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index b32c0e1c..3257e9bf 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -671,7 +671,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return outputs; } - public async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) { var data = JsonSerializer.Deserialize(response); return new RoleDialogModel(AgentRole.User, data.Transcript)