SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refine

Browse source
This commit is contained in:
Jicheng Lu 2025-05-15 22:59:57 -05:00
parent 350c479486
commit 0c2ba423d8
9 changed files with 105 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs
View file

@ -10,7 +10,6 @@ public class RealtimeHubConnection
public string KeypadInputBuffer { get; set; } = string.Empty;
public string CurrentAgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public string? PrevSessionId { get; set; }
public Func<string> OnModelReady { get; set; } = () => string.Empty;
public Func<string, string> OnModelMessageReceived { get; set; } = null!;
public Func<string> OnModelAudioResponseDone { get; set; } = null!;

16
src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs
View file

@ -73,14 +73,14 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook
return;
}
//if (message.StopCompletion)
//{
// await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
//}
//else
//{
// await hub.Completer.TriggerModelInference(instruction);
//}
if (message.StopCompletion)
{
await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
}
else
{
await hub.Completer.TriggerModelInference(instruction);
}
}
}
}

11
src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs → src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs
View file

@ -1,7 +1,7 @@
using BotSharp.Abstraction.Functions;
using System.Text.Json.Serialization;
namespace BotSharp.Core.Functions;
namespace BotSharp.Core.Demo.Functions;
public class GetWeatherFn : IFunctionCallback
{
@ -19,6 +19,7 @@ public class GetWeatherFn : IFunctionCallback
{
//var args = JsonSerializer.Deserialize<Location>(message.FunctionArgs, BotSharpOptions.defaultJsonOptions);
message.Content = $"It is a sunny day.";
//message.StopCompletion = true;
return true;
}
}
@ -27,12 +28,4 @@ class Location
{
[JsonPropertyName("city")]
public string? City { get; set; }
[JsonPropertyName("state")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? State { get; set; }
[JsonPropertyName("county")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? County { get; set; }
}

15
src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs
View file

@ -11,14 +11,17 @@ internal static class AiFunctionHelper
return null;
}
if (!tool.JsonSchema.TryGetProperty("properties", out var properties))
var properties = "{}";
var required = "[]";
if (tool.JsonSchema.TryGetProperty("properties", out var p))
{
properties = JsonDocument.Parse("{}").RootElement;
properties = p.GetRawText();
}
if (!tool.JsonSchema.TryGetProperty("required", out var required))
if (tool.JsonSchema.TryGetProperty("required", out var r))
{
required = JsonDocument.Parse("[]").RootElement;
required = r.GetRawText();
}
var funDef = new FunctionDef
@ -29,8 +32,8 @@ internal static class AiFunctionHelper
Parameters = new FunctionParametersDef
{
Type = "object",
Properties = JsonDocument.Parse(properties.GetRawText() ?? "{}"),
Required = JsonSerializer.Deserialize<List<string>>(required.GetRawText() ?? "[]") ?? []
Properties = JsonDocument.Parse(properties),
Required = JsonSerializer.Deserialize<List<string>>(required) ?? []
}
};

2
src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs
View file

@ -42,8 +42,8 @@ internal class RealtimeTranscriptionResponse : IDisposable
public void Clear()
{
_contentStream.SetLength(0);
_contentStream.Position = 0;
_contentStream.SetLength(0);
}
public void Dispose()

5
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs
View file

@ -1,13 +1,10 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Loggers;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Routing;
using LLMSharp.Google.Palm;
using LLMSharp.Google.Palm.DiscussService;
namespace BotSharp.Plugin.GoogleAi.Providers.Chat;
[Obsolete]
public class PalmChatCompletionProvider : IChatCompletion
{
private readonly IServiceProvider _services;

45
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -89,7 +89,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
var modelSettings = settingsService.GetSetting(Provider, _model);
Reset();
_inputStream = new();
_outputStream = new();
_session = new LlmRealtimeSession(_services, new ChatSessionOptions
@ -99,9 +98,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta");
await _session.ConnectAsync(uri: uri, cancellationToken: CancellationToken.None);
await onModelReady();
_ = ReceiveMessage();
}
@ -130,7 +127,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
else if (response.SessionResumptionUpdate != null)
{
_logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}");
_conn.PrevSessionId = response.SessionResumptionUpdate?.NewHandle;
}
else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty())
{
@ -227,7 +223,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task AppenAudioBuffer(string message)
{
await SendEventToModel(new BidiClientPayload
await SendEventToModel(new RealtimeClientPayload
{
RealtimeInput = new()
{
@ -239,7 +235,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
public async Task AppenAudioBuffer(ArraySegment<byte> data, int length)
{
var buffer = data.AsSpan(0, length).ToArray();
await SendEventToModel(new BidiClientPayload
await SendEventToModel(new RealtimeClientPayload
{
RealtimeInput = new()
{
@ -253,7 +249,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
if (string.IsNullOrWhiteSpace(instructions)) return;
var content = new Content(instructions, AgentRole.User);
await SendEventToModel(new BidiClientPayload
await SendEventToModel(new RealtimeClientPayload
{
ClientContent = new()
{
@ -284,7 +280,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
{
if (!isInit)
{
return null;
return string.Empty;
}
var agentService = _services.GetRequiredService<IAgentService>();
@ -294,18 +290,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
var (prompt, request) = PrepareOptions(agent, []);
var config = request.GenerationConfig ?? new();
if (config != null)
{
//Output Modality can either be text or audio
config.ResponseModalities = [Modality.AUDIO];
//Output Modality can either be text or audio
config.ResponseModalities = [Modality.AUDIO];
config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f);
config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens;
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id);
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id);
config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f);
config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens;
}
var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x =>
{
var fn = new FunctionDef
@ -331,7 +323,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
});
}
var payload = new RealtimeClientPayload
{
Setup = new RealtimeGenerateContentSetup()
@ -341,15 +332,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
SystemInstruction = request.SystemInstruction,
Tools = request.Tools?.ToArray(),
InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null,
SessionResumption = new()
{
Handle = _conn.PrevSessionId
}
OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null
}
};
Console.WriteLine($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}");
_logger.LogInformation($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}");
await SendEventToModel(payload);
return prompt;
@ -369,7 +356,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}
};
await SendEventToModel(new BidiClientPayload
await SendEventToModel(new RealtimeClientPayload
{
ToolResponse = new()
{
@ -379,7 +366,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}
else if (message.Role == AgentRole.Assistant)
{
await SendEventToModel(new BidiClientPayload
await SendEventToModel(new RealtimeClientPayload
{
ClientContent = new()
{
@ -390,7 +377,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
}
else if (message.Role == AgentRole.User)
{
await SendEventToModel(new BidiClientPayload
await SendEventToModel(new RealtimeClientPayload
{
ClientContent = new()
{
@ -613,7 +600,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion
};
}
private Uri BuildWebsocketUri(string apiKey, string version = "v1alpha")
private Uri BuildWebsocketUri(string apiKey, string version = "v1beta")
{
return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}");
}

5
src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs
View file

@ -1,9 +1,6 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Loggers;
namespace BotSharp.Plugin.GoogleAi.Providers.Text;
[Obsolete]
public class PalmTextCompletionProvider : ITextCompletion
{
private readonly IServiceProvider _services;

138
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -45,11 +45,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
_model = realtimeSettings.Model;
var settings = settingsService.GetSetting(Provider, _model);
if (_session != null)
{
_session.Dispose();
}
_session?.Dispose();
_session = new LlmRealtimeSession(_services, new ChatSessionOptions
{
JsonOptions = _botsharpOptions.JsonSerializerOptions
@ -77,72 +73,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
onInterruptionDetected);
}
public async Task Disconnect()
{
if (_session != null)
{
await _session.DisconnectAsync();
_session.Dispose();
}
}
public async Task AppenAudioBuffer(string message)
{
var audioAppend = new
{
type = "input_audio_buffer.append",
audio = message
};
await SendEventToModel(audioAppend);
}
public async Task AppenAudioBuffer(ArraySegment<byte> data, int length)
{
var message = Convert.ToBase64String(data.AsSpan(0, length).ToArray());
await AppenAudioBuffer(message);
}
public async Task TriggerModelInference(string? instructions = null)
{
// Triggering model inference
if (!string.IsNullOrEmpty(instructions))
{
await SendEventToModel(new
{
type = "response.create",
response = new
{
instructions
}
});
}
else
{
await SendEventToModel(new
{
type = "response.create"
});
}
}
public async Task CancelModelResponse()
{
await SendEventToModel(new
{
type = "response.cancel"
});
}
public async Task RemoveConversationItem(string itemId)
{
await SendEventToModel(new
{
type = "conversation.item.delete",
item_id = itemId
});
}
private async Task ReceiveMessage(
RealtimeModelSettings realtimeSettings,
RealtimeHubConnection conn,
@ -279,6 +209,72 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
_session.Dispose();
}
public async Task Disconnect()
{
if (_session != null)
{
await _session.DisconnectAsync();
_session.Dispose();
}
}
public async Task AppenAudioBuffer(string message)
{
var audioAppend = new
{
type = "input_audio_buffer.append",
audio = message
};
await SendEventToModel(audioAppend);
}
public async Task AppenAudioBuffer(ArraySegment<byte> data, int length)
{
var message = Convert.ToBase64String(data.AsSpan(0, length).ToArray());
await AppenAudioBuffer(message);
}
public async Task TriggerModelInference(string? instructions = null)
{
// Triggering model inference
if (!string.IsNullOrEmpty(instructions))
{
await SendEventToModel(new
{
type = "response.create",
response = new
{
instructions
}
});
}
else
{
await SendEventToModel(new
{
type = "response.create"
});
}
}
public async Task CancelModelResponse()
{
await SendEventToModel(new
{
type = "response.cancel"
});
}
public async Task RemoveConversationItem(string itemId)
{
await SendEventToModel(new
{
type = "conversation.item.delete",
item_id = itemId
});
}
public async Task SendEventToModel(object message)
{
if (_session == null) return;