SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'SciSharp:master' into master

Browse source
This commit is contained in:
geffzhang 2025-03-22 10:03:16 +08:00 committed by GitHub
parent 213d39349e 385b65e7fe
commit 994da497a4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
118 changed files with 1262 additions and 557 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Directory.Packages.props
View file

@ -1,7 +1,7 @@
<Project>
<PropertyGroup>
<MSExtensionsVersion>8.0.0</MSExtensionsVersion>
<AspNetCoreVersion>2.3.0</AspNetCoreVersion>
<AspNetCoreVersion>2.3.0</AspNetCoreVersion>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>
@ -13,7 +13,7 @@
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.3" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.0.0" />
@ -51,7 +51,7 @@
<PackageVersion Include="PdfPig" Version="0.1.8" />
<PackageVersion Include="TensorFlow.Keras" Version="0.15.0" />
<PackageVersion Include="LangChain.Providers.Google.VertexAI" Version="0.15.3-dev.58" />
<PackageVersion Include="LLamaSharp" Version="0.20.0" />
<PackageVersion Include="LLamaSharp" Version="0.21.0" />
<PackageVersion Include="FaissMask" Version="0.2.0" />
<PackageVersion Include="FastText.NetWrapper" Version="1.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.3.0-preview.1.25161.3" />
@ -108,7 +108,9 @@
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
</ItemGroup>
<ItemGroup>
<PackageVersion Include="BotSharp.Core" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Logger" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Core.Realtime" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.OpenAPI" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.Dashboard" Version="$(BotSharpVersion)" />
<PackageVersion Include="BotSharp.Plugin.AzureOpenAI" Version="$(BotSharpVersion)" />
@ -127,7 +129,6 @@
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.5" />
<PackageVersion Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="8.0.5" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'net6.0'">
<PackageVersion Include="Microsoft.AspNetCore.Authentication.Google" Version="6.0.27" />
<PackageVersion Include="AspNet.Security.OAuth.GitHub" Version="6.0.15" />

2
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs
View file

@ -20,7 +20,7 @@ public class PageActionArgs
public bool OpenNewTab { get; set; } = true;
[JsonPropertyName("open_blank_page")]
public bool OpenBlankPage { get; set; } = true;
[JsonPropertyName("enable_response_callback")]
public bool EnableResponseCallback { get; set; } = false;
/// <summary>

11
src/Infrastructure/BotSharp.Abstraction/Browsing/Models/WebPageResponseData.cs
View file

@ -7,9 +7,20 @@ public class WebPageResponseData
public string ResponseData { get; set; } = null!;
public bool ResponseInMemory { get; set; }
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
public string Method { get; set; }
public List<WebPageCookieData> Cookies { get; set; }
public int ResponseCode { get; set; }
public override string ToString()
{
return $"{Url} {ResponseData.Length}";
}
}
public class WebPageCookieData
{
public string Name { get; set; } = null!;
public string Value { get; set; } = null!;
public string Domain { get; set; } = null!;
public string Path { get; set; } = null!;
public float Expires { get; set; }
}

1
src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs
View file

@ -6,6 +6,7 @@ public class WebBrowsingSettings
public bool Headless { get; set; }
// Default timeout in milliseconds
public float DefaultTimeout { get; set; } = 30000;
public float DefaultNavigationTimeout { get; set; } = 30000;
public bool IsEnableScreenshot { get; set; }
// Default wait time in seconds after page is opened
public int DefaultWaitTime { get; set; } = 5;

2
src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs
View file

@ -9,7 +9,7 @@ public interface IConversationService
string ConversationId { get; }
Task<Conversation> NewConversation(Conversation conversation);
void SetConversationId(string conversationId, List<MessageState> states, bool isReadOnly = false);
Task<Conversation> GetConversation(string id);
Task<Conversation> GetConversation(string id, bool isLoadStates = false);
Task<PagedItems<Conversation>> GetConversations(ConversationFilter filter);
Task<Conversation> UpdateConversationTitle(string id, string title);
Task<Conversation> UpdateConversationTitleAlias(string id, string titleAlias);

1
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs
View file

@ -6,6 +6,7 @@ public class TokenStatsModel
public string Model { get; set; }
public string Prompt { get; set; }
public int PromptCount { get; set; }
public int CachedPromptCount { get; set; }
public int CompletionCount { get; set; }
public AgentLlmConfig LlmConfig { get; set; }
}

4
src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs
View file

@ -25,6 +25,10 @@ public class FunctionDef
[JsonPropertyName("parameters")]
public FunctionParametersDef Parameters { get; set; } = new FunctionParametersDef();
[JsonPropertyName("output")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Output { get; set; }
public override string ToString()
{
return $"{Name}: {Description}";

1
src/Infrastructure/BotSharp.Abstraction/Infrastructures/Enums/StateConst.cs
View file

@ -16,4 +16,5 @@ public class StateConst
public const string SUB_CONVERSATION_ID = "sub_conversation_id";
public const string ORIGIN_CONVERSATION_ID = "origin_conversation_id";
public const string WEB_DRIVER_TASK_ID = "web_driver_task_id";
}

6
src/Infrastructure/BotSharp.Abstraction/Instructs/InstructHookBase.cs
View file

@ -8,16 +8,16 @@ public class InstructHookBase : IInstructHook
public virtual async Task BeforeCompletion(Agent agent, RoleDialogModel message)
{
return;
await Task.CompletedTask;
}
public virtual async Task AfterCompletion(Agent agent, InstructResult result)
{
return;
await Task.CompletedTask;
}
public virtual async Task OnResponseGenerated(InstructResponseModel response)
{
return;
await Task.CompletedTask;
}
}

7
src/Infrastructure/BotSharp.Abstraction/Instructs/Models/ExecuteTemplateArgs.cs Normal file
View file

@ -0,0 +1,7 @@
namespace BotSharp.Abstraction.Instructs.Models;
public class ExecuteTemplateArgs
{
[JsonPropertyName("template_name")]
public string? TemplateName { get; set; }
}

8
src/Infrastructure/BotSharp.Abstraction/Instructs/Settings/InstructionSettings.cs
View file

@ -2,5 +2,11 @@ namespace BotSharp.Abstraction.Instructs.Settings;
public class InstructionSettings
{
public bool EnableLog { get; set; }
public InstructionLogSetting Logging { get; set; } = new();
}
public class InstructionLogSetting
{
public bool Enabled { get; set; } = true;
public List<string> ExcludedAgentIds { get; set; } = [];
}

1
src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs
View file

@ -5,6 +5,7 @@ namespace BotSharp.Abstraction.Loggers.Models;
public class InstructionLogModel
{
[JsonPropertyName("id")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string Id { get; set; } = default!;
[JsonPropertyName("agent_id")]

15
src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs
View file

@ -1,15 +0,0 @@
using System.IO;
namespace BotSharp.Abstraction.MLTasks;
public interface IAudioCompletion
{
string Provider { get; }
string Model { get; }
Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null);
Task<BinaryData> GenerateAudioFromTextAsync(string text);
void SetModelName(string model);
}

15
src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioSynthesis.cs Normal file
View file

@ -0,0 +1,15 @@
namespace BotSharp.Abstraction.MLTasks;
/// <summary>
/// Text to speech synthesis
/// </summary>
public interface IAudioSynthesis
{
string Provider { get; }
string Model { get; }
void SetModelName(string model);
Task<BinaryData> GenerateAudioAsync(string text, string? voice = "alloy", string? format = "mp3", string? instructions = null);
}

17
src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioTranscription.cs Normal file
View file

@ -0,0 +1,17 @@
using System.IO;
namespace BotSharp.Abstraction.MLTasks;
/// <summary>
/// Audio transcription service
/// </summary>
public interface IAudioTranscription
{
string Provider { get; }
string Model { get; }
Task<string> TranscriptTextAsync(Stream audio, string audioFileName, string? text = null);
void SetModelName(string model);
}

2
src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs
View file

@ -6,7 +6,7 @@ public interface ILlmProviderService
{
LlmModelSetting GetSetting(string provider, string model);
List<string> GetProviders();
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool? realTime = false, bool imageGenerate = false);
LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool realTime = false, bool imageGenerate = false);
List<LlmModelSetting> GetProviderModels(string provider);
List<LlmProviderSetting> GetLlmConfigs(LlmConfigOptions? options = null);
}

20
src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs
View file

@ -3,14 +3,14 @@ namespace BotSharp.Abstraction.MLTasks.Settings;
public class LlmModelSetting
{
/// <summary>
/// Model Id, like "gpt-3.5" and "gpt-4".
/// Model Id, like "gpt-4", "gpt-4o", "o1".
/// </summary>
public string? Id { get; set; }
public string Id { get; set; } = null!;
/// <summary>
/// Deployment model name
/// </summary>
public string Name { get; set; }
public string Name { get; set; } = null!;
/// <summary>
/// Model version
@ -28,8 +28,8 @@ public class LlmModelSetting
/// </summary>
public string? Group { get; set; }
public string ApiKey { get; set; }
public string Endpoint { get; set; }
public string ApiKey { get; set; } = null!;
public string? Endpoint { get; set; }
public LlmModelType Type { get; set; } = LlmModelType.Chat;
/// <summary>
@ -62,12 +62,22 @@ public class LlmModelSetting
/// </summary>
public int Dimension { get; set; }
public LlmCost AdditionalCost { get; set; } = new();
public override string ToString()
{
return $"[{Type}] {Name} {Endpoint}";
}
}
public class LlmCost
{
public float CachedPromptCost { get; set; } = 0f;
public float AudioPromptCost { get; set; } = 0f;
public float ReasoningCompletionCost { get; } = 0f;
public float AudioCompletionCost { get; } = 0f;
}
public enum LlmModelType
{
Text = 1,

10
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs
View file

@ -4,7 +4,13 @@ public class ModelTurnDetection
{
public int PrefixPadding { get; set; } = 300;
public int SilenceDuration { get; set; } = 800;
public int SilenceDuration { get; set; } = 500;
public float Threshold { get; set; } = 0.8f;
public float Threshold { get; set; } = 0.5f;
}
public class AudioTranscription
{
public string Model { get; set; } = "gpt-4o-mini-transcribe";
public string? Language { get; set; }
}

2
src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs
View file

@ -2,7 +2,9 @@ namespace BotSharp.Abstraction.Realtime.Models;
public class RealtimeModelSettings
{
public string Voice { get; set; } = "alloy";
public float Temperature { get; set; } = 0.8f;
public int MaxResponseOutputTokens { get; set; } = 512;
public AudioTranscription InputAudioTranscription { get; set; } = new();
public ModelTurnDetection TurnDetection { get; set; } = new();
}

2
src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs
View file

@ -27,6 +27,8 @@ public class ConversationFilter
public List<string>? Tags { get; set; }
public bool IsLoadLatestStates { get; set; }
public static ConversationFilter Empty()
{
return new ConversationFilter();

2
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -124,7 +124,7 @@ public interface IBotSharpRepository : IHaveServiceProvider
=> throw new NotImplementedException();
void UpdateConversationStatus(string conversationId, string status)
=> throw new NotImplementedException();
Conversation GetConversation(string conversationId)
Conversation GetConversation(string conversationId, bool isLoadStates = false)
=> throw new NotImplementedException();
PagedItems<Conversation> GetConversations(ConversationFilter filter)
=> throw new NotImplementedException();

11
src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
View file

@ -74,7 +74,7 @@ public class RealtimeHub : IRealtimeHub
if (!model.Contains("-realtime-"))
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name;
model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name;
}
_completer.SetModelName(model);
@ -92,6 +92,7 @@ public class RealtimeHub : IRealtimeHub
}
routing.Context.SetDialogs(dialogs);
routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
var states = _services.GetRequiredService<IConversationStateService>();
@ -115,13 +116,14 @@ public class RealtimeHub : IRealtimeHub
}
else
{
// Push dialogs into model context
// Append dialogs into model context
var history = "[CONVERSATION HISTORY]\r\n";
foreach (var message in dialogs)
{
await _completer.InsertConversationItem(message);
history += $"{message.Role}: {message.Content}\r\n";
}
await _completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user without repeating your previous statement.");
await _completer.TriggerModelInference($"{instruction}\r\n\r\n{history}\r\n\r\nAssist user without repeating your previous statement.");
}
},
onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
@ -188,6 +190,7 @@ public class RealtimeHub : IRealtimeHub
// append input audio transcript to conversation
dialogs.Add(message);
storage.Append(_conn.ConversationId, message);
routing.Context.SetMessageId(_conn.ConversationId, message.MessageId);
foreach (var hook in hookProvider.HooksOrderByPriority)
{

18
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -46,18 +46,22 @@
</PropertyGroup>
<ItemGroup>
<Compile Remove="build\**" />
<Compile Remove="packages\**" />
<Compile Remove="Planning\**" />
<Compile Remove="Translation\Models\**" />
<EmbeddedResource Remove="build\**" />
<EmbeddedResource Remove="packages\**" />
<EmbeddedResource Remove="Planning\**" />
<EmbeddedResource Remove="Translation\Models\**" />
<None Remove="build\**" />
<None Remove="packages\**" />
<None Remove="Planning\**" />
<None Remove="Translation\Models\**" />
</ItemGroup>
<ItemGroup>
<None Remove="Content.targets" />
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\database_knowledge.liquid" />
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\reasoner.hf.liquid" />
<None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\reasoner.naive.liquid" />
@ -90,6 +94,7 @@
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.reviewer.liquid" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
<None Remove="data\plugins\config.json" />
</ItemGroup>
@ -187,6 +192,12 @@
<Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-instruct-execute_template.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-instruct-execute_template.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\plugins\config.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
@ -213,4 +224,11 @@
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
<ItemGroup>
<Content Include="build\BotSharp.Core.targets" PackagePath="build\BotSharp.Core.targets" />
<Content Include="data/*.*">
<Pack>true</Pack>
</Content>
</ItemGroup>
</Project>

7
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs
View file

@ -71,7 +71,7 @@ public partial class ConversationService : IConversationService
return db.UpdateConversationMessage(conversationId, request);
}
public async Task<Conversation> GetConversation(string id)
public async Task<Conversation> GetConversation(string id, bool isLoadStates = false)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var conversation = db.GetConversation(id);
@ -80,6 +80,11 @@ public partial class ConversationService : IConversationService
public async Task<PagedItems<Conversation>> GetConversations(ConversationFilter filter)
{
if (filter == null)
{
filter = ConversationFilter.Empty();
}
var db = _services.GetRequiredService<IBotSharpRepository>();
var conversations = db.GetConversations(filter);
return conversations;

39
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
View file

@ -69,23 +69,34 @@ public class ConversationStateService : IConversationStateService
return this;
}
var defaultRound = -1;
var preValue = string.Empty;
var currentValue = value.ToString();
var hooks = _services.GetServices<IConversationHook>();
var curActiveRounds = activeRounds > 0 ? activeRounds : -1;
int? preActiveRounds = null;
var curActive = true;
StateKeyValue? pair = null;
StateValue? prevLeafNode = null;
var curActiveRounds = activeRounds > 0 ? activeRounds : defaultRound;
if (ContainsState(name) && _curStates.TryGetValue(name, out var pair))
if (ContainsState(name) && _curStates.TryGetValue(name, out pair))
{
var leafNode = pair?.Values?.LastOrDefault();
preActiveRounds = leafNode?.ActiveRounds;
preValue = leafNode?.Data ?? string.Empty;
prevLeafNode = pair?.Values?.LastOrDefault();
preValue = prevLeafNode?.Data ?? string.Empty;
}
_logger.LogInformation($"[STATE] {name} = {value}");
var routingCtx = _services.GetRequiredService<IRoutingContext>();
if (!ContainsState(name) || preValue != currentValue || preActiveRounds != curActiveRounds)
var isNoChange = ContainsState(name)
&& preValue == currentValue
&& prevLeafNode?.ActiveRounds == curActiveRounds
&& curActiveRounds == defaultRound
&& prevLeafNode?.Source == source
&& prevLeafNode?.DataType == valueType
&& prevLeafNode?.Active == curActive
&& pair?.Readonly == readOnly;
var hooks = _services.GetServices<IConversationHook>();
if (!ContainsState(name) || preValue != currentValue || prevLeafNode?.ActiveRounds != curActiveRounds)
{
foreach (var hook in hooks)
{
@ -95,7 +106,7 @@ public class ConversationStateService : IConversationStateService
MessageId = routingCtx.MessageId,
Name = name,
BeforeValue = preValue,
BeforeActiveRounds = preActiveRounds,
BeforeActiveRounds = prevLeafNode?.ActiveRounds,
AfterValue = currentValue,
AfterActiveRounds = curActiveRounds,
DataType = valueType,
@ -116,7 +127,7 @@ public class ConversationStateService : IConversationStateService
{
Data = currentValue,
MessageId = routingCtx.MessageId,
Active = true,
Active = curActive,
ActiveRounds = curActiveRounds,
DataType = valueType,
Source = source,
@ -128,6 +139,10 @@ public class ConversationStateService : IConversationStateService
newPair.Values = new List<StateValue> { newValue };
_curStates[name] = newPair;
}
else if (isNoChange)
{
// do nothing
}
else
{
_curStates[name].Values.Add(newValue);
@ -415,14 +430,14 @@ public class ConversationStateService : IConversationStateService
{
var values = _curStates.Values.ToList();
var copy = JsonSerializer.Deserialize<List<StateKeyValue>>(JsonSerializer.Serialize(values));
return new ConversationState(copy ?? new());
return new ConversationState(copy ?? []);
}
public void SetCurrentState(ConversationState state)
{
var values = _curStates.Values.ToList();
var copy = JsonSerializer.Deserialize<List<StateKeyValue>>(JsonSerializer.Serialize(values));
_curStates = new ConversationState(copy ?? new());
_curStates = new ConversationState(copy ?? []);
}
public void ResetCurrentState()

8
src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
View file

@ -41,9 +41,11 @@ public class TokenStatistics : ITokenStatistics
var settingsService = _services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(stats.Provider, _model);
var deltaPromptCost = stats.PromptCount / 1000f * settings.PromptCost;
var deltaPromptCost = (stats.PromptCount - stats.CachedPromptCount) / 1000f * settings.PromptCost;
var deltaCachedPromptCost = stats.CachedPromptCount / 1000f * (settings.AdditionalCost?.CachedPromptCost ?? 0f);
var deltaCompletionCost = stats.CompletionCount / 1000f * settings.CompletionCost;
var deltaTotal = deltaPromptCost + deltaCompletionCost;
var deltaTotal = deltaPromptCost + deltaCachedPromptCost + deltaCompletionCost;
_promptCost += deltaPromptCost;
_completionCost += deltaCompletionCost;
@ -53,6 +55,8 @@ public class TokenStatistics : ITokenStatistics
stat.SetState("prompt_total", stats.PromptCount + inputCount, isNeedVersion: false, source: StateSource.Application);
var outputCount = int.Parse(stat.GetState("completion_total", "0"));
stat.SetState("completion_total", stats.CompletionCount + outputCount, isNeedVersion: false, source: StateSource.Application);
var cachedCount = int.Parse(stat.GetState("cached_prompt_total", "0"));
stat.SetState("cached_prompt_total", stats.CachedPromptCount + cachedCount, isNeedVersion: false, source: StateSource.Application);
// Total cost
var total_cost = float.Parse(stat.GetState("llm_total_cost", "0"));

4
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs
View file

@ -6,14 +6,14 @@ public partial class FileInstructService
{
public async Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null)
{
var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
var audioBytes = await DownloadFile(audio);
using var stream = new MemoryStream();
stream.Write(audioBytes, 0, audioBytes.Length);
stream.Position = 0;
var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text);
var content = await completion.TranscriptTextAsync(stream, fileName, text);
stream.Close();
return content;
}

2
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs
View file

@ -27,7 +27,7 @@ public partial class FileInstructService
var innerAgentId = agentId ?? Guid.Empty.ToString();
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai",
model: model, modelId: modelId ?? "gpt-4", multiModal: true);
model: model, modelId: modelId ?? "gpt-4o", multiModal: true);
var message = await completion.GetChatCompletions(new Agent()
{
Id = innerAgentId,

2
src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs
View file

@ -93,7 +93,7 @@ public partial class FileInstructService
}
var providerName = options.Provider ?? "openai";
var modelId = options?.ModelId ?? "gpt-4";
var modelId = options?.ModelId ?? "gpt-4o";
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName);
var model = llmProviderService.GetProviderModel(provider: provider, id: modelId);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);

4
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs
View file

@ -32,6 +32,10 @@ public partial class LocalFileStorageService
public BinaryData GetSpeechFile(string conversationId, string fileName)
{
var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName);
if (!File.Exists(path))
{
return BinaryData.Empty;
}
using var fs = new FileStream(path, FileMode.Open, FileAccess.Read);
return BinaryData.FromStream(fs);
}

37
src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs
View file

@ -30,7 +30,7 @@ public class CompletionProvider
}
else if (settings.Type == LlmModelType.Audio)
{
return GetAudioCompletion(services, provider: provider, model: model);
return GetAudioTranscriber(services, provider: provider, model: model);
}
else
{
@ -126,20 +126,39 @@ public class CompletionProvider
return completer;
}
public static IAudioCompletion GetAudioCompletion(
public static IAudioTranscription GetAudioTranscriber(
IServiceProvider services,
string provider,
string model)
string? provider = null,
string? model = null)
{
var completions = services.GetServices<IAudioCompletion>();
var completer = completions.FirstOrDefault(x => x.Provider == provider);
var completions = services.GetServices<IAudioTranscription>();
var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai"));
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve audio-completion provider by {provider}");
logger.LogError($"Can't resolve audio-transcriber provider by {provider}");
return default!;
}
completer.SetModelName(model);
completer.SetModelName(model ?? "gpt-4o-mini-transcribe");
return completer;
}
public static IAudioSynthesis GetAudioSynthesizer(
IServiceProvider services,
string? provider = null,
string? model = null)
{
var completions = services.GetServices<IAudioSynthesis>();
var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai"));
if (completer == null)
{
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
logger.LogError($"Can't resolve audio-synthesizer provider by {provider}");
return default!;
}
completer.SetModelName(model ?? "gpt-4o-mini-tts");
return completer;
}
@ -172,7 +191,7 @@ public class CompletionProvider
string? model = null,
string? modelId = null,
bool? multiModal = null,
bool? realTime = null,
bool realTime = false,
bool imageGenerate = false,
AgentLlmConfig? agentConfig = null)
{

7
src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs
View file

@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService
?.Models ?? new List<LlmModelSetting>();
}
public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool? realTime = false, bool imageGenerate = false)
public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool realTime = false, bool imageGenerate = false)
{
var models = GetProviderModels(provider)
.Where(x => x.Id == id);
@ -54,10 +54,7 @@ public class LlmProviderService : ILlmProviderService
models = models.Where(x => x.MultiModal == multiModal);
}
if (realTime.HasValue)
{
models = models.Where(x => x.RealTime == realTime);
}
models = models.Where(x => x.RealTime == realTime);
models = models.Where(x => x.ImageGeneration == imageGenerate);

92
src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs Normal file
View file

@ -0,0 +1,92 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Instructs;
using BotSharp.Abstraction.Instructs.Models;
namespace BotSharp.Core.Instructs.Functions;
public class ExecuteTemplateFn : IFunctionCallback
{
public string Name => "util-instruct-execute_template";
private readonly IServiceProvider _services;
private readonly ILogger<ExecuteTemplateFn> _logger;
public ExecuteTemplateFn(
IServiceProvider services,
ILogger<ExecuteTemplateFn> logger)
{
_services = services;
_logger = logger;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<ExecuteTemplateArgs>(message.FunctionArgs);
if (string.IsNullOrEmpty(args.TemplateName))
{
message.Content = $"Invalid template name.";
return false;
}
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(message.CurrentAgentId);
var template = agent.Templates.FirstOrDefault(x => x.Name.IsEqualTo(args.TemplateName));
if (template == null)
{
message.Content = $"Cannot find template ({args.TemplateName}) in agent {agent.Name}";
return false;
}
var response = await GetAiResponse(agent, args.TemplateName);
message.Content = response;
return true;
}
private async Task<string> GetAiResponse(Agent agent, string templateName)
{
try
{
var agentService = _services.GetRequiredService<IAgentService>();
var text = agentService.RenderedTemplate(agent, templateName);
var completion = CompletionProvider.GetChatCompletion(_services, provider: agent.LlmConfig?.Provider, model: agent.LlmConfig?.Model);
var response = await completion.GetChatCompletions(new Agent()
{
Id = agent.Id
},
new List<RoleDialogModel>
{
new(AgentRole.User, text)
});
var hooks = _services.GetServices<IInstructHook>();
foreach (var hook in hooks)
{
if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agent.Id)
{
continue;
}
await hook.OnResponseGenerated(new InstructResponseModel
{
AgentId = agent.Id,
TemplateName = templateName,
Provider = completion.Provider,
Model = completion.Model,
UserMessage = text,
CompletionText = response.Content
});
}
return response.Content;
}
catch (Exception ex)
{
var error = $"Error when getting agent {agent.Name} instruction response.";
_logger.LogWarning($"{error} {ex.Message}\r\n{ex.InnerException}");
return error;
}
}
}

17
src/Infrastructure/BotSharp.Core/Instructs/Hooks/InstructUtilityHook.cs Normal file
View file

@ -0,0 +1,17 @@
namespace BotSharp.Core.Instructs.Hooks;
public class InstructUtilityHook : IAgentUtilityHook
{
private static string PREFIX = "util-instruct-";
private static string EXECUTE_TEMPLATE = $"{PREFIX}execute_template";
public void AddUtilities(List<AgentUtility> utilities)
{
utilities.Add(new AgentUtility
{
Name = "instruct.template",
Functions = [new($"{EXECUTE_TEMPLATE}")],
Templates = [new($"{EXECUTE_TEMPLATE}.fn")]
});
}
}

3
src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Instructs.Settings;
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Settings;
using BotSharp.Core.Instructs.Hooks;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Core.Instructs;
@ -18,6 +19,8 @@ public class InsturctionPlugin : IBotSharpPlugin
var settingService = provider.GetRequiredService<ISettingService>();
return settingService.Bind<InstructionSettings>("Instruction");
});
services.AddScoped<IAgentUtilityHook, InstructUtilityHook>();
}
public bool AttachMenu(List<PluginMenuDef> menu)

2
src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file

@ -80,7 +80,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
public bool DeleteConversations(IEnumerable<string> conversationIds)
=> throw new NotImplementedException();
public Conversation GetConversation(string conversationId)
public Conversation GetConversation(string conversationId, bool isLoadStates = false)
=> throw new NotImplementedException();
public PagedItems<Conversation> GetConversations(ConversationFilter filter)

38
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Abstraction.Users.Models;
using System;
using System.IO;
namespace BotSharp.Core.Repository;
@ -346,7 +347,7 @@ public partial class FileRepository
}
}
public Conversation GetConversation(string conversationId)
public Conversation GetConversation(string conversationId, bool isLoadStates = false)
{
var convDir = FindConversationDirectory(conversationId);
if (string.IsNullOrEmpty(convDir)) return null;
@ -361,18 +362,20 @@ public partial class FileRepository
record.Dialogs = CollectDialogElements(dialogFile);
}
var stateFile = Path.Combine(convDir, STATE_FILE);
if (record != null)
if (isLoadStates)
{
var states = CollectConversationStates(stateFile);
var curStates = new Dictionary<string, string>();
states.ForEach(x =>
var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE);
if (record != null && File.Exists(latestStateFile))
{
curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty;
});
record.States = curStates;
var stateJson = File.ReadAllText(latestStateFile);
var states = JsonSerializer.Deserialize<Dictionary<string, JsonDocument>>(stateJson, _options) ?? [];
record.States = states.ToDictionary(x => x.Key, x =>
{
var elem = x.Value.RootElement.GetProperty("data");
return elem.ValueKind != JsonValueKind.Null ? elem.ToString() : null;
});
}
}
return record;
}
@ -508,6 +511,21 @@ public partial class FileRepository
if (!matched) continue;
if (filter.IsLoadLatestStates)
{
var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE);
if (File.Exists(latestStateFile))
{
var stateJson = File.ReadAllText(latestStateFile);
var states = JsonSerializer.Deserialize<Dictionary<string, JsonDocument>>(stateJson, _options) ?? [];
record.States = states.ToDictionary(x => x.Key, x =>
{
var elem = x.Value.RootElement.GetProperty("data");
return elem.ValueKind != JsonValueKind.Null ? elem.ToString() : null;
});
}
}
records.Add(record);
}

6
src/Infrastructure/BotSharp.Core/Routing/Hooks/RoutingUtilityHook.cs
View file

@ -8,13 +8,11 @@ public class RoutingUtilityHook : IAgentUtilityHook
public void AddUtilities(List<AgentUtility> utilities)
{
var utility = new AgentUtility
utilities.Add(new AgentUtility
{
Name = "routing.tools",
Functions = [new($"{REDIRECT_TO_AGENT}"), new($"{FALLBACK_TO_ROUTER}")],
Templates = [new($"{REDIRECT_TO_AGENT}.fn"), new($"{FALLBACK_TO_ROUTER}.fn")]
};
utilities.Add(utility);
});
}
}

4
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs
View file

@ -101,8 +101,8 @@ public partial class RoutingService
Context.SetDialogs(dialogs);
// Send to Next LLM
var agentId = routing.Context.GetCurrentAgentId();
await InvokeAgent(agentId, dialogs);
var curAgentId = routing.Context.GetCurrentAgentId();
await InvokeAgent(curAgentId, dialogs);
}
}
else

66
src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Templating;
namespace BotSharp.Core.Routing;
public partial class RoutingService
@ -6,12 +8,20 @@ public partial class RoutingService
public async Task<bool> InvokeFunction(string name, RoleDialogModel message)
{
var function = _services.GetServices<IFunctionCallback>().FirstOrDefault(x => x.Name == name);
var isFillDummyContent = false;
var dummyFuncResponse = string.Empty;
if (function == null)
{
message.StopCompletion = true;
message.Content = $"Can't find function implementation of {name}.";
_logger.LogError(message.Content);
return false;
dummyFuncResponse = await GetDummyFunctionOutput(name, message);
isFillDummyContent = !string.IsNullOrEmpty(dummyFuncResponse);
if (!isFillDummyContent)
{
message.StopCompletion = true;
message.Content = $"Can't find function implementation of {name}.";
_logger.LogError(message.Content);
return false;
}
}
// Clone message
@ -25,7 +35,15 @@ public partial class RoutingService
var progressService = _services.GetService<IConversationProgressService>();
// Before executing functions
clonedMessage.Indication = await function.GetIndication(message);
if (!isFillDummyContent)
{
clonedMessage.Indication = await function.GetIndication(message);
}
else
{
clonedMessage.Indication = "Running";
}
if (progressService?.OnFunctionExecuting != null)
{
await progressService.OnFunctionExecuting(clonedMessage);
@ -40,7 +58,15 @@ public partial class RoutingService
try
{
result = await function.Execute(clonedMessage);
if (!isFillDummyContent)
{
result = await function.Execute(clonedMessage);
}
else
{
clonedMessage.Content = dummyFuncResponse;
result = true;
}
// After functions have been executed
foreach (var hook in hooks)
@ -87,4 +113,32 @@ public partial class RoutingService
return result;
}
private async Task<string?> GetDummyFunctionOutput(string functionName, RoleDialogModel message)
{
if (string.IsNullOrEmpty(message.CurrentAgentId))
{
return null;
}
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(message.CurrentAgentId);
var found = agent?.Functions?.FirstOrDefault(x => x.Name == functionName);
if (string.IsNullOrWhiteSpace(found?.Output))
{
return null;
}
var render = _services.GetRequiredService<ITemplateRender>();
var state = _services.GetRequiredService<IConversationStateService>();
var dict = new Dictionary<string, object>();
foreach (var item in state.GetStates())
{
dict[item.Key] = item.Value;
}
var text = render.Render(found.Output, dict);
return text;
}
}

10
src/Infrastructure/BotSharp.Core/build/BotSharp.Core.targets Normal file
View file

@ -0,0 +1,10 @@
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<!-- Define the files to be copied -->
<ContentFiles Include="$(MSBuildThisFileDirectory)..\contentFiles\any\**\*.*" />
</ItemGroup>
<Target Name="CopyContentFiles" AfterTargets="Build">
<!-- Copy the content files to the output directory -->
<Copy SourceFiles="@(ContentFiles)" DestinationFolder="$(TargetDir)%(RecursiveDir)" />
</Target>
</Project>

14
src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-instruct-execute_template.json Normal file
View file

@ -0,0 +1,14 @@
{
"name": "util-instruct-execute_template",
"description": "Select a specific template that can handle the user's request.",
"parameters": {
"type": "object",
"properties": {
"template_name": {
"type": "string",
"description": "The template name that is selected for handling the request."
}
},
"required": [ "template_name" ]
}
}

3
src/Infrastructure/BotSharp.Core/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-instruct-execute_template.fn.liquid Normal file
View file

@ -0,0 +1,3 @@
please call function util-routing-execute_template if user wants to use a template to fulfill a specific task.
Please ensure each template is executed only once.
Please output the template response directly without changing anthything.

12
src/Infrastructure/BotSharp.Logger/Hooks/InstructionLogHook.cs
View file

@ -26,7 +26,14 @@ public class InstructionLogHook : InstructHookBase
public override async Task OnResponseGenerated(InstructResponseModel response)
{
var settings = _services.GetRequiredService<InstructionSettings>();
if (!settings.EnableLog || response == null) return;
if (response == null
|| string.IsNullOrWhiteSpace(response.AgentId)
|| settings == null
|| !settings.Logging.Enabled
|| settings.Logging.ExcludedAgentIds.Contains(response.AgentId))
{
return;
}
var db = _services.GetRequiredService<IBotSharpRepository>();
var state = _services.GetRequiredService<IConversationStateService>();
@ -49,6 +56,7 @@ public class InstructionLogHook : InstructHookBase
UserId = user?.Id
}
});
return;
await base.OnResponseGenerated(response);
}
}

6
src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
View file

@ -138,7 +138,7 @@ public class ConversationController : ControllerBase
}
[HttpGet("/conversation/{conversationId}")]
public async Task<ConversationViewModel?> GetConversation([FromRoute] string conversationId)
public async Task<ConversationViewModel?> GetConversation([FromRoute] string conversationId, [FromQuery] bool isLoadStates = false)
{
var service = _services.GetRequiredService<IConversationService>();
var userService = _services.GetRequiredService<IUserService>();
@ -151,7 +151,8 @@ public class ConversationController : ControllerBase
var filter = new ConversationFilter
{
Id = conversationId,
UserId = !isAdmin ? user.Id : null
UserId = !isAdmin ? user.Id : null,
IsLoadLatestStates = isLoadStates
};
var conversations = await service.GetConversations(filter);
if (conversations.Items.IsNullOrEmpty())
@ -161,7 +162,6 @@ public class ConversationController : ControllerBase
var result = ConversationViewModel.FromSession(conversations.Items.First());
var state = _services.GetRequiredService<IConversationStateService>();
result.States = state.Load(conversationId, isReadOnly: true);
user = await userService.GetUser(result.User.Id);
result.User = UserViewModel.FromUser(user);

8
src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
View file

@ -499,8 +499,8 @@ public class InstructModeController : ControllerBase
file.CopyTo(stream);
stream.Position = 0;
var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text);
var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
var content = await completion.TranscriptTextAsync(stream, file.FileName, text);
viewModel.Content = content;
stream.Close();
return viewModel;
@ -520,8 +520,8 @@ public class InstructModeController : ControllerBase
var state = _services.GetRequiredService<IConversationStateService>();
input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1");
var binaryData = await completion.GenerateAudioFromTextAsync(input.Text);
var completion = CompletionProvider.GetAudioSynthesizer(_services, provider: input.Provider, model: input.Model);
var binaryData = await completion.GenerateAudioAsync(input.Text);
var stream = binaryData.ToStream();
stream.Position = 0;

2
src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs
View file

@ -22,7 +22,7 @@ public class RealtimeController : ControllerBase
[HttpGet("/agent/{agentId}/realtime/session")]
public async Task<RealtimeSession> CreateSession(string agentId)
{
var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4");
var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4o");
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(agentId);

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs
View file

@ -31,7 +31,7 @@ public class ConversationViewModel
public string? TaskId { get; set; }
public string Status { get; set; }
public Dictionary<string, string> States { get; set; }
public Dictionary<string, string> States { get; set; } = [];
public List<string> Tags { get; set; } = new();
@ -55,7 +55,8 @@ public class ConversationViewModel
Channel = sess.Channel,
Status = sess.Status,
TaskId = sess.TaskId,
Tags = sess.Tags ?? new(),
Tags = sess.Tags ?? [],
States = sess.States ?? [],
CreatedTime = sess.CreatedTime,
UpdatedTime = sess.UpdatedTime
};

2
src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs
View file

@ -16,7 +16,7 @@ public class AudioHandlerPlugin : IBotSharpPlugin
return settingService.Bind<AudioHandlerSettings>("AudioHandler");
});
services.AddScoped<IAudioCompletion, NativeWhisperProvider>();
services.AddScoped<IAudioTranscription, NativeWhisperProvider>();
services.AddScoped<IAgentUtilityHook, AudioHandlerUtilityHook>();
}
}

6
src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs
View file

@ -91,7 +91,7 @@ public class HandleAudioRequestFn : IFunctionCallback
using var stream = new MemoryStream(bytes);
stream.Position = 0;
var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName);
var result = await audioCompletion.TranscriptTextAsync(stream, fileName);
transcripts.Add(result);
stream.Close();
}
@ -104,9 +104,9 @@ public class HandleAudioRequestFn : IFunctionCallback
return string.Join("\r\n\r\n", transcripts);
}
private IAudioCompletion PrepareModel()
private IAudioTranscription PrepareModel()
{
return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1");
return CompletionProvider.GetAudioTranscriber(_serviceProvider);
}
private bool ParseAudioFileType(string fileName)

6
src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs
View file

@ -6,7 +6,7 @@ namespace BotSharp.Plugin.AudioHandler.Provider;
/// <summary>
/// Native Whisper provider for speech to text conversion
/// </summary>
public class NativeWhisperProvider : IAudioCompletion
public class NativeWhisperProvider : IAudioTranscription
{
private static WhisperProcessor _whisperProcessor;
@ -29,7 +29,7 @@ public class NativeWhisperProvider : IAudioCompletion
_logger = logger;
}
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
public async Task<string> TranscriptTextAsync(Stream audio, string audioFileName, string? text = null)
{
var textResult = new List<SegmentData>();
@ -49,7 +49,7 @@ public class NativeWhisperProvider : IAudioCompletion
return audioOutput.ToString();
}
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
public async Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3")
{
throw new NotImplementedException();
}

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs
View file

@ -31,6 +31,6 @@ public class AzureOpenAiPlugin : IBotSharpPlugin
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
services.AddScoped<IImageCompletion, ImageCompletionProvider>();
services.AddScoped<IAudioCompletion, AudioCompletionProvider>();
services.AddScoped<IAudioTranscription, AudioCompletionProvider>();
}
}

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs
View file

@ -4,7 +4,7 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
public async Task<string> TranscriptTextAsync(Stream audio, string audioFileName, string? text = null)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);

16
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs
View file

@ -4,27 +4,27 @@ namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider
{
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
public async Task<BinaryData> GenerateAudioFromTextAsync(string text, string? voice = "alloy", string? format = "mp3")
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format);
var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options);
return result.Value;
}
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions()
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format)
{
var state = _services.GetRequiredService<IConversationStateService>();
var voice = GetVoice(state.GetState("speech_generate_voice"));
var format = GetSpeechFormat(state.GetState("speech_generate_format"));
var speechVoice = GetVoice(voice ?? "alloy");
var responseFormat = GetSpeechFormat(format ?? "mp3");
var speed = GetSpeed(state.GetState("speech_generate_speed"));
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
SpeedRatio = speed
ResponseFormat = responseFormat,
SpeedRatio = speed,
};
return (voice, options);

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio;
public partial class AudioCompletionProvider : IAudioCompletion
public partial class AudioCompletionProvider : IAudioTranscription
{
private readonly IServiceProvider _services;

2
src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs
View file

@ -106,6 +106,7 @@ public class ChatHubConversationHook : ConversationHookBase
if (!AllowSendingMessage()) return;
var conv = _services.GetRequiredService<IConversationService>();
var state = _services.GetRequiredService<IConversationStateService>();
var json = JsonSerializer.Serialize(new ChatResponseModel()
{
ConversationId = conv.ConversationId,
@ -114,6 +115,7 @@ public class ChatHubConversationHook : ConversationHookBase
Function = message.FunctionName,
RichContent = message.SecondaryRichContent ?? message.RichContent,
Data = message.Data,
States = state.GetStates(),
Sender = new UserViewModel()
{
FirstName = "AI",

2
src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs
View file

@ -67,7 +67,7 @@ public class HandleEmailReaderFn : IFunctionCallback
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4");
var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4o");
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var convService = _services.GetRequiredService<IConversationService>();
var conversationId = convService.ConversationId;

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs
View file

@ -100,7 +100,7 @@ public class ReadImageFn : IFunctionCallback
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true);
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4o", multiModal: true);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var response = await completion.GetChatCompletions(agent, dialogs);
return response.Content;

2
src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs
View file

@ -78,7 +78,7 @@ public class ReadPdfFn : IFunctionCallback
{
var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4", multiModal: true);
var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4o", multiModal: true);
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
var response = await completion.GetChatCompletions(agent, dialogs);
return response.Content;

7
src/Plugins/BotSharp.Plugin.MongoStorage/Models/FunctionDefMongoElement.cs
View file

@ -12,6 +12,7 @@ public class FunctionDefMongoElement
public string? VisibilityExpression { get; set; }
public string? Impact { get; set; }
public FunctionParametersDefMongoElement Parameters { get; set; } = new();
public string? Output { get; set; }
public static FunctionDefMongoElement ToMongoElement(FunctionDef function)
{
@ -27,7 +28,8 @@ public class FunctionDefMongoElement
Type = function.Parameters.Type,
Properties = JsonSerializer.Serialize(function.Parameters.Properties),
Required = function.Parameters.Required,
}
},
Output = function.Output
};
}
@ -45,7 +47,8 @@ public class FunctionDefMongoElement
Type = function.Parameters.Type,
Properties = JsonSerializer.Deserialize<JsonDocument>(function.Parameters.Properties.IfNullOrEmptyAs("{}")),
Required = function.Parameters.Required,
}
},
Output = function.Output
};
}
}

2
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs
View file

@ -35,7 +35,7 @@ public partial class MongoRepository
UpdateAgentProfiles(agent.Id, agent.Profiles);
break;
case AgentField.Label:
UpdateAgentLabels(agent.Id, agent.Profiles);
UpdateAgentLabels(agent.Id, agent.Labels);
break;
case AgentField.RoutingRule:
UpdateAgentRoutingRules(agent.Id, agent.RoutingRules);

52
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs
View file

@ -299,26 +299,25 @@ public partial class MongoRepository
_dc.Conversations.UpdateOne(filter, update);
}
public Conversation GetConversation(string conversationId)
public Conversation GetConversation(string conversationId, bool isLoadStates = false)
{
if (string.IsNullOrEmpty(conversationId)) return null;
var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var filterState = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var conv = _dc.Conversations.Find(filterConv).FirstOrDefault();
var dialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault();
var states = _dc.ConversationStates.Find(filterState).FirstOrDefault();
if (conv == null) return null;
var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List<DialogElement>();
var curStates = new Dictionary<string, string>();
states.States.ForEach(x =>
var curStates = conv.LatestStates?.ToDictionary(x => x.Key, x =>
{
curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty;
});
var jsonDoc = JsonDocument.Parse(x.Value.ToJson());
var data = jsonDoc.RootElement.GetProperty("data");
return data.ValueKind != JsonValueKind.Null ? data.ToString() : null;
}) ?? [];
return new Conversation
{
@ -456,19 +455,34 @@ public partial class MongoRepository
var conversationDocs = _dc.Conversations.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList();
var count = _dc.Conversations.CountDocuments(filterDef);
var conversations = conversationDocs.Select(x => new Conversation
var conversations = conversationDocs.Select(x =>
{
Id = x.Id.ToString(),
AgentId = x.AgentId.ToString(),
UserId = x.UserId.ToString(),
TaskId = x.TaskId,
Title = x.Title,
Channel = x.Channel,
Status = x.Status,
DialogCount = x.DialogCount,
Tags = x.Tags ?? new(),
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
var states = new Dictionary<string, string>();
if (filter.IsLoadLatestStates)
{
states = x.LatestStates.ToDictionary(p => p.Key, p =>
{
var jsonDoc = JsonDocument.Parse(p.Value.ToJson());
var data = jsonDoc.RootElement.GetProperty("data");
return data.ValueKind != JsonValueKind.Null ? data.ToString() : null;
});
}
return new Conversation
{
Id = x.Id.ToString(),
AgentId = x.AgentId.ToString(),
UserId = x.UserId.ToString(),
TaskId = x.TaskId,
Title = x.Title,
Channel = x.Channel,
Status = x.Status,
DialogCount = x.DialogCount,
Tags = x.Tags ?? [],
States = states,
CreatedTime = x.CreatedTime,
UpdatedTime = x.UpdatedTime
};
}).ToList();
return new PagedItems<Conversation>

24
src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs
View file

@ -48,6 +48,9 @@ public class RealtimeSessionBody
[JsonPropertyName("turn_detection")]
public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new();
[JsonPropertyName("input_audio_noise_reduction")]
public InputAudioNoiseReduction InputAudioNoiseReduction { get; set; } = new();
}
public class RealtimeSessionTurnDetection
@ -58,28 +61,39 @@ public class RealtimeSessionTurnDetection
/// <summary>
/// Milliseconds
/// </summary>
[JsonPropertyName("prefix_padding_ms")]
/*[JsonPropertyName("prefix_padding_ms")]
public int PrefixPadding { get; set; } = 300;
[JsonPropertyName("silence_duration_ms")]
public int SilenceDuration { get; set; } = 500;
[JsonPropertyName("threshold")]
public float Threshold { get; set; } = 0.5f;
public float Threshold { get; set; } = 0.5f;*/
[JsonPropertyName("type")]
public string Type { get; set; } = "server_vad";
public string Type { get; set; } = "semantic_vad";
[JsonPropertyName("eagerness")]
public string eagerness { get;set; } = "auto";
}
public class InputAudioTranscription
{
[JsonPropertyName("model")]
public string Model { get; set; } = null!;
public string Model { get; set; } = "gpt-4o-transcribe";
[JsonPropertyName("language")]
public string Language { get; set; } = "en";
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Language { get; set; }
[JsonPropertyName("prompt")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Prompt { get; set; }
}
public class InputAudioNoiseReduction
{
[JsonPropertyName("type")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string Type { get; set; } = "far_field";
}

3
src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs
View file

@ -33,7 +33,8 @@ public class OpenAiPlugin : IBotSharpPlugin
services.AddScoped<IChatCompletion, ChatCompletionProvider>();
services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
services.AddScoped<IImageCompletion, ImageCompletionProvider>();
services.AddScoped<IAudioCompletion, AudioCompletionProvider>();
services.AddScoped<IAudioTranscription, AudioTranscriptionProvider>();
services.AddScoped<IAudioSynthesis, AudioSynthesisProvider>();
services.AddScoped<IRealTimeCompletion, RealTimeCompletionProvider>();
services.AddRefitClient<IOpenAiRealtimeApi>()

23
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs
View file

@ -1,23 +0,0 @@
using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider : IAudioCompletion
{
private readonly IServiceProvider _services;
public string Provider => "openai";
public string Model => _model;
private string _model;
public AudioCompletionProvider(IServiceProvider service)
{
_services = service;
}
public void SetModelName(string model)
{
_model = model;
}
}

38
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs → src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs
View file

@ -2,29 +2,45 @@ using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider
public class AudioSynthesisProvider : IAudioSynthesis
{
public async Task<BinaryData> GenerateAudioFromTextAsync(string text)
private readonly IServiceProvider _services;
public string Provider => "openai";
public string Model => _model;
private string _model;
public AudioSynthesisProvider(IServiceProvider service)
{
_services = service;
}
public void SetModelName(string model)
{
_model = model;
}
public async Task<BinaryData> GenerateAudioAsync(string text, string? voice = "alloy", string? format = "mp3", string? instructions = null)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format);
var result = await audioClient.GenerateSpeechAsync(text, speechVoice, options);
return result.Value;
}
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions()
private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions(string? voice, string? format)
{
var state = _services.GetRequiredService<IConversationStateService>();
var voice = GetVoice(state.GetState("speech_generate_voice"));
var format = GetSpeechFormat(state.GetState("speech_generate_format"));
var speechVoice = GetVoice(voice ?? "alloy");
var responseFormat = GetSpeechFormat(format ?? "mp3");
var speed = GetSpeed(state.GetState("speech_generate_speed"));
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
SpeedRatio = speed
ResponseFormat = responseFormat,
SpeedRatio = speed,
};
return (voice, options);
@ -32,10 +48,8 @@ public partial class AudioCompletionProvider
private GeneratedSpeechVoice GetVoice(string input)
{
var value = !string.IsNullOrEmpty(input) ? input : "alloy";
GeneratedSpeechVoice voice;
switch (value)
switch (input)
{
case "echo":
voice = GeneratedSpeechVoice.Echo;

21
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs → src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioTranscriptionProvider.cs
View file

@ -2,9 +2,26 @@ using OpenAI.Audio;
namespace BotSharp.Plugin.OpenAI.Providers.Audio;
public partial class AudioCompletionProvider
public class AudioTranscriptionProvider : IAudioTranscription
{
public async Task<string> GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null)
private readonly IServiceProvider _services;
public string Provider => "openai";
public string Model => _model;
private string _model;
public AudioTranscriptionProvider(IServiceProvider service)
{
_services = service;
}
public void SetModelName(string model)
{
_model = model;
}
public async Task<string> TranscriptTextAsync(Stream audio, string audioFileName, string? text = null)
{
var audioClient = ProviderHelper.GetClient(Provider, _model, _services)
.GetAudioClient(_model);

1
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -84,6 +84,7 @@ public class ChatCompletionProvider : IChatCompletion
Provider = Provider,
Model = _model,
PromptCount = response.Value?.Usage?.InputTokenCount ?? 0,
CachedPromptCount = response.Value?.Usage?.InputTokenDetails?.CachedTokenCount ?? 0,
CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0
});
}

46
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
View file

@ -96,14 +96,24 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
public async Task TriggerModelInference(string? instructions = null)
{
// Triggering model inference
await SendEventToModel(new
if (!string.IsNullOrEmpty(instructions))
{
type = "response.create",
response = new
await SendEventToModel(new
{
instructions
}
});
type = "response.create",
response = new
{
instructions
}
});
}
else
{
await SendEventToModel(new
{
type = "response.create"
});
}
}
public async Task CancelModelResponse()
@ -317,7 +327,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
var words = new List<string>();
HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
var realitmeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
var sessionUpdate = new
{
@ -328,23 +338,27 @@ public class RealTimeCompletionProvider : IRealTimeCompletion
OutputAudioFormat = "g711_ulaw",
InputAudioTranscription = new InputAudioTranscription
{
Model = "whisper-1",
Language = "en",
Model = realtimeModelSettings.InputAudioTranscription.Model,
Language = realtimeModelSettings.InputAudioTranscription.Language,
Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
},
Voice = "alloy",
Voice = realtimeModelSettings.Voice,
Instructions = instruction,
ToolChoice = "auto",
Tools = functions,
Modalities = [ "text", "audio" ],
Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens,
Temperature = Math.Max(options.Temperature ?? realtimeModelSettings.Temperature, 0.6f),
MaxResponseOutputTokens = realtimeModelSettings.MaxResponseOutputTokens,
TurnDetection = new RealtimeSessionTurnDetection
{
InterruptResponse = interruptResponse,
Threshold = realitmeModelSettings.TurnDetection.Threshold,
PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration
InterruptResponse = interruptResponse/*,
Threshold = realtimeModelSettings.TurnDetection.Threshold,
PrefixPadding = realtimeModelSettings.TurnDetection.PrefixPadding,
SilenceDuration = realtimeModelSettings.TurnDetection.SilenceDuration*/
},
InputAudioNoiseReduction = new InputAudioNoiseReduction
{
Type = "near_field"
}
}
};

17
src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
View file

@ -10,6 +10,19 @@
</PropertyGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-transfer_phone_call.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-hangup_phone_call.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-text_message.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-outbound_phone_call.json" />
</ItemGroup>
<ItemGroup>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-leave_voicemail.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-transfer_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-hangup_phone_call.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
@ -33,8 +46,4 @@
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
</ItemGroup>
</Project>

65
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs Normal file
View file

@ -0,0 +1,65 @@
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
namespace BotSharp.Plugin.Twilio.Controllers;
public class TwilioOutboundController : TwilioController
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
private readonly IHttpContextAccessor _context;
private readonly ILogger _logger;
public TwilioOutboundController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger<TwilioOutboundController> logger)
{
_settings = settings;
_services = services;
_context = context;
_logger = logger;
}
[ValidateRequest]
[HttpPost("twilio/voice/init-outbound-call")]
public async Task<TwiMLResult> InitiateOutboundCall(ConversationalVoiceRequest request)
{
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default!;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api")
{
response = new VoiceResponse();
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
}
else
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
ActionOnEmptyResult = true,
CallbackPath = $"twilio/voice/receive/1?agent-id={request.AgentId}&conversation-id={request.ConversationId}",
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
}
response = twilio.ReturnNoninterruptedInstructions(instruction);
}
return TwiML(response);
}
}

31
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
View file

@ -35,17 +35,9 @@ public class TwilioStreamController : TwilioController
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
}
var twilio = _services.GetRequiredService<TwilioService>();
VoiceResponse response = default!;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null)
{
response = new VoiceResponse();
response.Play(new Uri(request.InitAudioFile));
return TwiML(response);
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
@ -67,10 +59,25 @@ public class TwilioStreamController : TwilioController
});
request.ConversationId = await InitConversation(request);
instruction.ConversationId = request.ConversationId;
var twilio = _services.GetRequiredService<TwilioService>();
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api")
{
response = new VoiceResponse();
response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction);
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
{
await hook.OnVoicemailStarting(request);
});
var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
response.Play(new Uri(url));
}
else
{
response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction);
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
@ -91,7 +98,7 @@ public class TwilioStreamController : TwilioController
{
var conv = new Conversation
{
AgentId = request.AgentId ?? _settings.AgentId,
AgentId = request.AgentId,
Channel = ConversationChannel.Phone,
ChannelId = request.CallSid,
Title = $"Incoming phone call from {request.From}",

243
src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
@ -7,6 +6,7 @@ using BotSharp.Plugin.Twilio.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Twilio.Http;
using Task = System.Threading.Tasks.Task;
namespace BotSharp.Plugin.Twilio.Controllers;
@ -49,27 +49,26 @@ public class TwilioVoiceController : TwilioController
throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
}
VoiceResponse response = null;
VoiceResponse response = default!;
request.ConversationId = $"twilio_{request.CallSid}";
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = ["twilio/welcome.mp3"],
SpeechPaths = [$"twilio/welcome-{request.AgentId}.mp3"],
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreating(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
request.ConversationId = $"TwilioVoice_{request.CallSid}";
instruction.CallbackPath = $"twilio/voice/receive/0?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}";
var twilio = _services.GetRequiredService<TwilioService>();
if (string.IsNullOrWhiteSpace(request.Intent))
{
instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}";
response = twilio.ReturnNoninterruptedInstructions(instruction);
}
else
@ -80,6 +79,7 @@ public class TwilioVoiceController : TwilioController
await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent);
var callerMessage = new CallerMessage()
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SeqNumber = seqNum,
Content = request.Intent,
@ -88,15 +88,14 @@ public class TwilioVoiceController : TwilioController
};
await messageQueue.EnqueueAsync(callerMessage);
response = new VoiceResponse();
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post);
// delay 3 seconds to wait for the first message reply and caller is listening dudu sound
await Task.Delay(1000 * 3);
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"), HttpMethod.Post);
}
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnSessionCreated(request);
}, new HookEmitOption
{
OnlyOnce = true
});
return TwiML(response);
@ -114,21 +113,25 @@ public class TwilioVoiceController : TwilioController
var twilio = _services.GetRequiredService<TwilioService>();
var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
// Fetch all accumulated caller message.
var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(request.ConversationId, request.SeqNum);
string text = (request.SpeechResult + "\r\n" + request.Digits).Trim();
if (!string.IsNullOrWhiteSpace(text))
{
// Concanate with incoming message
messages.Add(text);
await sessionManager.StageCallerMessageAsync(request.ConversationId, request.SeqNum, text);
}
VoiceResponse response = null;
VoiceResponse response = default!;
if (messages.Any())
{
var messageContent = string.Join("\r\n", messages);
var callerMessage = new CallerMessage()
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SeqNumber = request.SeqNum,
Content = messageContent,
@ -141,14 +144,11 @@ public class TwilioVoiceController : TwilioController
await messageQueue.EnqueueAsync(callerMessage);
response = new VoiceResponse();
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post);
response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post);
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnReceivedUserMessage(request);
}, new HookEmitOption
{
OnlyOnce = true
});
}
else
@ -159,21 +159,19 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentHangUp(request);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.HangUp(null);
response = twilio.HangUp(string.Empty);
}
// keep waiting for user response
else
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = new List<string>(),
CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
CallbackPath = $"twilio/voice/receive/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
ActionOnEmptyResult = true
};
@ -185,9 +183,6 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnWaitingUserResponse(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
@ -210,9 +205,10 @@ public class TwilioVoiceController : TwilioController
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var twilio = _services.GetRequiredService<TwilioService>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
if (request.SpeechResult != null)
var text = (request.SpeechResult + "\r\n" + request.Digits).Trim();
if (!string.IsNullOrEmpty(text))
{
await sessionManager.StageCallerMessageAsync(request.ConversationId, nextSeqNum, request.SpeechResult);
await sessionManager.StageCallerMessageAsync(request.ConversationId, nextSeqNum, text);
}
var reply = await sessionManager.GetAssistantReplyAsync(request.ConversationId, request.SeqNum);
@ -225,112 +221,13 @@ public class TwilioVoiceController : TwilioController
{
request.AIResponseErrorMessage = $"AI response timeout: AIResponseWaitTime greater than {request.AIResponseWaitTime}, please check internal error log!";
await hook.OnAgentHangUp(request);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.HangUp($"twilio/error.mp3");
}
else if (reply == null)
{
var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum);
if (indication != null)
{
_logger.LogWarning($"Indication: {indication}");
var speechPaths = new List<string>();
int segIndex = 0;
foreach (var text in indication.Split('|'))
{
var seg = text.Trim();
if (seg.StartsWith('#'))
{
speechPaths.Add($"twilio/{seg.Substring(1)}.mp3");
}
else
{
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(seg);
// add hold-on
var holdOnIndex = Random.Shared.Next(1, 10);
if (holdOnIndex < 7)
{
speechPaths.Add($"twilio/hold-on-short-{holdOnIndex}.mp3");
}
var fileName = $"indication_{request.SeqNum}_{segIndex}.mp3";
fileStorage.SaveSpeechFile(request.ConversationId, fileName, data);
speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}");
// add typing
var typingIndex = Random.Shared.Next(1, 7);
if (typingIndex < 4)
{
speechPaths.Add($"twilio/typing-{typingIndex}.mp3");
}
segIndex++;
}
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnIndicationGenerated(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum);
}
else
{
var instructions = new List<string>
{
};
// add hold-on
var holdOnIndex = Random.Shared.Next(1, 15);
if (holdOnIndex < 9)
{
instructions.Add($"twilio/hold-on-long-{holdOnIndex}.mp3");
}
// add typing
var typingIndex = Random.Shared.Next(1, 7);
if (typingIndex < 4)
{
instructions.Add($"twilio/typing-{typingIndex}.mp3");
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
SpeechPaths = instructions,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnWaitingAgentResponse(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
}
response = await twilio.WaitingForAiResponse(request);
}
else
{
@ -339,9 +236,6 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentTransferring(request, _settings);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.DialCsrAgent($"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}");
@ -353,18 +247,16 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentHangUp(request);
}, new HookEmitOption
{
OnlyOnce = true
});
}
else
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"],
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}",
CallbackPath = $"twilio/voice/receive/{nextSeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}",
ActionOnEmptyResult = true,
Hints = reply.Hints
};
@ -372,9 +264,6 @@ public class TwilioVoiceController : TwilioController
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnAgentResponsing(request, instruction);
}, new HookEmitOption
{
OnlyOnce = true
});
response = twilio.ReturnInstructions(instruction);
@ -384,38 +273,6 @@ public class TwilioVoiceController : TwilioController
return TwiML(response);
}
[ValidateRequest]
[HttpPost("twilio/voice/init-outbound-call")]
public TwiMLResult InitiateOutboundCall(ConversationalVoiceRequest request)
{
VoiceResponse response = default!;
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null)
{
response = new VoiceResponse();
response.Play(new Uri(request.InitAudioFile));
return TwiML(response);
}
var instruction = new ConversationalVoiceResponse
{
ConversationId = request.ConversationId,
ActionOnEmptyResult = true,
CallbackPath = $"twilio/voice/receive/1?conversation-id={request.ConversationId}",
};
if (request.InitAudioFile != null)
{
instruction.CallbackPath += $"&init-audio-file={request.InitAudioFile}";
instruction.SpeechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}");
}
var twilio = _services.GetRequiredService<TwilioService>();
response = twilio.ReturnNoninterruptedInstructions(instruction);
return TwiML(response);
}
[ValidateRequest]
[HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")]
public async Task<FileContentResult> GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName)
@ -433,8 +290,40 @@ public class TwilioVoiceController : TwilioController
[HttpPost("twilio/voice/hang-up")]
public async Task<TwiMLResult> Hangup(ConversationalVoiceRequest request)
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
}
var twilio = _services.GetRequiredService<TwilioService>();
var response = twilio.HangUp("twilio/bye.mp3");
var response = twilio.HangUp(instruction);
return TwiML(response);
}
[ValidateRequest]
[HttpPost("twilio/voice/transfer-call")]
public async Task<TwiMLResult> TransferCall(ConversationalVoiceRequest request)
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
TransferTo = request.TransferTo
};
if (request.InitAudioFile != null)
{
instruction.SpeechPaths.Add(request.InitAudioFile);
}
var twilio = _services.GetRequiredService<TwilioService>();
var response = twilio.TransferCall(instruction);
return TwiML(response);
}
@ -445,8 +334,7 @@ public class TwilioVoiceController : TwilioController
if (request.CallStatus == "completed")
{
if (request.AnsweredBy == "machine_start" &&
request.Direction == "outbound-api" &&
request.InitAudioFile != null)
request.Direction == "outbound-api")
{
// voicemail
await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
@ -481,13 +369,4 @@ public class TwilioVoiceController : TwilioController
}
return result;
}
private string GenerateStatesParameter(List<string> states)
{
if (states is null || states.Count == 0)
{
return null;
}
return string.Join("&", states.Select(x => $"states={x}"));
}
}

1
src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
View file

@ -8,4 +8,5 @@ public interface ITwilioCallStatusHook
Task OnVoicemailLeft(ConversationalVoiceRequest request);
Task OnUserDisconnected(ConversationalVoiceRequest request);
Task OnRecordingCompleted(ConversationalVoiceRequest request);
Task OnVoicemailStarting(ConversationalVoiceRequest request);
}

9
src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs
View file

@ -45,15 +45,6 @@ public interface ITwilioSessionHook
Task OnWaitingUserResponse(ConversationalVoiceRequest request, ConversationalVoiceResponse response)
=> Task.CompletedTask;
/// <summary>
/// On agent generated indication
/// </summary>
/// <param name="request"></param>
/// <param name="response"></param>
/// <returns></returns>
Task OnIndicationGenerated(ConversationalVoiceRequest request, ConversationalVoiceResponse response)
=> Task.CompletedTask;
/// <summary>
/// Waiting agent response
/// </summary>

13
src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs
View file

@ -4,17 +4,18 @@ namespace BotSharp.Plugin.Twilio.Models
{
public class CallerMessage
{
public string ConversationId { get; set; }
public string AgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public int SeqNumber { get; set; }
public string Content { get; set; }
public string Digits { get; set; }
public string From { get; set; }
public string Content { get; set; } = null!;
public string? Digits { get; set; }
public string From { get; set; } = null!;
public Dictionary<string, string> States { get; set; } = new();
public KeyValuePair<string, StringValues>[] RequestHeaders { get; set; }
public KeyValuePair<string, StringValues>[] RequestHeaders { get; set; } = [];
public override string ToString()
{
return $"{ConversationId}-{SeqNumber}";
return $"{ConversationId}-{SeqNumber}: {Content}";
}
}
}

3
src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs
View file

@ -27,6 +27,9 @@ public class ConversationalVoiceRequest : VoiceRequest
[FromForm]
public string? CallbackSource { get; set; }
[FromQuery(Name = "transfer-to")]
public string? TransferTo { get; set; }
/// <summary>
/// machine_start
/// </summary>

6
src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs
View file

@ -2,6 +2,7 @@ namespace BotSharp.Plugin.Twilio.Models;
public class ConversationalVoiceResponse
{
public string AgentId { get; set; } = null!;
public string ConversationId { get; set; } = null!;
public List<string> SpeechPaths { get; set; } = [];
public string CallbackPath { get; set; }
@ -13,4 +14,9 @@ public class ConversationalVoiceResponse
public int Timeout { get; set; } = 3;
public string Hints { get; set; }
/// <summary>
/// The Phone Number to transfer to
/// </summary>
public string? TransferTo { get; set; }
}

36
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
@ -27,6 +29,7 @@ public class HangupPhoneCallFn : IFunctionCallback
{
var args = JsonSerializer.Deserialize<HangupPhoneCallArgs>(message.FunctionArgs);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var routing = _services.GetRequiredService<IRoutingService>();
var conversationId = routing.Context.ConversationId;
var states = _services.GetRequiredService<IConversationStateService>();
@ -34,26 +37,33 @@ public class HangupPhoneCallFn : IFunctionCallback
if (string.IsNullOrEmpty(callSid))
{
message.Content = "The call has not been initiated.";
message.Content = "Please hang up the phone directly.";
_logger.LogError(message.Content);
return false;
}
if (args.AnythingElseToHelp)
{
message.Content = "Tell me how I can help.";
}
else
{
var call = CallResource.Update(
url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"),
pathSid: callSid
);
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?agent-id={message.CurrentAgentId}&conversation-id={conversationId}";
message.Content = "The call is ending.";
message.StopCompletion = true;
// Generate initial assistant audio
string initAudioFile = null;
if (!string.IsNullOrEmpty(args.ResponseContent))
{
var completion = CompletionProvider.GetAudioSynthesizer(_services);
var data = await completion.GenerateAudioAsync(args.ResponseContent);
initAudioFile = "ending.mp3";
fileStorage.SaveSpeechFile(conversationId, initAudioFile, data);
processUrl += $"&init-audio-file={initAudioFile}";
}
var call = CallResource.Update(
url: new Uri(processUrl),
pathSid: callSid
);
message.Content = args.Reason;
message.StopCompletion = true;
return true;
}
}

59
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/LeaveVoicemailFn.cs Normal file
View file

@ -0,0 +1,59 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
public class LeaveVoicemailFn : IFunctionCallback
{
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private readonly TwilioSetting _setting;
public string Name => "util-twilio-leave_voicemail";
public string Indication => "leaving a voicemail";
public LeaveVoicemailFn(
IServiceProvider services,
ILogger<LeaveVoicemailFn> logger,
TwilioSetting setting)
{
_services = services;
_logger = logger;
_setting = setting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<LeaveVoicemailArgs>(message.FunctionArgs);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var routing = _services.GetRequiredService<IRoutingService>();
var conversationId = routing.Context.ConversationId;
var states = _services.GetRequiredService<IConversationStateService>();
var callSid = states.GetState("twilio_call_sid");
if (string.IsNullOrEmpty(callSid))
{
message.Content = "The call has not been initiated.";
_logger.LogError(message.Content);
return false;
}
// Generate voice message audio
string initAudioFile = null;
if (!string.IsNullOrEmpty(args.VoicemailMessage))
{
var completion = CompletionProvider.GetAudioSynthesizer(_services);
var data = await completion.GenerateAudioAsync(args.VoicemailMessage);
initAudioFile = "voicemail.mp3";
fileStorage.SaveSpeechFile(conversationId, initAudioFile, data);
}
message.Content = args.VoicemailMessage;
message.StopCompletion = true;
return true;
}
}

22
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
View file

@ -62,15 +62,15 @@ public class OutboundPhoneCallFn : IFunctionCallback
states.SetState(StateConst.SUB_CONVERSATION_ID, newConversationId);
var processUrl = $"{_twilioSetting.CallbackHost}/twilio";
var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?conversation-id={newConversationId}";
var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?conversation-id={newConversationId}";
var statusUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}";
var recordingStatusUrl = $"{_twilioSetting.CallbackHost}/twilio/recording/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}";
// Generate initial assistant audio
string initAudioFile = null;
if (!string.IsNullOrEmpty(args.InitialMessage))
{
var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
var data = await completion.GenerateAudioFromTextAsync(args.InitialMessage);
var completion = CompletionProvider.GetAudioSynthesizer(_services);
var data = await completion.GenerateAudioAsync(args.InitialMessage);
initAudioFile = "intial.mp3";
fileStorage.SaveSpeechFile(newConversationId, initAudioFile, data);
@ -94,7 +94,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
processUrl += "/voice/init-outbound-call";
}
processUrl += $"?conversation-id={newConversationId}";
processUrl += $"?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}";
if (!string.IsNullOrEmpty(initAudioFile))
{
processUrl += $"&init-audio-file={initAudioFile}";
@ -108,8 +108,9 @@ public class OutboundPhoneCallFn : IFunctionCallback
statusCallback: new Uri(statusUrl),
// https://www.twilio.com/docs/voice/answering-machine-detection
machineDetection: _twilioSetting.MachineDetection,
machineDetectionSilenceTimeout: _twilioSetting.MachineDetectionSilenceTimeout,
record: _twilioSetting.RecordingEnabled,
recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}");
recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?agent-id={message.CurrentAgentId}&conversation-id={newConversationId}");
var convService = _services.GetRequiredService<IConversationService>();
var routing = _services.GetRequiredService<IRoutingContext>();
@ -127,7 +128,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
string entryAgentId,
string originConversationId,
string newConversationId,
CallResource resource)
CallResource call)
{
// new scope service for isolated conversation
using var scope = _services.CreateScope();
@ -140,7 +141,7 @@ public class OutboundPhoneCallFn : IFunctionCallback
Id = newConversationId,
AgentId = entryAgentId,
Channel = ConversationChannel.Phone,
ChannelId = resource.Sid,
ChannelId = call.Sid,
Title = args.InitialMessage
});
@ -159,7 +160,10 @@ public class OutboundPhoneCallFn : IFunctionCallback
convService.SetConversationId(newConversationId,
[
new MessageState(StateConst.ORIGIN_CONVERSATION_ID, originConversationId),
new MessageState("phone_number", resource.To)
new MessageState("phone_from", call.From),
new MessageState("phone_direction", call.Direction),
new MessageState("phone_number", call.To),
new MessageState("twilio_call_sid", call.Sid)
]);
convService.SaveStates();
}

69
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs Normal file
View file

@ -0,0 +1,69 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Options;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
using Twilio.Rest.Api.V2010.Account;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
public class TransferPhoneCallFn : IFunctionCallback
{
private readonly IServiceProvider _services;
private readonly ILogger _logger;
private readonly BotSharpOptions _options;
private readonly TwilioSetting _twilioSetting;
public string Name => "util-twilio-transfer_phone_call";
public string Indication => "Transferring the active line";
public TransferPhoneCallFn(
IServiceProvider services,
ILogger<TransferPhoneCallFn> logger,
BotSharpOptions options,
TwilioSetting twilioSetting)
{
_services = services;
_logger = logger;
_options = options;
_twilioSetting = twilioSetting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<ForwardPhoneCallArgs>(message.FunctionArgs, _options.JsonSerializerOptions);
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var states = _services.GetRequiredService<IConversationStateService>();
var sid = states.GetState("twilio_call_sid");
if (string.IsNullOrEmpty(sid))
{
_logger.LogError("Twilio call sid is empty.");
message.Content = "There is an error when transferring the phone call.";
return false;
}
var routing = _services.GetRequiredService<IRoutingService>();
var conversationId = routing.Context.ConversationId;
var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?agent-id={routing.Context.EntryAgentId}&conversation-id={conversationId}&transfer-to={args.PhoneNumber}";
// Generate initial assistant audio
if (!string.IsNullOrEmpty(args.TransitionMessage))
{
var completion = CompletionProvider.GetAudioSynthesizer(_services);
var data = await completion.GenerateAudioAsync(args.TransitionMessage);
var initAudioFile = "transfer.mp3";
fileStorage.SaveSpeechFile(conversationId, initAudioFile, data);
processUrl += $"&init-audio-file={initAudioFile}";
}
// Transfer call
var call = CallResource.Update(
pathSid: sid,
url: new Uri(processUrl));
message.Content = args.TransitionMessage;
return true;
}
}

10
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs
View file

@ -7,8 +7,10 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
{
private static string PREFIX = "util-twilio-";
private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call";
private static string TRANSFER_PHONE_CALL_FN = $"{PREFIX}transfer_phone_call";
private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call";
public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";
private static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";
private static string LEAVE_VOICEMAIL_FN = $"{PREFIX}leave_voicemail";
public void AddUtilities(List<AgentUtility> utilities)
{
@ -18,13 +20,13 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
Functions =
[
new($"{OUTBOUND_PHONE_CALL_FN}"),
new($"{TRANSFER_PHONE_CALL_FN}"),
new($"{HANGUP_PHONE_CALL_FN}"),
new($"{TEXT_MESSAGE_FN}")
new($"{TEXT_MESSAGE_FN}"),
new($"{LEAVE_VOICEMAIL_FN}")
],
Templates =
[
new($"{OUTBOUND_PHONE_CALL_FN}.fn"),
new($"{HANGUP_PHONE_CALL_FN}.fn")
]
};

12
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs Normal file
View file

@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
public class ForwardPhoneCallArgs
{
[JsonPropertyName("phone_number")]
public string PhoneNumber { get; set; } = null!;
[JsonPropertyName("transition_message")]
public string TransitionMessage { get; set; } = null!;
}

7
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs
View file

@ -4,6 +4,9 @@ namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
public class HangupPhoneCallArgs
{
[JsonPropertyName("anything_else_to_help")]
public bool AnythingElseToHelp { get; set; } = true;
[JsonPropertyName("reason")]
public string Reason { get; set; } = null!;
[JsonPropertyName("response_content")]
public string ResponseContent { get; set; } = null!;
}

12
src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/LeaveVoicemailArgs.cs Normal file
View file

@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
public class LeaveVoicemailArgs
{
[JsonPropertyName("phone_number")]
public string PhoneNumber { get; set; } = null!;
[JsonPropertyName("voicemail_message")]
public string VoicemailMessage { get; set; } = null!;
}

9
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs
View file

@ -74,7 +74,7 @@ public class TwilioMessageQueueService : BackgroundService
}
httpContext.HttpContext.Request.Headers["X-Twilio-BotSharp"] = "LOST";
AssistantMessage reply = null;
AssistantMessage reply = default!;
var inputMsg = new RoleDialogModel(AgentRole.User, message.Content);
var conv = sp.GetRequiredService<IConversationService>();
@ -87,7 +87,7 @@ public class TwilioMessageQueueService : BackgroundService
// Need to consider Inbound and Outbound call
var conversation = await conv.GetConversation(message.ConversationId);
var agentId = string.IsNullOrWhiteSpace(conversation?.AgentId) ? config.AgentId : conversation.AgentId;
var agentId = message.AgentId;
var result = await conv.SendMessage(agentId,
inputMsg,
@ -105,7 +105,6 @@ public class TwilioMessageQueueService : BackgroundService
);
reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp);
reply.Hints = GetHints(reply);
reply.Content = null;
await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply);
}
@ -138,9 +137,9 @@ public class TwilioMessageQueueService : BackgroundService
private static async Task<string> GetReplySpeechFileName(string conversationId, AssistantMessage reply, IServiceProvider sp)
{
var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1");
var completion = CompletionProvider.GetAudioSynthesizer(sp);
var fileStorage = sp.GetRequiredService<IFileStorageService>();
var data = await completion.GenerateAudioFromTextAsync(reply.Content);
var data = await completion.GenerateAudioAsync(reply.Content);
var fileName = $"reply_{reply.MessageId}.mp3";
fileStorage.SaveSpeechFile(conversationId, fileName, data);
return fileName;

189
src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
View file

@ -1,4 +1,7 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Utilities;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.Twilio.Interfaces;
using BotSharp.Plugin.Twilio.Models;
using Twilio.Jwt.AccessToken;
using Token = Twilio.Jwt.AccessToken.Token;
@ -12,11 +15,13 @@ public class TwilioService
{
private readonly TwilioSetting _settings;
private readonly IServiceProvider _services;
public readonly ILogger _logger;
public TwilioService(TwilioSetting settings, IServiceProvider services)
public TwilioService(TwilioSetting settings, IServiceProvider services, ILogger<TwilioService> logger)
{
_settings = settings;
_services = services;
_logger = logger;
}
public string GetAccessToken()
@ -47,29 +52,6 @@ public class TwilioService
return token.ToJwt();
}
public VoiceResponse ReturnInstructions(string message)
{
var twilioSetting = _services.GetRequiredService<TwilioSetting>();
var response = new VoiceResponse();
var gather = new Gather()
{
Input = new List<Gather.InputEnum>()
{
Gather.InputEnum.Speech,
Gather.InputEnum.Dtmf
},
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"),
Enhanced = true,
SpeechModel = Gather.SpeechModelEnum.PhoneCall,
SpeechTimeout = "auto"
};
gather.Say(message);
response.Append(gather);
return response;
}
public VoiceResponse ReturnInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
@ -103,19 +85,13 @@ public class TwilioService
public VoiceResponse ReturnNoninterruptedInstructions(ConversationalVoiceResponse voiceResponse)
{
var response = new VoiceResponse();
var conversationId = voiceResponse.ConversationId;
if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in voiceResponse.SpeechPaths)
{
if (speechPath.StartsWith(_settings.CallbackHost))
{
response.Play(new Uri(speechPath));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
var uri = GetSpeechPath(conversationId, speechPath);
response.Play(new Uri(uri));
}
}
@ -149,6 +125,22 @@ public class TwilioService
return response;
}
public VoiceResponse HangUp(ConversationalVoiceResponse voiceResponse)
{
var response = new VoiceResponse();
var conversationId = voiceResponse.ConversationId;
if (voiceResponse.SpeechPaths != null && voiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in voiceResponse.SpeechPaths)
{
var uri = GetSpeechPath(conversationId, speechPath);
response.Play(new Uri(uri));
}
}
response.Hangup();
return response;
}
public VoiceResponse DialCsrAgent(string speechPath)
{
var response = new VoiceResponse();
@ -160,6 +152,23 @@ public class TwilioService
return response;
}
public VoiceResponse TransferCall(ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
var conversationId = conversationalVoiceResponse.ConversationId;
if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
{
var uri = GetSpeechPath(conversationId, speechPath);
response.Play(new Uri(uri));
}
}
response.Dial(conversationalVoiceResponse.TransferTo, answerOnBridge: true);
return response;
}
public VoiceResponse HoldOn(int interval, string message = null)
{
var twilioSetting = _services.GetRequiredService<TwilioSetting>();
@ -172,7 +181,7 @@ public class TwilioService
Gather.InputEnum.Speech,
Gather.InputEnum.Dtmf
},
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"),
Action = new Uri($"{_settings.CallbackHost}/twilio/voice/"),
ActionOnEmptyResult = true
};
@ -190,26 +199,17 @@ public class TwilioService
/// </summary>
/// <param name="conversationalVoiceResponse"></param>
/// <returns></returns>
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
{
var response = new VoiceResponse();
var conversationId = conversationalVoiceResponse.ConversationId;
if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
{
foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
{
if (speechPath.StartsWith("twilio/"))
{
response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
}
else if (speechPath.StartsWith(_settings.CallbackHost))
{
response.Play(new Uri(speechPath));
}
else
{
response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}"));
}
var uri = GetSpeechPath(conversationId, speechPath);
response.Play(new Uri(uri));
}
}
@ -220,4 +220,99 @@ public class TwilioService
return response;
}
public async Task<VoiceResponse> WaitingForAiResponse(ConversationalVoiceRequest request)
{
VoiceResponse response;
var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var indication = await sessionManager.GetReplyIndicationAsync(request.ConversationId, request.SeqNum);
if (indication != null)
{
_logger.LogWarning($"Indication ({request.SeqNum}): {indication}");
var speechPaths = new List<string>();
foreach (var text in indication.Split('|'))
{
var seg = text.Trim();
if (seg.StartsWith('#'))
{
speechPaths.Add($"twilio/{seg.Substring(1)}.mp3");
}
else
{
var hash = Utilities.HashTextMd5(seg);
var fileName = $"indication_{hash}.mp3";
var existing = fileStorage.GetSpeechFile(request.ConversationId, fileName);
if (existing == BinaryData.Empty)
{
var completion = CompletionProvider.GetAudioSynthesizer(_services);
var data = await completion.GenerateAudioAsync(seg);
fileStorage.SaveSpeechFile(request.ConversationId, fileName, data);
}
speechPaths.Add($"twilio/voice/speeches/{request.ConversationId}/{fileName}");
}
}
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = speechPaths,
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
response = ReturnInstructions(instruction);
await sessionManager.RemoveReplyIndicationAsync(request.ConversationId, request.SeqNum);
}
else
{
var instruction = new ConversationalVoiceResponse
{
AgentId = request.AgentId,
ConversationId = request.ConversationId,
SpeechPaths = [],
CallbackPath = $"twilio/voice/reply/{request.SeqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
ActionOnEmptyResult = true
};
await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
{
await hook.OnWaitingAgentResponse(request, instruction);
});
response = ReturnInstructions(instruction);
}
return response;
}
public string GetSpeechPath(string conversationId, string speechPath)
{
if (speechPath.StartsWith("twilio/"))
{
return $"{_settings.CallbackHost}/{speechPath}";
}
else if (speechPath.StartsWith(_settings.CallbackHost))
{
return speechPath;
}
else
{
return $"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}";
}
}
public string GenerateStatesParameter(List<string> states)
{
if (states is null || states.Count == 0)
{
return null;
}
return string.Join("&", states.Select(x => $"states={x}"));
}
}

6
src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
View file

@ -20,11 +20,6 @@ public class TwilioSetting
public string? MessagingShortCode { get; set; }
/// <summary>
/// Default Agent Id to handle inbound phone call
/// </summary>
public string? AgentId { get; set; }
/// <summary>
/// Human agent phone number if AI can't handle the call
/// </summary>
@ -33,6 +28,7 @@ public class TwilioSetting
public int MaxGatherAttempts { get; set; } = 4;
public string? MachineDetection { get; set; }
public int MachineDetectionSilenceTimeout { get; set; } = 2500;
public bool RecordingEnabled { get; set; } = false;
}

8
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json
View file

@ -9,11 +9,11 @@
"type": "string",
"description": "The reason why user wants to end the phone call."
},
"anything_else_to_help": {
"type": "boolean",
"description": "Check if user has anything else to help."
"response_content": {
"type": "string",
"description": "A statement said to the user when politely ending a conversation."
}
},
"required": [ "reason", "anything_else_to_help" ]
"required": [ "reason", "response_content" ]
}
}

19
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-leave_voicemail.json Normal file
View file

@ -0,0 +1,19 @@
{
"name": "util-twilio-leave_voicemail",
"description": "If the user wants you to leave a voicemail.",
"visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}",
"parameters": {
"type": "object",
"properties": {
"voicemail_message": {
"type": "string",
"description": "User voicemail with details."
},
"phone_number": {
"type": "string",
"description": "Phone number to callback."
}
},
"required": [ "voicemail_message", "phone_number" ]
}
}

19
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json Normal file
View file

@ -0,0 +1,19 @@
{
"name": "util-twilio-transfer_phone_call",
"description": "When user wants to transfer the phone call",
"visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}",
"parameters": {
"type": "object",
"properties": {
"transition_message": {
"type": "string",
"description": "Transition message when forwarding."
},
"phone_number": {
"type": "string",
"description": "Phone number transfer to."
}
},
"required": [ "transition_message", "phone_number" ]
}
}

3
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-hangup_phone_call.fn.liquid
View file

@ -1,3 +0,0 @@
{% if channel == 'phone' %}
** If user wants to end the phone call or conversation, ask user if there is anything else to help. If not, end the phone call.
{% endif %}

1
src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-twilio-outbound_phone_call.fn.liquid
View file

@ -1 +0,0 @@
** Please call util-twilio-outbound_phone_call if user wants to make an outbound call.

21
src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs
View file

@ -1,5 +1,6 @@
using Azure;
using BotSharp.Abstraction.Browsing.Settings;
using Microsoft.Playwright;
using System.IO;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
@ -44,6 +45,7 @@ public class PlaywrightInstance : IDisposable
public async Task<IBrowserContext> InitContext(string ctxId, BrowserActionArgs args)
{
var _webDriver = _services.GetRequiredService<WebBrowsingSettings>();
if (_contexts.ContainsKey(ctxId))
return _contexts[ctxId];
@ -82,6 +84,8 @@ public class PlaywrightInstance : IDisposable
// "--start-maximized"
]
});
_contexts[ctxId].SetDefaultTimeout(_webDriver.DefaultTimeout);
_contexts[ctxId].SetDefaultNavigationTimeout(_webDriver.DefaultNavigationTimeout);
}
_pages[ctxId] = new List<IPage>();
@ -131,7 +135,6 @@ public class PlaywrightInstance : IDisposable
{
return page;
}
page.Request += async (sender, e) =>
{
await HandleFetchRequest(e, message, args);
@ -154,7 +157,7 @@ public class PlaywrightInstance : IDisposable
public async Task HandleFetchResponse(IResponse response, MessageInfo message, PageActionArgs args)
{
if (response.Status != 204 &&
if (response.Status != 204 && response.Status != 302 &&
response.Headers.ContainsKey("content-type") &&
(response.Request.ResourceType == "fetch" || response.Request.ResourceType == "xhr") &&
(args.ExcludeResponseUrls == null || !args.ExcludeResponseUrls.Any(url => response.Url.ToLower().Contains(url))) &&
@ -164,11 +167,23 @@ public class PlaywrightInstance : IDisposable
try
{
var context = await GetContext(message.ContextId);
var cookies = await context.CookiesAsync(new string[] { response.Url });
var result = new WebPageResponseData
{
Url = response.Url.ToLower(),
PostData = response.Request?.PostData ?? string.Empty,
ResponseInMemory = args.ResponseInMemory
ResponseInMemory = args.ResponseInMemory,
Method = response.Request.Method,
ResponseCode = response.Status,
Cookies = cookies.Select(x => new WebPageCookieData
{
Name = x.Name,
Value = x.Value,
Domain = x.Domain,
Path = x.Path,
Expires = x.Expires
}).ToList()
};
var html = await response.TextAsync();

2
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs
View file

@ -34,7 +34,7 @@ public class ChangeCheckboxFn : IFunctionCallback
message.Data = await _browser.ScreenshotAsync(new MessageInfo
{
AgentId = message.CurrentAgentId,
ContextId = convService.ConversationId,
ContextId = webDriverService.GetMessageContext(message),
MessageId = message.MessageId
}, path);

2
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs
View file

@ -34,7 +34,7 @@ public class ChangeListValueFn : IFunctionCallback
message.Data = await _browser.ScreenshotAsync(new MessageInfo
{
AgentId = message.CurrentAgentId,
ContextId = convService.ConversationId,
ContextId = webDriverService.GetMessageContext(message),
MessageId = message.MessageId
}, path);

2
src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs
View file

@ -34,7 +34,7 @@ public class CheckRadioButtonFn : IFunctionCallback
message.Data = await _browser.ScreenshotAsync(new MessageInfo
{
AgentId = message.CurrentAgentId,
ContextId = convService.ConversationId,
ContextId = webDriverService.GetMessageContext(message),
MessageId = message.MessageId
}, path);

2
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs
View file

@ -34,7 +34,7 @@ public class ClickButtonFn : IFunctionCallback
message.Data = await _browser.ScreenshotAsync(new MessageInfo
{
AgentId = message.CurrentAgentId,
ContextId = convService.ConversationId,
ContextId = webDriverService.GetMessageContext(message),
MessageId = message.MessageId
}, path);

2
src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs
View file

@ -34,7 +34,7 @@ public class ClickElementFn : IFunctionCallback
message.Data = await _browser.ScreenshotAsync(new MessageInfo
{
AgentId = message.CurrentAgentId,
ContextId = convService.ConversationId,
ContextId = webDriverService.GetMessageContext(message),
MessageId = message.MessageId
}, path);

Some files were not shown because too many files have changed in this diff Show more