realtime test

src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(TargetFramework)</TargetFramework>

src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionDef.cs

@@ -2,6 +2,9 @@ namespace BotSharp.Abstraction.Functions.Models;
 
 public class FunctionDef
 {
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = "function";
+
     [JsonPropertyName("name")]
     public string Name { get; set; } = null!;
 

src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs

@@ -6,6 +6,6 @@ public interface ILlmProviderService
 {
     LlmModelSetting GetSetting(string provider, string model);
     List<string> GetProviders();
-    LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false);
+    LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool? realTime = false, bool imageGenerate = false);
     List<LlmModelSetting> GetProviderModels(string provider);
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -0,0 +1,12 @@
+using BotSharp.Abstraction.Realtime.Models;
+
+namespace BotSharp.Abstraction.MLTasks;
+
+public interface IRealTimeCompletion
+{
+    string Provider { get; }
+
+    void SetModelName(string model);
+
+    Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -37,6 +37,11 @@ public class LlmModelSetting
     /// </summary>
     public bool MultiModal { get; set; }
 
+    /// <summary>
+    /// If true, allow real-time interaction
+    /// </summary>
+    public bool RealTime { get; set; }
+
     /// <summary>
     /// If true, allow generating images
     /// </summary>

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSession.cs

@@ -0,0 +1,22 @@
+namespace BotSharp.Abstraction.Realtime.Models;
+
+public class RealtimeSession
+{
+    public string Id { get; set; } = null!;
+
+    public string Object { get; set;} = null!;
+    public string Model { get; set; } = null!;
+    public string Voice { get; set; } = null!;
+
+    [JsonPropertyName("client_secret")]
+    public RealtimeSessionClientSecret Secret { get; set; } = null!;
+}
+
+public class RealtimeSessionClientSecret
+{
+    [JsonPropertyName("value")]
+    public string Value { get; set; } = null!;
+
+    [JsonPropertyName("expires_at")]
+    public long Expires { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeSessionUpdate.cs

@@ -0,0 +1,11 @@
+namespace BotSharp.Abstraction.Realtime.Models;
+
+public class RealtimeSessionUpdate
+{
+    /// <summary>
+    /// Optional client-generated ID used to identify this event.
+    /// </summary>
+    public string EventId { get; set; } = null!;
+    public string Type { get; set; } = "session.update";
+    public RealtimeSession Session { get; set; } = null!;
+}

src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs

@@ -43,11 +43,13 @@ public class CompletionProvider
         string? model = null,
         string? modelId = null,
         bool? multiModal = null,
+        bool? realTime = null,
         AgentLlmConfig? agentConfig = null)
     {
         var completions = services.GetServices<IChatCompletion>();
         (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, 
-            multiModal: multiModal, agentConfig: agentConfig);
+            multiModal: multiModal,
+            agentConfig: agentConfig);
 
         var completer = completions.FirstOrDefault(x => x.Provider == provider);
         if (completer == null)
@@ -141,11 +143,36 @@ public class CompletionProvider
         return completer;
     }
 
+    public static IRealTimeCompletion GetRealTimeCompletion(IServiceProvider services,
+        string? provider = null,
+        string? model = null,
+        string? modelId = null,
+        bool? multiModal = null,
+        AgentLlmConfig? agentConfig = null)
+    {
+        var completions = services.GetServices<IRealTimeCompletion>();
+        (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId,
+            multiModal: multiModal,
+            realTime: true,
+            agentConfig: agentConfig);
+
+        var completer = completions.FirstOrDefault(x => x.Provider == provider);
+        if (completer == null)
+        {
+            var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
+            logger.LogError($"Can't resolve completion provider by {provider}");
+        }
+
+        completer?.SetModelName(model);
+        return completer;
+    }
+
     private static (string, string) GetProviderAndModel(IServiceProvider services,
         string? provider = null,
         string? model = null,
         string? modelId = null,
         bool? multiModal = null,
+        bool? realTime = null,
         bool imageGenerate = false,
         AgentLlmConfig? agentConfig = null)
     {
@@ -170,7 +197,9 @@ public class CompletionProvider
                 var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId;
                 var llmProviderService = services.GetRequiredService<ILlmProviderService>();
                 model = llmProviderService.GetProviderModel(provider, modelIdentity,
-                    multiModal: multiModal, imageGenerate: imageGenerate)?.Name;
+                    multiModal: multiModal, 
+                    realTime: realTime,
+                    imageGenerate: imageGenerate)?.Name;
             }
         }
 

src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs

@@ -44,7 +44,7 @@ public class LlmProviderService : ILlmProviderService
             ?.Models ?? new List<LlmModelSetting>();
     }
 
-    public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false)
+    public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool? realTime = false, bool imageGenerate = false)
     {
         var models = GetProviderModels(provider)
             .Where(x => x.Id == id);
@@ -54,6 +54,11 @@ public class LlmProviderService : ILlmProviderService
             models = models.Where(x => x.MultiModal == multiModal);
         }
 
+        if (realTime.HasValue)
+        {
+            models = models.Where(x => x.RealTime == realTime);
+        }
+
         models = models.Where(x => x.ImageGeneration == imageGenerate);
 
         var random = new Random();

src/Infrastructure/BotSharp.OpenAPI/Controllers/RealtimeController.cs

@@ -0,0 +1,48 @@
+using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Core.Infrastructures;
+
+namespace BotSharp.OpenAPI.Controllers;
+
+[Authorize]
+[ApiController]
+public class RealtimeController : ControllerBase
+{
+    private readonly IServiceProvider _services;
+
+    public RealtimeController(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    /// <summary>
+    /// Create an ephemeral API token for use in client-side applications with the Realtime API.
+    /// </summary>
+    /// <returns></returns>
+    [HttpGet("/agent/{agentId}/realtime/session")]
+    public async Task<RealtimeSession> CreateSession(string agentId)
+    {
+        var completion = CompletionProvider.GetRealTimeCompletion(_services, provider: "openai", modelId: "gpt-4");
+
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(agentId);
+
+        return await completion.CreateSession(agent, []);
+    }
+
+    [HttpPost("/agent/{agentId}/function/{functionName}/execute")]
+    public async Task<string> ExecuteFunction(string agentId, string functionName, [FromBody] JsonDocument args)
+    {
+        // var agentService = _services.GetRequiredService<IAgentService>();
+        // var agent = await agentService.LoadAgent(agentId);
+        var routing = _services.GetRequiredService<IRoutingService>();
+        // Call functions
+        var message = new RoleDialogModel(AgentRole.Function, "")
+        {
+            FunctionName = functionName,
+            FunctionArgs = JsonSerializer.Serialize(args)
+        };
+        await routing.InvokeFunction(functionName, message);
+        return message.Content;
+    }
+}

src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs

@@ -67,9 +67,9 @@ public class HandleEmailReaderFn : IFunctionCallback
 
                     var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
                     var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
-                    var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
+                    var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4");
                     var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
-                    var convService = _services.GetService<IConversationService>();
+                    var convService = _services.GetRequiredService<IConversationService>();
                     var conversationId = convService.ConversationId;
                     var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
                     var response = await completion.GetChatCompletions(agent, dialogs);

src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj

@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="LLamaSharp" Version="0.18.0" />
+    <PackageReference Include="LLamaSharp" Version="0.20.0" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs

@@ -41,7 +41,6 @@ public class ChatCompletionProvider : IChatCompletion
 
         var inferenceParams = new InferenceParams()
         {
-            Temperature = 0.1f,
             AntiPrompts = new List<string> { $"{AgentRole.User}:", "[/INST]" },
             MaxTokens = 128
         };
@@ -120,7 +119,6 @@ public class ChatCompletionProvider : IChatCompletion
 
         var inferenceParams = new InferenceParams()
         {
-            Temperature = 0.1f,
             AntiPrompts = new List<string> { $"{AgentRole.User}:", "[/INST]" },
             MaxTokens = 64
         };
@@ -170,7 +168,7 @@ public class ChatCompletionProvider : IChatCompletion
         llama.LoadModel(model);
 
         var executor = new StatelessExecutor(llama.Model, llama.Params);
-        var inferenceParams = new InferenceParams() { Temperature = 1.0f, AntiPrompts = new List<string> { $"{AgentRole.User}:" }, MaxTokens = 64 };
+        var inferenceParams = new InferenceParams() { AntiPrompts = new List<string> { $"{AgentRole.User}:" }, MaxTokens = 64 };
 
         var convSetting = _services.GetRequiredService<ConversationSetting>();
         if (convSetting.ShowVerboseLog)

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs

@@ -42,7 +42,7 @@ public class TextCompletionProvider : ITextCompletion
         llama.LoadModel(_model);
 
         var executor = new InstructExecutor(llama.Model.CreateContext(llama.Params));
-        var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 };
+        var inferenceParams = new InferenceParams() { MaxTokens = 128 };
 
         _tokenStatistics.StartTimer();
         string completion = "";

src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj

@@ -12,7 +12,8 @@
 
   <ItemGroup>
     <PackageReference Include="OpenAI" Version="2.1.0" />
-    <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
+    <PackageReference Include="Refit" Version="8.0.0" />
+    <PackageReference Include="Refit.HttpClientFactory" Version="8.0.0" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.OpenAI/Models/RealtimeSessionRequest.cs

@@ -0,0 +1,49 @@
+using BotSharp.Abstraction.Functions.Models;
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.OpenAI.Models;
+
+public class RealtimeSessionRequest
+{
+    [JsonPropertyName("model")]
+    public string Model { get; set; } = "gpt-4o-mini-realtime-preview-2024-12-17";
+
+    [JsonPropertyName("temperature")]
+    public float temperature { get; set; } = 0.8f;
+
+    [JsonPropertyName("modalities")]
+    public string[] Modalities { get; set; } = ["audio", "text"];
+
+    [JsonPropertyName("instructions")]
+    public string Instructions { get; set; } = "You are a friendly assistant.";
+
+    [JsonPropertyName("max_response_output_tokens")]
+    public int MaxResponseOutputTokens { get; set; } = 512;
+
+    [JsonPropertyName("tool_choice")]
+    public string ToolChoice { get; set; } = "auto";
+
+    [JsonPropertyName("tools")]
+    public FunctionDef[] Tools { get; set; } = [];
+
+    [JsonPropertyName("turn_detection")]
+    public RealtimeSessionTurnDetection TurnDetection { get; set; } = new();
+}
+
+public class RealtimeSessionTurnDetection
+{
+    /// <summary>
+    /// Milliseconds
+    /// </summary>
+    [JsonPropertyName("prefix_padding_ms")]
+    public int PrefixPadding { get; set; } = 300;
+
+    [JsonPropertyName("silence_duration_ms")]
+    public int SilenceDuration { get; set; } = 500;
+
+    [JsonPropertyName("threshold")]
+    public float Threshold { get; set; } = 0.5f;
+
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = "server_vad";
+}

src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs

@@ -6,6 +6,8 @@ using BotSharp.Plugin.OpenAI.Providers.Text;
 using BotSharp.Plugin.OpenAI.Providers.Chat;
 using BotSharp.Plugin.OpenAI.Providers.Audio;
 using Microsoft.Extensions.Configuration;
+using Refit;
+using BotSharp.Plugin.OpenAI.Providers.Realtime;
 
 namespace BotSharp.Plugin.OpenAI;
 
@@ -32,5 +34,9 @@ public class OpenAiPlugin : IBotSharpPlugin
         services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
         services.AddScoped<IImageCompletion, ImageCompletionProvider>();
         services.AddScoped<IAudioCompletion, AudioCompletionProvider>();
+        services.AddScoped<IRealTimeCompletion, RealTimeCompletionProvider>();
+
+        services.AddRefitClient<IOpenAiRealtimeApi>()
+            .ConfigureHttpClient(c => c.BaseAddress = new Uri("https://api.openai.com"));
     }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/IOpenAiRealtimeApi.cs

@@ -0,0 +1,10 @@
+using BotSharp.Abstraction.Realtime.Models;
+using Refit;
+
+namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
+
+public interface IOpenAiRealtimeApi
+{
+    [Post("/v1/realtime/sessions")]
+    Task<RealtimeSession> GetSessionAsync(RealtimeSessionRequest model, [Authorize("Bearer")] string token);
+}

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -0,0 +1,244 @@
+using BotSharp.Abstraction.Files.Utilities;
+using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Realtime.Models;
+using OpenAI.Chat;
+using System.Text.Json;
+
+namespace BotSharp.Plugin.OpenAI.Providers.Realtime;
+
+public class RealTimeCompletionProvider : IRealTimeCompletion
+{
+    public string Provider => "openai";
+
+    protected readonly OpenAiSettings _settings;
+    protected readonly IServiceProvider _services;
+    protected readonly ILogger<RealTimeCompletionProvider> _logger;
+
+    protected string _model = "gpt-4o-mini-realtime-preview-2024-12-17";
+
+    public RealTimeCompletionProvider(
+        OpenAiSettings settings,
+        ILogger<RealTimeCompletionProvider> logger,
+        IServiceProvider services)
+    {
+        _settings = settings;
+        _logger = logger;
+        _services = services;
+    }
+
+    public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations)
+    {
+        var contentHooks = _services.GetServices<IContentGeneratingHook>().ToList();
+
+        var client = ProviderHelper.GetClient(Provider, _model, _services);
+        var chatClient = client.GetChatClient(_model);
+        var (prompt, messages, options) = PrepareOptions(agent, conversations);
+
+        var args = new RealtimeSessionRequest
+        {
+            Instructions = prompt,
+            ToolChoice = "auto",
+            Tools = options.Tools.Select(x =>
+            {
+                var fn = new FunctionDef
+                {
+                    Name = x.FunctionName,
+                    Description = x.FunctionDescription
+                };
+                fn.Parameters = JsonSerializer.Deserialize<FunctionParametersDef>(x.FunctionParameters);
+                return fn;
+            }).ToArray(),
+        };
+
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
+        var settings = settingsService.GetSetting(Provider, args.Model);
+
+        var api = _services.GetRequiredService<IOpenAiRealtimeApi>();
+        var session = await api.GetSessionAsync(args, settings.ApiKey);
+        return session;
+    }
+
+    protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var state = _services.GetRequiredService<IConversationStateService>();
+        var fileStorage = _services.GetRequiredService<IFileStorageService>();
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
+        var settings = settingsService.GetSetting(Provider, _model);
+        var allowMultiModal = settings != null && settings.MultiModal;
+
+        var messages = new List<ChatMessage>();
+
+        var temperature = float.Parse(state.GetState("temperature", "0.0"));
+        var maxTokens = int.Parse(state.GetState("max_tokens", "1024"));
+        var options = new ChatCompletionOptions()
+        {
+            ToolChoice = ChatToolChoice.CreateAutoChoice(),
+            Temperature = temperature,
+            MaxOutputTokenCount = maxTokens
+        };
+
+        var functions = agent.Functions.Concat(agent.SecondaryFunctions ?? []);
+        foreach (var function in functions)
+        {
+            if (!agentService.RenderFunction(agent, function)) continue;
+
+            var property = agentService.RenderFunctionProperty(agent, function);
+
+            options.Tools.Add(ChatTool.CreateFunctionTool(
+                functionName: function.Name,
+                functionDescription: function.Description,
+                functionParameters: BinaryData.FromObjectAsJson(property)));
+        }
+
+        if (!string.IsNullOrEmpty(agent.Instruction) || !agent.SecondaryInstructions.IsNullOrEmpty())
+        {
+            var text = agentService.RenderedInstruction(agent);
+            messages.Add(new SystemChatMessage(text));
+        }
+
+        if (!string.IsNullOrEmpty(agent.Knowledges))
+        {
+            messages.Add(new SystemChatMessage(agent.Knowledges));
+        }
+
+        var samples = ProviderHelper.GetChatSamples(agent.Samples);
+        foreach (var sample in samples)
+        {
+            messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content));
+        }
+
+        var filteredMessages = conversations.Select(x => x).ToList();
+        var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User);
+        if (firstUserMsgIdx > 0)
+        {
+            filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
+        }
+
+        foreach (var message in filteredMessages)
+        {
+            if (message.Role == AgentRole.Function)
+            {
+                messages.Add(new AssistantChatMessage(new List<ChatToolCall>
+                {
+                    ChatToolCall.CreateFunctionToolCall(message.ToolCallId, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))
+                }));
+
+                messages.Add(new ToolChatMessage(message.ToolCallId, message.Content));
+            }
+            else if (message.Role == AgentRole.User)
+            {
+                var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
+                var textPart = ChatMessageContentPart.CreateTextPart(text);
+                var contentParts = new List<ChatMessageContentPart> { textPart };
+
+                if (allowMultiModal && !message.Files.IsNullOrEmpty())
+                {
+                    foreach (var file in message.Files)
+                    {
+                        if (!string.IsNullOrEmpty(file.FileData))
+                        {
+                            var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
+                            var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto);
+                            contentParts.Add(contentPart);
+                        }
+                        else if (!string.IsNullOrEmpty(file.FileStorageUrl))
+                        {
+                            var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
+                            var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
+                            var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Auto);
+                            contentParts.Add(contentPart);
+                        }
+                        else if (!string.IsNullOrEmpty(file.FileUrl))
+                        {
+                            var uri = new Uri(file.FileUrl);
+                            var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Auto);
+                            contentParts.Add(contentPart);
+                        }
+                    }
+                }
+                messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
+            }
+            else if (message.Role == AgentRole.Assistant)
+            {
+                messages.Add(new AssistantChatMessage(message.Content));
+            }
+        }
+
+        var prompt = GetPrompt(messages, options);
+        return (prompt, messages, options);
+    }
+
+
+    private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
+    {
+        var prompt = string.Empty;
+
+        if (!messages.IsNullOrEmpty())
+        {
+            // System instruction
+            var verbose = string.Join("\r\n", messages
+                .Select(x => x as SystemChatMessage)
+                .Where(x => x != null)
+                .Select(x =>
+                {
+                    if (!string.IsNullOrEmpty(x.ParticipantName))
+                    {
+                        // To display Agent name in log
+                        return $"[{x.ParticipantName}]: {x.Content.FirstOrDefault()?.Text ?? string.Empty}";
+                    }
+                    return $"{AgentRole.System}: {x.Content.FirstOrDefault()?.Text ?? string.Empty}";
+                }));
+            prompt += $"{verbose}\r\n";
+
+            prompt += "\r\n[CONVERSATION]";
+            verbose = string.Join("\r\n", messages
+                .Where(x => x as SystemChatMessage == null)
+                .Select(x =>
+                {
+                    var fnMessage = x as ToolChatMessage;
+                    if (fnMessage != null)
+                    {
+                        return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
+                    }
+
+                    var userMessage = x as UserChatMessage;
+                    if (userMessage != null)
+                    {
+                        var content = x.Content.FirstOrDefault()?.Text ?? string.Empty;
+                        return !string.IsNullOrEmpty(userMessage.ParticipantName) && userMessage.ParticipantName != "route_to_agent" ?
+                            $"{userMessage.ParticipantName}: {content}" :
+                            $"{AgentRole.User}: {content}";
+                    }
+
+                    var assistMessage = x as AssistantChatMessage;
+                    if (assistMessage != null)
+                    {
+                        var toolCall = assistMessage.ToolCalls?.FirstOrDefault();
+                        return toolCall != null ?
+                            $"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" :
+                            $"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
+                    }
+
+                    return string.Empty;
+                }));
+            prompt += $"\r\n{verbose}\r\n";
+        }
+
+        if (!options.Tools.IsNullOrEmpty())
+        {
+            var functions = string.Join("\r\n", options.Tools.Select(fn =>
+            {
+                return $"\r\n{fn.FunctionName}: {fn.FunctionDescription}\r\n{fn.FunctionParameters}";
+            }));
+            prompt += $"\r\n[FUNCTIONS]{functions}\r\n";
+        }
+
+        return prompt;
+    }
+
+    public void SetModelName(string model)
+    {
+        _model = model;
+    }
+}

src/WebStarter/appsettings.json

@@ -12,6 +12,7 @@
     "https://botsharp.scisharpstack.org",
     "https://chat.scisharpstack.org"
   ],
+
   "Jwt": {
     "Issuer": "botsharp",
     "Audience": "botsharp",
@@ -108,6 +109,51 @@
           "CompletionCost": 0.002
         }
       ]
+    },
+    {
+      "Provider": "openai",
+      "Models": [
+        {
+          "Id": "gpt-4",
+          "Name": "gpt-4o-mini",
+          "Version": "2024-07-18",
+          "ApiKey": "",
+          "Type": "chat",
+          "MultiModal": true,
+          "PromptCost": 0.00015,
+          "CompletionCost": 0.0006
+        },
+        {
+          "Id": "gpt-4",
+          "Name": "gpt-4o-2024-11-20",
+          "Version": "2024-11-20",
+          "ApiKey": "",
+          "Type": "chat",
+          "MultiModal": true,
+          "PromptCost": 0.0025,
+          "CompletionCost": 0.01
+        },
+        {
+          "Id": "gpt-4",
+          "Name": "gpt-4o-mini-realtime-preview-2024-12-17",
+          "Version": "2024-12-17",
+          "ApiKey": "",
+          "Type": "chat",
+          "MultiModal": true,
+          "RealTime": true,
+          "PromptCost": 0.0025,
+          "CompletionCost": 0.01
+        },
+        {
+          "Id": "text-embedding-3",
+          "Name": "text-embedding-3-small",
+          "Version": "3-small",
+          "ApiKey": "",
+          "Type": "embedding",
+          "Dimension": 1536,
+          "PromptCost": 0.00002
+        }
+      ]
     }
   ],
 
@@ -124,8 +170,8 @@
     "HostAgentId": "01e2fc5c-2c89-4ec7-8470-7688608b496c",
     "EnableTranslator": false,
     "LlmConfig": {
-      "Provider": "azure-openai",
-      "Model": "gpt-35-turbo"
+      "Provider": "openai",
+      "Model": "gpt-4o-mini"
     }
   },
 
@@ -245,6 +291,7 @@
     "Default": "FileRepository",
     "TablePrefix": "BotSharp",
     "BotSharpMongoDb": "",
+    "Redis": "botsharp.redis.cache.windows.net:6380,password=,ssl=True,abortConnect=False",
     "FileRepository": "data",
     "Assemblies": [ "BotSharp.Core" ]
   },