diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ChatAnnotation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ChatAnnotation.cs new file mode 100644 index 00000000..061e75a3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ChatAnnotation.cs @@ -0,0 +1,9 @@ +namespace BotSharp.Abstraction.Conversations.Models; + +public class ChatAnnotation +{ + public string Title { get; set; } + public string Url { get; set; } + public int StartIndex { get; set; } + public int EndIndex { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 6bc82f81..f4e75425 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -105,14 +105,21 @@ public class RoleDialogModel : ITrackableMessage /// /// Files to be used in conversation /// - public List Files { get; set; } = new List(); + public List? Files { get; set; } /// /// The images generated by AI /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("generated_images")] - public List GeneratedImages { get; set; } = new List(); + public List? GeneratedImages { get; set; } + + /// + /// The web annotations generated by AI + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("annotations")] + public List? Annotations { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public string RenderedInstruction { get; set; } = string.Empty; @@ -163,7 +170,8 @@ public class RoleDialogModel : ITrackableMessage StopCompletion = source.StopCompletion, Instruction = source.Instruction, Data = source.Data, - IsStreaming = source.IsStreaming + IsStreaming = source.IsStreaming, + Annotations = source.Annotations }; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 23e055d2..bae59659 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -52,6 +52,11 @@ public class LlmModelSetting /// public ReasoningSetting? Reasoning { get; set; } + /// + /// Settings for web search + /// + public WebSearchSetting? WebSearch { get; set; } + /// /// Settings for llm cost /// @@ -69,6 +74,11 @@ public class ReasoningSetting public string? EffortLevel { get; set; } } +public class WebSearchSetting +{ + public string? SearchContextSize { get; set; } +} + /// /// Cost per 1K tokens /// diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index de67fcab..a36d9a18 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -199,7 +199,7 @@ public class InstructModeController : ControllerBase TemplateName = input.TemplateName }); imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; return imageViewModel; } catch (Exception ex) @@ -235,7 +235,7 @@ public class InstructModeController : ControllerBase AgentId = input.AgentId }); imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; return imageViewModel; } @@ -273,7 +273,7 @@ public class InstructModeController : ControllerBase }); imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; return imageViewModel; } catch (Exception ex) @@ -307,7 +307,7 @@ public class InstructModeController : ControllerBase TemplateName = input.TemplateName }); imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; return imageViewModel; } catch (Exception ex) @@ -345,7 +345,7 @@ public class InstructModeController : ControllerBase }); imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; return imageViewModel; } @@ -382,7 +382,7 @@ public class InstructModeController : ControllerBase TemplateName = input.TemplateName }); imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; return imageViewModel; } catch (Exception ex) @@ -428,7 +428,7 @@ public class InstructModeController : ControllerBase }); imageViewModel.Content = message.Content; - imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageViewModel.Images = message.GeneratedImages?.Select(x => ImageViewModel.ToViewModel(x)) ?? []; return imageViewModel; } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 5a1c4a37..4b5ac042 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -4,6 +4,7 @@ using BotSharp.Abstraction.MessageHub.Models; using BotSharp.Core.Infrastructures.Streams; using BotSharp.Core.MessageHub; using OpenAI.Chat; +using Pipelines.Sockets.Unofficial.Arenas; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -75,7 +76,14 @@ public class ChatCompletionProvider : IChatCompletion { CurrentAgentId = agent.Id, MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - RenderedInstruction = string.Join("\r\n", renderedInstructions) + RenderedInstruction = string.Join("\r\n", renderedInstructions), + Annotations = value.Annotations?.Select(x => new ChatAnnotation + { + Title = x.WebResourceTitle, + Url = x.WebResourceUri.AbsoluteUri, + StartIndex = x.StartIndex, + EndIndex = x.EndIndex + })?.ToList() }; } @@ -330,16 +338,19 @@ public class ChatCompletionProvider : IChatCompletion } // Render functions - foreach (var function in functions) + if (options.WebSearchOptions == null) { - if (!agentService.RenderFunction(agent, function)) continue; + foreach (var function in functions) + { + if (!agentService.RenderFunction(agent, function)) continue; - var property = agentService.RenderFunctionProperty(agent, function); + var property = agentService.RenderFunctionProperty(agent, function); - options.Tools.Add(ChatTool.CreateFunctionTool( - functionName: function.Name, - functionDescription: function.Description, - functionParameters: BinaryData.FromObjectAsJson(property))); + options.Tools.Add(ChatTool.CreateFunctionTool( + functionName: function.Name, + functionDescription: function.Description, + functionParameters: BinaryData.FromObjectAsJson(property))); + } } if (!string.IsNullOrEmpty(agent.Knowledges)) @@ -488,8 +499,9 @@ public class ChatCompletionProvider : IChatCompletion var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); + // Reasoning effort ChatReasoningEffortLevel? reasoningEffortLevel = null; - var temperature = float.Parse(state.GetState("temperature", "0.0")); + float? temperature = float.Parse(state.GetState("temperature", "0.0")); if (settings?.Reasoning != null) { temperature = settings.Reasoning.Temperature; @@ -499,6 +511,15 @@ public class ChatCompletionProvider : IChatCompletion reasoningEffortLevel = ParseReasoningEffortLevel(level); } + // Web search + ChatWebSearchOptions? webSearchOptions = null; + if (settings?.WebSearch != null) + { + temperature = null; + reasoningEffortLevel = null; + webSearchOptions = new(); + } + var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) ? tokens : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; @@ -507,7 +528,8 @@ public class ChatCompletionProvider : IChatCompletion { Temperature = temperature, MaxOutputTokenCount = maxTokens, - ReasoningEffortLevel = reasoningEffortLevel + ReasoningEffortLevel = reasoningEffortLevel, + WebSearchOptions = webSearchOptions }; } @@ -541,11 +563,4 @@ public class ChatCompletionProvider : IChatCompletion { _model = model; } -} - - -class ToolCallData -{ - public ChatFinishReason? Reason { get; set; } - public List ToolCalls { get; set; } = []; } \ No newline at end of file