SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'SciSharp:master' into master

Browse source
This commit is contained in:
Haiping 2024-10-02 14:19:59 -05:00 committed by GitHub
parent 3439809d33 7b649c0248
commit 022c0500f6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 158 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeHook.cs
View file

@ -5,9 +5,9 @@ public interface IKnowledgeHook
Task<List<KnowledgeChunk>> CollectChunkedKnowledge()
=> Task.FromResult(new List<KnowledgeChunk>());
Task<List<string>> GetRelevantKnowledges(string text)
Task<List<string>> GetRelevantKnowledges(RoleDialogModel message, string text)
=> Task.FromResult(new List<string>());
Task<List<string>> GetGlobalKnowledges()
Task<List<string>> GetGlobalKnowledges(RoleDialogModel message)
=> Task.FromResult(new List<string>());
}

21
src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs
View file

@ -50,10 +50,10 @@ public class ChatCompletionProvider : IChatCompletion
{
var toolResult = response.Content.OfType<ToolUseContent>().First();
responseMessage = new RoleDialogModel(AgentRole.Function, response.FirstMessage?.Text)
responseMessage = new RoleDialogModel(AgentRole.Function, response.FirstMessage?.Text ?? string.Empty)
{
CurrentAgentId = agent.Id,
MessageId = conversations.Last().MessageId,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
ToolCallId = toolResult.Id,
FunctionName = toolResult.Name,
FunctionArgs = JsonSerializer.Serialize(toolResult.Input)
@ -62,10 +62,10 @@ public class ChatCompletionProvider : IChatCompletion
else
{
var message = response.FirstMessage;
responseMessage = new RoleDialogModel(AgentRole.Assistant, message.Text)
responseMessage = new RoleDialogModel(AgentRole.Assistant, message?.Text ?? string.Empty)
{
CurrentAgentId = agent.Id,
MessageId = conversations.Last().MessageId
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
};
}
@ -77,8 +77,8 @@ public class ChatCompletionProvider : IChatCompletion
Prompt = prompt,
Provider = Provider,
Model = _model,
PromptCount = response.Usage.InputTokens,
CompletionCount = response.Usage.OutputTokens
PromptCount = response.Usage?.InputTokens ?? 0,
CompletionCount = response.Usage?.OutputTokens ?? 0
});
}
@ -120,7 +120,14 @@ public class ChatCompletionProvider : IChatCompletion
prompt += "\r\n\r\n" + response_with_function;*/
var messages = new List<Message>();
foreach (var conv in conversations)
var filteredMessages = conversations.Select(x => x).ToList();
var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User);
if (firstUserMsgIdx > 0)
{
filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
}
foreach (var conv in filteredMessages)
{
if (conv.Role == AgentRole.User)
{

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj
View file

@ -11,7 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.OpenAI" Version="2.0.0-beta.2" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
</ItemGroup>

2
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs
View file

@ -24,7 +24,7 @@ public partial class AudioCompletionProvider
var options = new AudioTranscriptionOptions
{
ResponseFormat = format,
Granularities = granularity,
TimestampGranularities = granularity,
Temperature = temperature,
Prompt = text
};

4
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs
View file

@ -10,7 +10,7 @@ public partial class AudioCompletionProvider
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options);
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
return result.Value;
}
@ -24,7 +24,7 @@ public partial class AudioCompletionProvider
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
Speed = speed
SpeedRatio = speed
};
return (voice, options);

98
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -50,14 +50,15 @@ public class ChatCompletionProvider : IChatCompletion
var content = value.Content;
var text = content.FirstOrDefault()?.Text ?? string.Empty;
if (reason == ChatFinishReason.FunctionCall)
if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls)
{
var toolCall = value.ToolCalls.FirstOrDefault();
responseMessage = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
FunctionName = value.FunctionCall.FunctionName,
FunctionArgs = value.FunctionCall.FunctionArguments
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments?.ToString()
};
// Somethings LLM will generate a function name with agent name.
@ -66,17 +67,17 @@ public class ChatCompletionProvider : IChatCompletion
responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last();
}
}
else if (reason == ChatFinishReason.ToolCalls)
{
var toolCall = value.ToolCalls.FirstOrDefault();
responseMessage = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments
};
}
//else if (reason == ChatFinishReason.ToolCalls)
//{
// var toolCall = value.ToolCalls.FirstOrDefault();
// responseMessage = new RoleDialogModel(AgentRole.Function, text)
// {
// CurrentAgentId = agent.Id,
// MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
// FunctionName = toolCall?.FunctionName,
// FunctionArgs = toolCall?.FunctionArguments
// };
//}
else
{
responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
@ -113,8 +114,8 @@ public class ChatCompletionProvider : IChatCompletion
Prompt = prompt,
Provider = Provider,
Model = _model,
PromptCount = value?.Usage?.InputTokens ?? 0,
CompletionCount = value?.Usage?.OutputTokens ?? 0
PromptCount = value?.Usage?.InputTokenCount ?? 0,
CompletionCount = value?.Usage?.OutputTokenCount ?? 0
});
}
@ -157,20 +158,21 @@ public class ChatCompletionProvider : IChatCompletion
Prompt = prompt,
Provider = Provider,
Model = _model,
PromptCount = response.Value.Usage.InputTokens,
CompletionCount = response.Value.Usage.OutputTokens
PromptCount = response.Value?.Usage?.InputTokenCount ?? 0,
CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0
});
}
if (reason == ChatFinishReason.FunctionCall)
if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls)
{
_logger.LogInformation($"[{agent.Name}]: {value.FunctionCall.FunctionName}({value.FunctionCall.FunctionArguments})");
var toolCall = value.ToolCalls?.FirstOrDefault();
_logger.LogInformation($"[{agent.Name}]: {toolCall?.FunctionName}({toolCall?.FunctionArguments})");
var funcContextIn = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
FunctionName = value.FunctionCall?.FunctionName,
FunctionArgs = value.FunctionCall?.FunctionArguments
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments?.ToString()
};
// Somethings LLM will generate a function name with agent name.
@ -201,11 +203,12 @@ public class ChatCompletionProvider : IChatCompletion
await foreach (var choice in response)
{
if (choice.FinishReason == ChatFinishReason.FunctionCall)
if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls)
{
Console.Write(choice.FunctionCallUpdate?.FunctionArgumentsUpdate);
var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty;
Console.Write(update);
await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, choice.FunctionCallUpdate?.FunctionArgumentsUpdate));
await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update));
continue;
}
@ -213,7 +216,7 @@ public class ChatCompletionProvider : IChatCompletion
_logger.LogInformation(choice.ContentUpdate[0]?.Text);
await onMessageReceived(new RoleDialogModel(choice.Role.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty));
await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty));
}
return true;
@ -235,7 +238,7 @@ public class ChatCompletionProvider : IChatCompletion
var options = new ChatCompletionOptions()
{
Temperature = temperature,
MaxTokens = maxTokens
MaxOutputTokenCount = maxTokens
};
foreach (var function in agent.Functions)
@ -267,21 +270,35 @@ public class ChatCompletionProvider : IChatCompletion
messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content));
}
foreach (var message in conversations)
var filteredMessages = conversations.Select(x => x).ToList();
var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User);
if (firstUserMsgIdx > 0)
{
filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
}
foreach (var message in filteredMessages)
{
if (message.Role == AgentRole.Function)
{
messages.Add(new AssistantChatMessage(string.Empty)
{
FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty)
});
//messages.Add(new AssistantChatMessage(string.Empty)
//{
// FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty)
//});
messages.Add(new FunctionChatMessage(message.FunctionName, message.Content));
//messages.Add(new FunctionChatMessage(message.FunctionName, message.Content));
messages.Add(new AssistantChatMessage(new List<ChatToolCall>
{
ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))
}));
messages.Add(new ToolChatMessage(message.FunctionName, message.Content));
}
else if (message.Role == AgentRole.User)
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text);
var textPart = ChatMessageContentPart.CreateTextPart(text);
var contentParts = new List<ChatMessageContentPart> { textPart };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
@ -291,20 +308,20 @@ public class ChatCompletionProvider : IChatCompletion
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Low);
contentParts.Add(contentPart);
}
}
@ -347,7 +364,7 @@ public class ChatCompletionProvider : IChatCompletion
.Where(x => x as SystemChatMessage == null)
.Select(x =>
{
var fnMessage = x as FunctionChatMessage;
var fnMessage = x as ToolChatMessage;
if (fnMessage != null)
{
return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
@ -365,8 +382,9 @@ public class ChatCompletionProvider : IChatCompletion
var assistMessage = x as AssistantChatMessage;
if (assistMessage != null)
{
return assistMessage.FunctionCall != null ?
$"{AgentRole.Assistant}: Call function {assistMessage.FunctionCall.FunctionName}({assistMessage.FunctionCall.FunctionArguments})" :
var toolCall = assistMessage.ToolCalls?.FirstOrDefault();
return toolCall != null ?
$"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" :
$"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
}

4
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs
View file

@ -31,7 +31,7 @@ public class TextEmbeddingProvider : ITextEmbedding
var options = PrepareOptions();
var response = await embeddingClient.GenerateEmbeddingAsync(text, options);
var value = response.Value;
return value.Vector.ToArray();
return value.ToFloats().ToArray();
}
public async Task<List<float[]>> GetVectorsAsync(List<string> texts)
@ -41,7 +41,7 @@ public class TextEmbeddingProvider : ITextEmbedding
var options = PrepareOptions();
var response = await embeddingClient.GenerateEmbeddingsAsync(texts, options);
var value = response.Value;
return value.Select(x => x.Vector.ToArray()).ToList();
return value.Select(x => x.ToFloats().ToArray()).ToList();
}
public void SetModelName(string model)

4
src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ProviderHelper.cs
View file

@ -1,5 +1,5 @@
using Azure.AI.OpenAI;
using Azure;
using System.ClientModel;
namespace BotSharp.Plugin.AzureOpenAI.Providers;
@ -9,7 +9,7 @@ public class ProviderHelper
{
var settingsService = services.GetRequiredService<ILlmProviderService>();
var settings = settingsService.GetSetting(provider, model);
var client = new AzureOpenAIClient(new Uri(settings.Endpoint), new AzureKeyCredential(settings.ApiKey));
var client = new AzureOpenAIClient(new Uri(settings.Endpoint), new ApiKeyCredential(settings.ApiKey));
return client;
}

2
src/Plugins/BotSharp.Plugin.OpenAI/BotSharp.Plugin.OpenAI.csproj
View file

@ -11,7 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="OpenAI" Version="2.0.0-beta.5" />
<PackageReference Include="OpenAI" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
</ItemGroup>

2
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs
View file

@ -24,7 +24,7 @@ public partial class AudioCompletionProvider
var options = new AudioTranscriptionOptions
{
ResponseFormat = format,
Granularities = granularity,
TimestampGranularities = granularity,
Temperature = temperature,
Prompt = text
};

4
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs
View file

@ -10,7 +10,7 @@ public partial class AudioCompletionProvider
.GetAudioClient(_model);
var (voice, options) = PrepareGenerationOptions();
var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options);
var result = await audioClient.GenerateSpeechAsync(text, voice, options);
return result.Value;
}
@ -24,7 +24,7 @@ public partial class AudioCompletionProvider
var options = new SpeechGenerationOptions
{
ResponseFormat = format,
Speed = speed
SpeedRatio = speed
};
return (voice, options);

80
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
View file

@ -44,14 +44,16 @@ public class ChatCompletionProvider : IChatCompletion
var text = content.FirstOrDefault()?.Text ?? string.Empty;
RoleDialogModel responseMessage;
if (reason == ChatFinishReason.FunctionCall)
if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls)
{
var toolCall = value.ToolCalls.FirstOrDefault();
responseMessage = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
FunctionName = value.FunctionCall.FunctionName,
FunctionArgs = value.FunctionCall.FunctionArguments
ToolCallId = toolCall?.Id,
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments?.ToString()
};
// Somethings LLM will generate a function name with agent name.
@ -60,17 +62,6 @@ public class ChatCompletionProvider : IChatCompletion
responseMessage.FunctionName = responseMessage.FunctionName.Split('.').Last();
}
}
else if (reason == ChatFinishReason.ToolCalls)
{
var toolCall = value.ToolCalls.FirstOrDefault();
responseMessage = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty,
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments
};
}
else
{
responseMessage = new RoleDialogModel(AgentRole.Assistant, text)
@ -88,8 +79,8 @@ public class ChatCompletionProvider : IChatCompletion
Prompt = prompt,
Provider = Provider,
Model = _model,
PromptCount = response.Value.Usage.InputTokens,
CompletionCount = response.Value.Usage.OutputTokens
PromptCount = response.Value?.Usage?.InputTokenCount ?? 0,
CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0
});
}
@ -132,20 +123,22 @@ public class ChatCompletionProvider : IChatCompletion
Prompt = prompt,
Provider = Provider,
Model = _model,
PromptCount = response.Value.Usage.InputTokens,
CompletionCount = response.Value.Usage.OutputTokens
PromptCount = response.Value?.Usage?.InputTokenCount ?? 0,
CompletionCount = response.Value?.Usage?.OutputTokenCount ?? 0
});
}
if (reason == ChatFinishReason.FunctionCall)
if (reason == ChatFinishReason.FunctionCall || reason == ChatFinishReason.ToolCalls)
{
_logger.LogInformation($"[{agent.Name}]: {value.FunctionCall.FunctionName}({value.FunctionCall.FunctionArguments})");
var toolCall = value.ToolCalls?.FirstOrDefault();
_logger.LogInformation($"[{agent.Name}]: {toolCall?.FunctionName}({toolCall?.FunctionArguments})");
var funcContextIn = new RoleDialogModel(AgentRole.Function, text)
{
CurrentAgentId = agent.Id,
FunctionName = value.FunctionCall?.FunctionName,
FunctionArgs = value.FunctionCall?.FunctionArguments
ToolCallId = toolCall?.Id,
FunctionName = toolCall?.FunctionName,
FunctionArgs = toolCall?.FunctionArguments?.ToString()
};
// Somethings LLM will generate a function name with agent name.
@ -176,11 +169,12 @@ public class ChatCompletionProvider : IChatCompletion
await foreach (var choice in response)
{
if (choice.FinishReason == ChatFinishReason.FunctionCall)
if (choice.FinishReason == ChatFinishReason.FunctionCall || choice.FinishReason == ChatFinishReason.ToolCalls)
{
Console.Write(choice.FunctionCallUpdate?.FunctionArgumentsUpdate);
var update = choice.ToolCallUpdates?.FirstOrDefault()?.FunctionArgumentsUpdate?.ToString() ?? string.Empty;
_logger.LogInformation(update);
await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, choice.FunctionCallUpdate?.FunctionArgumentsUpdate));
await onMessageReceived(new RoleDialogModel(AgentRole.Assistant, update));
continue;
}
@ -188,7 +182,7 @@ public class ChatCompletionProvider : IChatCompletion
_logger.LogInformation(choice.ContentUpdate[0]?.Text);
await onMessageReceived(new RoleDialogModel(choice.Role.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty));
await onMessageReceived(new RoleDialogModel(choice.Role?.ToString() ?? ChatMessageRole.Assistant.ToString(), choice.ContentUpdate[0]?.Text ?? string.Empty));
}
return true;
@ -211,7 +205,7 @@ public class ChatCompletionProvider : IChatCompletion
var options = new ChatCompletionOptions()
{
Temperature = temperature,
MaxTokens = maxTokens
MaxOutputTokenCount = maxTokens
};
foreach (var function in agent.Functions)
@ -243,21 +237,28 @@ public class ChatCompletionProvider : IChatCompletion
messages.Add(sample.Role == AgentRole.User ? new UserChatMessage(sample.Content) : new AssistantChatMessage(sample.Content));
}
foreach (var message in conversations)
var filteredMessages = conversations.Select(x => x).ToList();
var firstUserMsgIdx = filteredMessages.FindIndex(x => x.Role == AgentRole.User);
if (firstUserMsgIdx > 0)
{
filteredMessages = filteredMessages.Where((_, idx) => idx >= firstUserMsgIdx).ToList();
}
foreach (var message in filteredMessages)
{
if (message.Role == AgentRole.Function)
{
messages.Add(new AssistantChatMessage(string.Empty)
messages.Add(new AssistantChatMessage(new List<ChatToolCall>
{
FunctionCall = new ChatFunctionCall(message.FunctionName, message.FunctionArgs ?? string.Empty)
});
ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty))
}));
messages.Add(new FunctionChatMessage(message.FunctionName, message.Content));
messages.Add(new ToolChatMessage(message.FunctionName, message.Content));
}
else if (message.Role == AgentRole.User)
{
var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text);
var textPart = ChatMessageContentPart.CreateTextPart(text);
var contentParts = new List<ChatMessageContentPart> { textPart };
if (allowMultiModal && !message.Files.IsNullOrEmpty())
@ -267,20 +268,20 @@ public class ChatCompletionProvider : IChatCompletion
if (!string.IsNullOrEmpty(file.FileData))
{
var (contentType, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileStorageUrl))
{
var contentType = FileUtility.GetFileContentType(file.FileStorageUrl);
var bytes = fileStorage.GetFileBytes(file.FileStorageUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
var contentPart = ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(bytes), contentType, ChatImageDetailLevel.Low);
contentParts.Add(contentPart);
}
else if (!string.IsNullOrEmpty(file.FileUrl))
{
var uri = new Uri(file.FileUrl);
var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
var contentPart = ChatMessageContentPart.CreateImagePart(uri, ChatImageDetailLevel.Low);
contentParts.Add(contentPart);
}
}
@ -324,7 +325,7 @@ public class ChatCompletionProvider : IChatCompletion
.Where(x => x as SystemChatMessage == null)
.Select(x =>
{
var fnMessage = x as FunctionChatMessage;
var fnMessage = x as ToolChatMessage;
if (fnMessage != null)
{
return $"{AgentRole.Function}: {fnMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
@ -342,8 +343,9 @@ public class ChatCompletionProvider : IChatCompletion
var assistMessage = x as AssistantChatMessage;
if (assistMessage != null)
{
return assistMessage.FunctionCall != null ?
$"{AgentRole.Assistant}: Call function {assistMessage.FunctionCall.FunctionName}({assistMessage.FunctionCall.FunctionArguments})" :
var toolCall = assistMessage.ToolCalls?.FirstOrDefault();
return toolCall != null ?
$"{AgentRole.Assistant}: Call function {toolCall?.FunctionName}({toolCall?.FunctionArguments})" :
$"{AgentRole.Assistant}: {assistMessage.Content.FirstOrDefault()?.Text ?? string.Empty}";
}

4
src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs
View file

@ -31,7 +31,7 @@ public class TextEmbeddingProvider : ITextEmbedding
var options = PrepareOptions();
var response = await embeddingClient.GenerateEmbeddingAsync(text, options);
var value = response.Value;
return value.Vector.ToArray();
return value.ToFloats().ToArray();
}
public async Task<List<float[]>> GetVectorsAsync(List<string> texts)
@ -41,7 +41,7 @@ public class TextEmbeddingProvider : ITextEmbedding
var options = PrepareOptions();
var response = await embeddingClient.GenerateEmbeddingsAsync(texts, options);
var value = response.Value;
return value.Select(x => x.Vector.ToArray()).ToList();
return value.Select(x => x.ToFloats().ToArray()).ToList();
}
public void SetModelName(string model)

17
src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs
View file

@ -19,27 +19,18 @@ public class PrimaryStagePlanFn : IFunctionCallback
{
var agentService = _services.GetRequiredService<IAgentService>();
var state = _services.GetRequiredService<IConversationStateService>();
// var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
// var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
state.SetState("max_tokens", "4096");
var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
// var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
// Get knowledge from vectordb
var hooks = _services.GetServices<IKnowledgeHook>();
var knowledges = new List<string>();
foreach (var question in task.Questions)
{
/*var list = await knowledgeService.SearchVectorKnowledge(question, collectionName, new VectorSearchOptions
{
Confidence = 0.4f
});
knowledges.Add(string.Join("\r\n\r\n=====\r\n", list.Select(x => x.ToQuestionAnswer())));*/
foreach (var hook in hooks)
{
var k = await hook.GetRelevantKnowledges(question);
var k = await hook.GetRelevantKnowledges(message, question);
knowledges.AddRange(k);
}
}
@ -47,7 +38,7 @@ public class PrimaryStagePlanFn : IFunctionCallback
// Get first stage planning prompt
var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
var firstPlanningPrompt = await GetFirstStagePlanPrompt(task.Requirements, knowledges);
var firstPlanningPrompt = await GetFirstStagePlanPrompt(message, task.Requirements, knowledges);
var plannerAgent = new Agent
{
Id = BuiltInAgentId.Planner,
@ -65,7 +56,7 @@ public class PrimaryStagePlanFn : IFunctionCallback
return true;
}
private async Task<string> GetFirstStagePlanPrompt(string taskDescription, List<string> relevantKnowledges)
private async Task<string> GetFirstStagePlanPrompt(RoleDialogModel message, string taskDescription, List<string> relevantKnowledges)
{
var agentService = _services.GetRequiredService<IAgentService>();
var render = _services.GetRequiredService<ITemplateRender>();
@ -83,7 +74,7 @@ public class PrimaryStagePlanFn : IFunctionCallback
var globalKnowledges = new List<string>();
foreach (var hook in knowledgeHooks)
{
var k = await hook.GetGlobalKnowledges();
var k = await hook.GetGlobalKnowledges(message);
globalKnowledges.AddRange(k);
}

14
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs
View file

@ -40,14 +40,18 @@ public class LookupDictionaryFn : IFunctionCallback
};
var response = await GetAiResponse(agent);
args = JsonSerializer.Deserialize<LookupDictionary>(response.Content);
args = response.Content.JsonContent<LookupDictionary>();
// check if need to instantely
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
var result = connection.Query(args.SqlStatement);
IEnumerable<dynamic>? result = null;
if (!string.IsNullOrWhiteSpace(args.SqlStatement))
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
result = connection.Query(args.SqlStatement);
}
if (result == null)
if (result.IsNullOrEmpty())
{
message.Content = "Record not found";
}

11
src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs
View file

@ -5,11 +5,16 @@ namespace BotSharp.Plugin.SqlDriver.Models;
public class LookupDictionary
{
[JsonPropertyName("sql_statement")]
public string? SqlStatement { get; set; }
public string SqlStatement { get; set; } = string.Empty;
[JsonPropertyName("reason")]
public string? Reason { get; set; }
public string Reason { get; set; } = string.Empty;
[JsonPropertyName("table")]
public string? Table { get; set; }
public string Table { get; set; } = string.Empty;
public override string ToString()
{
return $"Sql: {SqlStatement}, Table: {Table}, Reason: {Reason}";
}
}

8
src/WebStarter/Program.cs
View file

@ -4,6 +4,7 @@ using BotSharp.Logger;
using BotSharp.Plugin.ChatHub;
using Serilog;
using BotSharp.Abstraction.Messaging.JsonConverters;
using StackExchange.Redis;
var builder = WebApplication.CreateBuilder(args);
@ -28,7 +29,12 @@ string[] allowedOrigins = builder.Configuration.GetSection("AllowedOrigins").Get
builder.AddServiceDefaults();
// Add SignalR for WebSocket
builder.Services.AddSignalR();
builder.Services.AddSignalR()
// Enable Redis backplane for SignalR
/*.AddStackExchangeRedis("127.0.0.1", o =>
{
o.Configuration.ChannelPrefix = RedisChannel.Literal("botsharp");
})*/;
var app = builder.Build();

1
src/WebStarter/WebStarter.csproj
View file

@ -21,6 +21,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="8.0.8" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.20.1" />
</ItemGroup>