SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

merge by master

Browse source
This commit is contained in:
jason.wang 2024-09-19 08:57:01 +08:00
parent b0e3349ae7 a12685213f
commit cadec5c2a7
62 changed files with 806 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -63,16 +63,17 @@ public interface IFileStorageService
#endregion
#region Knowledge
bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData);
bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData);
/// <summary>
/// Delete files in a knowledge collection. If fileId is null, remove all files in the collection.
/// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection.
/// </summary>
/// <param name="collectionName"></param>
/// <param name="vectorStoreProvider"></param>
/// <param name="fileId"></param>
/// <returns></returns>
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null);
string GetKnowledgeBaseFileUrl(string collectionName, string fileId);
FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId);
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
#endregion
}

3
src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs
View file

@ -2,9 +2,10 @@ namespace BotSharp.Abstraction.Files.Models;
public class KnowledgeFileModel
{
public string FileId { get; set; }
public Guid FileId { get; set; }
public string FileName { get; set; }
public string FileExtension { get; set; }
public string ContentType { get; set; }
public string FileUrl { get; set; }
public DocMetaRefData? RefData { get; set; }
}

5
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -12,6 +12,7 @@ public interface IKnowledgeService
Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
Task<bool> DeleteVectorCollectionData(string collectionName, string id);
Task<bool> DeleteVectorCollectionAllData(string collectionName);
Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update);
#endregion
@ -22,9 +23,9 @@ public interface IKnowledgeService
#region Document
Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId);
Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId);
Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
#endregion
#region Common

21
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs
View file

@ -6,7 +6,7 @@ public class KnowledgeDocMetaData
public string Collection { get; set; }
[JsonPropertyName("file_id")]
public string FileId { get; set; }
public Guid FileId { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
@ -23,6 +23,9 @@ public class KnowledgeDocMetaData
[JsonPropertyName("vector_data_ids")]
public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
[JsonPropertyName("ref_data")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public DocMetaRefData? RefData { get; set; }
[JsonPropertyName("create_date")]
public DateTime CreateDate { get; set; } = DateTime.UtcNow;
@ -30,3 +33,19 @@ public class KnowledgeDocMetaData
[JsonPropertyName("create_user_id")]
public string CreateUserId { get; set; }
}
public class DocMetaRefData
{
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("name")]
public string Name { get; set; }
[JsonPropertyName("url")]
public string Url { get; set; }
[JsonPropertyName("json_content")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? JsonContent { get; set; }
}

2
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs
View file

@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.Knowledges.Models;
public class KnowledgeFileFilter : Pagination
{
public IEnumerable<string>? FileIds { get; set; }
public IEnumerable<Guid>? FileIds { get; set; }
}

9
src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs Normal file
View file

@ -0,0 +1,9 @@
namespace BotSharp.Abstraction.Planning;
public interface IPlanningHook
{
Task<string> GetSummaryAdditionalRequirements(string planner)
=> Task.FromResult(string.Empty);
Task OnPlanningCompleted(string planner, RoleDialogModel msg)
=> Task.CompletedTask;
}

12
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -113,7 +113,15 @@ public interface IBotSharpRepository
bool DeleteKnowledgeCollectionConfig(string collectionName);
IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
/// <summary>
/// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection.
/// </summary>
/// <param name="collectionName"></param>
/// <param name="vectorStoreProvider"></param>
/// <param name="fileId"></param>
/// <returns></returns>
bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null);
PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
#endregion
}

4
src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserRole.cs
View file

@ -13,9 +13,9 @@ public class UserRole
public const string CSR = "csr";
/// <summary>
/// Client
/// Authorized user
/// </summary>
public const string Client = "client";
public const string User = "user";
/// <summary>
/// Back office operations

8
src/Infrastructure/BotSharp.Abstraction/Users/Enums/UserType.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Abstraction.Users.Enums;
public class UserType
{
public const string Internal = "internal";
public const string Client = "client";
public const string Affiliate = "affiliate";
}

1
src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs
View file

@ -16,5 +16,6 @@ public interface IUserService
Task<bool> ResetUserPassword(User user);
Task<bool> ModifyUserEmail(string email);
Task<bool> ModifyUserPhone(string phone);
Task<bool> UpdatePassword(string newPassword, string verificationCode);
Task<DateTime> GetUserTokenExpires();
}

6
src/Infrastructure/BotSharp.Abstraction/Users/Models/User.cs
View file

@ -14,7 +14,11 @@ public class User
public string Password { get; set; } = string.Empty;
public string Source { get; set; } = "internal";
public string? ExternalId { get; set; }
public string Role { get; set; } = UserRole.Client;
/// <summary>
/// internal, client, affiliate
/// </summary>
public string Type { get; set; } = UserType.Client;
public string Role { get; set; } = UserRole.User;
public string? VerificationCode { get; set; }
public bool Verified { get; set; }
public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;

1
src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs
View file

@ -14,4 +14,5 @@ public interface IVectorDb
Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields, int limit = 5, float confidence = 0.5f, bool withVector = false);
Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids);
Task<bool> DeleteCollectionAllData(string collectionName);
}

2
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
View file

@ -126,7 +126,7 @@ public class ConversationStateService : IConversationStateService, IDisposable
_historyStates = _db.GetConversationStates(conversationId);
var dialogs = _db.GetConversationDialogs(conversationId);
var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.Client)
var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User || x.MetaData?.Role == UserRole.User)
.GroupBy(x => x.MetaData?.MessageId)
.Select(g => g.First())
.OrderBy(x => x.MetaData?.CreateTime)

61
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
View file

@ -1,23 +1,21 @@
using BotSharp.Abstraction.Knowledges.Models;
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData)
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
try
{
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var dir = Path.Combine(docDir, fileId);
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var dir = Path.Combine(docDir, fileId.ToString());
if (ExistDirectory(dir))
{
Directory.Delete(dir);
@ -40,7 +38,7 @@ public partial class LocalFileStorageService
}
}
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
@ -48,16 +46,16 @@ public partial class LocalFileStorageService
return false;
}
var dir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!ExistDirectory(dir)) return false;
if (string.IsNullOrEmpty(fileId))
if (fileId == null)
{
Directory.Delete(dir, true);
}
else
{
var fileDir = Path.Combine(dir, fileId);
var fileDir = Path.Combine(dir, fileId.ToString());
if (ExistDirectory(fileDir))
{
Directory.Delete(fileDir, true);
@ -67,10 +65,18 @@ public partial class LocalFileStorageService
return true;
}
public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return string.Empty;
}
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var file = Path.Combine(docDir, fileId.ToString(), fileName);
if (!File.Exists(file))
{
return string.Empty;
}
@ -78,39 +84,34 @@ public partial class LocalFileStorageService
return $"/knowledge/document/{collectionName}/file/{fileId}";
}
public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(fileName))
{
return null;
return BinaryData.Empty;
}
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var fileDir = Path.Combine(docDir, fileId);
if (!ExistDirectory(fileDir)) return null;
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var file = Path.Combine(docDir, fileId.ToString(), fileName);
var metaFile = Path.Combine(fileDir, KNOWLEDGE_DOC_META_FILE);
var content = File.ReadAllText(metaFile);
var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
var file = Path.Combine(fileDir, metaData.FileName);
if (!File.Exists(file))
{
return BinaryData.Empty;
}
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
stream.Position = 0;
return new FileBinaryDataModel
{
FileName = metaData.FileName,
ContentType = metaData.ContentType,
FileBinaryData = BinaryData.FromStream(stream)
};
return BinaryData.FromStream(stream);
}
#region Private methods
private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider)
private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
{
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr());
}
#endregion
}

1
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs
View file

@ -21,7 +21,6 @@ public partial class LocalFileStorageService : IFileStorageService
private const string TEXT_TO_SPEECH_FOLDER = "speeches";
private const string KNOWLEDGE_FOLDER = "knowledgebase";
private const string KNOWLEDGE_DOC_FOLDER = "document";
private const string KNOWLEDGE_DOC_META_FILE = "meta.json";
private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions
{

7
src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs
View file

@ -182,8 +182,11 @@ public class PluginLoader
foreach (var agentId in plugin.AgentIds)
{
var agent = agentService.LoadAgent(agentId).Result;
agent.Disabled = true;
agentService.UpdateAgent(agent, AgentField.Disabled);
if (agent != null)
{
agent.Disabled = true;
agentService.UpdateAgent(agent, AgentField.Disabled);
}
}
}
return plugin;

5
src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file

@ -234,7 +234,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
throw new NotImplementedException();
#endregion
#region Knowledge
#region KnowledgeBase
public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false) =>
throw new NotImplementedException();
@ -247,6 +247,9 @@ public class BotSharpDbContext : Database, IBotSharpRepository
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) =>
throw new NotImplementedException();
public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) =>
throw new NotImplementedException();
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) =>
throw new NotImplementedException();
#endregion

40
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs
View file

@ -110,14 +110,13 @@ public partial class FileRepository
{
if (metaData == null
|| string.IsNullOrWhiteSpace(metaData.Collection)
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
|| string.IsNullOrWhiteSpace(metaData.FileId))
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeDocumentDir(metaData.Collection.CleanStr(), metaData.VectorStoreProvider.CleanStr());
var docDir = Path.Combine(dir, metaData.FileId);
var dir = BuildKnowledgeCollectionFileDir(metaData.Collection, metaData.VectorStoreProvider);
var docDir = Path.Combine(dir, metaData.FileId.ToString());
if (!Directory.Exists(docDir))
{
Directory.CreateDirectory(docDir);
@ -129,6 +128,33 @@ public partial class FileRepository
return true;
}
public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!Directory.Exists(dir)) return false;
if (fileId == null)
{
Directory.Delete(dir, true);
}
else
{
var fileDir = Path.Combine(dir, fileId.ToString());
if (Directory.Exists(fileDir))
{
Directory.Delete(fileDir, true);
}
}
return true;
}
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
{
if (string.IsNullOrWhiteSpace(collectionName)
@ -137,7 +163,7 @@ public partial class FileRepository
return new PagedItems<KnowledgeDocMetaData>();
}
var dir = BuildKnowledgeDocumentDir(collectionName, vectorStoreProvider);
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!Directory.Exists(dir))
{
return new PagedItems<KnowledgeDocMetaData>();
@ -181,9 +207,9 @@ public partial class FileRepository
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
}
private string BuildKnowledgeDocumentDir(string collectionName, string vectorStoreProvider)
private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
{
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr());
}
#endregion
}

54
src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Users.Enums;
using BotSharp.Abstraction.Infrastructures;
using BotSharp.Abstraction.Users.Models;
using BotSharp.Abstraction.Users.Settings;
@ -84,6 +85,27 @@ public class UserService : IUserService
return record;
}
public async Task<bool> UpdatePassword(string password, string verificationCode)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var record = db.GetUserByUserName(_user.UserName);
if (record == null)
{
return false;
}
if (record.VerificationCode != verificationCode)
{
return false;
}
var newPassword = Utilities.HashTextMd5($"{password}{record.Salt}");
db.UpdateUserPassword(record.Id, newPassword);
return true;
}
public async Task<Token?> GetToken(string authorization)
{
var base64 = Encoding.UTF8.GetString(Convert.FromBase64String(authorization));
@ -138,6 +160,7 @@ public class UserService : IUserService
Source = user.Source,
ExternalId = user.ExternalId,
Password = user.Password,
Type = user.Type,
};
await CreateUser(record);
}
@ -191,6 +214,7 @@ public class UserService : IUserService
new Claim(JwtRegisteredClaimNames.FamilyName, user?.LastName ?? string.Empty),
new Claim("source", user.Source),
new Claim("external_id", user.ExternalId ?? string.Empty),
new Claim("type", user.Type ?? UserType.Client),
new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()),
new Claim("phone", user.Phone ?? string.Empty)
};
@ -343,24 +367,32 @@ public class UserService : IUserService
public async Task<bool> SendVerificationCodeResetPassword(User user)
{
if (!string.IsNullOrEmpty(user.Email) && !string.IsNullOrEmpty(user.Phone))
{
return false;
}
var db = _services.GetRequiredService<IBotSharpRepository>();
User? record = null;
if (!string.IsNullOrEmpty(user.Email))
if (!string.IsNullOrWhiteSpace(_user.Id))
{
record = db.GetUserByEmail(user.Email);
record = db.GetUserById(_user.Id);
}
else
{
if (!string.IsNullOrEmpty(user.Email) && !string.IsNullOrEmpty(user.Phone))
{
return false;
}
if (!string.IsNullOrEmpty(user.Email))
{
record = db.GetUserByEmail(user.Email);
}
if (!string.IsNullOrEmpty(user.Phone))
{
record = db.GetUserByPhone(user.Phone);
}
}
if (!string.IsNullOrEmpty(user.Phone))
{
record = db.GetUserByPhone(user.Phone);
}
if (record == null)
{
return false;

10
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -102,6 +102,12 @@ public class KnowledgeBaseController : ControllerBase
{
return await _knowledgeService.DeleteVectorCollectionData(collection, id);
}
[HttpDelete("/knowledge/vector/{collection}/data")]
public async Task<bool> DeleteVectorCollectionAllData([FromRoute] string collection)
{
return await _knowledgeService.DeleteVectorCollectionAllData(collection);
}
#endregion
@ -137,7 +143,7 @@ public class KnowledgeBaseController : ControllerBase
}
[HttpDelete("/knowledge/document/{collection}/delete/{fileId}")]
public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId)
{
var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId);
return response;
@ -160,7 +166,7 @@ public class KnowledgeBaseController : ControllerBase
}
[HttpGet("/knowledge/document/{collection}/file/{fileId}")]
public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId)
{
var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId);
var stream = file.FileBinaryData.ToStream();

10
src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs
View file

@ -121,6 +121,16 @@ public class UserController : ControllerBase
return await _userService.ResetUserPassword(user.ToUser());
}
[HttpPost("/user/updatepassword")]
public async Task<bool> UpdatePassword([FromBody] User user)
{
if (string.IsNullOrWhiteSpace(user.Password) || string.IsNullOrWhiteSpace(user.VerificationCode))
{
return false;
}
return await _userService.UpdatePassword(user.Password, user.VerificationCode);
}
[HttpPost("/user/email/modify")]
public async Task<bool> ModifyUserEmail([FromQuery] string email)
{

9
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs
View file

@ -5,7 +5,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class KnowledgeFileViewModel
{
[JsonPropertyName("file_id")]
public string FileId { get; set; }
public Guid FileId { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
@ -19,6 +19,10 @@ public class KnowledgeFileViewModel
[JsonPropertyName("file_url")]
public string FileUrl { get; set; }
[JsonPropertyName("ref_data")]
public DocMetaRefData? RefData { get; set; }
public static KnowledgeFileViewModel From(KnowledgeFileModel model)
{
return new KnowledgeFileViewModel
@ -27,7 +31,8 @@ public class KnowledgeFileViewModel
FileName = model.FileName,
FileExtension = model.FileExtension,
ContentType = model.ContentType,
FileUrl = model.FileUrl
FileUrl = model.FileUrl,
RefData = model.RefData
};
}
}

6
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserCreationModel.cs
View file

@ -10,7 +10,8 @@ public class UserCreationModel
public string? Email { get; set; }
public string? Phone { get; set; }
public string Password { get; set; } = string.Empty;
public string Role { get; set; } = UserRole.Client;
public string Type { get; set; } = UserType.Client;
public string Role { get; set; } = UserRole.User;
public User ToUser()
{
@ -22,7 +23,8 @@ public class UserCreationModel
Email = Email,
Phone = Phone,
Password = Password,
Role = Role
Role = Role,
Type = Type
};
}
}

5
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserViewModel.cs
View file

@ -14,7 +14,8 @@ public class UserViewModel
public string? LastName { get; set; }
public string? Email { get; set; }
public string? Phone { get; set; }
public string Role { get; set; } = UserRole.Client;
public string Type { get; set; } = UserType.Client;
public string Role { get; set; } = UserRole.User;
[JsonPropertyName("full_name")]
public string FullName => $"{FirstName} {LastName}".Trim();
public string Source { get; set; }
@ -34,6 +35,7 @@ public class UserViewModel
{
FirstName = "Unknown",
LastName = "Anonymous",
Type = UserType.Client,
Role = AgentRole.User
};
}
@ -46,6 +48,7 @@ public class UserViewModel
LastName = user.LastName,
Email = user.Email,
Phone = user.Phone,
Type = user.Type,
Role = user.Role,
Source = user.Source,
ExternalId = user.ExternalId,

6
src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs
View file

@ -44,11 +44,7 @@ public class WelcomeHook : ConversationHookBase
foreach (var message in messages)
{
var richContent = new RichContent<IRichMessage>(message)
{
Editor = message.RichType == RichTypeEnum.QuickReply ? EditorTypeEnum.None : EditorTypeEnum.Text,
};
var richContent = new RichContent<IRichMessage>(message);
var json = JsonSerializer.Serialize(new ChatResponseModel()
{
ConversationId = conversation.Id,

5
src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs
View file

@ -78,4 +78,9 @@ public class MemoryVectorDb : IVectorDb
{
return await Task.FromResult(false);
}
public async Task<bool> DeleteCollectionAllData(string collectionName)
{
return await Task.FromResult(false);
}
}

60
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -42,7 +42,7 @@ public partial class KnowledgeService
var contents = await GetFileContent(contentType, bytes);
// Save document
var fileId = Guid.NewGuid().ToString();
var fileId = Guid.NewGuid();
var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes);
if (!saved)
{
@ -89,9 +89,9 @@ public partial class KnowledgeService
}
public async Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId)
public async Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId)
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
if (string.IsNullOrWhiteSpace(collectionName))
{
return false;
}
@ -104,21 +104,23 @@ public partial class KnowledgeService
var vectorStoreProvider = _settings.VectorDb.Provider;
// Get doc meta data
var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
{
FileIds = new[] { fileId }
FileIds = [ fileId ],
Size = 1
});
// Delete doc
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
var found = pagedData?.Items?.FirstOrDefault();
fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider, fileId);
var found = pageData?.Items?.FirstOrDefault();
if (found != null && !found.VectorDataIds.IsNullOrEmpty())
{
var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList();
await vectorDb.DeleteCollectionData(collectionName, guids);
}
db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider, fileId);
return true;
}
catch (Exception ex)
@ -154,7 +156,8 @@ public partial class KnowledgeService
FileName = x.FileName,
FileExtension = Path.GetExtension(x.FileName),
ContentType = x.ContentType,
FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, x.FileId)
FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName),
RefData = x.RefData
})?.ToList() ?? new List<KnowledgeFileModel>();
return new PagedItems<KnowledgeFileModel>
@ -164,14 +167,37 @@ public partial class KnowledgeService
};
}
public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId)
public async Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var vectorStoreProvider = _settings.VectorDb.Provider;
// Get doc binary data
var file = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
return file;
var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
{
FileIds = [ fileId ],
Size = 1
});
var metaData = pageData?.Items?.FirstOrDefault();
if (metaData == null)
{
return new FileBinaryDataModel
{
FileName = "error.txt",
ContentType = "text/plain",
FileBinaryData = BinaryData.Empty
};
};
var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName);
return new FileBinaryDataModel
{
FileName = metaData.FileName,
ContentType = metaData.ContentType,
FileBinaryData = binaryData
};
}
@ -246,16 +272,16 @@ public partial class KnowledgeService
#endregion
private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes)
private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, byte[] bytes)
{
var fileStoreage = _services.GetRequiredService<IFileStorageService>();
var data = BinaryData.FromBytes(bytes);
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, fileName, data);
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, data);
return saved;
}
private async Task<IEnumerable<string>> SaveToVectorDb(
string collectionName, string fileId, string fileName, IEnumerable<string> contents,
string collectionName, Guid fileId, string fileName, IEnumerable<string> contents,
string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File)
{
if (contents.IsNullOrEmpty())
@ -275,7 +301,7 @@ public partial class KnowledgeService
var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
{
{ KnowledgePayloadName.DataSource, vectorDataSource },
{ KnowledgePayloadName.FileId, fileId },
{ KnowledgePayloadName.FileId, fileId.ToString() },
{ KnowledgePayloadName.FileName, fileName },
{ KnowledgePayloadName.FileSource, fileSource },
{ "textNumber", $"{i + 1}" }

19
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.VectorStorage.Enums;
using System;
namespace BotSharp.Plugin.KnowledgeBase.Services;
@ -92,7 +93,8 @@ public partial class KnowledgeService
var vectorStoreProvider = _settings.VectorDb.Provider;
db.DeleteKnowledgeCollectionConfig(collectionName);
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider);
db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider);
}
return deleted;
@ -181,6 +183,21 @@ public partial class KnowledgeService
}
}
public async Task<bool> DeleteVectorCollectionAllData(string collectionName)
{
try
{
var db = GetVectorDb();
return await db.DeleteCollectionAllData(collectionName);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting vector collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter)
{
try

4
src/Plugins/BotSharp.Plugin.MetaAI/MetaAiPlugin.cs
View file

@ -1,13 +1,10 @@
using BotSharp.Abstraction.Knowledges.Settings;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Plugins;
using BotSharp.Abstraction.Settings;
using BotSharp.Abstraction.VectorStorage;
using BotSharp.Plugin.MetaAI.Providers;
using BotSharp.Plugin.MetaAI.Settings;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using System;
namespace BotSharp.Plugin.MetaAI;
@ -32,6 +29,5 @@ public class MetaAiPlugin : IBotSharpPlugin
});
services.AddSingleton<ITextEmbedding, fastTextEmbeddingProvider>();
services.AddSingleton<IVectorDb, FaissDb>();
}
}

55
src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs
View file

@ -1,55 +0,0 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Abstraction.VectorStorage;
using BotSharp.Abstraction.VectorStorage.Models;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace BotSharp.Plugin.MetaAI.Providers;
public class FaissDb : IVectorDb
{
public string Provider => "Faiss";
public Task<bool> CreateCollection(string collectionName, int dimension)
{
throw new NotImplementedException();
}
public Task<bool> DeleteCollection(string collectionName)
{
throw new NotImplementedException();
}
public Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
{
throw new NotImplementedException();
}
public Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
bool withPayload = false, bool withVector = false)
{
throw new NotImplementedException();
}
public Task<IEnumerable<string>> GetCollections()
{
throw new NotImplementedException();
}
public Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector,
IEnumerable<string>? fields, int limit = 10, float confidence = 0.5f, bool withVector = false)
{
throw new NotImplementedException();
}
public Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
throw new NotImplementedException();
}
public Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
{
throw new NotImplementedException();
}
}

5
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs → src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs
View file

@ -1,14 +1,15 @@
namespace BotSharp.Plugin.MongoStorage.Collections;
public class KnowledgeCollectionFileDocument : MongoBase
public class KnowledgeCollectionFileMetaDocument : MongoBase
{
public string Collection { get; set; }
public string FileId { get; set; }
public Guid FileId { get; set; }
public string FileName { get; set; }
public string FileSource { get; set; }
public string ContentType { get; set; }
public string VectorStoreProvider { get; set; }
public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
public KnowledgeFileMetaRefMongoModel? RefData { get; set; }
public DateTime CreateDate { get; set; }
public string CreateUserId { get; set; }
}

3
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/UserDocument.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Users.Enums;
using BotSharp.Abstraction.Users.Models;
namespace BotSharp.Plugin.MongoStorage.Collections;
@ -13,6 +14,7 @@ public class UserDocument : MongoBase
public string Password { get; set; } = null!;
public string Source { get; set; } = "internal";
public string? ExternalId { get; set; }
public string Type { get; set; } = UserType.Client;
public string Role { get; set; } = null!;
public string? VerificationCode { get; set; }
public bool Verified { get; set; }
@ -33,6 +35,7 @@ public class UserDocument : MongoBase
Salt = Salt,
Source = Source,
ExternalId = ExternalId,
Type = Type,
Role = Role,
VerificationCode = VerificationCode,
Verified = Verified,

37
src/Plugins/BotSharp.Plugin.MongoStorage/Models/KnowledgeFileMetaRefMongoModel.cs Normal file
View file

@ -0,0 +1,37 @@
using BotSharp.Abstraction.Knowledges.Models;
namespace BotSharp.Plugin.MongoStorage.Models;
public class KnowledgeFileMetaRefMongoModel
{
public string Id { get; set; }
public string Name { get; set; }
public string Url { get; set; }
public string? JsonContent { get; set; }
public static KnowledgeFileMetaRefMongoModel? ToMongoModel(DocMetaRefData? model)
{
if (model == null) return null;
return new KnowledgeFileMetaRefMongoModel
{
Id = model.Id,
Name = model.Name,
Url = model.Url,
JsonContent = model.JsonContent
};
}
public static DocMetaRefData? ToDomainModel(KnowledgeFileMetaRefMongoModel? model)
{
if (model == null) return null;
return new DocMetaRefData
{
Id = model.Id,
Name = model.Name,
Url = model.Url,
JsonContent = model.JsonContent
};
}
}

4
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -157,6 +157,6 @@ public class MongoDbContext
public IMongoCollection<KnowledgeCollectionConfigDocument> KnowledgeCollectionConfigs
=> Database.GetCollection<KnowledgeCollectionConfigDocument>($"{_collectionPrefix}_KnowledgeCollectionConfigs");
public IMongoCollection<KnowledgeCollectionFileDocument> KnowledgeCollectionFiles
=> Database.GetCollection<KnowledgeCollectionFileDocument>($"{_collectionPrefix}_KnowledgeCollectionFiles");
public IMongoCollection<KnowledgeCollectionFileMetaDocument> KnowledgeCollectionFileMeta
=> Database.GetCollection<KnowledgeCollectionFileMetaDocument>($"{_collectionPrefix}_KnowledgeCollectionFileMeta");
}

46
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs
View file

@ -120,13 +120,12 @@ public partial class MongoRepository
{
if (metaData == null
|| string.IsNullOrWhiteSpace(metaData.Collection)
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
|| string.IsNullOrWhiteSpace(metaData.FileId))
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider))
{
return false;
}
var doc = new KnowledgeCollectionFileDocument
var doc = new KnowledgeCollectionFileMetaDocument
{
Id = Guid.NewGuid().ToString(),
Collection = metaData.Collection,
@ -136,14 +135,39 @@ public partial class MongoRepository
ContentType = metaData.ContentType,
VectorStoreProvider = metaData.VectorStoreProvider,
VectorDataIds = metaData.VectorDataIds,
RefData = KnowledgeFileMetaRefMongoModel.ToMongoModel(metaData.RefData),
CreateDate = metaData.CreateDate,
CreateUserId = metaData.CreateUserId
};
_dc.KnowledgeCollectionFiles.InsertOne(doc);
_dc.KnowledgeCollectionFileMeta.InsertOne(doc);
return true;
}
public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var builder = Builders<KnowledgeCollectionFileMetaDocument>.Filter;
var filters = new List<FilterDefinition<KnowledgeCollectionFileMetaDocument>>()
{
builder.Eq(x => x.Collection, collectionName),
builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider)
};
if (fileId != null)
{
filters.Add(builder.Eq(x => x.FileId, fileId));
}
var res = _dc.KnowledgeCollectionFileMeta.DeleteMany(builder.And(filters));
return res.DeletedCount > 0;
}
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
{
if (string.IsNullOrWhiteSpace(collectionName)
@ -152,9 +176,8 @@ public partial class MongoRepository
return new PagedItems<KnowledgeDocMetaData>();
}
var builder = Builders<KnowledgeCollectionFileDocument>.Filter;
var docFilters = new List<FilterDefinition<KnowledgeCollectionFileDocument>>()
var builder = Builders<KnowledgeCollectionFileMetaDocument>.Filter;
var docFilters = new List<FilterDefinition<KnowledgeCollectionFileMetaDocument>>()
{
builder.Eq(x => x.Collection, collectionName),
builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider)
@ -167,9 +190,9 @@ public partial class MongoRepository
}
var filterDef = builder.And(docFilters);
var sortDef = Builders<KnowledgeCollectionFileDocument>.Sort.Descending(x => x.CreateDate);
var docs = _dc.KnowledgeCollectionFiles.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.KnowledgeCollectionFiles.CountDocuments(filterDef);
var sortDef = Builders<KnowledgeCollectionFileMetaDocument>.Sort.Descending(x => x.CreateDate);
var docs = _dc.KnowledgeCollectionFileMeta.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.KnowledgeCollectionFileMeta.CountDocuments(filterDef);
var files = docs?.Select(x => new KnowledgeDocMetaData
{
@ -180,9 +203,10 @@ public partial class MongoRepository
ContentType = x.ContentType,
VectorStoreProvider = x.VectorStoreProvider,
VectorDataIds = x.VectorDataIds,
RefData = KnowledgeFileMetaRefMongoModel.ToDomainModel(x.RefData),
CreateDate = x.CreateDate,
CreateUserId = x.CreateUserId
})?.ToList() ?? new List<KnowledgeDocMetaData>();
})?.ToList() ?? new();
return new PagedItems<KnowledgeDocMetaData>
{

1
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs
View file

@ -46,6 +46,7 @@ public partial class MongoRepository
Source = user.Source,
ExternalId = user.ExternalId,
Role = user.Role,
Type = user.Type,
VerificationCode = user.VerificationCode,
Verified = user.Verified,
CreatedTime = DateTime.UtcNow,

12
src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj
View file

@ -16,8 +16,10 @@
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\functions\plan_secondary_stage.json" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\functions\plan_summary.json" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\instructions\instruction.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.1st.next.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.MySql.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.SqlServer.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.2nd.plan.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.next.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.summarize.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\planner_prompt.two_stage.1st.plan.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\plan_primary_stage.fn.liquid" />
@ -41,7 +43,13 @@
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\instructions\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.1st.next.liquid">
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.mysql.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.sqlserver.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.next.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.2nd.plan.liquid">

2
src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs
View file

@ -5,7 +5,7 @@ namespace BotSharp.Plugin.Planner.Functions;
public class PrimaryStagePlanFn : IFunctionCallback
{
public string Name => "plan_primary_stage";
public string Indication => "Currently analyzing and breaking down user requirements.";
private readonly IServiceProvider _services;
private readonly ILogger<PrimaryStagePlanFn> _logger;

2
src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs
View file

@ -5,7 +5,7 @@ namespace BotSharp.Plugin.Planner.Functions;
public class SecondaryStagePlanFn : IFunctionCallback
{
public string Name => "plan_secondary_stage";
public string Indication => "Further analyzing and breaking down user sub-needs.";
private readonly IServiceProvider _services;
private readonly ILogger<SecondaryStagePlanFn> _logger;

24
src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Planning;
using BotSharp.Plugin.Planner.TwoStaging;
using BotSharp.Plugin.Planner.TwoStaging.Models;
namespace BotSharp.Plugin.Planner.Functions;
@ -5,7 +7,7 @@ namespace BotSharp.Plugin.Planner.Functions;
public class SummaryPlanFn : IFunctionCallback
{
public string Name => "plan_summary";
public string Indication => "Organizing and summarizing the final output results.";
private readonly IServiceProvider _services;
private readonly ILogger<SummaryPlanFn> _logger;
@ -62,7 +64,9 @@ public class SummaryPlanFn : IFunctionCallback
var summary = await GetAiResponse(plannerAgent);
message.Content = summary.Content;
message.StopCompletion = true;
await HookEmitter.Emit<IPlanningHook>(_services, x =>
x.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message));
return true;
}
@ -74,18 +78,20 @@ public class SummaryPlanFn : IFunctionCallback
var agent = await agentService.GetAgent(BuiltInAgentId.Planner);
var template = agent.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty;
var responseFormat = JsonSerializer.Serialize(new FirstStagePlan
var additionalRequirements = new List<string>();
await HookEmitter.Emit<IPlanningHook>(_services, async x =>
{
Parameters = [JsonDocument.Parse("{}")],
Results = [""]
var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner));
additionalRequirements.Add(requirement);
});
return render.Render(template, new Dictionary<string, object>
{
{ "table_structure", ddlStatement },
{ "task_description", taskDescription },
{ "summary_requirements", string.Join("\r\n",additionalRequirements) },
{ "relevant_knowledges", relevantKnowledge },
{ "response_format", responseFormat }
{ "table_structure", ddlStatement },
});
}
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
@ -94,8 +100,8 @@ public class SummaryPlanFn : IFunctionCallback
var wholeDialogs = conv.GetDialogHistory();
// Append text
wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;";
wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs";
wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id).\nFor example, you should use SET @id = select max(id) from table;";
wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs.";
var completion = CompletionProvider.GetChatCompletion(_services,
provider: plannerAgent.LlmConfig.Provider,

6
src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs
View file

@ -3,7 +3,7 @@ namespace BotSharp.Plugin.Planner.TwoStaging.Models;
public class SecondStagePlan
{
[JsonPropertyName("related_tables")]
public string[] Tables { get; set; } = new string[0];
public string[] Tables { get; set; } = [];
[JsonPropertyName("description")]
public string Description { get; set; } = "";
@ -12,8 +12,8 @@ public class SecondStagePlan
public string Tool { get; set; } = "";
[JsonPropertyName("input_args")]
public JsonDocument[] Parameters { get; set; } = new JsonDocument[0];
public JsonDocument[] Parameters { get; set; } = [];
[JsonPropertyName("output_results")]
public string[] Results { get; set; } = new string[0];
public string[] Results { get; set; } = [];
}

4
src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs
View file

@ -18,8 +18,8 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner
public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs)
{
var nextStepPrompt = await GetNextStepPrompt(router);
var inst = new FunctionCallFromLlm();
var nextStepPrompt = await GetNextStepPrompt(router);
// chat completion
var completion = CompletionProvider.GetChatCompletion(_services,
@ -125,7 +125,7 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner
{
var agentService = _services.GetRequiredService<IAgentService>();
var planner = await agentService.LoadAgent(BuiltInAgentId.Planner);
var template = planner.Templates.First(x => x.Name == "two_stage.1st.next").Content;
var template = planner.Templates.First(x => x.Name == "two_stage.next").Content;
var states = _services.GetRequiredService<IConversationStateService>();
var render = _services.GetRequiredService<ITemplateRender>();
return render.Render(template, new Dictionary<string, object>

4
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json
View file

@ -11,8 +11,8 @@
"profiles": [ "planning" ],
"utilities": [ "two-stage-planner" ],
"llmConfig": {
"provider": "anthropic",
"model": "claude-3-5-sonnet-20240620",
"provider": "azure-openai",
"model": "gpt-4o",
"max_recursion_depth": 10
}
}

6
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid
View file

@ -1,7 +1,11 @@
Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly.
1. call plan_primary_stage to generate the primary plan.
2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage.
3. You must call plan_summary as the last planning step to summarize the final query.
3. You must call plan_summary for you final planned output.
*** IMPORTANT ***
Don't run the planning process repeatedly if you have already got the result of user's request.
{% if global_knowledges != empty -%}
=====

20
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid Normal file
View file

@ -0,0 +1,20 @@
Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters.
The parameters can be extracted from the original task.
If not, generate the query step by step based on the planning.
The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information.
Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0.
Don't use the sql statement that specify target table for update in FROM clause.
For example, you CAN'T write query as below:
INSERT INTO data_Service (Id, Name)
VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC');
If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.
For example, you should use SET @id = select max(id) from table;
*** the alias of the table name in the sql query should be identical. ***
*** the generated sql query MUST be basedd on the provided table structure. ***
*** All queries return a maximum of 20 records. ***
*** Only select user friendly columns. ***

12
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid Normal file
View file

@ -0,0 +1,12 @@
Try if you can generate a SQL Server single query to fulfill the needs. The step should contains all needed parameters.
The parameters can be extracted from the original task.
If not, generate the query step by step based on the planning.
The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information.
Note: Output should be only the sql query with sql comments that can be directly run in SQL Server.
*** the alias of the table name in the sql query should be identical. ***
*** The generated sql query MUST be based on the provided table structure. ***
*** All queries return a maximum of 10 records. ***
*** Only select user friendly columns. ***

0
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.next.liquid → src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.next.liquid
View file

26
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid
View file

@ -1,25 +1,7 @@
You are a planning summarizer and sql generator. You will convert the requirement into the excutable MySQL query statement based on the task description and related table structure and relationship.
You are a planning summarizer. You will generate the final output in JSON format based on the task description, knowledge and related table structure and relationship.
Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters.
The parameters can be extracted from the original task.
If not, generate the query step by step based on the planning.
The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information.
Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0.
Don't use the sql statement that specify target table for update in FROM clause.
For example, you CAN'T write query as below:
INSERT INTO data_Service (Id, Name)
VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC');
If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.
For example, you should use SET @id = select max(id) from table;
Additional Requirements:
* the alias of the table name in the sql query should be identical.
*** the generated sql query MUST be basedd on the provided table structure. ***
Requirements:
{{ summary_requirements }}
=====
Task description:
@ -31,4 +13,4 @@ Relevant Knowledges:
=====
Table Structure:
{{ table_structure }}
{{ table_structure }}

20
src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Utilities;
using BotSharp.Abstraction.VectorStorage.Models;
using Google.Protobuf.WellKnownTypes;
using Microsoft.Extensions.Logging;
using Qdrant.Client;
using Qdrant.Client.Grpc;
@ -246,10 +245,29 @@ public class QdrantDb : IVectorDb
if (ids.IsNullOrEmpty()) return false;
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
if (!exist)
{
return false;
}
var result = await client.DeleteAsync(collectionName, ids);
return result.Status == UpdateStatus.Completed;
}
public async Task<bool> DeleteCollectionAllData(string collectionName)
{
var client = GetClient();
var exist = await DoesCollectionExist(client, collectionName);
if (!exist)
{
return false;
}
var result = await client.DeleteAsync(collectionName, new Filter());
return result.Status == UpdateStatus.Completed;
}
private async Task<bool> DoesCollectionExist(QdrantClient client, string collectionName)
{

5
src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs
View file

@ -95,5 +95,10 @@ namespace BotSharp.Plugin.SemanticKernel
await _memoryStore.RemoveBatchAsync(collectionName, ids.Select(x => x.ToString()));
return true;
}
public async Task<bool> DeleteCollectionAllData(string collectionName)
{
return await Task.FromResult(false);
}
}
}

10
src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj
View file

@ -10,12 +10,19 @@
<OutputPath>$(SolutionDir)packages</OutputPath>
</PropertyGroup>
<ItemGroup>
<Compile Remove="packages\**" />
<EmbeddedResource Remove="packages\**" />
<None Remove="packages\**" />
</ItemGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\get_table_definition.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_select.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\get_table_definition.fn.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_executor.fn.liquid" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\agent.json" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\execute_sql.json" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\lookup_dictionary.json" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_insert.json" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_select.json" />
@ -33,6 +40,9 @@
<Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\agent.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\execute_sql.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\instructions\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

41
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs
View file

@ -1,25 +1,50 @@
using BotSharp.Plugin.SqlDriver.Models;
using Dapper;
using Microsoft.Data.SqlClient;
using MySqlConnector;
namespace BotSharp.Plugin.SqlDriver.Functions;
public class ExecuteQueryFn : IFunctionCallback
{
public string Name => "execute_sql";
public string Indication => "Performing data retrieval operation.";
private readonly SqlDriverSetting _setting;
private readonly IServiceProvider _services;
public ExecuteQueryFn(SqlDriverSetting setting)
public ExecuteQueryFn(IServiceProvider services, SqlDriverSetting setting)
{
_services = services;
_setting = setting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
message.Content = "Executed";
/*using var connection = new MySqlConnection(_setting.MySqlConnectionString);
message.Content = JsonSerializer.Serialize(connection.Query(args.SqlStatement), new JsonSerializerOptions
var args = JsonSerializer.Deserialize<ExecuteQueryArgs>(message.FunctionArgs);
var settings = _services.GetRequiredService<SqlDriverSetting>();
var results = settings.DatabaseType switch
{
WriteIndented = true,
});*/
// message.StopCompletion = true;
"MySql" => RunQueryInMySql(args.SqlStatements),
"SqlServer" => RunQueryInSqlServer(args.SqlStatements),
_ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.")
};
message.Content = JsonSerializer.Serialize(results);
return true;
}
private IEnumerable<dynamic> RunQueryInMySql(string[] sqlTexts)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString ?? settings.MySqlConnectionString);
return connection.Query(string.Join(";\r\n", sqlTexts));
}
private IEnumerable<dynamic> RunQueryInSqlServer(string[] sqlTexts)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);
var dictionary = new Dictionary<string, object>();
return connection.Query(string.Join("\r\n", sqlTexts));
}
}

75
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Plugin.SqlDriver.Models;
using Fluid.Ast.BinaryExpressions;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Logging;
using MySqlConnector;
@ -7,6 +9,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions;
public class GetTableDefinitionFn : IFunctionCallback
{
public string Name => "get_table_definition";
public string Indication => "Obtain the relevant data structure definitions.";
private readonly IServiceProvider _services;
private readonly ILogger<GetTableDefinitionFn> _logger;
@ -23,11 +26,24 @@ public class GetTableDefinitionFn : IFunctionCallback
var args = JsonSerializer.Deserialize<SqlStatement>(message.FunctionArgs);
var tables = new string[] { args.Table };
var agentService = _services.GetRequiredService<IAgentService>();
var sqlDriver = _services.GetRequiredService<SqlDriverService>();
var settings = _services.GetRequiredService<SqlDriverSetting>();
// Get table DDL from database
var tableDdls = settings.DatabaseType switch
{
"MySql" => GetDdlFromMySql(tables),
"SqlServer" => GetDdlFromSqlServer(tables),
_ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.")
};
message.Content = string.Join("\r\n\r\n", tableDdls);
return true;
}
private List<string> GetDdlFromMySql(string[] tables)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
var tableDdls = new List<string>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
connection.Open();
@ -57,7 +73,60 @@ public class GetTableDefinitionFn : IFunctionCallback
}
connection.Close();
message.Content = string.Join("\r\n\r\n", tableDdls);
return true;
return tableDdls;
}
private List<string> GetDdlFromSqlServer(string[] tables)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
var tableDdls = new List<string>();
using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);
connection.Open();
foreach (var table in tables)
{
try
{
var sql = @$"DECLARE @TableName NVARCHAR(128) = '{table}';
DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' (';
SELECT @SQL = @SQL + '
' + COLUMN_NAME + ' ' +
DATA_TYPE +
CASE
WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%'
THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'
WHEN DATA_TYPE IN ('decimal', 'numeric')
THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'
ELSE ''
END + ' ' +
CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ','
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = @TableName
ORDER BY ORDINAL_POSITION;
-- Remove the last comma and add closing parenthesis
SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');';
SELECT @SQL;";
using var command = new SqlCommand(sql, connection);
using var reader = command.ExecuteReader();
if (reader.Read())
{
var result = reader.GetString(0);
tableDdls.Add(result);
}
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}");
}
}
connection.Close();
return tableDdls;
}
}

36
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Plugin.SqlDriver.Models;
using Microsoft.Data.SqlClient;
using MySqlConnector;
using static Dapper.SqlMapper;
@ -26,13 +27,12 @@ public class SqlSelect : IFunctionCallback
// check if need to instantely
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
var dictionary = new Dictionary<string, object>();
foreach(var p in args.Parameters)
var result = settings.DatabaseType switch
{
dictionary["@" + p.Name] = p.Value;
}
var result = connection.Query(args.Statement, dictionary);
"MySql" => RunQueryInMySql(args),
"SqlServer" => RunQueryInSqlServer(args),
_ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.")
};
if (result == null)
{
@ -46,4 +46,28 @@ public class SqlSelect : IFunctionCallback
return true;
}
private IEnumerable<dynamic> RunQueryInMySql(SqlStatement args)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
var dictionary = new Dictionary<string, object>();
foreach (var p in args.Parameters)
{
dictionary["@" + p.Name] = p.Value;
}
return connection.Query(args.Statement, dictionary);
}
private IEnumerable<dynamic> RunQueryInSqlServer(SqlStatement args)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);
var dictionary = new Dictionary<string, object>();
foreach (var p in args.Parameters)
{
dictionary["@" + p.Name] = p.Value;
}
return connection.Query(args.Statement, dictionary);
}
}

53
src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs Normal file
View file

@ -0,0 +1,53 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Planning;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Agents.Services;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.SqlDriver.Hooks;
public class SqlDriverPlanningHook : IPlanningHook
{
private readonly IServiceProvider _services;
public SqlDriverPlanningHook(IServiceProvider services)
{
_services = services;
}
public async Task<string> GetSummaryAdditionalRequirements(string planner)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(BuiltInAgentId.Planner);
return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{settings.DatabaseType.ToLower()}")?.Content ?? string.Empty;
}
public async Task OnPlanningCompleted(string planner, RoleDialogModel msg)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
if (!settings.ExecuteSqlSelectAutonomous)
{
return;
}
var conv = _services.GetRequiredService<IConversationService>();
var wholeDialogs = conv.GetDialogHistory();
wholeDialogs.Add(RoleDialogModel.From(msg));
wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, "use execute_sql to run query"));
var agent = await _services.GetRequiredService<IAgentService>().LoadAgent("beda4c12-e1ec-4b4b-b328-3df4a6687c4f");
var completion = CompletionProvider.GetChatCompletion(_services,
provider: agent.LlmConfig.Provider,
model: agent.LlmConfig.Model);
var response = await completion.GetChatCompletions(agent, wholeDialogs);
var routing = _services.GetRequiredService<IRoutingService>();
await routing.InvokeFunction(response.FunctionName, response);
msg.CurrentAgentId = agent.Id;
msg.FunctionName = response.FunctionName;
msg.FunctionArgs = response.FunctionArgs;
msg.Content = response.Content;
}
}

9
src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs Normal file
View file

@ -0,0 +1,9 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.SqlDriver.Models;
public class ExecuteQueryArgs
{
[JsonPropertyName("sql_statements")]
public string[] SqlStatements { get; set; } = [];
}

3
src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs
View file

@ -2,8 +2,11 @@ namespace BotSharp.Plugin.SqlHero.Settings;
public class SqlDriverSetting
{
public string DatabaseType { get; set; } = "MySql";
public string MySqlConnectionString { get; set; } = null!;
public string MySqlExecutionConnectionString { get; set; } = null!;
public string SqlServerConnectionString { get; set; } = null!;
public string SqlServerExecutionConnectionString { get; set; } = null!;
public string SqlLiteConnectionString { get; set; } = null!;
public bool ExecuteSqlSelectAutonomous { get; set; } = false;
}

3
src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Planning;
namespace BotSharp.Plugin.SqlDriver;
public class SqlDriverPlugin : IBotSharpPlugin
@ -20,5 +22,6 @@ public class SqlDriverPlugin : IBotSharpPlugin
services.AddScoped<IKnowledgeHook, SqlDriverKnowledgeHook>();
services.AddScoped<IAgentHook, SqlExecutorHook>();
services.AddScoped<IAgentUtilityHook, SqlExecutorUtilityHook>();
services.AddScoped<IPlanningHook, SqlDriverPlanningHook>();
}
}

2
src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json
View file

@ -9,7 +9,7 @@
"isPublic": true,
"profiles": [ "database" ],
"llmConfig": {
"provider": "openai",
"provider": "azure-openai",
"model": "gpt-4o-mini"
},
"routingRules": [

18
src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json Normal file
View file

@ -0,0 +1,18 @@
{
"name": "execute_sql",
"description": "Run the sql statements provided in the last converastion",
"parameters": {
"type": "object",
"properties": {
"sql_statements": {
"type": "array",
"description": "raw sql statements",
"items": {
"type": "string",
"description": "sql statement"
}
}
},
"required": [ "sql_statement" ]
}
}

115
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs
View file

@ -2,23 +2,122 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData)
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
try
{
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var dir = $"{docDir}/{fileId}";
if (ExistDirectory(dir))
{
_cosClient.BucketClient.DeleteDir(dir);
}
var file = $"{dir}/{fileName}";
var res = _cosClient.BucketClient.UploadBytes(file, fileData.ToArray());
return res;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving knowledge file " +
$"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." +
$"\r\n{ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!ExistDirectory(dir)) return false;
if (fileId == null)
{
_cosClient.BucketClient.DeleteDir(dir);
}
else
{
var fileDir = $"{dir}/{fileId}";
if (ExistDirectory(fileDir))
{
_cosClient.BucketClient.DeleteDir(fileDir);
}
}
return true;
}
public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return string.Empty;
}
var docDir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
var fileDir = $"{docDir}/{fileId}";
if (!ExistDirectory(fileDir))
{
return string.Empty;
}
return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{fileDir}/{fileName}"; ;
}
public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
public BinaryData GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return BinaryData.Empty;
}
try
{
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var fileDir = $"{docDir}/{fileId}";
if (!ExistDirectory(fileDir))
{
return BinaryData.Empty;
}
var file = $"{fileDir}/{fileName}";
var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
if (bytes == null)
{
return BinaryData.Empty;
}
return BinaryData.FromBytes(bytes);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when downloading collection file ({collectionName}-{vectorStoreProvider}-{fileId}-{fileName})" +
$"\r\n{ex.Message}\r\n{ex.InnerException}");
return BinaryData.Empty;
}
}
#region Private methods
private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
{
return $"{KNOWLEDGE_FOLDER}/{KNOWLEDGE_DOC_FOLDER}/{vectorStoreProvider.CleanStr()}/{collectionName.CleanStr()}";
}
#endregion
}

2
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs
View file

@ -28,6 +28,8 @@ public partial class TencentCosService : IFileStorageService
private const string USER_AVATAR_FOLDER = "avatar";
private const string SESSION_FOLDER = "sessions";
private const string TEXT_TO_SPEECH_FOLDER = "speeches";
private const string KNOWLEDGE_FOLDER = "knowledgebase";
private const string KNOWLEDGE_DOC_FOLDER = "document";
public TencentCosService(
TencentCosSettings settings,