SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into lida_dev

Browse source
This commit is contained in:
AnonymousDotNet 2024-09-18 17:55:36 +08:00
parent 81dfb1f84a 6dfdda7434
commit bad946b073
43 changed files with 570 additions and 162 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs
View file

@ -63,16 +63,17 @@ public interface IFileStorageService
#endregion
#region Knowledge
bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData);
bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData);
/// <summary>
/// Delete files in a knowledge collection. If fileId is null, remove all files in the collection.
/// Delete files in a knowledge collection, given the vector store provider. If "fileId" is null, delete all files in the collection.
/// </summary>
/// <param name="collectionName"></param>
/// <param name="vectorStoreProvider"></param>
/// <param name="fileId"></param>
/// <returns></returns>
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null);
string GetKnowledgeBaseFileUrl(string collectionName, string fileId);
FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId);
bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null);
string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName);
#endregion
}

2
src/Infrastructure/BotSharp.Abstraction/Files/Models/KnowledgeFileModel.cs
View file

@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Models;
public class KnowledgeFileModel
{
public string FileId { get; set; }
public Guid FileId { get; set; }
public string FileName { get; set; }
public string FileExtension { get; set; }
public string ContentType { get; set; }

4
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -22,9 +22,9 @@ public interface IKnowledgeService
#region Document
Task<UploadKnowledgeResponse> UploadKnowledgeDocuments(string collectionName, IEnumerable<ExternalFileModel> files);
Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId);
Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId);
Task<PagedItems<KnowledgeFileModel>> GetPagedKnowledgeDocuments(string collectionName, KnowledgeFileFilter filter);
Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId);
Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId);
#endregion
#region Common

5
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeDocMetaData.cs
View file

@ -6,7 +6,7 @@ public class KnowledgeDocMetaData
public string Collection { get; set; }
[JsonPropertyName("file_id")]
public string FileId { get; set; }
public Guid FileId { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }
@ -23,6 +23,9 @@ public class KnowledgeDocMetaData
[JsonPropertyName("vector_data_ids")]
public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
[JsonPropertyName("web_url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? WebUrl { get; set; }
[JsonPropertyName("create_date")]
public DateTime CreateDate { get; set; } = DateTime.UtcNow;

2
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFileFilter.cs
View file

@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.Knowledges.Models;
public class KnowledgeFileFilter : Pagination
{
public IEnumerable<string>? FileIds { get; set; }
public IEnumerable<Guid>? FileIds { get; set; }
}

9
src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs Normal file
View file

@ -0,0 +1,9 @@
namespace BotSharp.Abstraction.Planning;
public interface IPlanningHook
{
Task<string> GetSummaryAdditionalRequirements(string planner)
=> Task.FromResult(string.Empty);
Task OnPlanningCompleted(string planner, RoleDialogModel msg)
=> Task.CompletedTask;
}

12
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -113,7 +113,15 @@ public interface IBotSharpRepository
bool DeleteKnowledgeCollectionConfig(string collectionName);
IEnumerable<VectorCollectionConfig> GetKnowledgeCollectionConfigs(VectorCollectionConfigFilter filter);
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData);
/// <summary>
/// Delete file meta data in a knowledge collection, given the vector store provider. If "fileId" is null, delete all in the collection.
/// </summary>
/// <param name="collectionName"></param>
/// <param name="vectorStoreProvider"></param>
/// <param name="fileId"></param>
/// <returns></returns>
bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null);
PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
#endregion
}

46
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
View file

@ -1,23 +1,21 @@
using BotSharp.Abstraction.Knowledges.Models;
using System.IO;
namespace BotSharp.Core.Files.Services;
public partial class LocalFileStorageService
{
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData)
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
try
{
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var dir = Path.Combine(docDir, fileId);
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var dir = Path.Combine(docDir, fileId.ToString());
if (ExistDirectory(dir))
{
Directory.Delete(dir);
@ -40,7 +38,7 @@ public partial class LocalFileStorageService
}
}
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
@ -48,16 +46,16 @@ public partial class LocalFileStorageService
return false;
}
var dir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!ExistDirectory(dir)) return false;
if (string.IsNullOrEmpty(fileId))
if (fileId == null)
{
Directory.Delete(dir, true);
}
else
{
var fileDir = Path.Combine(dir, fileId);
var fileDir = Path.Combine(dir, fileId.ToString());
if (ExistDirectory(fileDir))
{
Directory.Delete(fileDir, true);
@ -67,10 +65,10 @@ public partial class LocalFileStorageService
return true;
}
public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return string.Empty;
}
@ -78,39 +76,31 @@ public partial class LocalFileStorageService
return $"/knowledge/document/{collectionName}/file/{fileId}";
}
public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileId))
|| string.IsNullOrWhiteSpace(fileName))
{
return null;
}
var docDir = BuildKnowledgeCollectionDocumentDir(collectionName, vectorStoreProvider);
var fileDir = Path.Combine(docDir, fileId);
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var fileDir = Path.Combine(docDir, fileId.ToString());
if (!ExistDirectory(fileDir)) return null;
var metaFile = Path.Combine(fileDir, KNOWLEDGE_DOC_META_FILE);
var content = File.ReadAllText(metaFile);
var metaData = JsonSerializer.Deserialize<KnowledgeDocMetaData>(content, _jsonOptions);
var file = Path.Combine(fileDir, metaData.FileName);
var file = Path.Combine(fileDir, fileName);
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read);
stream.Position = 0;
return new FileBinaryDataModel
{
FileName = metaData.FileName,
ContentType = metaData.ContentType,
FileBinaryData = BinaryData.FromStream(stream)
};
return BinaryData.FromStream(stream);
}
#region Private methods
private string BuildKnowledgeCollectionDocumentDir(string collectionName, string vectorStoreProvider)
private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
{
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
return Path.Combine(_baseDir, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr());
}
#endregion
}

1
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs
View file

@ -21,7 +21,6 @@ public partial class LocalFileStorageService : IFileStorageService
private const string TEXT_TO_SPEECH_FOLDER = "speeches";
private const string KNOWLEDGE_FOLDER = "knowledgebase";
private const string KNOWLEDGE_DOC_FOLDER = "document";
private const string KNOWLEDGE_DOC_META_FILE = "meta.json";
private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions
{

5
src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file

@ -234,7 +234,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
throw new NotImplementedException();
#endregion
#region Knowledge
#region KnowledgeBase
public bool AddKnowledgeCollectionConfigs(List<VectorCollectionConfig> configs, bool reset = false) =>
throw new NotImplementedException();
@ -247,6 +247,9 @@ public class BotSharpDbContext : Database, IBotSharpRepository
public bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData) =>
throw new NotImplementedException();
public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null) =>
throw new NotImplementedException();
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter) =>
throw new NotImplementedException();
#endregion

40
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.KnowledgeBase.cs
View file

@ -110,14 +110,13 @@ public partial class FileRepository
{
if (metaData == null
|| string.IsNullOrWhiteSpace(metaData.Collection)
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
|| string.IsNullOrWhiteSpace(metaData.FileId))
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeDocumentDir(metaData.Collection.CleanStr(), metaData.VectorStoreProvider.CleanStr());
var docDir = Path.Combine(dir, metaData.FileId);
var dir = BuildKnowledgeCollectionFileDir(metaData.Collection, metaData.VectorStoreProvider);
var docDir = Path.Combine(dir, metaData.FileId.ToString());
if (!Directory.Exists(docDir))
{
Directory.CreateDirectory(docDir);
@ -129,6 +128,33 @@ public partial class FileRepository
return true;
}
public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!Directory.Exists(dir)) return false;
if (fileId == null)
{
Directory.Delete(dir, true);
}
else
{
var fileDir = Path.Combine(dir, fileId.ToString());
if (Directory.Exists(fileDir))
{
Directory.Delete(fileDir, true);
}
}
return true;
}
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
{
if (string.IsNullOrWhiteSpace(collectionName)
@ -137,7 +163,7 @@ public partial class FileRepository
return new PagedItems<KnowledgeDocMetaData>();
}
var dir = BuildKnowledgeDocumentDir(collectionName, vectorStoreProvider);
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!Directory.Exists(dir))
{
return new PagedItems<KnowledgeDocMetaData>();
@ -181,9 +207,9 @@ public partial class FileRepository
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, VECTOR_FOLDER);
}
private string BuildKnowledgeDocumentDir(string collectionName, string vectorStoreProvider)
private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
{
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider, collectionName);
return Path.Combine(_dbSettings.FileRepository, KNOWLEDGE_FOLDER, KNOWLEDGE_DOC_FOLDER, vectorStoreProvider.CleanStr(), collectionName.CleanStr());
}
#endregion
}

4
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -137,7 +137,7 @@ public class KnowledgeBaseController : ControllerBase
}
[HttpDelete("/knowledge/document/{collection}/delete/{fileId}")]
public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
public async Task<bool> DeleteKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId)
{
var response = await _knowledgeService.DeleteKnowledgeDocument(collection, fileId);
return response;
@ -160,7 +160,7 @@ public class KnowledgeBaseController : ControllerBase
}
[HttpGet("/knowledge/document/{collection}/file/{fileId}")]
public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] string fileId)
public async Task<IActionResult> GetKnowledgeDocument([FromRoute] string collection, [FromRoute] Guid fileId)
{
var file = await _knowledgeService.GetKnowledgeDocumentBinaryData(collection, fileId);
var stream = file.FileBinaryData.ToStream();

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeFileViewModel.cs
View file

@ -5,7 +5,7 @@ namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class KnowledgeFileViewModel
{
[JsonPropertyName("file_id")]
public string FileId { get; set; }
public Guid FileId { get; set; }
[JsonPropertyName("file_name")]
public string FileName { get; set; }

6
src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs
View file

@ -44,11 +44,7 @@ public class WelcomeHook : ConversationHookBase
foreach (var message in messages)
{
var richContent = new RichContent<IRichMessage>(message)
{
Editor = message.RichType == RichTypeEnum.QuickReply ? EditorTypeEnum.None : EditorTypeEnum.Text,
};
var richContent = new RichContent<IRichMessage>(message);
var json = JsonSerializer.Serialize(new ChatResponseModel()
{
ConversationId = conversation.Id,

67
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -42,7 +42,7 @@ public partial class KnowledgeService
var contents = await GetFileContent(contentType, bytes);
// Save document
var fileId = Guid.NewGuid().ToString();
var fileId = Guid.NewGuid();
var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes);
if (!saved)
{
@ -89,9 +89,9 @@ public partial class KnowledgeService
}
public async Task<bool> DeleteKnowledgeDocument(string collectionName, string fileId)
public async Task<bool> DeleteKnowledgeDocument(string collectionName, Guid fileId)
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(fileId))
if (string.IsNullOrWhiteSpace(collectionName))
{
return false;
}
@ -104,21 +104,23 @@ public partial class KnowledgeService
var vectorStoreProvider = _settings.VectorDb.Provider;
// Get doc meta data
var pagedData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
{
FileIds = new[] { fileId }
FileIds = [ fileId ],
Size = 1
});
// Delete doc
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
var found = pagedData?.Items?.FirstOrDefault();
fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider, fileId);
var found = pageData?.Items?.FirstOrDefault();
if (found != null && !found.VectorDataIds.IsNullOrEmpty())
{
var guids = found.VectorDataIds.Where(x => Guid.TryParse(x, out _)).Select(x => Guid.Parse(x)).ToList();
await vectorDb.DeleteCollectionData(collectionName, guids);
}
db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider, fileId);
return true;
}
catch (Exception ex)
@ -148,13 +150,19 @@ public partial class KnowledgeService
Size = filter.Size
});
var files = pagedData.Items?.Select(x => new KnowledgeFileModel
var files = pagedData.Items?.Select(x =>
{
FileId = x.FileId,
FileName = x.FileName,
FileExtension = Path.GetExtension(x.FileName),
ContentType = x.ContentType,
FileUrl = fileStorage.GetKnowledgeBaseFileUrl(collectionName, x.FileId)
var fileUrl = x.ContentType == MediaTypeNames.Text.Html ?
x.WebUrl : fileStorage.GetKnowledgeBaseFileUrl(collectionName, vectorStoreProvider, x.FileId, x.FileName);
return new KnowledgeFileModel
{
FileId = x.FileId,
FileName = x.FileName,
FileExtension = Path.GetExtension(x.FileName),
ContentType = x.ContentType,
FileUrl = fileUrl
};
})?.ToList() ?? new List<KnowledgeFileModel>();
return new PagedItems<KnowledgeFileModel>
@ -164,14 +172,29 @@ public partial class KnowledgeService
};
}
public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, string fileId)
public async Task<FileBinaryDataModel?> GetKnowledgeDocumentBinaryData(string collectionName, Guid fileId)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var fileStorage = _services.GetRequiredService<IFileStorageService>();
var vectorStoreProvider = _settings.VectorDb.Provider;
// Get doc binary data
var file = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
return file;
var pageData = db.GetKnowledgeBaseFileMeta(collectionName, vectorStoreProvider, new KnowledgeFileFilter
{
FileIds = [ fileId ],
Size = 1
});
var metaData = pageData?.Items?.FirstOrDefault();
if (metaData == null) return null;
var binaryData = fileStorage.GetKnowledgeBaseFileBinaryData(collectionName, vectorStoreProvider, fileId, metaData.FileName);
return new FileBinaryDataModel
{
FileName = metaData.FileName,
ContentType = metaData.ContentType,
FileBinaryData = binaryData
};
}
@ -246,16 +269,16 @@ public partial class KnowledgeService
#endregion
private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes)
private bool SaveDocument(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, byte[] bytes)
{
var fileStoreage = _services.GetRequiredService<IFileStorageService>();
var data = BinaryData.FromBytes(bytes);
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, fileName, data);
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName, vectorStoreProvider, fileId, fileName, data);
return saved;
}
private async Task<IEnumerable<string>> SaveToVectorDb(
string collectionName, string fileId, string fileName, IEnumerable<string> contents,
string collectionName, Guid fileId, string fileName, IEnumerable<string> contents,
string fileSource = KnowledgeDocSource.Api, string vectorDataSource = VectorDataSource.File)
{
if (contents.IsNullOrEmpty())
@ -275,7 +298,7 @@ public partial class KnowledgeService
var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
{
{ KnowledgePayloadName.DataSource, vectorDataSource },
{ KnowledgePayloadName.FileId, fileId },
{ KnowledgePayloadName.FileId, fileId.ToString() },
{ KnowledgePayloadName.FileName, fileName },
{ KnowledgePayloadName.FileSource, fileSource },
{ "textNumber", $"{i + 1}" }

3
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs
View file

@ -92,7 +92,8 @@ public partial class KnowledgeService
var vectorStoreProvider = _settings.VectorDb.Provider;
db.DeleteKnowledgeCollectionConfig(collectionName);
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr());
fileStorage.DeleteKnowledgeFile(collectionName, vectorStoreProvider);
db.DeleteKnolwedgeBaseFileMeta(collectionName, vectorStoreProvider);
}
return deleted;

5
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileDocument.cs → src/Plugins/BotSharp.Plugin.MongoStorage/Collections/KnowledgeCollectionFileMetaDocument.cs
View file

@ -1,14 +1,15 @@
namespace BotSharp.Plugin.MongoStorage.Collections;
public class KnowledgeCollectionFileDocument : MongoBase
public class KnowledgeCollectionFileMetaDocument : MongoBase
{
public string Collection { get; set; }
public string FileId { get; set; }
public Guid FileId { get; set; }
public string FileName { get; set; }
public string FileSource { get; set; }
public string ContentType { get; set; }
public string VectorStoreProvider { get; set; }
public IEnumerable<string> VectorDataIds { get; set; } = new List<string>();
public string? WebUrl { get; set; }
public DateTime CreateDate { get; set; }
public string CreateUserId { get; set; }
}

4
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -157,6 +157,6 @@ public class MongoDbContext
public IMongoCollection<KnowledgeCollectionConfigDocument> KnowledgeCollectionConfigs
=> Database.GetCollection<KnowledgeCollectionConfigDocument>($"{_collectionPrefix}_KnowledgeCollectionConfigs");
public IMongoCollection<KnowledgeCollectionFileDocument> KnowledgeCollectionFiles
=> Database.GetCollection<KnowledgeCollectionFileDocument>($"{_collectionPrefix}_KnowledgeCollectionFiles");
public IMongoCollection<KnowledgeCollectionFileMetaDocument> KnowledgeCollectionFileMeta
=> Database.GetCollection<KnowledgeCollectionFileMetaDocument>($"{_collectionPrefix}_KnowledgeCollectionFileMeta");
}

44
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.KnowledgeBase.cs
View file

@ -120,13 +120,12 @@ public partial class MongoRepository
{
if (metaData == null
|| string.IsNullOrWhiteSpace(metaData.Collection)
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider)
|| string.IsNullOrWhiteSpace(metaData.FileId))
|| string.IsNullOrWhiteSpace(metaData.VectorStoreProvider))
{
return false;
}
var doc = new KnowledgeCollectionFileDocument
var doc = new KnowledgeCollectionFileMetaDocument
{
Id = Guid.NewGuid().ToString(),
Collection = metaData.Collection,
@ -136,14 +135,39 @@ public partial class MongoRepository
ContentType = metaData.ContentType,
VectorStoreProvider = metaData.VectorStoreProvider,
VectorDataIds = metaData.VectorDataIds,
WebUrl = metaData.WebUrl,
CreateDate = metaData.CreateDate,
CreateUserId = metaData.CreateUserId
};
_dc.KnowledgeCollectionFiles.InsertOne(doc);
_dc.KnowledgeCollectionFileMeta.InsertOne(doc);
return true;
}
public bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var builder = Builders<KnowledgeCollectionFileMetaDocument>.Filter;
var filters = new List<FilterDefinition<KnowledgeCollectionFileMetaDocument>>()
{
builder.Eq(x => x.Collection, collectionName),
builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider)
};
if (fileId != null)
{
filters.Add(builder.Eq(x => x.FileId, fileId));
}
var res = _dc.KnowledgeCollectionFileMeta.DeleteMany(builder.And(filters));
return res.DeletedCount > 0;
}
public PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
{
if (string.IsNullOrWhiteSpace(collectionName)
@ -152,9 +176,8 @@ public partial class MongoRepository
return new PagedItems<KnowledgeDocMetaData>();
}
var builder = Builders<KnowledgeCollectionFileDocument>.Filter;
var docFilters = new List<FilterDefinition<KnowledgeCollectionFileDocument>>()
var builder = Builders<KnowledgeCollectionFileMetaDocument>.Filter;
var docFilters = new List<FilterDefinition<KnowledgeCollectionFileMetaDocument>>()
{
builder.Eq(x => x.Collection, collectionName),
builder.Eq(x => x.VectorStoreProvider, vectorStoreProvider)
@ -167,9 +190,9 @@ public partial class MongoRepository
}
var filterDef = builder.And(docFilters);
var sortDef = Builders<KnowledgeCollectionFileDocument>.Sort.Descending(x => x.CreateDate);
var docs = _dc.KnowledgeCollectionFiles.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.KnowledgeCollectionFiles.CountDocuments(filterDef);
var sortDef = Builders<KnowledgeCollectionFileMetaDocument>.Sort.Descending(x => x.CreateDate);
var docs = _dc.KnowledgeCollectionFileMeta.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.KnowledgeCollectionFileMeta.CountDocuments(filterDef);
var files = docs?.Select(x => new KnowledgeDocMetaData
{
@ -180,6 +203,7 @@ public partial class MongoRepository
ContentType = x.ContentType,
VectorStoreProvider = x.VectorStoreProvider,
VectorDataIds = x.VectorDataIds,
WebUrl = x.WebUrl,
CreateDate = x.CreateDate,
CreateUserId = x.CreateUserId
})?.ToList() ?? new List<KnowledgeDocMetaData>();

12
src/Plugins/BotSharp.Plugin.Planner/BotSharp.Plugin.Planner.csproj
View file

@ -16,8 +16,10 @@
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\functions\plan_secondary_stage.json" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\functions\plan_summary.json" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\instructions\instruction.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.1st.next.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.MySql.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.SqlServer.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.2nd.plan.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.next.liquid" />
<None Remove="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.summarize.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\planner_prompt.two_stage.1st.plan.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\plan_primary_stage.fn.liquid" />
@ -41,7 +43,13 @@
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\instructions\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.1st.next.liquid">
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.mysql.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\database.summarize.sqlserver.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.next.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\282a7128-69a1-44b0-878c-a9159b88f3b9\templates\two_stage.2nd.plan.liquid">

2
src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs
View file

@ -5,7 +5,7 @@ namespace BotSharp.Plugin.Planner.Functions;
public class PrimaryStagePlanFn : IFunctionCallback
{
public string Name => "plan_primary_stage";
public string Indication => "Currently analyzing and breaking down user requirements.";
private readonly IServiceProvider _services;
private readonly ILogger<PrimaryStagePlanFn> _logger;

2
src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs
View file

@ -5,7 +5,7 @@ namespace BotSharp.Plugin.Planner.Functions;
public class SecondaryStagePlanFn : IFunctionCallback
{
public string Name => "plan_secondary_stage";
public string Indication => "Further analyzing and breaking down user sub-needs.";
private readonly IServiceProvider _services;
private readonly ILogger<SecondaryStagePlanFn> _logger;

24
src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Planning;
using BotSharp.Plugin.Planner.TwoStaging;
using BotSharp.Plugin.Planner.TwoStaging.Models;
namespace BotSharp.Plugin.Planner.Functions;
@ -5,7 +7,7 @@ namespace BotSharp.Plugin.Planner.Functions;
public class SummaryPlanFn : IFunctionCallback
{
public string Name => "plan_summary";
public string Indication => "Organizing and summarizing the final output results.";
private readonly IServiceProvider _services;
private readonly ILogger<SummaryPlanFn> _logger;
@ -62,7 +64,9 @@ public class SummaryPlanFn : IFunctionCallback
var summary = await GetAiResponse(plannerAgent);
message.Content = summary.Content;
message.StopCompletion = true;
await HookEmitter.Emit<IPlanningHook>(_services, x =>
x.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message));
return true;
}
@ -74,18 +78,20 @@ public class SummaryPlanFn : IFunctionCallback
var agent = await agentService.GetAgent(BuiltInAgentId.Planner);
var template = agent.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty;
var responseFormat = JsonSerializer.Serialize(new FirstStagePlan
var additionalRequirements = new List<string>();
await HookEmitter.Emit<IPlanningHook>(_services, async x =>
{
Parameters = [JsonDocument.Parse("{}")],
Results = [""]
var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner));
additionalRequirements.Add(requirement);
});
return render.Render(template, new Dictionary<string, object>
{
{ "table_structure", ddlStatement },
{ "task_description", taskDescription },
{ "summary_requirements", string.Join("\r\n",additionalRequirements) },
{ "relevant_knowledges", relevantKnowledge },
{ "response_format", responseFormat }
{ "table_structure", ddlStatement },
});
}
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
@ -94,8 +100,8 @@ public class SummaryPlanFn : IFunctionCallback
var wholeDialogs = conv.GetDialogHistory();
// Append text
wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;";
wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs";
wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id).\nFor example, you should use SET @id = select max(id) from table;";
wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs.";
var completion = CompletionProvider.GetChatCompletion(_services,
provider: plannerAgent.LlmConfig.Provider,

6
src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs
View file

@ -3,7 +3,7 @@ namespace BotSharp.Plugin.Planner.TwoStaging.Models;
public class SecondStagePlan
{
[JsonPropertyName("related_tables")]
public string[] Tables { get; set; } = new string[0];
public string[] Tables { get; set; } = [];
[JsonPropertyName("description")]
public string Description { get; set; } = "";
@ -12,8 +12,8 @@ public class SecondStagePlan
public string Tool { get; set; } = "";
[JsonPropertyName("input_args")]
public JsonDocument[] Parameters { get; set; } = new JsonDocument[0];
public JsonDocument[] Parameters { get; set; } = [];
[JsonPropertyName("output_results")]
public string[] Results { get; set; } = new string[0];
public string[] Results { get; set; } = [];
}

4
src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs
View file

@ -18,8 +18,8 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner
public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId, List<RoleDialogModel> dialogs)
{
var nextStepPrompt = await GetNextStepPrompt(router);
var inst = new FunctionCallFromLlm();
var nextStepPrompt = await GetNextStepPrompt(router);
// chat completion
var completion = CompletionProvider.GetChatCompletion(_services,
@ -125,7 +125,7 @@ public partial class TwoStageTaskPlanner : IRoutingPlaner
{
var agentService = _services.GetRequiredService<IAgentService>();
var planner = await agentService.LoadAgent(BuiltInAgentId.Planner);
var template = planner.Templates.First(x => x.Name == "two_stage.1st.next").Content;
var template = planner.Templates.First(x => x.Name == "two_stage.next").Content;
var states = _services.GetRequiredService<IConversationStateService>();
var render = _services.GetRequiredService<ITemplateRender>();
return render.Render(template, new Dictionary<string, object>

4
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json
View file

@ -11,8 +11,8 @@
"profiles": [ "planning" ],
"utilities": [ "two-stage-planner" ],
"llmConfig": {
"provider": "anthropic",
"model": "claude-3-5-sonnet-20240620",
"provider": "azure-openai",
"model": "gpt-4o",
"max_recursion_depth": 10
}
}

4
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid
View file

@ -3,6 +3,10 @@ Use the TwoStagePlanner approach to plan the overall implementation steps, follo
2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage.
3. You must call plan_summary as the last planning step to summarize the final query.
*** IMPORTANT ***
Don't run the planning process repeatedly if you have already got the result of user's request.
{% if global_knowledges != empty -%}
=====
Global Knowledge:

18
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.mysql.liquid Normal file
View file

@ -0,0 +1,18 @@
Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters.
The parameters can be extracted from the original task.
If not, generate the query step by step based on the planning.
The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information.
Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0.
Don't use the sql statement that specify target table for update in FROM clause.
For example, you CAN'T write query as below:
INSERT INTO data_Service (Id, Name)
VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC');
If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.
For example, you should use SET @id = select max(id) from table;
* the alias of the table name in the sql query should be identical.
*** the generated sql query MUST be basedd on the provided table structure. ***

12
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/database.summarize.sqlserver.liquid Normal file
View file

@ -0,0 +1,12 @@
Try if you can generate a SQL Server single query to fulfill the needs. The step should contains all needed parameters.
The parameters can be extracted from the original task.
If not, generate the query step by step based on the planning.
The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information.
Note: Output should be only the sql query with sql comments that can be directly run in SQL Server.
*** the alias of the table name in the sql query should be identical. ***
*** The generated sql query MUST be basedd on the provided table structure. ***
*** All queries return a maximum of 10 records. ***
*** Only select user friendly columns. ***

0
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.next.liquid → src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.next.liquid
View file

26
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid
View file

@ -1,25 +1,7 @@
You are a planning summarizer and sql generator. You will convert the requirement into the excutable MySQL query statement based on the task description and related table structure and relationship.
You are a planning summarizer. You will generate the final output in JSON format based on the task description, knowledge and related table structure and relationship.
Try if you can generate a single query to fulfill the needs. The step should contains all needed parameters.
The parameters can be extracted from the original task.
If not, generate the query step by step based on the planning.
The query must exactly based on the provided table structure. And carefully review the foreign keys to make sure you include all the accurate information.
Note: Output should be only the sql query with sql comments that can be directly run in mysql database with version 8.0.
Don't use the sql statement that specify target table for update in FROM clause.
For example, you CAN'T write query as below:
INSERT INTO data_Service (Id, Name)
VALUES ((SELECT MAX(Id) + 1 FROM data_Service), 'HVAC');
If the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.
For example, you should use SET @id = select max(id) from table;
Additional Requirements:
* the alias of the table name in the sql query should be identical.
*** the generated sql query MUST be basedd on the provided table structure. ***
Requirements:
{{ summary_requirements }}
=====
Task description:
@ -31,4 +13,4 @@ Relevant Knowledges:
=====
Table Structure:
{{ table_structure }}
{{ table_structure }}

10
src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj
View file

@ -10,12 +10,19 @@
<OutputPath>$(SolutionDir)packages</OutputPath>
</PropertyGroup>
<ItemGroup>
<Compile Remove="packages\**" />
<EmbeddedResource Remove="packages\**" />
<None Remove="packages\**" />
</ItemGroup>
<ItemGroup>
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\get_table_definition.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\sql_select.json" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\get_table_definition.fn.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\sql_executor.fn.liquid" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\agent.json" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\execute_sql.json" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\lookup_dictionary.json" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_insert.json" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\sql_select.json" />
@ -33,6 +40,9 @@
<Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\agent.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\functions\execute_sql.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\instructions\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>

41
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs
View file

@ -1,25 +1,50 @@
using BotSharp.Plugin.SqlDriver.Models;
using Dapper;
using Microsoft.Data.SqlClient;
using MySqlConnector;
namespace BotSharp.Plugin.SqlDriver.Functions;
public class ExecuteQueryFn : IFunctionCallback
{
public string Name => "execute_sql";
public string Indication => "Performing data retrieval operation.";
private readonly SqlDriverSetting _setting;
private readonly IServiceProvider _services;
public ExecuteQueryFn(SqlDriverSetting setting)
public ExecuteQueryFn(IServiceProvider services, SqlDriverSetting setting)
{
_services = services;
_setting = setting;
}
public async Task<bool> Execute(RoleDialogModel message)
{
message.Content = "Executed";
/*using var connection = new MySqlConnection(_setting.MySqlConnectionString);
message.Content = JsonSerializer.Serialize(connection.Query(args.SqlStatement), new JsonSerializerOptions
var args = JsonSerializer.Deserialize<ExecuteQueryArgs>(message.FunctionArgs);
var settings = _services.GetRequiredService<SqlDriverSetting>();
var results = settings.DatabaseType switch
{
WriteIndented = true,
});*/
// message.StopCompletion = true;
"MySql" => RunQueryInMySql(args.SqlStatements),
"SqlServer" => RunQueryInSqlServer(args.SqlStatements),
_ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.")
};
message.Content = JsonSerializer.Serialize(results);
return true;
}
private IEnumerable<dynamic> RunQueryInMySql(string[] sqlTexts)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
return connection.Query(string.Join(";\r\n", sqlTexts));
}
private IEnumerable<dynamic> RunQueryInSqlServer(string[] sqlTexts)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);
var dictionary = new Dictionary<string, object>();
return connection.Query(string.Join("\r\n", sqlTexts));
}
}

75
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs
View file

@ -1,4 +1,6 @@
using BotSharp.Plugin.SqlDriver.Models;
using Fluid.Ast.BinaryExpressions;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Logging;
using MySqlConnector;
@ -7,6 +9,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions;
public class GetTableDefinitionFn : IFunctionCallback
{
public string Name => "get_table_definition";
public string Indication => "Obtain the relevant data structure definitions.";
private readonly IServiceProvider _services;
private readonly ILogger<GetTableDefinitionFn> _logger;
@ -23,11 +26,24 @@ public class GetTableDefinitionFn : IFunctionCallback
var args = JsonSerializer.Deserialize<SqlStatement>(message.FunctionArgs);
var tables = new string[] { args.Table };
var agentService = _services.GetRequiredService<IAgentService>();
var sqlDriver = _services.GetRequiredService<SqlDriverService>();
var settings = _services.GetRequiredService<SqlDriverSetting>();
// Get table DDL from database
var tableDdls = settings.DatabaseType switch
{
"MySql" => GetDdlFromMySql(tables),
"SqlServer" => GetDdlFromSqlServer(tables),
_ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.")
};
message.Content = string.Join("\r\n\r\n", tableDdls);
return true;
}
private List<string> GetDdlFromMySql(string[] tables)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
var tableDdls = new List<string>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
connection.Open();
@ -57,7 +73,60 @@ public class GetTableDefinitionFn : IFunctionCallback
}
connection.Close();
message.Content = string.Join("\r\n\r\n", tableDdls);
return true;
return tableDdls;
}
private List<string> GetDdlFromSqlServer(string[] tables)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
var tableDdls = new List<string>();
using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);
connection.Open();
foreach (var table in tables)
{
try
{
var sql = @$"DECLARE @TableName NVARCHAR(128) = '{table}';
DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' (';
SELECT @SQL = @SQL + '
' + COLUMN_NAME + ' ' +
DATA_TYPE +
CASE
WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%'
THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'
WHEN DATA_TYPE IN ('decimal', 'numeric')
THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'
ELSE ''
END + ' ' +
CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ','
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = @TableName
ORDER BY ORDINAL_POSITION;
-- Remove the last comma and add closing parenthesis
SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');';
SELECT @SQL;";
using var command = new SqlCommand(sql, connection);
using var reader = command.ExecuteReader();
if (reader.Read())
{
var result = reader.GetString(0);
tableDdls.Add(result);
}
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting ddl statement of table {table}. {ex.Message}\r\n{ex.InnerException}");
}
}
connection.Close();
return tableDdls;
}
}

36
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/SqlSelect.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Plugin.SqlDriver.Models;
using Microsoft.Data.SqlClient;
using MySqlConnector;
using static Dapper.SqlMapper;
@ -26,13 +27,12 @@ public class SqlSelect : IFunctionCallback
// check if need to instantely
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
var dictionary = new Dictionary<string, object>();
foreach(var p in args.Parameters)
var result = settings.DatabaseType switch
{
dictionary["@" + p.Name] = p.Value;
}
var result = connection.Query(args.Statement, dictionary);
"MySql" => RunQueryInMySql(args),
"SqlServer" => RunQueryInSqlServer(args),
_ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.")
};
if (result == null)
{
@ -46,4 +46,28 @@ public class SqlSelect : IFunctionCallback
return true;
}
private IEnumerable<dynamic> RunQueryInMySql(SqlStatement args)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
var dictionary = new Dictionary<string, object>();
foreach (var p in args.Parameters)
{
dictionary["@" + p.Name] = p.Value;
}
return connection.Query(args.Statement, dictionary);
}
private IEnumerable<dynamic> RunQueryInSqlServer(SqlStatement args)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);
var dictionary = new Dictionary<string, object>();
foreach (var p in args.Parameters)
{
dictionary["@" + p.Name] = p.Value;
}
return connection.Query(args.Statement, dictionary);
}
}

53
src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs Normal file
View file

@ -0,0 +1,53 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Planning;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Agents.Services;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.SqlDriver.Hooks;
public class SqlDriverPlanningHook : IPlanningHook
{
private readonly IServiceProvider _services;
public SqlDriverPlanningHook(IServiceProvider services)
{
_services = services;
}
public async Task<string> GetSummaryAdditionalRequirements(string planner)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.GetAgent(BuiltInAgentId.Planner);
return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{settings.DatabaseType.ToLower()}")?.Content ?? string.Empty;
}
public async Task OnPlanningCompleted(string planner, RoleDialogModel msg)
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
if (!settings.ExecuteSqlSelectAutonomous)
{
return;
}
var conv = _services.GetRequiredService<IConversationService>();
var wholeDialogs = conv.GetDialogHistory();
wholeDialogs.Add(RoleDialogModel.From(msg));
wholeDialogs.Add(RoleDialogModel.From(msg, AgentRole.User, "use execute_sql to run query"));
var agent = await _services.GetRequiredService<IAgentService>().LoadAgent("beda4c12-e1ec-4b4b-b328-3df4a6687c4f");
var completion = CompletionProvider.GetChatCompletion(_services,
provider: agent.LlmConfig.Provider,
model: agent.LlmConfig.Model);
var response = await completion.GetChatCompletions(agent, wholeDialogs);
var routing = _services.GetRequiredService<IRoutingService>();
await routing.InvokeFunction(response.FunctionName, response);
msg.CurrentAgentId = agent.Id;
msg.FunctionName = response.FunctionName;
msg.FunctionArgs = response.FunctionArgs;
msg.Content = response.Content;
}
}

9
src/Plugins/BotSharp.Plugin.SqlDriver/Models/ExecuteQueryArgs.cs Normal file
View file

@ -0,0 +1,9 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.SqlDriver.Models;
public class ExecuteQueryArgs
{
[JsonPropertyName("sql_statements")]
public string[] SqlStatements { get; set; } = [];
}

3
src/Plugins/BotSharp.Plugin.SqlDriver/Settings/SqlDriverSetting.cs
View file

@ -2,8 +2,11 @@ namespace BotSharp.Plugin.SqlHero.Settings;
public class SqlDriverSetting
{
public string DatabaseType { get; set; } = "MySql";
public string MySqlConnectionString { get; set; } = null!;
public string MySqlExecutionConnectionString { get; set; } = null!;
public string SqlServerConnectionString { get; set; } = null!;
public string SqlServerExecutionConnectionString { get; set; } = null!;
public string SqlLiteConnectionString { get; set; } = null!;
public bool ExecuteSqlSelectAutonomous { get; set; } = false;
}

3
src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs
View file

@ -1,3 +1,5 @@
using BotSharp.Abstraction.Planning;
namespace BotSharp.Plugin.SqlDriver;
public class SqlDriverPlugin : IBotSharpPlugin
@ -20,5 +22,6 @@ public class SqlDriverPlugin : IBotSharpPlugin
services.AddScoped<IKnowledgeHook, SqlDriverKnowledgeHook>();
services.AddScoped<IAgentHook, SqlExecutorHook>();
services.AddScoped<IAgentUtilityHook, SqlExecutorUtilityHook>();
services.AddScoped<IPlanningHook, SqlDriverPlanningHook>();
}
}

2
src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json
View file

@ -9,7 +9,7 @@
"isPublic": true,
"profiles": [ "database" ],
"llmConfig": {
"provider": "openai",
"provider": "azure-openai",
"model": "gpt-4o-mini"
},
"routingRules": [

18
src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/functions/execute_sql.json Normal file
View file

@ -0,0 +1,18 @@
{
"name": "execute_sql",
"description": "Run the sql statements provided in the last converastion",
"parameters": {
"type": "object",
"properties": {
"sql_statements": {
"type": "array",
"description": "raw sql statements",
"items": {
"type": "string",
"description": "sql statement"
}
}
},
"required": [ "sql_statement" ]
}
}

94
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.KnowledgeBase.cs
View file

@ -2,23 +2,101 @@ namespace BotSharp.Plugin.TencentCos.Services;
public partial class TencentCosService
{
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, string fileId, string fileName, BinaryData fileData)
public bool SaveKnowledgeBaseFile(string collectionName, string vectorStoreProvider, Guid fileId, string fileName, BinaryData fileData)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
try
{
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var dir = $"{docDir}/{fileId}";
if (ExistDirectory(dir))
{
_cosClient.BucketClient.DeleteDir(dir);
}
var file = $"{dir}/{fileName}";
var res = _cosClient.BucketClient.UploadBytes(file, fileData.ToArray());
return res;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving knowledge file " +
$"(Vector store provider: {vectorStoreProvider}, Collection: {collectionName}, File name: {fileName})." +
$"\r\n{ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, string? fileId = null)
public bool DeleteKnowledgeFile(string collectionName, string vectorStoreProvider, Guid? fileId = null)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider))
{
return false;
}
var dir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
if (!ExistDirectory(dir)) return false;
if (fileId == null)
{
_cosClient.BucketClient.DeleteDir(dir);
}
else
{
var fileDir = $"{dir}/{fileId}";
if (ExistDirectory(fileDir))
{
_cosClient.BucketClient.DeleteDir(fileDir);
}
}
return true;
}
public string GetKnowledgeBaseFileUrl(string collectionName, string fileId)
public string GetKnowledgeBaseFileUrl(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return string.Empty;
}
var dir = BuildKnowledgeCollectionFileDir(vectorStoreProvider, collectionName);
return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{dir}/{fileId}/{fileName}"; ;
}
public FileBinaryDataModel? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, string fileId)
public BinaryData? GetKnowledgeBaseFileBinaryData(string collectionName, string vectorStoreProvider, Guid fileId, string fileName)
{
throw new NotImplementedException();
if (string.IsNullOrWhiteSpace(collectionName)
|| string.IsNullOrWhiteSpace(vectorStoreProvider)
|| string.IsNullOrWhiteSpace(fileName))
{
return null;
}
var docDir = BuildKnowledgeCollectionFileDir(collectionName, vectorStoreProvider);
var fileDir = $"{docDir}/{fileId}";
if (!ExistDirectory(fileDir)) return null;
var file = $"{fileDir}/{fileName}";
var bytes = _cosClient.BucketClient.DownloadFileBytes(file);
if (bytes == null) return null;
return BinaryData.FromBytes(bytes);
}
#region Private methods
private string BuildKnowledgeCollectionFileDir(string collectionName, string vectorStoreProvider)
{
return $"{KNOWLEDGE_FOLDER}/{KNOWLEDGE_DOC_FOLDER}/{vectorStoreProvider.CleanStr()}/{collectionName.CleanStr()}";
}
#endregion
}

2
src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs
View file

@ -28,6 +28,8 @@ public partial class TencentCosService : IFileStorageService
private const string USER_AVATAR_FOLDER = "avatar";
private const string SESSION_FOLDER = "sessions";
private const string TEXT_TO_SPEECH_FOLDER = "speeches";
private const string KNOWLEDGE_FOLDER = "knowledgebase";
private const string KNOWLEDGE_DOC_FOLDER = "document";
public TencentCosService(
TencentCosSettings settings,