SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

clean code

Browse source
This commit is contained in:
Jicheng Lu 2024-09-09 10:00:46 -05:00
parent 75bba73ccb
commit 9a7f2821a0
12 changed files with 224 additions and 335 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
View file

@ -19,6 +19,8 @@ public interface IKnowledgeService
#region Graph
Task<GraphSearchResult> SearchGraphKnowledge(string query, GraphSearchOptions options);
Task<KnowledgeSearchResult> SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions);
#endregion
#region Document
#endregion
}

10
src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeSearchResult.cs
View file

@ -1,10 +0,0 @@
using BotSharp.Abstraction.Graph.Models;
using BotSharp.Abstraction.VectorStorage.Models;
namespace BotSharp.Abstraction.Knowledges.Models;
public class KnowledgeSearchResult
{
public IEnumerable<VectorSearchResult> VectorResult { get; set; }
public GraphSearchResult GraphResult { get; set; }
}

26
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs
View file

@ -143,29 +143,7 @@ public class KnowledgeBaseController : ControllerBase
#endregion
#region Knowledge
[HttpPost("/knowledge/search")]
public async Task<KnowledgeSearchViewModel> SearchKnowledge([FromBody] SearchKnowledgeRequest request)
{
var vectorOptions = new VectorSearchOptions
{
Fields = request.VectorParams.Fields,
Limit = request.VectorParams.Limit ?? 5,
Confidence = request.VectorParams.Confidence ?? 0.5f,
WithVector = request.VectorParams.WithVector
};
var graphOptions = new GraphSearchOptions
{
Method = request.GraphParams.Method
};
var result = await _knowledgeService.SearchKnowledge(request.Text, request.VectorParams.Collection, vectorOptions, graphOptions);
return new KnowledgeSearchViewModel
{
VectorResult = result?.VectorResult?.Select(x => VectorKnowledgeViewModel.From(x)),
GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null
};
}
#region Document
#endregion
}

12
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/KnowledgeSearchViewModel.cs
View file

@ -1,12 +0,0 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class KnowledgeSearchViewModel
{
[JsonPropertyName("vector_result")]
public IEnumerable<VectorKnowledgeViewModel>? VectorResult { get; set; }
[JsonPropertyName("graph_result")]
public GraphKnowledgeViewModel? GraphResult { get; set; }
}

43
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/SearchKnowledgeRequest.cs
View file

@ -1,43 +0,0 @@
using System.Text.Json.Serialization;
namespace BotSharp.OpenAPI.ViewModels.Knowledges;
public class SearchKnowledgeRequest
{
[JsonPropertyName("text")]
public string Text { get; set; } = string.Empty;
#region Vector
[JsonPropertyName("vector_params")]
public VectorParam VectorParams { get; set; }
#endregion
#region Graph
[JsonPropertyName("graph_params")]
public GraphParam GraphParams { get; set; }
#endregion
}
public class VectorParam
{
[JsonPropertyName("collection")]
public string Collection { get; set; }
[JsonPropertyName("fields")]
public IEnumerable<string>? Fields { get; set; }
[JsonPropertyName("limit")]
public int? Limit { get; set; } = 5;
[JsonPropertyName("confidence")]
public float? Confidence { get; set; } = 0.5f;
[JsonPropertyName("with_vector")]
public bool WithVector { get; set; }
}
public class GraphParam
{
[JsonPropertyName("method")]
public string Method { get; set; } = string.Empty;
}

69
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs
View file

@ -1,69 +0,0 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge)
{
var index = 0;
var lines = TextChopper.Chop(knowledge.Content, new ChunkOption
{
Size = 1024,
Conjunction = 32,
SplitByWord = true,
});
var db = GetVectorDb();
var textEmbedding = GetTextEmbedding(collectionName);
await db.CreateCollection(collectionName, textEmbedding.GetDimension());
foreach (var line in lines)
{
var vec = await textEmbedding.GetVectorAsync(line);
await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
index++;
Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
}
}
public async Task<bool> CreateVectorCollection(string collectionName, int dimension)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName))
{
return false;
}
var db = GetVectorDb();
return await db.CreateCollection(collectionName, dimension);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text))
{
return false;
}
var textEmbedding = GetTextEmbedding(collectionName);
var vector = await textEmbedding.GetVectorAsync(create.Text);
var db = GetVectorDb();
var guid = Guid.NewGuid();
return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
}

42
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs
View file

@ -1,42 +0,0 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
public async Task<bool> DeleteVectorCollection(string collectionName)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName))
{
return false;
}
var db = GetVectorDb();
return await db.DeleteCollection(collectionName);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task<bool> DeleteVectorCollectionData(string collectionName, string id)
{
try
{
if (!Guid.TryParse(id, out var guid))
{
return false;
}
var db = GetVectorDb();
return await db.DeleteCollectionData(collectionName, guid);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
}

27
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs Normal file
View file

@ -0,0 +1,27 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge)
{
var index = 0;
var lines = TextChopper.Chop(knowledge.Content, new ChunkOption
{
Size = 1024,
Conjunction = 32,
SplitByWord = true,
});
var db = GetVectorDb();
var textEmbedding = GetTextEmbedding(collectionName);
await db.CreateCollection(collectionName, textEmbedding.GetDimension());
foreach (var line in lines)
{
var vec = await textEmbedding.GetVectorAsync(line);
await db.Upsert(collectionName, Guid.NewGuid(), vec, line);
index++;
Console.WriteLine($"Saved vector {index}/{lines.Count}: {line}\n");
}
}
}

103
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs
View file

@ -1,103 +0,0 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
public async Task<IEnumerable<string>> GetVectorCollections()
{
try
{
var db = GetVectorDb();
return await db.GetCollections();
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}");
return Enumerable.Empty<string>();
}
}
public async Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter)
{
try
{
var db = GetVectorDb();
var pagedResult = await db.GetPagedCollectionData(collectionName, filter);
return new StringIdPagedItems<VectorSearchResult>
{
Count = pagedResult.Count,
Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)),
NextId = pagedResult.NextId,
};
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return new StringIdPagedItems<VectorSearchResult>();
}
}
public async Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options)
{
try
{
var textEmbedding = GetTextEmbedding(collectionName);
var vector = await textEmbedding.GetVectorAsync(query);
// Vector search
var db = GetVectorDb();
var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList();
return results;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return new List<VectorSearchResult>();
}
}
public async Task<GraphSearchResult> SearchGraphKnowledge(string query, GraphSearchOptions options)
{
try
{
var db = GetGraphDb();
var found = await db.Search(query, options);
return new GraphSearchResult
{
Result = found.Result
};
}
catch (Exception ex)
{
_logger.LogWarning($"Error when searching graph knowledge (Query: {query}). {ex.Message}\r\n{ex.InnerException}");
return new GraphSearchResult();
}
}
public async Task<KnowledgeSearchResult> SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions)
{
try
{
var textEmbedding = GetTextEmbedding(collectionName);
var vector = await textEmbedding.GetVectorAsync(query);
var vectorDb = GetVectorDb();
var vectorRes = await vectorDb.Search(collectionName, vector, vectorOptions.Fields, limit: vectorOptions.Limit ?? 5,
confidence: vectorOptions.Confidence ?? 0.5f, withVector: vectorOptions.WithVector);
var graphDb = GetGraphDb();
var graphRes = await graphDb.Search(query, graphOptions);
return new KnowledgeSearchResult
{
VectorResult = vectorRes.Select(x => VectorSearchResult.CopyFrom(x)),
GraphResult = new GraphSearchResult { Result = graphRes.Result }
};
}
catch (Exception ex)
{
_logger.LogWarning($"Error when searching knowledge (Vector collection: {collectionName}) (Query: {query}). {ex.Message}\r\n{ex.InnerException}");
return new KnowledgeSearchResult();
}
}
}

22
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Graph.cs Normal file
View file

@ -0,0 +1,22 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
public async Task<GraphSearchResult> SearchGraphKnowledge(string query, GraphSearchOptions options)
{
try
{
var db = GetGraphDb();
var found = await db.Search(query, options);
return new GraphSearchResult
{
Result = found.Result
};
}
catch (Exception ex)
{
_logger.LogWarning($"Error when searching graph knowledge (Query: {query}). {ex.Message}\r\n{ex.InnerException}");
return new GraphSearchResult();
}
}
}

31
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs
View file

@ -1,31 +0,0 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
public async Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid))
{
return false;
}
var db = GetVectorDb();
var found = await db.GetCollectionData(collectionName, new List<Guid> { guid });
if (found.IsNullOrEmpty())
{
return false;
}
var textEmbedding = GetTextEmbedding(collectionName);
var vector = await textEmbedding.GetVectorAsync(update.Text);
return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
}

170
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs Normal file
View file

@ -0,0 +1,170 @@
namespace BotSharp.Plugin.KnowledgeBase.Services;
public partial class KnowledgeService
{
#region Collection
public async Task<bool> CreateVectorCollection(string collectionName, int dimension)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName))
{
return false;
}
var db = GetVectorDb();
return await db.CreateCollection(collectionName, dimension);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task<IEnumerable<string>> GetVectorCollections()
{
try
{
var db = GetVectorDb();
return await db.GetCollections();
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting vector db collections. {ex.Message}\r\n{ex.InnerException}");
return Enumerable.Empty<string>();
}
}
public async Task<bool> DeleteVectorCollection(string collectionName)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName))
{
return false;
}
var db = GetVectorDb();
return await db.DeleteCollection(collectionName);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
#endregion
#region Collection data
public async Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(create.Text))
{
return false;
}
var textEmbedding = GetTextEmbedding(collectionName);
var vector = await textEmbedding.GetVectorAsync(create.Text);
var db = GetVectorDb();
var guid = Guid.NewGuid();
return await db.Upsert(collectionName, guid, vector, create.Text, create.Payload);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when creating vector collection data. {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task<bool> UpdateVectorCollectionData(string collectionName, VectorUpdateModel update)
{
try
{
if (string.IsNullOrWhiteSpace(collectionName) || string.IsNullOrWhiteSpace(update.Text) || !Guid.TryParse(update.Id, out var guid))
{
return false;
}
var db = GetVectorDb();
var found = await db.GetCollectionData(collectionName, new List<Guid> { guid });
if (found.IsNullOrEmpty())
{
return false;
}
var textEmbedding = GetTextEmbedding(collectionName);
var vector = await textEmbedding.GetVectorAsync(update.Text);
return await db.Upsert(collectionName, guid, vector, update.Text, update.Payload);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when updating vector collection data. {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task<bool> DeleteVectorCollectionData(string collectionName, string id)
{
try
{
if (!Guid.TryParse(id, out var guid))
{
return false;
}
var db = GetVectorDb();
return await db.DeleteCollectionData(collectionName, guid);
}
catch (Exception ex)
{
_logger.LogWarning($"Error when deleting vector collection data ({collectionName}-{id}). {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public async Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter)
{
try
{
var db = GetVectorDb();
var pagedResult = await db.GetPagedCollectionData(collectionName, filter);
return new StringIdPagedItems<VectorSearchResult>
{
Count = pagedResult.Count,
Items = pagedResult.Items.Select(x => VectorSearchResult.CopyFrom(x)),
NextId = pagedResult.NextId,
};
}
catch (Exception ex)
{
_logger.LogWarning($"Error when getting vector knowledge collection data ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return new StringIdPagedItems<VectorSearchResult>();
}
}
public async Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options)
{
try
{
var textEmbedding = GetTextEmbedding(collectionName);
var vector = await textEmbedding.GetVectorAsync(query);
// Vector search
var db = GetVectorDb();
var found = await db.Search(collectionName, vector, options.Fields, limit: options.Limit ?? 5, confidence: options.Confidence ?? 0.5f, withVector: options.WithVector);
var results = found.Select(x => VectorSearchResult.CopyFrom(x)).ToList();
return results;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when searching vector knowledge ({collectionName}). {ex.Message}\r\n{ex.InnerException}");
return new List<VectorSearchResult>();
}
}
#endregion
}