SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

relocate text chopper

Browse source
This commit is contained in:
Jicheng Lu 2024-08-31 01:05:53 -05:00
parent 614aa3e199
commit 48be12a920
4 changed files with 8 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs
View file

@ -1,11 +0,0 @@
using BotSharp.Abstraction.Knowledges.Models;
namespace BotSharp.Abstraction.Knowledges;
/// <summary>
/// Chop large content into chunks
/// </summary>
public interface ITextChopper
{
List<string> Chop(string content, ChunkOption option);
}

11
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs → src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs
View file

@ -1,17 +1,18 @@
using BotSharp.Abstraction.Knowledges.Models;
using System.Text.RegularExpressions;
namespace BotSharp.Plugin.KnowledgeBase.Services;
namespace BotSharp.Core.Knowledges.Helpers;
public class TextChopperService : ITextChopper
public static class TextChopper
{
public List<string> Chop(string content, ChunkOption option)
public static List<string> Chop(string content, ChunkOption option)
{
content = Regex.Replace(content, @"\.{2,}", " ");
content = Regex.Replace(content, @"_{2,}", " ");
return option.SplitByWord ? ChopByWord(content, option) : ChopByChar(content, option);
}
private List<string> ChopByWord(string content, ChunkOption option)
private static List<string> ChopByWord(string content, ChunkOption option)
{
var chunks = new List<string>();
@ -34,7 +35,7 @@ public class TextChopperService : ITextChopper
return chunks;
}
private List<string> ChopByChar(string content, ChunkOption option)
private static List<string> ChopByChar(string content, ChunkOption option)
{
var chunks = new List<string>();
var currentPos = 0;

5
src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.VectorStorage.Models;
using BotSharp.Core.Knowledges.Helpers;
namespace BotSharp.Core.Knowledges.Services;
@ -8,9 +9,7 @@ public partial class KnowledgeService
public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge)
{
var index = 0;
var textChopper = _services.GetRequiredService<ITextChopper>();
var lines = textChopper.Chop(knowledge.Content, new ChunkOption
var lines = TextChopper.Chop(knowledge.Content, new ChunkOption
{
Size = 1024,
Conjunction = 32,

1
src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs
View file

@ -20,7 +20,6 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
return settingService.Bind<KnowledgeBaseSettings>("KnowledgeBase");
});
services.AddScoped<ITextChopper, TextChopperService>();
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
services.AddScoped<IAgentUtilityHook, KnowledgeBaseUtilityHook>();
services.AddScoped<IAgentHook, KnowledgeBaseAgentHook>();