using System.Text.RegularExpressions; namespace BotSharp.Plugin.KnowledgeBase.Services; public class TextChopperService : ITextChopper { public List Chop(string content, ChunkOption option) { content = Regex.Replace(content, @"\.{2,}", " "); content = Regex.Replace(content, @"_{2,}", " "); return option.SplitByWord ? ChopByWord(content, option) : ChopByChar(content, option); } private List ChopByWord(string content, ChunkOption option) { var chunks = new List(); var words = content.Split(' ') .Where(x => !string.IsNullOrEmpty(x)) .ToList(); var chunk = ""; for (int i = 0; i < words.Count; i++) { chunk += words[i] + " "; if (chunk.Length > option.Size) { chunks.Add(chunk.Trim()); chunk = ""; i -= option.Conjunction; } } return chunks; } private List ChopByChar(string content, ChunkOption option) { var chunks = new List(); var currentPos = 0; while (currentPos < content.Length) { var len = content.Length - currentPos > option.Size ? option.Size : content.Length - currentPos; var chunk = content.Substring(currentPos, len); chunks.Add(chunk); // move backward currentPos += option.Size - option.Conjunction; } return chunks; } }