From 1f9f8767d495d2b06fce57282085d7e1e9cb008c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 18 Jun 2023 13:15:00 -0500 Subject: [PATCH] Add chunk chopping #70 --- .../Knowledges/IKnowledgeService.cs | 2 +- .../Knowledges/ITextChopper.cs | 11 ++++ .../Knowledges/IVectorDb.cs | 2 +- .../Knowledges/Models/ChunkOption.cs | 14 +++++ .../Knowledges/Models/KnowledgeFeedModel.cs | 1 + .../Models/KnowledgeRetrievalModel.cs | 7 +++ .../BotSharpServiceCollectionExtensions.cs | 2 + .../BotSharp.Core/Knowledges/KnowledgeBase.cs | 2 +- .../Knowledges/KnowledgeController.cs | 15 +++-- .../Knowledges/Services/KnowledgeService.cs | 59 +++++++++++++------ .../Knowledges/Services/TextChopperService.cs | 23 ++++++++ .../Providers/FaissDb.cs | 2 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 6 +- 13 files changed, 116 insertions(+), 30 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ChunkOption.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs create mode 100644 src/Infrastructure/BotSharp.Core/Knowledges/Services/TextChopperService.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index ed70bf1c..b707ad6d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -5,5 +5,5 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { Task Feed(KnowledgeFeedModel knowledge); - Task GetAnswer(string question); + Task GetAnswer(KnowledgeRetrievalModel retrievalModel); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs new file mode 100644 index 00000000..7d06a8a4 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs @@ -0,0 +1,11 @@ +using BotSharp.Abstraction.Knowledges.Models; + +namespace BotSharp.Abstraction.Knowledges; + +/// +/// Chop large content into chunks +/// +public interface ITextChopper +{ + List Chop(string content, ChunkOption option); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IVectorDb.cs index 7fe53c75..cf799979 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IVectorDb.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Knowledges; public interface IVectorDb { Task> GetCollections(); - Task CreateCollection(string collectionName); + Task CreateCollection(string collectionName, int dim); Task Upsert(string collectionName, int id, float[] vector); Task> Search(string collectionName, float[] vector, int limit = 10); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ChunkOption.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ChunkOption.cs new file mode 100644 index 00000000..8d6a59eb --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ChunkOption.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class ChunkOption +{ + /// + /// Chunk size + /// + public int Size { get; set; } + + /// + /// Overlap length in between two chunks + /// + public int Conjunction { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs index 7e3a315e..75c69782 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs @@ -3,5 +3,6 @@ namespace BotSharp.Abstraction.Knowledges.Models; public class KnowledgeFeedModel { public string AgentId { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; public string Content { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs new file mode 100644 index 00000000..03f66eb5 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs @@ -0,0 +1,7 @@ +namespace BotSharp.Abstraction.Knowledges.Models; + +public class KnowledgeRetrievalModel +{ + public string AgentId { get; set; } = string.Empty; + public string Question { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs index 82c09c38..e78d242b 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs @@ -12,6 +12,8 @@ public static class BotSharpServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeBase.cs b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeBase.cs index 5b0335c2..b1cdb8aa 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeBase.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeBase.cs @@ -2,7 +2,7 @@ namespace BotSharp.Core.Knowledges; public class KnowledgeBase : IVectorDb { - public Task CreateCollection(string collectionName) + public Task CreateCollection(string collectionName, int dim) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeController.cs b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeController.cs index ca90d767..dec3ef3e 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeController.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeController.cs @@ -20,14 +20,18 @@ public class KnowledgeController : ControllerBase, IApiAdapter _knowledgeService = knowledgeService; } - [HttpGet("/knowledge")] - public async Task GetAnswer([FromQuery(Name = "q")] string question) + [HttpGet("/knowledge/{agentId}")] + public async Task RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question) { - return await _knowledgeService.GetAnswer(question); + return await _knowledgeService.GetAnswer(new KnowledgeRetrievalModel + { + AgentId = agentId, + Question = question + }); } - [HttpPost("/knowledge/feed/{agentId}")] - public async Task FeedKnowledge([FromRoute] string agentId, List files) + [HttpPost("/knowledge/{agentId}")] + public async Task FeedKnowledge([FromRoute] string agentId, [FromForm] string name, List files) { long size = files.Sum(f => f.Length); @@ -58,6 +62,7 @@ public class KnowledgeController : ControllerBase, IApiAdapter await _knowledgeService.Feed(new KnowledgeFeedModel { AgentId = agentId, + Name = name, Content = content }); } diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs index 81ba2d26..d54a6e02 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs @@ -1,7 +1,5 @@ -using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.Knowledges.Models; using System.IO; -using System.Collections; using BotSharp.Abstraction.MLTasks; namespace BotSharp.Core.Knowledges.Services; @@ -10,49 +8,74 @@ public class KnowledgeService : IKnowledgeService { private readonly ITextEmbedding _textEmbedding; private readonly ITextCompletion _textCompletion; + private readonly ITextChopper _textChopper; private readonly IVectorDb _db; - string collectionName = "my_collection"; + public KnowledgeService(ITextEmbedding textEmbedding, ITextCompletion textCompletion, + ITextChopper textChopper, IVectorDb db) { _textEmbedding = textEmbedding; _textCompletion = textCompletion; + _textChopper = textChopper; _db = db; } public async Task Feed(KnowledgeFeedModel knowledge) { var idStart = 0; - var lines = knowledge.Content.Split(". "); - lines = lines.Select((x, i) => $"{i+1} {x}").ToArray(); - File.WriteAllLines(collectionName + ".txt", lines); + var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + { + Size = 256, + Conjunction = 32 + }); + // Store chunks in local file system + var knowledgeStoreDir = Path.Combine("knowledge_chunks", knowledge.AgentId); + if(!Directory.Exists(knowledgeStoreDir)) + { + Directory.CreateDirectory(knowledgeStoreDir); + } + + var knowledgePath = Path.Combine(knowledgeStoreDir, knowledge.Name); + File.WriteAllLines(knowledgePath + ".txt", lines); + + await _db.CreateCollection(knowledge.Name, _textEmbedding.Dimension); foreach (var line in lines) { + await _db.Upsert(knowledge.Name, idStart, _textEmbedding.GetVector(line)); idStart++; - await _db.Upsert(collectionName, idStart, _textEmbedding.GetVector(line)); } } - public async Task GetAnswer(string question) + public async Task GetAnswer(KnowledgeRetrievalModel retrievalModel) { - var vector = _textEmbedding.GetVector(question); + var vector = _textEmbedding.GetVector(retrievalModel.Question); - // Vector search - var result = await _db.Search(collectionName, vector); + // Scan local knowledge directory + var knowledgeName = ""; + var chunks = new string[0]; - var prompt = ""; - var lines = File.ReadAllLines(collectionName + ".txt"); - foreach (var r in result) + foreach (var file in Directory.GetFiles(Path.Combine("knowledge_chunks", retrievalModel.AgentId))) { - prompt += lines[r - 1] + "\n"; + knowledgeName = new FileInfo(file).Name.Split('.').First(); + chunks = File.ReadAllLines(file); } - prompt += "###\r\n"; + // Vector search + var result = await _db.Search(knowledgeName, vector); + + // Restore + var prompt = ""; + foreach (var r in result) + { + prompt += chunks[r] + "\n"; + } + + prompt += "\r\n###\r\n"; prompt += "Answer the user's question based on the content provided above, and your reply should be as concise and organized as possible.\r\n"; - prompt += "Q: how to turn on Hood Light? \r\nA: Press the Hood Light keypad to turn the light beneath the hood on or off.\r\n"; - prompt += $"Q: {question}\r\nA: "; + prompt += $"Question: {retrievalModel.Question}\r\nAnswer: "; var completion = await _textCompletion.GetCompletion(prompt); return completion; diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/TextChopperService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/TextChopperService.cs new file mode 100644 index 00000000..c803f988 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/TextChopperService.cs @@ -0,0 +1,23 @@ +using BotSharp.Abstraction.Knowledges.Models; + +namespace BotSharp.Core.Knowledges.Services; + +public class TextChopperService : ITextChopper +{ + public List Chop(string content, ChunkOption option) + { + var chunks = new List(); + var currentPos = 0; + while (currentPos < content.Length) + { + var len = content.Length - currentPos > option.Size ? + option.Size : + content.Length - currentPos; + var chunk = content.Substring(currentPos, len); + chunks.Add(chunk); + // move backward + currentPos += option.Size - option.Conjunction; + } + return chunks; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 6110fbe5..54e02882 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -7,7 +7,7 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { - public Task CreateCollection(string collectionName) + public Task CreateCollection(string collectionName, int dim) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 2846cf85..627aab8d 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -30,18 +30,18 @@ public class QdrantDb : IVectorDb return collections.Result.Collections.Select(x => x.Name).ToList(); } - public async Task CreateCollection(string collectionName) + public async Task CreateCollection(string collectionName, int dim) { var collections = await GetCollections(); if (!collections.Contains(collectionName)) { // Create a new collection - await _client.CreateCollection(collectionName, new VectorParams(size: 300, distance: Distance.COSINE)); + await _client.CreateCollection(collectionName, new VectorParams(size: dim, distance: Distance.COSINE)); } // Get collection info var collectionInfo = await _client.GetCollection(collectionName); - if(collectionInfo == null) + if (collectionInfo == null) { throw new Exception($"Create {collectionName} failed."); }