Add chunk chopping #70

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -5,5 +5,5 @@ namespace BotSharp.Abstraction.Knowledges;
 public interface IKnowledgeService
 {
     Task Feed(KnowledgeFeedModel knowledge);
-    Task<string> GetAnswer(string question);
+    Task<string> GetAnswer(KnowledgeRetrievalModel retrievalModel);
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs

@@ -0,0 +1,11 @@
+using BotSharp.Abstraction.Knowledges.Models;
+
+namespace BotSharp.Abstraction.Knowledges;
+
+/// <summary>
+/// Chop large content into chunks
+/// </summary>
+public interface ITextChopper
+{
+    List<string> Chop(string content, ChunkOption option);
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/IVectorDb.cs

@@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Knowledges;
 public interface IVectorDb
 {
     Task<List<string>> GetCollections();
-    Task CreateCollection(string collectionName);
+    Task CreateCollection(string collectionName, int dim);
     Task Upsert(string collectionName, int id, float[] vector);
     Task<List<int>> Search(string collectionName, float[] vector, int limit = 10);
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ChunkOption.cs

@@ -0,0 +1,14 @@
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class ChunkOption
+{
+    /// <summary>
+    /// Chunk size
+    /// </summary>
+    public int Size { get; set; }
+
+    /// <summary>
+    /// Overlap length in between two chunks
+    /// </summary>
+    public int Conjunction { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeFeedModel.cs

@@ -3,5 +3,6 @@ namespace BotSharp.Abstraction.Knowledges.Models;
 public class KnowledgeFeedModel
 {
     public string AgentId { get; set; } = string.Empty;
+    public string Name { get; set; } = string.Empty;
     public string Content { get; set; } = string.Empty;
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/KnowledgeRetrievalModel.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Knowledges.Models;
+
+public class KnowledgeRetrievalModel
+{
+    public string AgentId { get; set; } = string.Empty;
+    public string Question { get; set; } = string.Empty;
+}

src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs

@@ -12,6 +12,8 @@ public static class BotSharpServiceCollectionExtensions
         services.AddScoped<IAgentService, AgentService>();
         services.AddScoped<ISessionService, SessionService>();
         services.AddScoped<IConversationService, ConversationService>();
+
+        services.AddScoped<ITextChopper, TextChopperService>();
         services.AddScoped<IKnowledgeService, KnowledgeService>();
 
         services.AddScoped<IContentTransfer, ContentTransfer>();

src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeBase.cs

@@ -2,7 +2,7 @@ namespace BotSharp.Core.Knowledges;
 
 public class KnowledgeBase : IVectorDb
 {
-    public Task CreateCollection(string collectionName)
+    public Task CreateCollection(string collectionName, int dim)
     {
         throw new NotImplementedException();
     }

src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeController.cs

@@ -20,14 +20,18 @@ public class KnowledgeController : ControllerBase, IApiAdapter
         _knowledgeService = knowledgeService;
     }
 
-    [HttpGet("/knowledge")]
-    public async Task<string> GetAnswer([FromQuery(Name = "q")] string question)
+    [HttpGet("/knowledge/{agentId}")]
+    public async Task<string> RetrieveKnowledge([FromRoute] string agentId, [FromQuery(Name = "q")] string question)
     {
-        return await _knowledgeService.GetAnswer(question);
+        return await _knowledgeService.GetAnswer(new KnowledgeRetrievalModel
+        {
+            AgentId = agentId,
+            Question = question
+        });
     }
 
-    [HttpPost("/knowledge/feed/{agentId}")]
-    public async Task<IActionResult> FeedKnowledge([FromRoute] string agentId, List<IFormFile> files)
+    [HttpPost("/knowledge/{agentId}")]
+    public async Task<IActionResult> FeedKnowledge([FromRoute] string agentId, [FromForm] string name, List<IFormFile> files)
     {
         long size = files.Sum(f => f.Length);
 
@@ -58,6 +62,7 @@ public class KnowledgeController : ControllerBase, IApiAdapter
             await _knowledgeService.Feed(new KnowledgeFeedModel
             {
                 AgentId = agentId,
+                Name = name,
                 Content = content
             });
         }

src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs

@@ -1,7 +1,5 @@
-using BotSharp.Abstraction.Knowledges;
 using BotSharp.Abstraction.Knowledges.Models;
 using System.IO;
-using System.Collections;
 using BotSharp.Abstraction.MLTasks;
 
 namespace BotSharp.Core.Knowledges.Services;
@@ -10,49 +8,74 @@ public class KnowledgeService : IKnowledgeService
 {
     private readonly ITextEmbedding _textEmbedding;
     private readonly ITextCompletion _textCompletion;
+    private readonly ITextChopper _textChopper;
     private readonly IVectorDb _db;
-    string collectionName = "my_collection";
+    
     public KnowledgeService(ITextEmbedding textEmbedding, 
         ITextCompletion textCompletion, 
+        ITextChopper textChopper,
         IVectorDb db)
     {
         _textEmbedding = textEmbedding;
         _textCompletion = textCompletion;
+        _textChopper = textChopper;
         _db = db;
     }
 
     public async Task Feed(KnowledgeFeedModel knowledge)
     {
         var idStart = 0;
-        var lines = knowledge.Content.Split(". ");
-        lines = lines.Select((x, i) => $"{i+1} {x}").ToArray();
-        File.WriteAllLines(collectionName + ".txt", lines);
+        var lines = _textChopper.Chop(knowledge.Content, new ChunkOption
+        {
+            Size = 256,
+            Conjunction = 32
+        });
 
+        // Store chunks in local file system
+        var knowledgeStoreDir = Path.Combine("knowledge_chunks", knowledge.AgentId);
+        if(!Directory.Exists(knowledgeStoreDir))
+        {
+            Directory.CreateDirectory(knowledgeStoreDir);
+        }
+
+        var knowledgePath = Path.Combine(knowledgeStoreDir, knowledge.Name);
+        File.WriteAllLines(knowledgePath + ".txt", lines);
+
+        await _db.CreateCollection(knowledge.Name, _textEmbedding.Dimension);
         foreach (var line in lines)
         {
+            await _db.Upsert(knowledge.Name, idStart, _textEmbedding.GetVector(line));
             idStart++;
-            await _db.Upsert(collectionName, idStart, _textEmbedding.GetVector(line));
         }
     }
 
-    public async Task<string> GetAnswer(string question)
+    public async Task<string> GetAnswer(KnowledgeRetrievalModel retrievalModel)
     {
-        var vector = _textEmbedding.GetVector(question);
+        var vector = _textEmbedding.GetVector(retrievalModel.Question);
 
-        // Vector search
-        var result = await _db.Search(collectionName, vector);
+        // Scan local knowledge directory
+        var knowledgeName = "";
+        var chunks = new string[0];
 
-        var prompt = "";
-        var lines = File.ReadAllLines(collectionName + ".txt");
-        foreach (var r in result)
+        foreach (var file in Directory.GetFiles(Path.Combine("knowledge_chunks", retrievalModel.AgentId)))
         {
-            prompt += lines[r - 1] + "\n";
+            knowledgeName = new FileInfo(file).Name.Split('.').First();
+            chunks = File.ReadAllLines(file);
         }
 
-        prompt += "###\r\n";
+        // Vector search
+        var result = await _db.Search(knowledgeName, vector);
+
+        // Restore 
+        var prompt = "";
+        foreach (var r in result)
+        {
+            prompt += chunks[r] + "\n";
+        }
+
+        prompt += "\r\n###\r\n";
         prompt += "Answer the user's question based on the content provided above, and your reply should be as concise and organized as possible.\r\n";
-        prompt += "Q: how to turn on Hood Light? \r\nA: Press the Hood Light keypad to turn the light beneath the hood on or off.\r\n";
-        prompt += $"Q: {question}\r\nA: ";
+        prompt += $"Question: {retrievalModel.Question}\r\nAnswer: ";
 
         var completion = await _textCompletion.GetCompletion(prompt);
         return completion;

src/Infrastructure/BotSharp.Core/Knowledges/Services/TextChopperService.cs

@@ -0,0 +1,23 @@
+using BotSharp.Abstraction.Knowledges.Models;
+
+namespace BotSharp.Core.Knowledges.Services;
+
+public class TextChopperService : ITextChopper
+{
+    public List<string> Chop(string content, ChunkOption option)
+    {
+        var chunks = new List<string>();
+        var currentPos = 0;
+        while (currentPos < content.Length)
+        {
+            var len = content.Length - currentPos > option.Size ? 
+                option.Size : 
+                content.Length - currentPos;
+            var chunk = content.Substring(currentPos, len);
+            chunks.Add(chunk);
+            // move backward
+            currentPos += option.Size - option.Conjunction;
+        }
+        return chunks;
+    }
+}

src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs

@@ -7,7 +7,7 @@ namespace BotSharp.Plugin.MetaAI.Providers;
 
 public class FaissDb : IVectorDb
 {
-    public Task CreateCollection(string collectionName)
+    public Task CreateCollection(string collectionName, int dim)
     {
         throw new NotImplementedException();
     }

src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs

@@ -30,18 +30,18 @@ public class QdrantDb : IVectorDb
         return collections.Result.Collections.Select(x => x.Name).ToList();
     }
 
-    public async Task CreateCollection(string collectionName)
+    public async Task CreateCollection(string collectionName, int dim)
     {
         var collections = await GetCollections();
         if (!collections.Contains(collectionName))
         {
             // Create a new collection
-            await _client.CreateCollection(collectionName, new VectorParams(size: 300, distance: Distance.COSINE));
+            await _client.CreateCollection(collectionName, new VectorParams(size: dim, distance: Distance.COSINE));
         }
 
         // Get collection info
         var collectionInfo = await _client.GetCollection(collectionName);
-        if(collectionInfo == null)
+        if (collectionInfo == null)
         {
             throw new Exception($"Create {collectionName} failed.");
         }