SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chop text

Browse source
This commit is contained in:
Jicheng Lu 2024-09-11 15:48:05 -05:00
parent 6bfb2523f5
commit 39571b2f9b
3 changed files with 101 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.KnowledgeBase.cs
View file

@ -1,8 +1,4 @@
using AspectInjector.Broker;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Knowledges.Models;
using StackExchange.Redis;
using System.IO;
namespace BotSharp.Core.Files.Services;
@ -29,8 +25,10 @@ public partial class LocalFileStorageService
Directory.CreateDirectory(dir);
var filePath = Path.Combine(dir, fileName);
using var fs = File.Create(filePath);
using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write);
stream.CopyTo(fs);
fs.Flush();
fs.Close();
return true;
}
catch (Exception ex)

28
src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs
View file

@ -14,40 +14,48 @@ public static class TextChopper
private static List<string> ChopByWord(string content, ChunkOption option)
{
var chunks = new List<string>();
var words = content.Split(' ').Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
var words = content.Split(' ')
.Where(x => !string.IsNullOrWhiteSpace(x))
.ToList();
var chunk = "";
var chunk = string.Empty;
for (int i = 0; i < words.Count; i++)
{
chunk += words[i] + " ";
if (chunk.Length > option.Size)
{
chunks.Add(chunk.Trim());
chunk = "";
chunk = string.Empty;
i -= option.Conjunction;
}
}
if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk))
{
chunks.Add(chunk);
}
return chunks;
}
private static List<string> ChopByChar(string content, ChunkOption option)
{
var chunks = new List<string>();
var chunk = string.Empty;
var currentPos = 0;
while (currentPos < content.Length)
{
var len = content.Length - currentPos > option.Size ?
option.Size :
content.Length - currentPos;
var chunk = content.Substring(currentPos, len);
var len = content.Length - currentPos > option.Size ? option.Size : content.Length - currentPos;
chunk = content.Substring(currentPos, len);
chunks.Add(chunk);
// move backward
currentPos += option.Size - option.Conjunction;
}
if (chunks.IsNullOrEmpty() && !string.IsNullOrEmpty(chunk))
{
chunks.Add(chunk);
}
return chunks;
}
}

108
src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
View file

@ -2,6 +2,7 @@ using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Files.Models;
using BotSharp.Abstraction.Files.Utilities;
using System.Net.Http;
using System.Net.Mime;
namespace BotSharp.Plugin.KnowledgeBase.Services;
@ -34,43 +35,23 @@ public partial class KnowledgeService
try
{
var dataIds = new List<string>();
// Chop text (to do)
// Get document info
var (contentType, bytes) = await GetFileInfo(file);
using var stream = new MemoryStream(bytes);
using var reader = new StreamReader(stream);
var content = await reader.ReadToEndAsync();
// Save file
var contents = await GetFileContent(contentType, bytes);
// Save document
var fileId = Guid.NewGuid().ToString();
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, file.FileName, stream);
reader.Close();
stream.Close();
var saved = SaveDocument(collectionName, vectorStoreProvider, fileId, file.FileName, bytes);
if (!saved)
{
failedFiles.Add(file.FileName);
continue;
}
// Text embedding
var vectorDb = GetVectorDb();
var textEmbedding = GetTextEmbedding(collectionName);
var vector = await textEmbedding.GetVectorAsync(content);
// Save to vector db
var dataId = Guid.NewGuid();
saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
var dataIds = await SaveToVectorDb(collectionName, fileId, file.FileName, contents);
if (!dataIds.IsNullOrEmpty())
{
{ "fileName", file.FileName },
{ "fileId", fileId },
{ "page", "0" }
});
if (saved)
{
dataIds.Add(dataId.ToString());
fileStoreage.SaveKnolwedgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, new KnowledgeDocMetaData
{
Collection = collectionName,
@ -117,8 +98,8 @@ public partial class KnowledgeService
var vectorDb = GetVectorDb();
var vectorStoreProvider = _settings.VectorDb.Provider;
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
var metaData = fileStorage.GetKnowledgeBaseFileMeta(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
fileStorage.DeleteKnowledgeFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId);
if (metaData != null && !metaData.VectorDataIds.IsNullOrEmpty())
{
@ -212,5 +193,76 @@ public partial class KnowledgeService
return (string.Empty, new byte[0]);
}
private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[] bytes)
{
var results = new List<string>();
if (contentType.IsEqualTo(MediaTypeNames.Text.Plain))
{
using var stream = new MemoryStream(bytes);
using var reader = new StreamReader(stream);
var content = await reader.ReadToEndAsync();
var lines = TextChopper.Chop(content, new ChunkOption
{
Size = 1024,
Conjunction = 32,
SplitByWord = true,
});
reader.Close();
stream.Close();
results.AddRange(lines);
}
else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf))
{
// to do
}
return results;
}
private bool SaveDocument(string collectionName, string vectorStoreProvider, string fileId, string fileName, byte[] bytes)
{
var fileStoreage = _services.GetRequiredService<IFileStorageService>();
using var stream = new MemoryStream(bytes);
stream.Position = 0;
var saved = fileStoreage.SaveKnowledgeBaseFile(collectionName.CleanStr(), vectorStoreProvider.CleanStr(), fileId, fileName, stream);
stream.Close();
return saved;
}
private async Task<IEnumerable<string>> SaveToVectorDb(string collectionName, string fileId, string fileName, IEnumerable<string> contents)
{
if (contents.IsNullOrEmpty())
{
return Enumerable.Empty<string>();
}
var dataIds = new List<string>();
var vectorDb = GetVectorDb();
var textEmbedding = GetTextEmbedding(collectionName);
for (int i = 0; i < contents.Count(); i++)
{
var content = contents.ElementAt(i);
var vector = await textEmbedding.GetVectorAsync(content);
var dataId = Guid.NewGuid();
var saved = await vectorDb.Upsert(collectionName, dataId, vector, content, new Dictionary<string, string>
{
{ "fileName", fileName },
{ "fileId", fileId },
{ "textNumber", $"{i + 1}" }
});
if (!saved) continue;
dataIds.Add(dataId.ToString());
}
return dataIds;
}
#endregion
}