SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add translation memory

Browse source
This commit is contained in:
Jicheng Lu 2024-06-14 15:39:56 -05:00
parent 7d99ad845b
commit 64dca85d77
13 changed files with 342 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -2,6 +2,7 @@ using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Repositories.Filters;
using BotSharp.Abstraction.Tasks.Models;
using BotSharp.Abstraction.Translation.Models;
using BotSharp.Abstraction.Users.Models;
namespace BotSharp.Abstraction.Repositories;
@ -88,4 +89,10 @@ public interface IBotSharpRepository
#region Statistics
void IncrementConversationCount();
#endregion
#region Translation
IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries);
bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs);
#endregion
}

25
src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs Normal file
View file

@ -0,0 +1,25 @@
namespace BotSharp.Abstraction.Translation.Models;
public class TranslationMemory
{
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("original_text")]
public string OriginalText { get; set; }
[JsonPropertyName("hash_text")]
public string HashText { get; set; }
[JsonPropertyName("memories")]
public List<TranslationMemoryItem> Memories { get; set; } = new List<TranslationMemoryItem>();
}
public class TranslationMemoryItem
{
[JsonPropertyName("translated_text")]
public string TranslatedText { get; set; }
[JsonPropertyName("language")]
public string Language { get; set; }
}

43
src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs Normal file
View file

@ -0,0 +1,43 @@
namespace BotSharp.Abstraction.Translation.Models;
public class TranslationMemoryQuery
{
public string OriginalText { get; set; }
public string HashText { get; set; }
public string Language { get; set; }
public TranslationMemoryQuery()
{
}
}
public class TranslationMemoryInput : TranslationMemoryQuery
{
public string TranslatedText { get; set; }
public TranslationMemoryInput()
{
}
public override string ToString()
{
return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})";
}
}
public class TranslationMemoryOutput: TranslationMemoryQuery
{
public string TranslatedText { get; set; }
public TranslationMemoryOutput()
{
}
public override string ToString()
{
return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})";
}
}

4
src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
View file

@ -172,4 +172,8 @@
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Translation\Models\" />
</ItemGroup>
</Project>

17
src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs
View file

@ -4,11 +4,11 @@ namespace BotSharp.Core.Infrastructures;
public static class Utilities
{
public static string HashText(string password, string salt)
public static string HashTextMd5(string text)
{
using var md5 = System.Security.Cryptography.MD5.Create();
var data = md5.ComputeHash(Encoding.UTF8.GetBytes(password + salt));
var data = md5.ComputeHash(Encoding.UTF8.GetBytes(text));
var sb = new StringBuilder();
foreach (var c in data)
{
@ -17,6 +17,19 @@ public static class Utilities
return sb.ToString();
}
public static string HashTextSha256(string text)
{
using var sha256 = System.Security.Cryptography.SHA256.Create();
var data = sha256.ComputeHash(Encoding.UTF8.GetBytes(text));
var sb = new StringBuilder();
foreach(var c in data)
{
sb.Append(c.ToString("x2"));
}
return sb.ToString();
}
public static (string, string) SplitAsTuple(this string str, string sep)
{
var splits = str.Split(sep);

8
src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Tasks.Models;
using BotSharp.Abstraction.Translation.Models;
using BotSharp.Abstraction.Users.Models;
using Microsoft.EntityFrameworkCore.Infrastructure;
@ -224,4 +225,11 @@ public class BotSharpDbContext : Database, IBotSharpRepository
throw new NotImplementedException();
}
#endregion
#region Translation
public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
=> throw new NotImplementedException();
public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs) =>
throw new NotImplementedException();
#endregion
}

156
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs Normal file
View file

@ -0,0 +1,156 @@
using BotSharp.Abstraction.Translation.Models;
using System.IO;
namespace BotSharp.Core.Repository;
public partial class FileRepository
{
public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
{
var list = new List<TranslationMemoryOutput>();
if (queries.IsNullOrEmpty())
{
return list;
}
var dir = Path.Combine(_dbSettings.FileRepository, "translation");
var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE);
if (!Directory.Exists(dir) || !File.Exists(file))
{
return list;
}
var content = File.ReadAllText(file);
if (string.IsNullOrWhiteSpace(content))
{
return list;
}
var memories = ReadTranslationMemoryContent(content);
foreach (var query in queries)
{
if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language))
{
continue;
}
var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText));
if (foundMemory == null) continue;
var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(query.Language));
if (foundItem == null) continue;
list.Add(new TranslationMemoryOutput
{
OriginalText = query.OriginalText,
TranslatedText = foundItem.TranslatedText,
HashText = foundMemory.HashText,
Language = foundItem.Language,
});
}
return list;
}
public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs)
{
if (inputs.IsNullOrEmpty()) return false;
try
{
var dir = Path.Combine(_dbSettings.FileRepository, "translation");
var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
var content = string.Empty;
if (File.Exists(file))
{
content = File.ReadAllText(file);
}
var memories = ReadTranslationMemoryContent(content);
foreach (var input in inputs)
{
if (string.IsNullOrWhiteSpace(input.OriginalText) ||
string.IsNullOrWhiteSpace(input.TranslatedText) ||
string.IsNullOrWhiteSpace(input.HashText) ||
string.IsNullOrWhiteSpace(input.Language))
{
continue;
}
var newItem = new TranslationMemoryItem
{
TranslatedText = input.TranslatedText,
Language = input.Language
};
var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText));
if (foundMemory == null)
{
var newMemory = new TranslationMemory
{
Id = Guid.NewGuid().ToString(),
OriginalText = input.OriginalText,
HashText = input.HashText,
Memories = new List<TranslationMemoryItem> { newItem }
};
if (memories == null)
{
memories = new List<TranslationMemory> { newMemory };
}
else
{
memories.Add(newMemory);
}
}
else
{
var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(input.Language));
if (foundItem != null) continue;
if (foundMemory.Memories == null)
{
foundMemory.Memories = new List<TranslationMemoryItem> { newItem };
}
else
{
foundMemory.Memories.Add(newItem);
}
}
}
var json = JsonSerializer.Serialize(memories, _options);
File.WriteAllText(file, json);
return true;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving translation memories: {ex.Message}");
return false;
}
}
private List<TranslationMemory> ReadTranslationMemoryContent(string? content)
{
var memories = new List<TranslationMemory>();
try
{
if (string.IsNullOrWhiteSpace(content))
{
return memories;
}
memories = JsonSerializer.Deserialize<List<TranslationMemory>>(content, _options) ?? new List<TranslationMemory>();
}
catch {}
return memories;
}
}

6
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs
View file

@ -17,6 +17,7 @@ public partial class FileRepository : IBotSharpRepository
private readonly AgentSettings _agentSettings;
private readonly ConversationSetting _conversationSettings;
private readonly StatisticsSettings _statisticsSetting;
private readonly ILogger<FileRepository> _logger;
private JsonSerializerOptions _options;
private const string AGENT_FILE = "agent.json";
@ -34,19 +35,22 @@ public partial class FileRepository : IBotSharpRepository
private const string PLUGIN_CONFIG_FILE = "config.json";
private const string AGENT_TASK_PREFIX = "#metadata";
private const string AGENT_TASK_SUFFIX = "/metadata";
private const string TRANSLATION_MEMORY_FILE = "memory.json";
public FileRepository(
IServiceProvider services,
BotSharpDatabaseSettings dbSettings,
AgentSettings agentSettings,
ConversationSetting conversationSettings,
StatisticsSettings statisticsSettings)
StatisticsSettings statisticsSettings,
ILogger<FileRepository> logger)
{
_services = services;
_dbSettings = dbSettings;
_agentSettings = agentSettings;
_conversationSettings = conversationSettings;
_statisticsSetting = statisticsSettings;
_logger = logger;
_options = new JsonSerializerOptions
{

2
src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs
View file

@ -32,7 +32,7 @@ public partial class TwoStagePlanner : IPlaner
if (_plan1st.IsNullOrEmpty() && _plan2nd.IsNullOrEmpty())
{
Directory.CreateDirectory(tempDir);
_md5 = Utilities.HashText(string.Join(".", dialogs.Where(x => x.Role == AgentRole.User)), "botsharp");
_md5 = Utilities.HashTextMd5($"{string.Join(".", dialogs.Where(x => x.Role == AgentRole.User))}{"botsharp"}");
var filePath = Path.Combine(tempDir, $"{_md5}-1st.json");
FirstStagePlan[] items = new FirstStagePlan[0];
if (File.Exists(filePath))

89
src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs
View file

@ -1,4 +1,3 @@
using Amazon.Runtime.Internal.Transform;
using BotSharp.Abstraction.Infrastructures.Enums;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Options;
@ -13,17 +12,21 @@ namespace BotSharp.Core.Translation;
public class TranslationService : ITranslationService
{
private readonly IServiceProvider _services;
private readonly IBotSharpRepository _db;
private readonly ILogger<TranslationService> _logger;
private readonly BotSharpOptions _options;
private Agent _router;
private string _messageId;
private IChatCompletion _completion;
public TranslationService(IServiceProvider services,
public TranslationService(
IServiceProvider services,
IBotSharpRepository db,
ILogger<TranslationService> logger,
BotSharpOptions options)
{
_services = services;
_db = db;
_logger = logger;
_options = options;
}
@ -56,41 +59,73 @@ public class TranslationService : ITranslationService
model: _router?.LlmConfig?.Model);
var template = _router.Templates.First(x => x.Name == "translation_prompt").Content;
var map = new Dictionary<string, string>();
var keys = unique.ToArray();
var texts = unique.ToArray()
#region Search memory
var queries = keys.Select(x => new TranslationMemoryQuery
{
OriginalText = x,
HashText = Utilities.HashTextSha256(x),
Language = language
}).ToList();
var memories = _db.GetTranslationMemories(queries);
var memoryHashes = memories.Select(x => x.HashText).ToList();
foreach (var memory in memories)
{
map[memory.OriginalText] = memory.TranslatedText;
}
var outOfMemoryList = queries.Where(x => !memoryHashes.Contains(x.HashText)).ToList();
#endregion
var texts = outOfMemoryList.ToArray()
.Select((text, i) => new TranslationInput
{
Id = i + 1,
Text = text
Text = text.OriginalText
}).ToList();
try
{
var translatedStringList = await InnerTranslate(texts, language, template);
int retry = 0;
while (translatedStringList.Texts.Length != texts.Count && retry < 3)
if (!texts.IsNullOrEmpty())
{
translatedStringList = await InnerTranslate(texts, language, template);
retry++;
var translatedStringList = await InnerTranslate(texts, language, template);
int retry = 0;
while (translatedStringList.Texts.Length != texts.Count && retry < 3)
{
translatedStringList = await InnerTranslate(texts, language, template);
retry++;
}
// Override language if it's Unknown, it's used to output the corresponding language.
var states = _services.GetRequiredService<IConversationStateService>();
if (!states.ContainsState(StateConst.LANGUAGE))
{
var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage;
states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1);
}
var translatedTexts = translatedStringList.Texts;
var memoryInputs = new List<TranslationMemoryInput>();
for (var i = 0; i < texts.Count; i++)
{
map[outOfMemoryList[i].OriginalText] = translatedTexts[i].Text;
memoryInputs.Add(new TranslationMemoryInput
{
OriginalText = outOfMemoryList[i].OriginalText,
HashText = outOfMemoryList[i].HashText,
TranslatedText = translatedTexts[i].Text,
Language = language
});
}
;
_db.SaveTranslationMemories(memoryInputs);
}
// Override language if it's Unknown, it's used to output the corresponding language.
var states = _services.GetRequiredService<IConversationStateService>();
if (!states.ContainsState(StateConst.LANGUAGE))
{
var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage;
states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1);
}
var translatedTexts = translatedStringList.Texts;
var map = new Dictionary<string, string>();
for (var i = 0; i < texts.Count; i++)
{
map[keys[i]] = translatedTexts[i].Text;
}
clonedData = Assign(clonedData, map);
}
catch (Exception ex)

2
src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
View file

@ -61,7 +61,7 @@ public class UserService : IUserService
record.Phone = "+" + Regex.Match(user.Phone, @"\d+").Value;
}
record.Salt = Guid.NewGuid().ToString("N");
record.Password = Utilities.HashText(user.Password, record.Salt);
record.Password = Utilities.HashTextMd5($"{user.Password}{record.Salt}");
if (_setting.NewUserVerification)
{

14
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs Normal file
View file

@ -0,0 +1,14 @@
namespace BotSharp.Plugin.MongoStorage.Repository;
public partial class MongoRepository
{
public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
{
throw new NotImplementedException();
}
public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs)
{
throw new NotImplementedException();
}
}

1
src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs
View file

@ -7,6 +7,7 @@ global using BotSharp.Abstraction.Repositories;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Utilities;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Translation.Models;
global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using MongoDB.Bson;