From 64dca85d77650112ecc80d6b3b4b445b39539e3a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 14 Jun 2024 15:39:56 -0500 Subject: [PATCH] add translation memory --- .../Repositories/IBotSharpRepository.cs | 7 + .../Translation/Models/TranslationMemory.cs | 25 +++ .../Models/TranslationMemoryQuery.cs | 43 +++++ .../BotSharp.Core/BotSharp.Core.csproj | 4 + .../Infrastructures/Utilities.cs | 17 +- .../Repository/BotSharpDbContext.cs | 8 + .../FileRepository.Translation.cs | 156 ++++++++++++++++++ .../FileRepository/FileRepository.cs | 6 +- .../TwoStagePlanner/TwoStagePlanner.cs | 2 +- .../Translation/TranslationService.cs | 89 +++++++--- .../Users/Services/UserService.cs | 2 +- .../Repository/MongoRepository.Translation.cs | 14 ++ .../BotSharp.Plugin.MongoStorage/Using.cs | 1 + 13 files changed, 342 insertions(+), 32 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs create mode 100644 src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 6adcc8ad..8c3a0e81 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Tasks.Models; +using BotSharp.Abstraction.Translation.Models; using BotSharp.Abstraction.Users.Models; namespace BotSharp.Abstraction.Repositories; @@ -88,4 +89,10 @@ public interface IBotSharpRepository #region Statistics void IncrementConversationCount(); #endregion + + #region Translation + IEnumerable GetTranslationMemories(IEnumerable queries); + bool SaveTranslationMemories(IEnumerable inputs); + + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs new file mode 100644 index 00000000..73e1f9c0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs @@ -0,0 +1,25 @@ +namespace BotSharp.Abstraction.Translation.Models; + +public class TranslationMemory +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("original_text")] + public string OriginalText { get; set; } + + [JsonPropertyName("hash_text")] + public string HashText { get; set; } + + [JsonPropertyName("memories")] + public List Memories { get; set; } = new List(); +} + +public class TranslationMemoryItem +{ + [JsonPropertyName("translated_text")] + public string TranslatedText { get; set; } + + [JsonPropertyName("language")] + public string Language { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs new file mode 100644 index 00000000..66f1df8e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs @@ -0,0 +1,43 @@ +namespace BotSharp.Abstraction.Translation.Models; + +public class TranslationMemoryQuery +{ + public string OriginalText { get; set; } + public string HashText { get; set; } + public string Language { get; set; } + + public TranslationMemoryQuery() + { + + } +} + +public class TranslationMemoryInput : TranslationMemoryQuery +{ + public string TranslatedText { get; set; } + + public TranslationMemoryInput() + { + + } + + public override string ToString() + { + return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})"; + } +} + +public class TranslationMemoryOutput: TranslationMemoryQuery +{ + public string TranslatedText { get; set; } + + public TranslationMemoryOutput() + { + + } + + public override string ToString() + { + return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})"; + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 7bdd8464..eb01f6a0 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -172,4 +172,8 @@ + + + + diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs index 449d2c3f..4cb26888 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs @@ -4,11 +4,11 @@ namespace BotSharp.Core.Infrastructures; public static class Utilities { - public static string HashText(string password, string salt) + public static string HashTextMd5(string text) { using var md5 = System.Security.Cryptography.MD5.Create(); - var data = md5.ComputeHash(Encoding.UTF8.GetBytes(password + salt)); + var data = md5.ComputeHash(Encoding.UTF8.GetBytes(text)); var sb = new StringBuilder(); foreach (var c in data) { @@ -17,6 +17,19 @@ public static class Utilities return sb.ToString(); } + public static string HashTextSha256(string text) + { + using var sha256 = System.Security.Cryptography.SHA256.Create(); + + var data = sha256.ComputeHash(Encoding.UTF8.GetBytes(text)); + var sb = new StringBuilder(); + foreach(var c in data) + { + sb.Append(c.ToString("x2")); + } + return sb.ToString(); + } + public static (string, string) SplitAsTuple(this string str, string sep) { var splits = str.Split(sep); diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 6bba6c2a..5ce76901 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Tasks.Models; +using BotSharp.Abstraction.Translation.Models; using BotSharp.Abstraction.Users.Models; using Microsoft.EntityFrameworkCore.Infrastructure; @@ -224,4 +225,11 @@ public class BotSharpDbContext : Database, IBotSharpRepository throw new NotImplementedException(); } #endregion + + #region Translation + public IEnumerable GetTranslationMemories(IEnumerable queries) + => throw new NotImplementedException(); + public bool SaveTranslationMemories(IEnumerable inputs) => + throw new NotImplementedException(); + #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs new file mode 100644 index 00000000..8d8a14e7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs @@ -0,0 +1,156 @@ +using BotSharp.Abstraction.Translation.Models; +using System.IO; + +namespace BotSharp.Core.Repository; + +public partial class FileRepository +{ + public IEnumerable GetTranslationMemories(IEnumerable queries) + { + var list = new List(); + if (queries.IsNullOrEmpty()) + { + return list; + } + + var dir = Path.Combine(_dbSettings.FileRepository, "translation"); + var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE); + if (!Directory.Exists(dir) || !File.Exists(file)) + { + return list; + } + + var content = File.ReadAllText(file); + if (string.IsNullOrWhiteSpace(content)) + { + return list; + } + + var memories = ReadTranslationMemoryContent(content); + foreach (var query in queries) + { + if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language)) + { + continue; + } + + var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText)); + if (foundMemory == null) continue; + + var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(query.Language)); + if (foundItem == null) continue; + + list.Add(new TranslationMemoryOutput + { + OriginalText = query.OriginalText, + TranslatedText = foundItem.TranslatedText, + HashText = foundMemory.HashText, + Language = foundItem.Language, + }); + } + + return list; + } + + public bool SaveTranslationMemories(IEnumerable inputs) + { + if (inputs.IsNullOrEmpty()) return false; + + try + { + var dir = Path.Combine(_dbSettings.FileRepository, "translation"); + var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + var content = string.Empty; + if (File.Exists(file)) + { + content = File.ReadAllText(file); + } + + var memories = ReadTranslationMemoryContent(content); + + foreach (var input in inputs) + { + if (string.IsNullOrWhiteSpace(input.OriginalText) || + string.IsNullOrWhiteSpace(input.TranslatedText) || + string.IsNullOrWhiteSpace(input.HashText) || + string.IsNullOrWhiteSpace(input.Language)) + { + continue; + } + + var newItem = new TranslationMemoryItem + { + TranslatedText = input.TranslatedText, + Language = input.Language + }; + + var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText)); + if (foundMemory == null) + { + var newMemory = new TranslationMemory + { + Id = Guid.NewGuid().ToString(), + OriginalText = input.OriginalText, + HashText = input.HashText, + Memories = new List { newItem } + }; + + if (memories == null) + { + memories = new List { newMemory }; + } + else + { + memories.Add(newMemory); + } + } + else + { + var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(input.Language)); + if (foundItem != null) continue; + + if (foundMemory.Memories == null) + { + foundMemory.Memories = new List { newItem }; + } + else + { + foundMemory.Memories.Add(newItem); + } + } + } + + var json = JsonSerializer.Serialize(memories, _options); + File.WriteAllText(file, json); + return true; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving translation memories: {ex.Message}"); + return false; + } + } + + private List ReadTranslationMemoryContent(string? content) + { + var memories = new List(); + + try + { + if (string.IsNullOrWhiteSpace(content)) + { + return memories; + } + + memories = JsonSerializer.Deserialize>(content, _options) ?? new List(); + } + catch {} + + return memories; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 2094b821..a30142be 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -17,6 +17,7 @@ public partial class FileRepository : IBotSharpRepository private readonly AgentSettings _agentSettings; private readonly ConversationSetting _conversationSettings; private readonly StatisticsSettings _statisticsSetting; + private readonly ILogger _logger; private JsonSerializerOptions _options; private const string AGENT_FILE = "agent.json"; @@ -34,19 +35,22 @@ public partial class FileRepository : IBotSharpRepository private const string PLUGIN_CONFIG_FILE = "config.json"; private const string AGENT_TASK_PREFIX = "#metadata"; private const string AGENT_TASK_SUFFIX = "/metadata"; + private const string TRANSLATION_MEMORY_FILE = "memory.json"; public FileRepository( IServiceProvider services, BotSharpDatabaseSettings dbSettings, AgentSettings agentSettings, ConversationSetting conversationSettings, - StatisticsSettings statisticsSettings) + StatisticsSettings statisticsSettings, + ILogger logger) { _services = services; _dbSettings = dbSettings; _agentSettings = agentSettings; _conversationSettings = conversationSettings; _statisticsSetting = statisticsSettings; + _logger = logger; _options = new JsonSerializerOptions { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs index 6df03a08..d815ee5a 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs @@ -32,7 +32,7 @@ public partial class TwoStagePlanner : IPlaner if (_plan1st.IsNullOrEmpty() && _plan2nd.IsNullOrEmpty()) { Directory.CreateDirectory(tempDir); - _md5 = Utilities.HashText(string.Join(".", dialogs.Where(x => x.Role == AgentRole.User)), "botsharp"); + _md5 = Utilities.HashTextMd5($"{string.Join(".", dialogs.Where(x => x.Role == AgentRole.User))}{"botsharp"}"); var filePath = Path.Combine(tempDir, $"{_md5}-1st.json"); FirstStagePlan[] items = new FirstStagePlan[0]; if (File.Exists(filePath)) diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index ec33f609..d72bef2a 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -1,4 +1,3 @@ -using Amazon.Runtime.Internal.Transform; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Options; @@ -13,17 +12,21 @@ namespace BotSharp.Core.Translation; public class TranslationService : ITranslationService { private readonly IServiceProvider _services; + private readonly IBotSharpRepository _db; private readonly ILogger _logger; private readonly BotSharpOptions _options; private Agent _router; private string _messageId; private IChatCompletion _completion; - public TranslationService(IServiceProvider services, + public TranslationService( + IServiceProvider services, + IBotSharpRepository db, ILogger logger, BotSharpOptions options) { _services = services; + _db = db; _logger = logger; _options = options; } @@ -56,41 +59,73 @@ public class TranslationService : ITranslationService model: _router?.LlmConfig?.Model); var template = _router.Templates.First(x => x.Name == "translation_prompt").Content; + var map = new Dictionary(); var keys = unique.ToArray(); - var texts = unique.ToArray() + + #region Search memory + var queries = keys.Select(x => new TranslationMemoryQuery + { + OriginalText = x, + HashText = Utilities.HashTextSha256(x), + Language = language + }).ToList(); + var memories = _db.GetTranslationMemories(queries); + var memoryHashes = memories.Select(x => x.HashText).ToList(); + + foreach (var memory in memories) + { + map[memory.OriginalText] = memory.TranslatedText; + } + + var outOfMemoryList = queries.Where(x => !memoryHashes.Contains(x.HashText)).ToList(); + #endregion + + var texts = outOfMemoryList.ToArray() .Select((text, i) => new TranslationInput { Id = i + 1, - Text = text + Text = text.OriginalText }).ToList(); try { - var translatedStringList = await InnerTranslate(texts, language, template); - - int retry = 0; - while (translatedStringList.Texts.Length != texts.Count && retry < 3) + if (!texts.IsNullOrEmpty()) { - translatedStringList = await InnerTranslate(texts, language, template); - retry++; + var translatedStringList = await InnerTranslate(texts, language, template); + + int retry = 0; + while (translatedStringList.Texts.Length != texts.Count && retry < 3) + { + translatedStringList = await InnerTranslate(texts, language, template); + retry++; + } + + // Override language if it's Unknown, it's used to output the corresponding language. + var states = _services.GetRequiredService(); + if (!states.ContainsState(StateConst.LANGUAGE)) + { + var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage; + states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1); + } + + var translatedTexts = translatedStringList.Texts; + var memoryInputs = new List(); + + for (var i = 0; i < texts.Count; i++) + { + map[outOfMemoryList[i].OriginalText] = translatedTexts[i].Text; + memoryInputs.Add(new TranslationMemoryInput + { + OriginalText = outOfMemoryList[i].OriginalText, + HashText = outOfMemoryList[i].HashText, + TranslatedText = translatedTexts[i].Text, + Language = language + }); + } +; + _db.SaveTranslationMemories(memoryInputs); } - - // Override language if it's Unknown, it's used to output the corresponding language. - var states = _services.GetRequiredService(); - if (!states.ContainsState(StateConst.LANGUAGE)) - { - var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage; - states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1); - } - - var translatedTexts = translatedStringList.Texts; - var map = new Dictionary(); - - for (var i = 0; i < texts.Count; i++) - { - map[keys[i]] = translatedTexts[i].Text; - } - + clonedData = Assign(clonedData, map); } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 81a7a79c..aadeb2e1 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -61,7 +61,7 @@ public class UserService : IUserService record.Phone = "+" + Regex.Match(user.Phone, @"\d+").Value; } record.Salt = Guid.NewGuid().ToString("N"); - record.Password = Utilities.HashText(user.Password, record.Salt); + record.Password = Utilities.HashTextMd5($"{user.Password}{record.Salt}"); if (_setting.NewUserVerification) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs new file mode 100644 index 00000000..9f2f98a1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + public IEnumerable GetTranslationMemories(IEnumerable queries) + { + throw new NotImplementedException(); + } + + public bool SaveTranslationMemories(IEnumerable inputs) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs index ca13f204..f40df94a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs @@ -7,6 +7,7 @@ global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Translation.Models; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using MongoDB.Bson;