Merge pull request #501 from iceljc/features/add-translation-memory

Features/add translation memory

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Tasks.Models;
+using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
 
 namespace BotSharp.Abstraction.Repositories;
@@ -88,4 +89,10 @@ public interface IBotSharpRepository
     #region Statistics
     void IncrementConversationCount();
     #endregion
+
+    #region Translation
+    IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries);
+    bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs);
+
+    #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs

@@ -0,0 +1,25 @@
+namespace BotSharp.Abstraction.Translation.Models;
+
+public class TranslationMemory
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("original_text")]
+    public string OriginalText { get; set; }
+
+    [JsonPropertyName("hash_text")]
+    public string HashText { get; set; }
+
+    [JsonPropertyName("translations")]
+    public List<TranslationMemoryItem> Translations { get; set; } = new List<TranslationMemoryItem>();
+}
+
+public class TranslationMemoryItem
+{
+    [JsonPropertyName("translated_text")]
+    public string TranslatedText { get; set; }
+
+    [JsonPropertyName("language")]
+    public string Language { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs

@@ -0,0 +1,48 @@
+namespace BotSharp.Abstraction.Translation.Models;
+
+public class TranslationMemoryQuery
+{
+    public string OriginalText { get; set; }
+    public string HashText { get; set; }
+    public string Language { get; set; }
+
+    public TranslationMemoryQuery()
+    {
+        
+    }
+
+    public override string ToString()
+    {
+        return $"[Origin: {OriginalText}] translates to [{Language}]";
+    }
+}
+
+public class TranslationMemoryInput : TranslationMemoryQuery
+{
+    public string TranslatedText { get; set; }
+
+    public TranslationMemoryInput()
+    {
+        
+    }
+
+    public override string ToString()
+    {
+        return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})";
+    }
+}
+
+public class TranslationMemoryOutput: TranslationMemoryQuery
+{
+    public string TranslatedText { get; set; }
+
+    public TranslationMemoryOutput()
+    {
+        
+    }
+
+    public override string ToString()
+    {
+        return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})";
+    }
+}

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -172,4 +172,8 @@
     <ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Translation\Models\" />
+  </ItemGroup>
+
 </Project>

src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs

@@ -4,11 +4,11 @@ namespace BotSharp.Core.Infrastructures;
 
 public static class Utilities
 {
-    public static string HashText(string password, string salt)
+    public static string HashTextMd5(string text)
     {
         using var md5 = System.Security.Cryptography.MD5.Create();
 
-        var data = md5.ComputeHash(Encoding.UTF8.GetBytes(password + salt));
+        var data = md5.ComputeHash(Encoding.UTF8.GetBytes(text));
         var sb = new StringBuilder();
         foreach (var c in data)
         {
@@ -17,6 +17,19 @@ public static class Utilities
         return sb.ToString();
     }
 
+    public static string HashTextSha256(string text)
+    {
+        using var sha256 = System.Security.Cryptography.SHA256.Create();
+
+        var data = sha256.ComputeHash(Encoding.UTF8.GetBytes(text));
+        var sb = new StringBuilder();
+        foreach(var c in data)
+        {
+            sb.Append(c.ToString("x2"));
+        }
+        return sb.ToString();
+    }
+
     public static (string, string) SplitAsTuple(this string str, string sep)
     {
         var splits = str.Split(sep);

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Tasks.Models;
+using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 
@@ -224,4 +225,11 @@ public class BotSharpDbContext : Database, IBotSharpRepository
         throw new NotImplementedException();
     }
     #endregion
+
+    #region Translation
+    public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
+        => throw new NotImplementedException();
+    public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs) =>
+        throw new NotImplementedException();
+    #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs

@@ -0,0 +1,156 @@
+using BotSharp.Abstraction.Translation.Models;
+using System.IO;
+
+namespace BotSharp.Core.Repository;
+
+public partial class FileRepository
+{
+    public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
+    {
+        var list = new List<TranslationMemoryOutput>();
+        if (queries.IsNullOrEmpty())
+        {
+            return list;
+        }
+
+        var dir = Path.Combine(_dbSettings.FileRepository, "translation");
+        var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE);
+        if (!Directory.Exists(dir) || !File.Exists(file))
+        {
+            return list;
+        }
+
+        var content = File.ReadAllText(file);
+        if (string.IsNullOrWhiteSpace(content))
+        {
+            return list;
+        }
+
+        var memories = ReadTranslationMemoryContent(content);
+        foreach (var query in queries)
+        {
+            if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language))
+            {
+                continue;
+            }
+
+            var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText));
+            if (foundMemory == null) continue;
+
+            var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(query.Language));
+            if (foundItem == null) continue;
+
+            list.Add(new TranslationMemoryOutput
+            {
+                OriginalText = query.OriginalText,
+                TranslatedText = foundItem.TranslatedText,
+                HashText = foundMemory.HashText,
+                Language = foundItem.Language,
+            });
+        }
+
+        return list;
+    }
+
+    public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs)
+    {
+        if (inputs.IsNullOrEmpty()) return false;
+
+        try
+        {
+            var dir = Path.Combine(_dbSettings.FileRepository, "translation");
+            var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE);
+            if (!Directory.Exists(dir))
+            {
+                Directory.CreateDirectory(dir);
+            }
+
+            var content = string.Empty;
+            if (File.Exists(file))
+            {
+                content = File.ReadAllText(file);
+            }
+            
+            var memories = ReadTranslationMemoryContent(content);
+
+            foreach (var input in inputs)
+            {
+                if (string.IsNullOrWhiteSpace(input.OriginalText) ||
+                    string.IsNullOrWhiteSpace(input.TranslatedText) ||
+                    string.IsNullOrWhiteSpace(input.HashText) ||
+                    string.IsNullOrWhiteSpace(input.Language))
+                {
+                    continue;
+                }
+
+                var newItem = new TranslationMemoryItem
+                {
+                    TranslatedText = input.TranslatedText,
+                    Language = input.Language
+                };
+
+                var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText));
+                if (foundMemory == null)
+                {
+                    var newMemory = new TranslationMemory
+                    {
+                        Id = Guid.NewGuid().ToString(),
+                        OriginalText = input.OriginalText,
+                        HashText = input.HashText,
+                        Translations = new List<TranslationMemoryItem> { newItem }
+                    };
+
+                    if (memories == null)
+                    {
+                        memories = new List<TranslationMemory> { newMemory };
+                    }
+                    else
+                    {
+                        memories.Add(newMemory);
+                    }
+                }
+                else
+                {
+                    var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(input.Language));
+                    if (foundItem != null) continue;
+
+                    if (foundMemory.Translations == null)
+                    {
+                        foundMemory.Translations = new List<TranslationMemoryItem> { newItem };
+                    }
+                    else
+                    {
+                        foundMemory.Translations.Add(newItem);
+                    }
+                }
+            }
+
+            var json = JsonSerializer.Serialize(memories, _options);
+            File.WriteAllText(file, json);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when saving translation memories: {ex.Message}");
+            return false;
+        }
+    }
+
+    private List<TranslationMemory> ReadTranslationMemoryContent(string? content)
+    {
+        var memories = new List<TranslationMemory>();
+
+        try
+        {
+            if (string.IsNullOrWhiteSpace(content))
+            {
+                return memories;
+            }
+
+           memories = JsonSerializer.Deserialize<List<TranslationMemory>>(content, _options) ?? new List<TranslationMemory>();
+        }
+        catch {}
+
+        return memories;
+    }
+}

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs

@@ -17,6 +17,7 @@ public partial class FileRepository : IBotSharpRepository
     private readonly AgentSettings _agentSettings;
     private readonly ConversationSetting _conversationSettings;
     private readonly StatisticsSettings _statisticsSetting;
+    private readonly ILogger<FileRepository> _logger;
     private JsonSerializerOptions _options;
 
     private const string AGENT_FILE = "agent.json";
@@ -34,19 +35,22 @@ public partial class FileRepository : IBotSharpRepository
     private const string PLUGIN_CONFIG_FILE = "config.json";
     private const string AGENT_TASK_PREFIX = "#metadata";
     private const string AGENT_TASK_SUFFIX = "/metadata";
+    private const string TRANSLATION_MEMORY_FILE = "memory.json";
 
     public FileRepository(
         IServiceProvider services,
         BotSharpDatabaseSettings dbSettings,
         AgentSettings agentSettings,
         ConversationSetting conversationSettings,
-        StatisticsSettings statisticsSettings)
+        StatisticsSettings statisticsSettings,
+         ILogger<FileRepository> logger)
     {
         _services = services;
         _dbSettings = dbSettings;
         _agentSettings = agentSettings;
         _conversationSettings = conversationSettings;
         _statisticsSetting = statisticsSettings;
+        _logger = logger;
 
         _options = new JsonSerializerOptions
         {

src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs

@@ -32,7 +32,7 @@ public partial class TwoStagePlanner : IPlaner
         if (_plan1st.IsNullOrEmpty() && _plan2nd.IsNullOrEmpty())
         {
             Directory.CreateDirectory(tempDir);
-            _md5 = Utilities.HashText(string.Join(".", dialogs.Where(x => x.Role == AgentRole.User)), "botsharp");
+            _md5 = Utilities.HashTextMd5($"{string.Join(".", dialogs.Where(x => x.Role == AgentRole.User))}{"botsharp"}");
             var filePath = Path.Combine(tempDir, $"{_md5}-1st.json");
             FirstStagePlan[] items = new FirstStagePlan[0];
             if (File.Exists(filePath))

src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs

@@ -1,4 +1,3 @@
-using Amazon.Runtime.Internal.Transform;
 using BotSharp.Abstraction.Infrastructures.Enums;
 using BotSharp.Abstraction.MLTasks;
 using BotSharp.Abstraction.Options;
@@ -13,17 +12,21 @@ namespace BotSharp.Core.Translation;
 public class TranslationService : ITranslationService
 {
     private readonly IServiceProvider _services;
+    private readonly IBotSharpRepository _db;
     private readonly ILogger<TranslationService> _logger;
     private readonly BotSharpOptions _options;
     private Agent _router;
     private string _messageId;
     private IChatCompletion _completion;
 
-    public TranslationService(IServiceProvider services,
+    public TranslationService(
+        IServiceProvider services,
+        IBotSharpRepository db,
         ILogger<TranslationService> logger,
         BotSharpOptions options)
     {
         _services = services;
+        _db = db;
         _logger = logger;
         _options = options;
     }
@@ -56,41 +59,73 @@ public class TranslationService : ITranslationService
             model: _router?.LlmConfig?.Model);
         var template = _router.Templates.First(x => x.Name == "translation_prompt").Content;
 
+        var map = new Dictionary<string, string>();
         var keys = unique.ToArray();
-        var texts = unique.ToArray()
+
+        #region Search memory
+        var queries = keys.Select(x => new TranslationMemoryQuery
+        {
+            OriginalText = x,
+            HashText = Utilities.HashTextSha256(x),
+            Language = language
+        }).ToList();
+        var memories = _db.GetTranslationMemories(queries);
+        var memoryHashes = memories.Select(x => x.HashText).ToList();
+
+        foreach (var memory in memories)
+        {
+            map[memory.OriginalText] = memory.TranslatedText;
+        }
+
+        var outOfMemoryList = queries.Where(x => !memoryHashes.Contains(x.HashText)).ToList();
+        #endregion
+
+        var texts = outOfMemoryList.ToArray()
             .Select((text, i) => new TranslationInput
             {
                 Id = i + 1,
-                Text = text
+                Text = text.OriginalText
             }).ToList();
 
         try
         {
-            var translatedStringList = await InnerTranslate(texts, language, template);
-
-            int retry = 0;
-            while (translatedStringList.Texts.Length != texts.Count && retry < 3)
+            if (!texts.IsNullOrEmpty())
             {
-                translatedStringList = await InnerTranslate(texts, language, template);
-                retry++;
+                var translatedStringList = await InnerTranslate(texts, language, template);
+
+                int retry = 0;
+                while (translatedStringList.Texts.Length != texts.Count && retry < 3)
+                {
+                    translatedStringList = await InnerTranslate(texts, language, template);
+                    retry++;
+                }
+
+                // Override language if it's Unknown, it's used to output the corresponding language.
+                var states = _services.GetRequiredService<IConversationStateService>();
+                if (!states.ContainsState(StateConst.LANGUAGE))
+                {
+                    var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage;
+                    states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1);
+                }
+
+                var translatedTexts = translatedStringList.Texts;
+                var memoryInputs = new List<TranslationMemoryInput>();
+
+                for (var i = 0; i < texts.Count; i++)
+                {
+                    map[outOfMemoryList[i].OriginalText] = translatedTexts[i].Text;
+                    memoryInputs.Add(new TranslationMemoryInput
+                    {
+                        OriginalText = outOfMemoryList[i].OriginalText,
+                        HashText = outOfMemoryList[i].HashText,
+                        TranslatedText = translatedTexts[i].Text,
+                        Language = language
+                    });
+                }
+
+                _db.SaveTranslationMemories(memoryInputs);
             }
-
-            // Override language if it's Unknown, it's used to output the corresponding language.
-            var states = _services.GetRequiredService<IConversationStateService>();
-            if (!states.ContainsState(StateConst.LANGUAGE))
-            {
-                var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage;
-                states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1);
-            }
-
-            var translatedTexts = translatedStringList.Texts;
-            var map = new Dictionary<string, string>();
-
-            for (var i = 0; i < texts.Count; i++)
-            {
-                map[keys[i]] = translatedTexts[i].Text;
-            }
-
+            
             clonedData = Assign(clonedData, map);
         }
         catch (Exception ex)

src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

@@ -61,7 +61,7 @@ public class UserService : IUserService
             record.Phone = "+" + Regex.Match(user.Phone, @"\d+").Value;
         }
         record.Salt = Guid.NewGuid().ToString("N");
-        record.Password = Utilities.HashText(user.Password, record.Salt);
+        record.Password = Utilities.HashTextMd5($"{user.Password}{record.Salt}");
 
         if (_setting.NewUserVerification)
         {

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/TranslationMemoryDocument.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Plugin.MongoStorage.Collections;
+
+public class TranslationMemoryDocument : MongoBase
+{
+    public string OriginalText { get; set; }
+    public string HashText { get; set; }
+    public List<TranslationMemoryMongoElement> Translations { get; set; } = new List<TranslationMemoryMongoElement>();
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs

@@ -0,0 +1,25 @@
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class TranslationMemoryMongoElement
+{
+    public string TranslatedText { get; set; }
+    public string Language { get; set; }
+
+    public static TranslationMemoryMongoElement ToMongoElement(TranslationMemoryItem item)
+    {
+        return new TranslationMemoryMongoElement
+        {
+            TranslatedText = item.TranslatedText,
+            Language = item.Language
+        };
+    }
+
+    public static TranslationMemoryItem ToDomainElement(TranslationMemoryMongoElement element)
+    {
+        return new TranslationMemoryItem
+        {
+            TranslatedText = element.TranslatedText,
+            Language = element.Language
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs

@@ -1,5 +1,3 @@
-using BotSharp.Plugin.MongoStorage.Collections;
-
 namespace BotSharp.Plugin.MongoStorage;
 
 public class MongoDbContext
@@ -117,4 +115,7 @@ public class MongoDbContext
 
     public IMongoCollection<PluginDocument> Plugins
         => Database.GetCollection<PluginDocument>($"{_collectionPrefix}_Plugins");
+
+    public IMongoCollection<TranslationMemoryDocument> TranslationMemories
+        => Database.GetCollection<TranslationMemoryDocument>($"{_collectionPrefix}_TranslationMemories");
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs

@@ -2,8 +2,6 @@ using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Routing.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
-using BotSharp.Plugin.MongoStorage.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs

@@ -1,6 +1,5 @@
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Tasks.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -1,9 +1,5 @@
 using BotSharp.Abstraction.Conversations.Models;
-using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.Repositories.Filters;
-using BotSharp.Abstraction.Repositories.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
-using BotSharp.Plugin.MongoStorage.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs

@@ -1,6 +1,4 @@
 using BotSharp.Abstraction.Loggers.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
-using BotSharp.Plugin.MongoStorage.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Plugins.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs

@@ -1,8 +1,5 @@
 using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Users.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
-using BotSharp.Plugin.MongoStorage.Models;
-
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs

@@ -0,0 +1,125 @@
+using Microsoft.Extensions.Logging;
+using System.Threading;
+
+namespace BotSharp.Plugin.MongoStorage.Repository;
+
+public partial class MongoRepository
+{
+    public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
+    {
+        var list = new List<TranslationMemoryOutput>();
+        if (queries.IsNullOrEmpty())
+        {
+            return list;
+        }
+
+        var hashTexts = queries.Where(x => !string.IsNullOrEmpty(x.HashText)).Select(x => x.HashText).ToList();
+        var filter = Builders<TranslationMemoryDocument>.Filter.In(x => x.HashText, hashTexts);
+        var memories = _dc.TranslationMemories.Find(filter).ToList();
+        if (memories.IsNullOrEmpty()) return list;
+
+        foreach (var query in queries)
+        {
+            if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language))
+            {
+                continue;
+            }
+
+            var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText));
+            if (foundMemory == null) continue;
+
+            var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(query.Language));
+            if (foundItem == null) continue;
+
+            list.Add(new TranslationMemoryOutput
+            {
+                OriginalText = query.OriginalText,
+                TranslatedText = foundItem.TranslatedText,
+                HashText = foundMemory.HashText,
+                Language = foundItem.Language,
+            });
+        }
+
+        return list;
+    }
+
+    public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs)
+    {
+        if (inputs.IsNullOrEmpty()) return false;
+
+        var hashTexts = inputs.Where(x => !string.IsNullOrEmpty(x.HashText)).Select(x => x.HashText).ToList();
+        var filter = Builders<TranslationMemoryDocument>.Filter.In(x => x.HashText, hashTexts);
+        var memories = _dc.TranslationMemories.Find(filter)?.ToList() ?? new List<TranslationMemoryDocument>();
+
+        var newMemories = new List<TranslationMemoryDocument>();
+        var updateMemories = new List<TranslationMemoryDocument>();
+
+        try
+        {
+            foreach (var input in inputs)
+            {
+                if (string.IsNullOrWhiteSpace(input.OriginalText) ||
+                    string.IsNullOrWhiteSpace(input.TranslatedText) ||
+                    string.IsNullOrWhiteSpace(input.HashText) ||
+                    string.IsNullOrWhiteSpace(input.Language))
+                {
+                    continue;
+                }
+
+                var newItem = new TranslationMemoryMongoElement
+                {
+                    TranslatedText = input.TranslatedText,
+                    Language = input.Language
+                };
+
+                var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText));
+                if (foundMemory == null)
+                {
+                    newMemories.Add(new TranslationMemoryDocument
+                    {
+                        Id = Guid.NewGuid().ToString(),
+                        OriginalText = input.OriginalText,
+                        HashText = input.HashText,
+                        Translations = new List<TranslationMemoryMongoElement> { newItem }
+                    });
+                }
+                else
+                {
+                    var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(input.Language));
+                    if (foundItem != null) continue;
+
+                    if (foundMemory.Translations == null)
+                    {
+                        foundMemory.Translations = new List<TranslationMemoryMongoElement> { newItem };
+                    }
+                    else
+                    {
+                        foundMemory.Translations.Add(newItem);
+                    }
+                    updateMemories.Add(foundMemory);
+                }
+            }
+
+            if (!newMemories.IsNullOrEmpty())
+            {
+                _dc.TranslationMemories.InsertMany(newMemories);
+            }
+
+            if (!updateMemories.IsNullOrEmpty())
+            {
+                foreach (var mem in updateMemories)
+                {
+                    var updateFilter = Builders<TranslationMemoryDocument>.Filter.Eq(x => x.Id, mem.Id);
+                    _dc.TranslationMemories.ReplaceOne(updateFilter, mem);
+                    Thread.Sleep(50);
+                }
+            }
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when saving translation memories: {ex.Message}");
+            return false;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Users.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Users.Models;
+using Microsoft.Extensions.Logging;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 
@@ -8,12 +9,17 @@ public partial class MongoRepository : IBotSharpRepository
 {
     private readonly MongoDbContext _dc;
     private readonly IServiceProvider _services;
+    private readonly ILogger<MongoRepository> _logger;
     private UpdateOptions _options;
 
-    public MongoRepository(MongoDbContext dc, IServiceProvider services)
+    public MongoRepository(
+        MongoDbContext dc,
+        IServiceProvider services,
+        ILogger<MongoRepository> logger)
     {
         _dc = dc;
         _services = services;
+        _logger = logger;
         _options = new UpdateOptions
         {
             IsUpsert = true,

src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs

@@ -7,8 +7,11 @@ global using BotSharp.Abstraction.Repositories;
 global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Utilities;
 global using BotSharp.Abstraction.Plugins;
+global using BotSharp.Abstraction.Translation.Models;
 global using Microsoft.Extensions.Configuration;
 global using Microsoft.Extensions.DependencyInjection;
 global using MongoDB.Bson;
 global using MongoDB.Driver;
-global using MongoDB.Bson.Serialization.Attributes;
+global using MongoDB.Bson.Serialization.Attributes;
+global using BotSharp.Plugin.MongoStorage.Collections;
+global using BotSharp.Plugin.MongoStorage.Models;