Merge branch 'master' of https://github.com/SciSharp/BotSharp into features/upgrade-open-ai

src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Agents.Enums;
+
+public class BuiltInAgentId
+{
+    public const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a";
+    public const string Chatbot = "01e2fc5c-2c89-4ec7-8470-7688608b496c";
+    public const string HumanSupport = "01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b";
+}

src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj

@@ -10,6 +10,12 @@
     <OutputPath>$(SolutionDir)packages</OutputPath>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Remove="packages\**" />
+    <EmbeddedResource Remove="packages\**" />
+    <None Remove="packages\**" />
+  </ItemGroup>
+
   <ItemGroup>
     <None Include="..\..\..\arts\Icon.png">
       <Pack>True</Pack>

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionArgs.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Browsing.Models;
+
+public class BrowserActionArgs
+{
+
+}

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs

@@ -8,6 +8,7 @@ public class BrowserActionResult
     public string Selector { get; set; }
     public string Body { get; set; }
     public bool IsHighlighted { get; set; }
+    public DateTime? ExecutedAt { get; set; } = DateTime.UtcNow;
 
     public override string ToString()
     {

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs

@@ -31,4 +31,5 @@ public class ElementLocatingArgs
     /// Draw outline around the element
     /// </summary>
     public bool Highlight { get; set; }
+    public string HighlightColor { get; set; } = "red";
 }

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/PageActionArgs.cs

@@ -11,4 +11,11 @@ public class PageActionArgs
 
     public string Url { get; set; } = null!;
     public bool OpenNewTab { get; set; } = false;
+
+    /// <summary>
+    /// On page data fetched
+    /// </summary>
+    public DataFetched? OnDataFetched { get; set; }
 }
+
+public delegate void DataFetched(string url, string data);

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -2,6 +2,7 @@ using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Tasks.Models;
+using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
 
 namespace BotSharp.Abstraction.Repositories;
@@ -88,4 +89,10 @@ public interface IBotSharpRepository
     #region Statistics
     void IncrementConversationCount();
     #endregion
+
+    #region Translation
+    IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries);
+    bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs);
+
+    #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs

@@ -0,0 +1,25 @@
+namespace BotSharp.Abstraction.Translation.Models;
+
+public class TranslationMemory
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("original_text")]
+    public string OriginalText { get; set; }
+
+    [JsonPropertyName("hash_text")]
+    public string HashText { get; set; }
+
+    [JsonPropertyName("translations")]
+    public List<TranslationMemoryItem> Translations { get; set; } = new List<TranslationMemoryItem>();
+}
+
+public class TranslationMemoryItem
+{
+    [JsonPropertyName("translated_text")]
+    public string TranslatedText { get; set; }
+
+    [JsonPropertyName("language")]
+    public string Language { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs

@@ -0,0 +1,48 @@
+namespace BotSharp.Abstraction.Translation.Models;
+
+public class TranslationMemoryQuery
+{
+    public string OriginalText { get; set; }
+    public string HashText { get; set; }
+    public string Language { get; set; }
+
+    public TranslationMemoryQuery()
+    {
+        
+    }
+
+    public override string ToString()
+    {
+        return $"[Origin: {OriginalText}] translates to [{Language}]";
+    }
+}
+
+public class TranslationMemoryInput : TranslationMemoryQuery
+{
+    public string TranslatedText { get; set; }
+
+    public TranslationMemoryInput()
+    {
+        
+    }
+
+    public override string ToString()
+    {
+        return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})";
+    }
+}
+
+public class TranslationMemoryOutput: TranslationMemoryQuery
+{
+    public string TranslatedText { get; set; }
+
+    public TranslationMemoryOutput()
+    {
+        
+    }
+
+    public override string ToString()
+    {
+        return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})";
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs

@@ -37,6 +37,8 @@ public class Pagination
     {
         get { return (Page - 1) * Size; }
     }
+
+    public bool ReturnTotal { get; set; } = true;
 }
 
 public class PagedItems<T>

src/Infrastructure/BotSharp.Core/Agents/AgentPlugin.cs

@@ -17,9 +17,9 @@ public class AgentPlugin : IBotSharpPlugin
 
     public string[] AgentIds => new string[]
     {
-        "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",
-        "01e2fc5c-2c89-4ec7-8470-7688608b496c",
-        "01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b"
+        BuiltInAgentId.AIAssistant,
+        BuiltInAgentId.Chatbot,
+        BuiltInAgentId.HumanSupport
     };
 
     public object GetNewSettingsInstance() =>

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -180,8 +180,7 @@
 
   <ItemGroup>
     <PackageReference Include="Aspects.Cache" Version="2.0.4" />
-    <PackageReference Include="Colorful.Console" Version="1.2.15" />
-    <PackageReference Include="EntityFrameworkCore.BootKit" Version="8.3.1" />
+    <PackageReference Include="EntityFrameworkCore.BootKit" Version="8.4.2" />
     <PackageReference Include="Fluid.Core" Version="2.8.0" />
     <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
     <PackageReference Include="Nanoid" Version="3.0.0" />
@@ -192,4 +191,8 @@
     <ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Translation\Models\" />
+  </ItemGroup>
+
 </Project>

src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs

@@ -4,11 +4,11 @@ namespace BotSharp.Core.Infrastructures;
 
 public static class Utilities
 {
-    public static string HashText(string password, string salt)
+    public static string HashTextMd5(string text)
     {
         using var md5 = System.Security.Cryptography.MD5.Create();
 
-        var data = md5.ComputeHash(Encoding.UTF8.GetBytes(password + salt));
+        var data = md5.ComputeHash(Encoding.UTF8.GetBytes(text));
         var sb = new StringBuilder();
         foreach (var c in data)
         {
@@ -17,6 +17,19 @@ public static class Utilities
         return sb.ToString();
     }
 
+    public static string HashTextSha256(string text)
+    {
+        using var sha256 = System.Security.Cryptography.SHA256.Create();
+
+        var data = sha256.ComputeHash(Encoding.UTF8.GetBytes(text));
+        var sb = new StringBuilder();
+        foreach(var c in data)
+        {
+            sb.Append(c.ToString("x2"));
+        }
+        return sb.ToString();
+    }
+
     public static (string, string) SplitAsTuple(this string str, string sep)
     {
         var splits = str.Split(sep);

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Tasks.Models;
+using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 
@@ -224,4 +225,11 @@ public class BotSharpDbContext : Database, IBotSharpRepository
         throw new NotImplementedException();
     }
     #endregion
+
+    #region Translation
+    public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
+        => throw new NotImplementedException();
+    public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs) =>
+        throw new NotImplementedException();
+    #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs

@@ -0,0 +1,156 @@
+using BotSharp.Abstraction.Translation.Models;
+using System.IO;
+
+namespace BotSharp.Core.Repository;
+
+public partial class FileRepository
+{
+    public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
+    {
+        var list = new List<TranslationMemoryOutput>();
+        if (queries.IsNullOrEmpty())
+        {
+            return list;
+        }
+
+        var dir = Path.Combine(_dbSettings.FileRepository, "translation");
+        var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE);
+        if (!Directory.Exists(dir) || !File.Exists(file))
+        {
+            return list;
+        }
+
+        var content = File.ReadAllText(file);
+        if (string.IsNullOrWhiteSpace(content))
+        {
+            return list;
+        }
+
+        var memories = ReadTranslationMemoryContent(content);
+        foreach (var query in queries)
+        {
+            if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language))
+            {
+                continue;
+            }
+
+            var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText));
+            if (foundMemory == null) continue;
+
+            var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(query.Language));
+            if (foundItem == null) continue;
+
+            list.Add(new TranslationMemoryOutput
+            {
+                OriginalText = query.OriginalText,
+                TranslatedText = foundItem.TranslatedText,
+                HashText = foundMemory.HashText,
+                Language = foundItem.Language,
+            });
+        }
+
+        return list;
+    }
+
+    public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs)
+    {
+        if (inputs.IsNullOrEmpty()) return false;
+
+        try
+        {
+            var dir = Path.Combine(_dbSettings.FileRepository, "translation");
+            var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE);
+            if (!Directory.Exists(dir))
+            {
+                Directory.CreateDirectory(dir);
+            }
+
+            var content = string.Empty;
+            if (File.Exists(file))
+            {
+                content = File.ReadAllText(file);
+            }
+            
+            var memories = ReadTranslationMemoryContent(content);
+
+            foreach (var input in inputs)
+            {
+                if (string.IsNullOrWhiteSpace(input.OriginalText) ||
+                    string.IsNullOrWhiteSpace(input.TranslatedText) ||
+                    string.IsNullOrWhiteSpace(input.HashText) ||
+                    string.IsNullOrWhiteSpace(input.Language))
+                {
+                    continue;
+                }
+
+                var newItem = new TranslationMemoryItem
+                {
+                    TranslatedText = input.TranslatedText,
+                    Language = input.Language
+                };
+
+                var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText));
+                if (foundMemory == null)
+                {
+                    var newMemory = new TranslationMemory
+                    {
+                        Id = Guid.NewGuid().ToString(),
+                        OriginalText = input.OriginalText,
+                        HashText = input.HashText,
+                        Translations = new List<TranslationMemoryItem> { newItem }
+                    };
+
+                    if (memories == null)
+                    {
+                        memories = new List<TranslationMemory> { newMemory };
+                    }
+                    else
+                    {
+                        memories.Add(newMemory);
+                    }
+                }
+                else
+                {
+                    var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(input.Language));
+                    if (foundItem != null) continue;
+
+                    if (foundMemory.Translations == null)
+                    {
+                        foundMemory.Translations = new List<TranslationMemoryItem> { newItem };
+                    }
+                    else
+                    {
+                        foundMemory.Translations.Add(newItem);
+                    }
+                }
+            }
+
+            var json = JsonSerializer.Serialize(memories, _options);
+            File.WriteAllText(file, json);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when saving translation memories: {ex.Message}");
+            return false;
+        }
+    }
+
+    private List<TranslationMemory> ReadTranslationMemoryContent(string? content)
+    {
+        var memories = new List<TranslationMemory>();
+
+        try
+        {
+            if (string.IsNullOrWhiteSpace(content))
+            {
+                return memories;
+            }
+
+           memories = JsonSerializer.Deserialize<List<TranslationMemory>>(content, _options) ?? new List<TranslationMemory>();
+        }
+        catch {}
+
+        return memories;
+    }
+}

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs

@@ -17,6 +17,7 @@ public partial class FileRepository : IBotSharpRepository
     private readonly AgentSettings _agentSettings;
     private readonly ConversationSetting _conversationSettings;
     private readonly StatisticsSettings _statisticsSetting;
+    private readonly ILogger<FileRepository> _logger;
     private JsonSerializerOptions _options;
 
     private const string AGENT_FILE = "agent.json";
@@ -34,19 +35,22 @@ public partial class FileRepository : IBotSharpRepository
     private const string PLUGIN_CONFIG_FILE = "config.json";
     private const string AGENT_TASK_PREFIX = "#metadata";
     private const string AGENT_TASK_SUFFIX = "/metadata";
+    private const string TRANSLATION_MEMORY_FILE = "memory.json";
 
     public FileRepository(
         IServiceProvider services,
         BotSharpDatabaseSettings dbSettings,
         AgentSettings agentSettings,
         ConversationSetting conversationSettings,
-        StatisticsSettings statisticsSettings)
+        StatisticsSettings statisticsSettings,
+         ILogger<FileRepository> logger)
     {
         _services = services;
         _dbSettings = dbSettings;
         _agentSettings = agentSettings;
         _conversationSettings = conversationSettings;
         _statisticsSetting = statisticsSettings;
+        _logger = logger;
 
         _options = new JsonSerializerOptions
         {

src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs

@@ -32,7 +32,7 @@ public partial class TwoStagePlanner : IPlaner
         if (_plan1st.IsNullOrEmpty() && _plan2nd.IsNullOrEmpty())
         {
             Directory.CreateDirectory(tempDir);
-            _md5 = Utilities.HashText(string.Join(".", dialogs.Where(x => x.Role == AgentRole.User)), "botsharp");
+            _md5 = Utilities.HashTextMd5($"{string.Join(".", dialogs.Where(x => x.Role == AgentRole.User))}{"botsharp"}");
             var filePath = Path.Combine(tempDir, $"{_md5}-1st.json");
             FirstStagePlan[] items = new FirstStagePlan[0];
             if (File.Exists(filePath))

src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs

@@ -1,4 +1,3 @@
-using Amazon.Runtime.Internal.Transform;
 using BotSharp.Abstraction.Infrastructures.Enums;
 using BotSharp.Abstraction.MLTasks;
 using BotSharp.Abstraction.Options;
@@ -13,17 +12,21 @@ namespace BotSharp.Core.Translation;
 public class TranslationService : ITranslationService
 {
     private readonly IServiceProvider _services;
+    private readonly IBotSharpRepository _db;
     private readonly ILogger<TranslationService> _logger;
     private readonly BotSharpOptions _options;
     private Agent _router;
     private string _messageId;
     private IChatCompletion _completion;
 
-    public TranslationService(IServiceProvider services,
+    public TranslationService(
+        IServiceProvider services,
+        IBotSharpRepository db,
         ILogger<TranslationService> logger,
         BotSharpOptions options)
     {
         _services = services;
+        _db = db;
         _logger = logger;
         _options = options;
     }
@@ -56,41 +59,73 @@ public class TranslationService : ITranslationService
             model: _router?.LlmConfig?.Model);
         var template = _router.Templates.First(x => x.Name == "translation_prompt").Content;
 
+        var map = new Dictionary<string, string>();
         var keys = unique.ToArray();
-        var texts = unique.ToArray()
+
+        #region Search memory
+        var queries = keys.Select(x => new TranslationMemoryQuery
+        {
+            OriginalText = x,
+            HashText = Utilities.HashTextSha256(x),
+            Language = language
+        }).ToList();
+        var memories = _db.GetTranslationMemories(queries);
+        var memoryHashes = memories.Select(x => x.HashText).ToList();
+
+        foreach (var memory in memories)
+        {
+            map[memory.OriginalText] = memory.TranslatedText;
+        }
+
+        var outOfMemoryList = queries.Where(x => !memoryHashes.Contains(x.HashText)).ToList();
+        #endregion
+
+        var texts = outOfMemoryList.ToArray()
             .Select((text, i) => new TranslationInput
             {
                 Id = i + 1,
-                Text = text
+                Text = text.OriginalText
             }).ToList();
 
         try
         {
-            var translatedStringList = await InnerTranslate(texts, language, template);
-
-            int retry = 0;
-            while (translatedStringList.Texts.Length != texts.Count && retry < 3)
+            if (!texts.IsNullOrEmpty())
             {
-                translatedStringList = await InnerTranslate(texts, language, template);
-                retry++;
+                var translatedStringList = await InnerTranslate(texts, language, template);
+
+                int retry = 0;
+                while (translatedStringList.Texts.Length != texts.Count && retry < 3)
+                {
+                    translatedStringList = await InnerTranslate(texts, language, template);
+                    retry++;
+                }
+
+                // Override language if it's Unknown, it's used to output the corresponding language.
+                var states = _services.GetRequiredService<IConversationStateService>();
+                if (!states.ContainsState(StateConst.LANGUAGE))
+                {
+                    var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage;
+                    states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1);
+                }
+
+                var translatedTexts = translatedStringList.Texts;
+                var memoryInputs = new List<TranslationMemoryInput>();
+
+                for (var i = 0; i < texts.Count; i++)
+                {
+                    map[outOfMemoryList[i].OriginalText] = translatedTexts[i].Text;
+                    memoryInputs.Add(new TranslationMemoryInput
+                    {
+                        OriginalText = outOfMemoryList[i].OriginalText,
+                        HashText = outOfMemoryList[i].HashText,
+                        TranslatedText = translatedTexts[i].Text,
+                        Language = language
+                    });
+                }
+
+                _db.SaveTranslationMemories(memoryInputs);
             }
-
-            // Override language if it's Unknown, it's used to output the corresponding language.
-            var states = _services.GetRequiredService<IConversationStateService>();
-            if (!states.ContainsState(StateConst.LANGUAGE))
-            {
-                var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage;
-                states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1);
-            }
-
-            var translatedTexts = translatedStringList.Texts;
-            var map = new Dictionary<string, string>();
-
-            for (var i = 0; i < texts.Count; i++)
-            {
-                map[keys[i]] = translatedTexts[i].Text;
-            }
-
+            
             clonedData = Assign(clonedData, map);
         }
         catch (Exception ex)

src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs

@@ -61,7 +61,7 @@ public class UserService : IUserService
             record.Phone = "+" + Regex.Match(user.Phone, @"\d+").Value;
         }
         record.Salt = Guid.NewGuid().ToString("N");
-        record.Password = Utilities.HashText(user.Password, record.Salt);
+        record.Password = Utilities.HashTextMd5($"{user.Password}{record.Salt}");
 
         if (_setting.NewUserVerification)
         {

src/Infrastructure/BotSharp.Core/Using.cs

@@ -36,4 +36,3 @@ global using BotSharp.Core.Conversations.Services;
 global using BotSharp.Core.Infrastructures;
 global using BotSharp.Core.Users.Services;
 global using Aspects.Cache;
-global using Console = Colorful.Console;

src/Infrastructure/BotSharp.OpenAPI/Controllers/TranslationController.cs

@@ -0,0 +1,30 @@
+using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Abstraction.Translation;
+using BotSharp.OpenAPI.ViewModels.Translations;
+
+namespace BotSharp.OpenAPI.Controllers;
+
+[Authorize]
+[ApiController]
+public class TranslationController : ControllerBase
+{
+    private readonly IServiceProvider _services;
+
+    public TranslationController(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    [HttpPost("/translate")]
+    public async Task<TranslationResponseModel> Translate([FromBody] TranslationRequestModel model)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.LoadAgent(BuiltInAgentId.AIAssistant);
+        var translator = _services.GetRequiredService<ITranslationService>();
+        var text = await translator.Translate(agent, Guid.NewGuid().ToString(), model.Text, language: model.ToLang);
+        return new TranslationResponseModel
+        {
+            Text = text
+        };
+    }
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Translations/TranslationRequestModel.cs

@@ -0,0 +1,9 @@
+using BotSharp.Abstraction.Infrastructures.Enums;
+
+namespace BotSharp.OpenAPI.ViewModels.Translations;
+
+public class TranslationRequestModel
+{
+    public string Text { get; set; } = null!;
+    public string ToLang { get; set; } = LanguageType.CHINESE;
+}

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Translations/TranslationResponseModel.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.OpenAPI.ViewModels.Translations;
+
+public class TranslationResponseModel
+{
+    public string Text { get; set; } = null!;
+
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string FromLang { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/TranslationMemoryDocument.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Plugin.MongoStorage.Collections;
+
+public class TranslationMemoryDocument : MongoBase
+{
+    public string OriginalText { get; set; }
+    public string HashText { get; set; }
+    public List<TranslationMemoryMongoElement> Translations { get; set; } = new List<TranslationMemoryMongoElement>();
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs

@@ -0,0 +1,25 @@
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class TranslationMemoryMongoElement
+{
+    public string TranslatedText { get; set; }
+    public string Language { get; set; }
+
+    public static TranslationMemoryMongoElement ToMongoElement(TranslationMemoryItem item)
+    {
+        return new TranslationMemoryMongoElement
+        {
+            TranslatedText = item.TranslatedText,
+            Language = item.Language
+        };
+    }
+
+    public static TranslationMemoryItem ToDomainElement(TranslationMemoryMongoElement element)
+    {
+        return new TranslationMemoryItem
+        {
+            TranslatedText = element.TranslatedText,
+            Language = element.Language
+        };
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs

@@ -1,5 +1,3 @@
-using BotSharp.Plugin.MongoStorage.Collections;
-
 namespace BotSharp.Plugin.MongoStorage;
 
 public class MongoDbContext
@@ -117,4 +115,7 @@ public class MongoDbContext
 
     public IMongoCollection<PluginDocument> Plugins
         => Database.GetCollection<PluginDocument>($"{_collectionPrefix}_Plugins");
+
+    public IMongoCollection<TranslationMemoryDocument> TranslationMemories
+        => Database.GetCollection<TranslationMemoryDocument>($"{_collectionPrefix}_TranslationMemories");
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs

@@ -2,8 +2,6 @@ using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Routing.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
-using BotSharp.Plugin.MongoStorage.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs

@@ -1,6 +1,5 @@
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Tasks.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -1,9 +1,5 @@
 using BotSharp.Abstraction.Conversations.Models;
-using BotSharp.Abstraction.Files;
 using BotSharp.Abstraction.Repositories.Filters;
-using BotSharp.Abstraction.Repositories.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
-using BotSharp.Plugin.MongoStorage.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs

@@ -1,6 +1,4 @@
 using BotSharp.Abstraction.Loggers.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
-using BotSharp.Plugin.MongoStorage.Models;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Plugins.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs

@@ -1,8 +1,5 @@
 using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Users.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
-using BotSharp.Plugin.MongoStorage.Models;
-
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs

@@ -0,0 +1,125 @@
+using Microsoft.Extensions.Logging;
+using System.Threading;
+
+namespace BotSharp.Plugin.MongoStorage.Repository;
+
+public partial class MongoRepository
+{
+    public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
+    {
+        var list = new List<TranslationMemoryOutput>();
+        if (queries.IsNullOrEmpty())
+        {
+            return list;
+        }
+
+        var hashTexts = queries.Where(x => !string.IsNullOrEmpty(x.HashText)).Select(x => x.HashText).ToList();
+        var filter = Builders<TranslationMemoryDocument>.Filter.In(x => x.HashText, hashTexts);
+        var memories = _dc.TranslationMemories.Find(filter).ToList();
+        if (memories.IsNullOrEmpty()) return list;
+
+        foreach (var query in queries)
+        {
+            if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language))
+            {
+                continue;
+            }
+
+            var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText));
+            if (foundMemory == null) continue;
+
+            var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(query.Language));
+            if (foundItem == null) continue;
+
+            list.Add(new TranslationMemoryOutput
+            {
+                OriginalText = query.OriginalText,
+                TranslatedText = foundItem.TranslatedText,
+                HashText = foundMemory.HashText,
+                Language = foundItem.Language,
+            });
+        }
+
+        return list;
+    }
+
+    public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs)
+    {
+        if (inputs.IsNullOrEmpty()) return false;
+
+        var hashTexts = inputs.Where(x => !string.IsNullOrEmpty(x.HashText)).Select(x => x.HashText).ToList();
+        var filter = Builders<TranslationMemoryDocument>.Filter.In(x => x.HashText, hashTexts);
+        var memories = _dc.TranslationMemories.Find(filter)?.ToList() ?? new List<TranslationMemoryDocument>();
+
+        var newMemories = new List<TranslationMemoryDocument>();
+        var updateMemories = new List<TranslationMemoryDocument>();
+
+        try
+        {
+            foreach (var input in inputs)
+            {
+                if (string.IsNullOrWhiteSpace(input.OriginalText) ||
+                    string.IsNullOrWhiteSpace(input.TranslatedText) ||
+                    string.IsNullOrWhiteSpace(input.HashText) ||
+                    string.IsNullOrWhiteSpace(input.Language))
+                {
+                    continue;
+                }
+
+                var newItem = new TranslationMemoryMongoElement
+                {
+                    TranslatedText = input.TranslatedText,
+                    Language = input.Language
+                };
+
+                var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText));
+                if (foundMemory == null)
+                {
+                    newMemories.Add(new TranslationMemoryDocument
+                    {
+                        Id = Guid.NewGuid().ToString(),
+                        OriginalText = input.OriginalText,
+                        HashText = input.HashText,
+                        Translations = new List<TranslationMemoryMongoElement> { newItem }
+                    });
+                }
+                else
+                {
+                    var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(input.Language));
+                    if (foundItem != null) continue;
+
+                    if (foundMemory.Translations == null)
+                    {
+                        foundMemory.Translations = new List<TranslationMemoryMongoElement> { newItem };
+                    }
+                    else
+                    {
+                        foundMemory.Translations.Add(newItem);
+                    }
+                    updateMemories.Add(foundMemory);
+                }
+            }
+
+            if (!newMemories.IsNullOrEmpty())
+            {
+                _dc.TranslationMemories.InsertMany(newMemories);
+            }
+
+            if (!updateMemories.IsNullOrEmpty())
+            {
+                foreach (var mem in updateMemories)
+                {
+                    var updateFilter = Builders<TranslationMemoryDocument>.Filter.Eq(x => x.Id, mem.Id);
+                    _dc.TranslationMemories.ReplaceOne(updateFilter, mem);
+                    Thread.Sleep(50);
+                }
+            }
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when saving translation memories: {ex.Message}");
+            return false;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Users.Models;
-using BotSharp.Plugin.MongoStorage.Collections;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Agents.Models;
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Users.Models;
+using Microsoft.Extensions.Logging;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 
@@ -8,12 +9,17 @@ public partial class MongoRepository : IBotSharpRepository
 {
     private readonly MongoDbContext _dc;
     private readonly IServiceProvider _services;
+    private readonly ILogger<MongoRepository> _logger;
     private UpdateOptions _options;
 
-    public MongoRepository(MongoDbContext dc, IServiceProvider services)
+    public MongoRepository(
+        MongoDbContext dc,
+        IServiceProvider services,
+        ILogger<MongoRepository> logger)
     {
         _dc = dc;
         _services = services;
+        _logger = logger;
         _options = new UpdateOptions
         {
             IsUpsert = true,

src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs

@@ -7,8 +7,11 @@ global using BotSharp.Abstraction.Repositories;
 global using BotSharp.Abstraction.Agents.Enums;
 global using BotSharp.Abstraction.Utilities;
 global using BotSharp.Abstraction.Plugins;
+global using BotSharp.Abstraction.Translation.Models;
 global using Microsoft.Extensions.Configuration;
 global using Microsoft.Extensions.DependencyInjection;
 global using MongoDB.Bson;
 global using MongoDB.Driver;
-global using MongoDB.Bson.Serialization.Attributes;
+global using MongoDB.Bson.Serialization.Attributes;
+global using BotSharp.Plugin.MongoStorage.Collections;
+global using BotSharp.Plugin.MongoStorage.Models;

src/Plugins/BotSharp.Plugin.SqlDriver/Using.cs

@@ -21,4 +21,3 @@ global using BotSharp.Plugin.SqlDriver.Hooks;
 global using BotSharp.Plugin.SqlDriver.Services;
 global using BotSharp.Plugin.SqlHero.Settings;
 global using System.Drawing;
-global using Console = Colorful.Console;

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs

@@ -1,3 +1,4 @@
+using Microsoft.Playwright;
 using System.IO;
 
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
@@ -6,31 +7,41 @@ public class PlaywrightInstance : IDisposable
 {
     IPlaywright _playwright;
     Dictionary<string, IBrowserContext> _contexts = new Dictionary<string, IBrowserContext>();
+    Dictionary<string, List<IPage>> _pages = new Dictionary<string, List<IPage>>();
+
+    /// <summary>
+    /// ContextId and BrowserContext
+    /// </summary>
     public Dictionary<string, IBrowserContext> Contexts => _contexts;
 
-    public IPage GetPage(string id)
+    /// <summary>
+    /// ContextId and Pages
+    /// </summary>
+    public Dictionary<string, List<IPage>> Pages => _pages;
+
+    public IPage GetPage(string id, string? pattern = null)
     {
         InitInstance(id).Wait();
         return _contexts[id].Pages.LastOrDefault();
     }
 
-    public async Task<IBrowserContext> InitInstance(string id)
+    public async Task<IBrowserContext> InitInstance(string ctxId)
     {
         if (_playwright == null)
         {
             _playwright = await Playwright.CreateAsync();
         }
-        return await InitContext(id);
+        return await InitContext(ctxId);
     }
 
-    public async Task<IBrowserContext> InitContext(string id)
+    public async Task<IBrowserContext> InitContext(string ctxId)
     {
-        if (_contexts.ContainsKey(id))
-            return _contexts[id];
+        if (_contexts.ContainsKey(ctxId))
+            return _contexts[ctxId];
 
-        string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{id}";
+        string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{ctxId}";
 
-        _contexts[id] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
+        _contexts[ctxId] = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
         {
 #if DEBUG
             Headless = false,
@@ -42,63 +53,110 @@ public class PlaywrightInstance : IDisposable
             [
                 "--enable-automation",
             ],
-            Args = 
+            Args =
             [
                 "--disable-infobars",
                 "--test-type"
                 // "--start-maximized"
             ]
         });
+        _pages[ctxId] = new List<IPage>();
 
-        _contexts[id].Page += async (sender, e) =>
+        _contexts[ctxId].Page += async (sender, page) =>
         {
-            e.Close += async (sender, e) =>
+            _pages[ctxId].Add(page);
+            page.Close += async (sender, e) =>
             {
+                _pages[ctxId].Remove(e);
                 Serilog.Log.Information($"Page is closed: {e.Url}");
             };
-            Serilog.Log.Information($"New page is created: {e.Url}");
-            await e.SetViewportSizeAsync(1280, 800);
+            Serilog.Log.Information($"New page is created: {page.Url}");
+            if (!page.IsClosed)
+            {
+                await page.SetViewportSizeAsync(1600, 900);
+            }
+
+            /*page.Response += async (sender, e) =>
+            {
+                Serilog.Log.Information($"Response: {e.Url}");
+                if (e.Headers.ContainsKey("content-type") && e.Headers["content-type"].Contains("application/json"))
+                {
+                    var json = await e.JsonAsync();
+                    Serilog.Log.Information(json.ToString());
+                }
+            };*/
         };
 
-        _contexts[id].Close += async (sender, e) =>
+        _contexts[ctxId].Close += async (sender, e) =>
         {
             Serilog.Log.Warning($"Playwright browser context is closed");
-            _contexts.Remove(id);
+            _pages.Remove(ctxId);
+            _contexts.Remove(ctxId);
         };
 
-        return _contexts[id];
+        return _contexts[ctxId];
     }
 
-    public async Task<IPage> NewPage(string id)
+    public async Task<IPage> NewPage(string ctxId, DataFetched? fetched)
     {
-        await InitContext(id);
-        return await _contexts[id].NewPageAsync();
-    }
+        await InitContext(ctxId);
+        var page = await _contexts[ctxId].NewPageAsync();
 
-    public async Task Wait(string id)
-    {
-        if (_contexts.ContainsKey(id))
+        if (fetched != null)
+        {
+            page.Response += async (sender, e) =>
+            {
+                if (e.Headers.ContainsKey("content-type") &&
+                    e.Headers["content-type"].Contains("application/json") &&
+                    e.Request.ResourceType == "fetch")
+                {
+                    Serilog.Log.Information($"Response: {e.Url}");
+                    var json = await e.JsonAsync();
+                    fetched(e.Url.ToLower(), JsonSerializer.Serialize(json));
+                }
+            };
+        }
+
+        return page;
+    }
+
+    /// <summary>
+    /// Wait page and network until timeout in seconds
+    /// </summary>
+    /// <param name="ctxId"></param>
+    /// <param name="timeout">seconds</param>
+    /// <returns></returns>
+    public async Task Wait(string ctxId, int timeout = 60)
+    {
+        foreach (var page in _pages[ctxId])
         {
-            var page = _contexts[id].Pages.Last();
             await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
-            await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
+            await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new PageWaitForLoadStateOptions
+            {
+                Timeout = 1000 * timeout
+            });
         }
         await Task.Delay(100);
     }
 
-    public async Task Close(string id)
+    public async Task Close(string ctxId)
     {
-        if (_contexts.ContainsKey(id))
+        if (_contexts.ContainsKey(ctxId))
         {
-            await _contexts[id].CloseAsync();
+            await _contexts[ctxId].CloseAsync();
         }
     }
 
-    public async Task CloseCurrentPage(string id)
+    public async Task CloseCurrentPage(string ctxId)
     {
-        if (_contexts.ContainsKey(id))
+        var pages = _pages[ctxId].ToArray();
+        for (var i = 0; i < pages.Length; i++)
         {
-            await GetPage(id).CloseAsync();
+            var page = _pages[ctxId].FirstOrDefault();
+            if (page != null)
+            {
+                await page.CloseAsync();
+            }
         }
     }
 

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs

@@ -1,3 +1,5 @@
+using System.Linq;
+
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
 public partial class PlaywrightWebDriver
@@ -9,7 +11,7 @@ public partial class PlaywrightWebDriver
         try
         {
             // Check if the page is already open
-            if (!args.OpenNewTab && context.Pages.Count > 0)
+            /*if (!args.OpenNewTab && context.Pages.Count > 0)
             {
                 foreach (var p in context.Pages)
                 {
@@ -18,14 +20,15 @@ public partial class PlaywrightWebDriver
                         // Disable this due to performance issue, some page is too large
                         // result.Body = await p.ContentAsync();
                         result.IsSuccess = true;
-                        await p.BringToFrontAsync();
+                        // await p.BringToFrontAsync();
                         return result;
                     }
                 }
-            }
+            }*/
 
-            var page = args.OpenNewTab ? await _instance.NewPage(message.ContextId) : 
+            var page = args.OpenNewTab ? await _instance.NewPage(message.ContextId, fetched: args.OnDataFetched) : 
                 _instance.GetPage(message.ContextId);
+
             var response = await page.GotoAsync(args.Url);
             await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
             await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
@@ -50,4 +53,14 @@ public partial class PlaywrightWebDriver
         
         return result;
     }
+
+    private void Page_Response1(object sender, IResponse e)
+    {
+        throw new NotImplementedException();
+    }
+
+    private void Page_Response(object sender, IResponse e)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs

@@ -1,4 +1,4 @@
-using System.Xml.Linq;
+using System.Web;
 
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
@@ -88,7 +88,8 @@ public partial class PlaywrightWebDriver
             }*/
 
             var html = await locator.InnerHTMLAsync();
-            result.Body = html;
+            // fix if html has &
+            result.Body = HttpUtility.HtmlDecode(html);
             result.IsSuccess = true;
         }
         else if (count > 1)
@@ -129,7 +130,7 @@ public partial class PlaywrightWebDriver
 
             await page.EvaluateAsync($@"
                 (element) => {{
-                    element.style.outline = '2px solid red';
+                    element.style.outline = '2px solid {location.HighlightColor}';
                 }}", handle);
 
             result.IsHighlighted = true;