SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add mongo translation memory

Browse source
This commit is contained in:
Jicheng Lu 2024-06-14 17:10:25 -05:00
parent 64dca85d77
commit 54b551ae44
17 changed files with 175 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs
View file

@ -11,8 +11,8 @@ public class TranslationMemory
[JsonPropertyName("hash_text")]
public string HashText { get; set; }
[JsonPropertyName("memories")]
public List<TranslationMemoryItem> Memories { get; set; } = new List<TranslationMemoryItem>();
[JsonPropertyName("translations")]
public List<TranslationMemoryItem> Translations { get; set; } = new List<TranslationMemoryItem>();
}
public class TranslationMemoryItem

9
src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs
View file

@ -10,6 +10,11 @@ public class TranslationMemoryQuery
{
}
public override string ToString()
{
return $"[Origin: {OriginalText}] translates to [{Language}]";
}
}
public class TranslationMemoryInput : TranslationMemoryQuery
@ -23,7 +28,7 @@ public class TranslationMemoryInput : TranslationMemoryQuery
public override string ToString()
{
return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})";
return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})";
}
}
@ -38,6 +43,6 @@ public class TranslationMemoryOutput: TranslationMemoryQuery
public override string ToString()
{
return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})";
return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})";
}
}

12
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs
View file

@ -37,7 +37,7 @@ public partial class FileRepository
var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText));
if (foundMemory == null) continue;
var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(query.Language));
var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(query.Language));
if (foundItem == null) continue;
list.Add(new TranslationMemoryOutput
@ -97,7 +97,7 @@ public partial class FileRepository
Id = Guid.NewGuid().ToString(),
OriginalText = input.OriginalText,
HashText = input.HashText,
Memories = new List<TranslationMemoryItem> { newItem }
Translations = new List<TranslationMemoryItem> { newItem }
};
if (memories == null)
@ -111,16 +111,16 @@ public partial class FileRepository
}
else
{
var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(input.Language));
var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(input.Language));
if (foundItem != null) continue;
if (foundMemory.Memories == null)
if (foundMemory.Translations == null)
{
foundMemory.Memories = new List<TranslationMemoryItem> { newItem };
foundMemory.Translations = new List<TranslationMemoryItem> { newItem };
}
else
{
foundMemory.Memories.Add(newItem);
foundMemory.Translations.Add(newItem);
}
}
}

2
src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs
View file

@ -122,7 +122,7 @@ public class TranslationService : ITranslationService
Language = language
});
}
;
_db.SaveTranslationMemories(memoryInputs);
}

8
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/TranslationMemoryDocument.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Plugin.MongoStorage.Collections;
public class TranslationMemoryDocument : MongoBase
{
public string OriginalText { get; set; }
public string HashText { get; set; }
public List<TranslationMemoryMongoElement> Translations { get; set; } = new List<TranslationMemoryMongoElement>();
}

25
src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs Normal file
View file

@ -0,0 +1,25 @@
namespace BotSharp.Plugin.MongoStorage.Models;
public class TranslationMemoryMongoElement
{
public string TranslatedText { get; set; }
public string Language { get; set; }
public static TranslationMemoryMongoElement ToMongoElement(TranslationMemoryItem item)
{
return new TranslationMemoryMongoElement
{
TranslatedText = item.TranslatedText,
Language = item.Language
};
}
public static TranslationMemoryItem ToDomainElement(TranslationMemoryMongoElement element)
{
return new TranslationMemoryItem
{
TranslatedText = element.TranslatedText,
Language = element.Language
};
}
}

5
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Plugin.MongoStorage.Collections;
namespace BotSharp.Plugin.MongoStorage;
public class MongoDbContext
@ -117,4 +115,7 @@ public class MongoDbContext
public IMongoCollection<PluginDocument> Plugins
=> Database.GetCollection<PluginDocument>($"{_collectionPrefix}_Plugins");
public IMongoCollection<TranslationMemoryDocument> TranslationMemories
=> Database.GetCollection<TranslationMemoryDocument>($"{_collectionPrefix}_TranslationMemories");
}

2
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs
View file

@ -2,8 +2,6 @@ using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories.Filters;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Plugin.MongoStorage.Collections;
using BotSharp.Plugin.MongoStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Repository;

1
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Repositories.Filters;
using BotSharp.Abstraction.Tasks.Models;
using BotSharp.Plugin.MongoStorage.Collections;
namespace BotSharp.Plugin.MongoStorage.Repository;

4
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs
View file

@ -1,9 +1,5 @@
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Files;
using BotSharp.Abstraction.Repositories.Filters;
using BotSharp.Abstraction.Repositories.Models;
using BotSharp.Plugin.MongoStorage.Collections;
using BotSharp.Plugin.MongoStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Repository;

2
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs
View file

@ -1,6 +1,4 @@
using BotSharp.Abstraction.Loggers.Models;
using BotSharp.Plugin.MongoStorage.Collections;
using BotSharp.Plugin.MongoStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Repository;

1
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Plugin.MongoStorage.Collections;
namespace BotSharp.Plugin.MongoStorage.Repository;

3
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs
View file

@ -1,8 +1,5 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Users.Models;
using BotSharp.Plugin.MongoStorage.Collections;
using BotSharp.Plugin.MongoStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Repository;

115
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs
View file

@ -1,14 +1,125 @@
using Microsoft.Extensions.Logging;
using System.Threading;
namespace BotSharp.Plugin.MongoStorage.Repository;
public partial class MongoRepository
{
public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
{
throw new NotImplementedException();
var list = new List<TranslationMemoryOutput>();
if (queries.IsNullOrEmpty())
{
return list;
}
var hashTexts = queries.Where(x => !string.IsNullOrEmpty(x.HashText)).Select(x => x.HashText).ToList();
var filter = Builders<TranslationMemoryDocument>.Filter.In(x => x.HashText, hashTexts);
var memories = _dc.TranslationMemories.Find(filter).ToList();
if (memories.IsNullOrEmpty()) return list;
foreach (var query in queries)
{
if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language))
{
continue;
}
var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText));
if (foundMemory == null) continue;
var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(query.Language));
if (foundItem == null) continue;
list.Add(new TranslationMemoryOutput
{
OriginalText = query.OriginalText,
TranslatedText = foundItem.TranslatedText,
HashText = foundMemory.HashText,
Language = foundItem.Language,
});
}
return list;
}
public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs)
{
throw new NotImplementedException();
if (inputs.IsNullOrEmpty()) return false;
var hashTexts = inputs.Where(x => !string.IsNullOrEmpty(x.HashText)).Select(x => x.HashText).ToList();
var filter = Builders<TranslationMemoryDocument>.Filter.In(x => x.HashText, hashTexts);
var memories = _dc.TranslationMemories.Find(filter)?.ToList() ?? new List<TranslationMemoryDocument>();
var newMemories = new List<TranslationMemoryDocument>();
var updateMemories = new List<TranslationMemoryDocument>();
try
{
foreach (var input in inputs)
{
if (string.IsNullOrWhiteSpace(input.OriginalText) ||
string.IsNullOrWhiteSpace(input.TranslatedText) ||
string.IsNullOrWhiteSpace(input.HashText) ||
string.IsNullOrWhiteSpace(input.Language))
{
continue;
}
var newItem = new TranslationMemoryMongoElement
{
TranslatedText = input.TranslatedText,
Language = input.Language
};
var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText));
if (foundMemory == null)
{
newMemories.Add(new TranslationMemoryDocument
{
Id = Guid.NewGuid().ToString(),
OriginalText = input.OriginalText,
HashText = input.HashText,
Translations = new List<TranslationMemoryMongoElement> { newItem }
});
}
else
{
var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(input.Language));
if (foundItem != null) continue;
if (foundMemory.Translations == null)
{
foundMemory.Translations = new List<TranslationMemoryMongoElement> { newItem };
}
else
{
foundMemory.Translations.Add(newItem);
}
updateMemories.Add(foundMemory);
}
}
if (!newMemories.IsNullOrEmpty())
{
_dc.TranslationMemories.InsertMany(newMemories);
}
if (!updateMemories.IsNullOrEmpty())
{
foreach (var mem in updateMemories)
{
var updateFilter = Builders<TranslationMemoryDocument>.Filter.Eq(x => x.Id, mem.Id);
_dc.TranslationMemories.ReplaceOne(updateFilter, mem);
Thread.Sleep(50);
}
}
return true;
}
catch (Exception ex)
{
_logger.LogWarning($"Error when saving translation memories: {ex.Message}");
return false;
}
}
}

1
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Abstraction.Users.Models;
using BotSharp.Plugin.MongoStorage.Collections;
namespace BotSharp.Plugin.MongoStorage.Repository;

8
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Users.Models;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.MongoStorage.Repository;
@ -8,12 +9,17 @@ public partial class MongoRepository : IBotSharpRepository
{
private readonly MongoDbContext _dc;
private readonly IServiceProvider _services;
private readonly ILogger<MongoRepository> _logger;
private UpdateOptions _options;
public MongoRepository(MongoDbContext dc, IServiceProvider services)
public MongoRepository(
MongoDbContext dc,
IServiceProvider services,
ILogger<MongoRepository> logger)
{
_dc = dc;
_services = services;
_logger = logger;
_options = new UpdateOptions
{
IsUpsert = true,

4
src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs
View file

@ -12,4 +12,6 @@ global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using MongoDB.Bson;
global using MongoDB.Driver;
global using MongoDB.Bson.Serialization.Attributes;
global using MongoDB.Bson.Serialization.Attributes;
global using BotSharp.Plugin.MongoStorage.Collections;
global using BotSharp.Plugin.MongoStorage.Models;