SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #214 from iceljc/features/split-conversation-dialog

Browse source 
parse conv dialog
This commit is contained in:
Haiping 2023-11-21 19:34:39 -06:00 committed by GitHub
parent 8aaa3c6b50 37f506687d
commit 8a6593dc01
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 164 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs
View file

@ -10,7 +10,7 @@ public class Conversation
public string Title { get; set; } = string.Empty;
[JsonIgnore]
public string Dialog { get; set; } = string.Empty;
public List<DialogElement> Dialogs { get; set; } = new List<DialogElement>();
[JsonIgnore]
public ConversationState States { get; set; } = new ConversationState();
@ -22,3 +22,20 @@ public class Conversation
public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;
public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
}
public class DialogElement
{
public string MetaData { get; set; }
public string Content { get; set; }
public DialogElement()
{
}
public DialogElement(string meta, string content)
{
MetaData = meta;
Content = content;
}
}

4
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -28,8 +28,8 @@ public interface IBotSharpRepository
#region Conversation
void CreateNewConversation(Conversation conversation);
string GetConversationDialog(string conversationId);
void UpdateConversationDialog(string conversationId, string dialogs);
List<DialogElement> GetConversationDialogs(string conversationId);
void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs);
List<StateKeyValue> GetConversationStates(string conversationId);
void UpdateConversationStates(string conversationId, List<StateKeyValue> states);
void UpdateConversationStatus(string conversationId, string status);

1
src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.Repositories;
using System.IO;

50
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Abstraction.Repositories;
using System;
using System.IO;
namespace BotSharp.Core.Conversations.Services;
@ -19,49 +20,44 @@ public class ConversationStorage : IConversationStorage
{
var agentId = dialog.CurrentAgentId;
var db = _services.GetRequiredService<IBotSharpRepository>();
var dialogText = db.GetConversationDialog(conversationId);
var sb = new StringBuilder(dialogText);
var dialogElements = new List<DialogElement>();
if (dialog.Role == AgentRole.Function)
{
// var args = dialog.FunctionArgs.RemoveNewLine();
sb.AppendLine($"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{dialog.MessageId}|{dialog.FunctionName}");
var content = dialog.Content;
content = content.RemoveNewLine();
if (string.IsNullOrEmpty(content))
{
return;
}
sb.AppendLine($" - {content}");
}
else
{
sb.AppendLine($"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{dialog.MessageId}|{dialog.SenderId}");
var meta = $"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{dialog.MessageId}|{dialog.FunctionName}";
var content = dialog.Content.RemoveNewLine();
if (string.IsNullOrEmpty(content))
{
return;
}
sb.AppendLine($" - {content}");
dialogElements.Add(new DialogElement(meta, content));
}
else
{
var meta = $"{dialog.CreatedAt}|{dialog.Role}|{agentId}|{dialog.MessageId}|{dialog.SenderId}";
var content = dialog.Content.RemoveNewLine();
if (string.IsNullOrEmpty(content))
{
return;
}
dialogElements.Add(new DialogElement(meta, content));
}
var updatedDialogs = sb.ToString();
db.UpdateConversationDialog(conversationId, updatedDialogs);
db.AppendConversationDialogs(conversationId, dialogElements);
}
public List<RoleDialogModel> GetDialogs(string conversationId)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var dialogText = db.GetConversationDialog(conversationId);
var dialogs = dialogText.SplitByNewLine();
var dialogs = db.GetConversationDialogs(conversationId);
var results = new List<RoleDialogModel>();
for (int i = 0; i < dialogs.Length; i += 2)
foreach (var dialog in dialogs)
{
var meta = dialogs[i];
var dialog = dialogs[i + 1];
var meta = dialog.MetaData;
var content = dialog.Content;
var blocks = meta.Split('|');
var createdAt = DateTime.Parse(blocks[0]);
var role = blocks[1];
@ -69,13 +65,11 @@ public class ConversationStorage : IConversationStorage
var messageId = blocks[3];
var senderId = role == AgentRole.Function ? currentAgentId : blocks[4];
var function = role == AgentRole.Function ? blocks[4] : null;
var text = dialog.Substring(4);
results.Add(new RoleDialogModel(role, text)
results.Add(new RoleDialogModel(role, content)
{
CurrentAgentId = currentAgentId,
MessageId = messageId,
Content = text,
CreatedAt = createdAt,
SenderId = senderId,
FunctionName = function

4
src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file

@ -136,7 +136,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
throw new NotImplementedException();
}
public string GetConversationDialog(string conversationId)
public List<DialogElement> GetConversationDialogs(string conversationId)
{
throw new NotImplementedException();
}
@ -146,7 +146,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
throw new NotImplementedException();
}
public void UpdateConversationDialog(string conversationId, string dialogs)
public void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs)
{
throw new NotImplementedException();
}

95
src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs
View file

@ -625,22 +625,20 @@ public class FileRepository : IBotSharpRepository
}
}
public string GetConversationDialog(string conversationId)
public List<DialogElement> GetConversationDialogs(string conversationId)
{
var dialogs = new List<DialogElement>();
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var dialogDir = Path.Combine(convDir, "dialogs.txt");
if (File.Exists(dialogDir))
{
return File.ReadAllText(dialogDir);
}
dialogs = CollectDialogElements(dialogDir);
}
return string.Empty;
return dialogs;
}
public void UpdateConversationDialog(string conversationId, string dialogs)
public void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs)
{
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
@ -648,7 +646,8 @@ public class FileRepository : IBotSharpRepository
var dialogDir = Path.Combine(convDir, "dialogs.txt");
if (File.Exists(dialogDir))
{
File.WriteAllText(dialogDir, dialogs);
var texts = ParseDialogElements(dialogs);
File.AppendAllLines(dialogDir, texts);
}
}
@ -662,15 +661,7 @@ public class FileRepository : IBotSharpRepository
if (!string.IsNullOrEmpty(convDir))
{
var stateDir = Path.Combine(convDir, "state.dict");
if (File.Exists(stateDir))
{
var dict = File.ReadAllLines(stateDir);
foreach (var line in dict)
{
var data = line.Split('=');
curStates.Add(new StateKeyValue(data[0], data[1]));
}
}
curStates = CollectConversationStates(stateDir);
}
return curStates;
@ -721,16 +712,16 @@ public class FileRepository : IBotSharpRepository
var record = JsonSerializer.Deserialize<Conversation>(content, _options);
var dialogFile = Path.Combine(convDir, "dialogs.txt");
if (record != null && File.Exists(dialogFile))
if (record != null)
{
record.Dialog = File.ReadAllText(dialogFile);
record.Dialogs = CollectDialogElements(dialogFile);
}
var stateFile = Path.Combine(convDir, "state.dict");
if (record != null && File.Exists(stateFile))
if (record != null)
{
var states = File.ReadLines(stateFile);
record.States = new ConversationState(states.Select(x => new StateKeyValue(x.Split('=')[0], x.Split('=')[1])).ToList());
var states = CollectConversationStates(stateFile);
record.States = new ConversationState(states);
}
return record;
@ -772,14 +763,13 @@ public class FileRepository : IBotSharpRepository
var json = File.ReadAllText(path);
var record = JsonSerializer.Deserialize<Conversation>(json, _options);
if (record != null)
{
records.Add(record);
}
if (record == null) continue;
records.Add(record);
}
return records.GroupBy(r => r.UserId)
.Select(g => g.OrderByDescending(x => x.CreatedTime).First())
.ToList();
.Select(g => g.OrderByDescending(x => x.CreatedTime).First())
.ToList();
}
public void AddExectionLogs(string conversationId, List<string> logs)
@ -944,5 +934,54 @@ public class FileRepository : IBotSharpRepository
return null;
}
private List<DialogElement> CollectDialogElements(string dialogDir)
{
var dialogs = new List<DialogElement>();
if (!File.Exists(dialogDir)) return dialogs;
var rawDialogs = File.ReadAllLines(dialogDir);
if (!rawDialogs.IsNullOrEmpty())
{
for (int i = 0; i < rawDialogs.Count(); i += 2)
{
var meta = rawDialogs[i];
var content = rawDialogs[i + 1];
var trimmed = content.Substring(4);
dialogs.Add(new DialogElement(meta, trimmed));
}
}
return dialogs;
}
private List<string> ParseDialogElements(List<DialogElement> dialogs)
{
var dialogTexts = new List<string>();
if (dialogs.IsNullOrEmpty()) return dialogTexts;
foreach (var element in dialogs)
{
dialogTexts.Add(element.MetaData);
var content = $" - {element.Content}";
dialogTexts.Add(content);
}
return dialogTexts;
}
private List<StateKeyValue> CollectConversationStates(string stateDir)
{
var states = new List<StateKeyValue>();
if (!File.Exists(stateDir)) return states;
var dict = File.ReadAllLines(stateDir);
foreach (var line in dict)
{
var data = line.Split('=');
states.Add(new StateKeyValue(data[0], data[1]));
}
return states;
}
#endregion
}

4
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogCollection.cs
View file

@ -1,7 +1,9 @@
using BotSharp.Plugin.MongoStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Collections;
public class ConversationDialogCollection : MongoBase
{
public string ConversationId { get; set; }
public string Dialog { get; set; }
public List<DialogMongoElement> Dialogs { get; set; }
}

32
src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs Normal file
View file

@ -0,0 +1,32 @@
using BotSharp.Abstraction.Conversations.Models;
namespace BotSharp.Plugin.MongoStorage.Models;
public class DialogMongoElement
{
public string MetaData { get; set; }
public string Content { get; set; }
public DialogMongoElement()
{
}
public static DialogMongoElement ToMongoElement(DialogElement dialog)
{
return new DialogMongoElement
{
MetaData = dialog.MetaData,
Content = dialog.Content
};
}
public static DialogElement ToDomainElement(DialogMongoElement dialog)
{
return new DialogElement
{
MetaData = dialog.MetaData,
Content = dialog.Content
};
}
}

30
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs
View file

@ -620,25 +620,27 @@ public class MongoRepository : IBotSharpRepository
{
Id = Guid.NewGuid().ToString(),
ConversationId = conv.Id,
Dialog = string.Empty
Dialogs = new List<DialogMongoElement>()
};
_dc.Conversations.InsertOne(conv);
_dc.ConversationDialogs.InsertOne(dialog);
}
public string GetConversationDialog(string conversationId)
public List<DialogElement> GetConversationDialogs(string conversationId)
{
if (string.IsNullOrEmpty(conversationId)) return string.Empty;
var dialogs = new List<DialogElement>();
if (string.IsNullOrEmpty(conversationId)) return dialogs;
var filter = Builders<ConversationDialogCollection>.Filter.Eq(x => x.ConversationId, conversationId);
var foundDialog = _dc.ConversationDialogs.Find(filter).FirstOrDefault();
if (foundDialog == null) return string.Empty;
if (foundDialog == null) return dialogs;
return foundDialog.Dialog;
var formattedDialog = foundDialog.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList();
return formattedDialog ?? new List<DialogElement>();
}
public void UpdateConversationDialog(string conversationId, string dialogs)
public void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs)
{
if (string.IsNullOrEmpty(conversationId)) return;
@ -650,7 +652,8 @@ public class MongoRepository : IBotSharpRepository
var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault();
if (foundDialog == null) return;
var updateDialog = Builders<ConversationDialogCollection>.Update.Set(x => x.Dialog, dialogs);
var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList();
var updateDialog = Builders<ConversationDialogCollection>.Update.PushEach(x => x.Dialogs, dialogElements);
var updateConv = Builders<ConversationCollection>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow);
_dc.ConversationDialogs.UpdateOne(filterDialog, updateDialog);
@ -710,6 +713,8 @@ public class MongoRepository : IBotSharpRepository
if (conv == null) return null;
var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List<DialogElement>();
return new Conversation
{
Id = conv.Id.ToString(),
@ -717,7 +722,7 @@ public class MongoRepository : IBotSharpRepository
UserId = conv.UserId.ToString(),
Title = conv.Title,
Status = conv.Status,
Dialog = dialog?.Dialog ?? string.Empty,
Dialogs = dialogElements,
States = new ConversationState(conv.States ?? new List<StateKeyValue>()),
CreatedTime = conv.CreatedTime,
UpdatedTime = conv.UpdatedTime
@ -754,9 +759,8 @@ public class MongoRepository : IBotSharpRepository
{
var records = new List<Conversation>();
var conversations = _dc.Conversations.Aggregate()
.Group(c => c.UserId,
g => g.OrderByDescending(x => x.CreatedTime).First())
.ToList();
.Group(c => c.UserId, g => g.OrderByDescending(x => x.CreatedTime).First())
.ToList();
return conversations.Select(c => new Conversation()
{
Id = c.Id.ToString(),
@ -775,8 +779,8 @@ public class MongoRepository : IBotSharpRepository
var filter = Builders<ExectionLogCollection>.Filter.Eq(x => x.ConversationId, conversationId);
var update = Builders<ExectionLogCollection>.Update
.SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
.PushEach(x => x.Logs, logs);
.SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
.PushEach(x => x.Logs, logs);
_dc.ExectionLogs.UpdateOne(filter, update, _options);
}