SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

add breakpoint

Browse source
This commit is contained in:
Jicheng Lu 2024-03-26 12:10:34 -05:00
parent 098aa92be2
commit 1ee3c4eb14
12 changed files with 206 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs
View file

@ -28,12 +28,6 @@ public class Conversation
public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;
public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
/// <summary>
/// The default value will be same as CreatedTime
/// It used to insert a breakpoint in the conversation to hide the previous dialogs.
/// </summary>
public DateTime Breakpoint { get; set; } = DateTime.UtcNow.AddMilliseconds(-100);
}
public class DialogElement

13
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationBreakpoint.cs Normal file
View file

@ -0,0 +1,13 @@
namespace BotSharp.Abstraction.Conversations.Models;
public class ConversationBreakpoint
{
[JsonPropertyName("message_id")]
public string? MessageId { get; set; }
[JsonPropertyName("breakpoint")]
public DateTime Breakpoint { get; set; }
[JsonPropertyName("created_time")]
public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
}

3
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -58,7 +58,8 @@ public interface IBotSharpRepository
Conversation GetConversation(string conversationId);
PagedItems<Conversation> GetConversations(ConversationFilter filter);
void UpdateConversationTitle(string conversationId, string title);
void UpdateConversationBreakpoint(string conversationId, DateTime breakpoint);
void UpdateConversationBreakpoint(string conversationId, string messageId, DateTime breakpoint);
DateTime GetConversationBreakpoint(string conversationId);
List<Conversation> GetLastConversations();
List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours);
bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false);

4
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs
View file

@ -5,7 +5,9 @@ public partial class ConversationService : IConversationService
public async Task UpdateBreakpoint(bool resetStates = false)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
db.UpdateConversationBreakpoint(_conversationId, DateTime.UtcNow);
var routingCtx = _services.GetRequiredService<IRoutingContext>();
var messageId = routingCtx.MessageId;
db.UpdateConversationBreakpoint(_conversationId, messageId, DateTime.UtcNow);
// Reset states
if (resetStates)

4
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs
View file

@ -111,8 +111,8 @@ public partial class ConversationService : IConversationService
if (fromBreakpoint)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var conversation = db.GetConversation(_conversationId);
dialogs = dialogs.Where(x => x.CreatedAt >= conversation.Breakpoint).ToList();
var breakpoint = db.GetConversationBreakpoint(_conversationId);
dialogs = dialogs.Where(x => x.CreatedAt >= breakpoint).ToList();
}
return dialogs

7
src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file

@ -189,9 +189,12 @@ public class BotSharpDbContext : Database, IBotSharpRepository
public void UpdateConversationTitle(string conversationId, string title)
=> new NotImplementedException();
public void UpdateConversationBreakpoint(string conversationId, DateTime breakpoint)
public void UpdateConversationBreakpoint(string conversationId, string messageId, DateTime breakpoint)
=> new NotImplementedException();
public DateTime GetConversationBreakpoint(string conversationId)
=> throw new NotImplementedException();
public void UpdateConversationStates(string conversationId, List<StateKeyValue> states)
=> new NotImplementedException();

113
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
View file

@ -10,6 +10,10 @@ namespace BotSharp.Core.Repository
{
public void CreateNewConversation(Conversation conversation)
{
var utcNow = DateTime.UtcNow;
conversation.CreatedTime = utcNow;
conversation.UpdatedTime = utcNow;
var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversation.Id);
if (!Directory.Exists(dir))
{
@ -42,6 +46,20 @@ namespace BotSharp.Core.Repository
}).ToList();
File.WriteAllText(stateFile, JsonSerializer.Serialize(initialStates, _options));
}
var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE);
if (!File.Exists(breakpointFile))
{
var initialBreakpoints = new List<ConversationBreakpoint>
{
new ConversationBreakpoint()
{
Breakpoint = utcNow.AddMilliseconds(-100),
CreatedTime = DateTime.UtcNow
}
};
File.WriteAllText(breakpointFile, JsonSerializer.Serialize(initialBreakpoints, _options));
}
}
public bool DeleteConversations(IEnumerable<string> conversationIds)
@ -142,23 +160,63 @@ namespace BotSharp.Core.Repository
}
}
public void UpdateConversationBreakpoint(string conversationId, DateTime breakpoint)
public void UpdateConversationBreakpoint(string conversationId, string messageId, DateTime breakpoint)
{
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var convFile = Path.Combine(convDir, CONVERSATION_FILE);
var content = File.ReadAllText(convFile);
var record = JsonSerializer.Deserialize<Conversation>(content, _options);
if (record != null)
var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE);
if (!File.Exists(breakpointFile))
{
record.UpdatedTime = DateTime.UtcNow;
record.Breakpoint = breakpoint;
File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options));
File.Create(breakpointFile);
}
var content = File.ReadAllText(breakpointFile);
var records = JsonSerializer.Deserialize<List<ConversationBreakpoint>>(content, _options);
var newBreakpoint = new List<ConversationBreakpoint>()
{
new ConversationBreakpoint
{
MessageId = messageId,
Breakpoint = breakpoint,
CreatedTime = DateTime.UtcNow
}
};
if (records != null && !records.IsNullOrEmpty())
{
records = records.Concat(newBreakpoint).ToList();
}
else
{
records = newBreakpoint;
}
File.WriteAllText(breakpointFile, JsonSerializer.Serialize(records, _options));
}
}
public DateTime GetConversationBreakpoint(string conversationId)
{
var convDir = FindConversationDirectory(conversationId);
if (string.IsNullOrEmpty(convDir))
{
return default;
}
var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE);
if (!File.Exists(breakpointFile))
{
File.Create(breakpointFile);
}
var content = File.ReadAllText(breakpointFile);
var records = JsonSerializer.Deserialize<List<ConversationBreakpoint>>(content, _options);
return records?.LastOrDefault()?.Breakpoint ?? default;
}
public ConversationState GetConversationStates(string conversationId)
{
var states = new List<StateKeyValue>();
@ -425,6 +483,11 @@ namespace BotSharp.Core.Repository
var states = CollectConversationStates(stateDir);
isSaved = HandleTruncatedStates(stateDir, states, refTime);
// Handle truncated breakpoints
var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE);
var breakpoints = CollectConversationBreakpoints(breakpointDir);
isSaved = HandleTruncatedBreakpoints(breakpointDir, breakpoints, messageId);
// Remove logs
if (cleanLog)
{
@ -485,7 +548,7 @@ namespace BotSharp.Core.Repository
foreach (var element in dialogs)
{
var meta = element.MetaData;
var createTime = meta.CreateTime.ToString("MM/dd/yyyy hh:mm:ss.fff tt", CultureInfo.InvariantCulture);
var createTime = meta.CreateTime.ToString("MM/dd/yyyy hh:mm:ss.ffffff tt", CultureInfo.InvariantCulture);
var metaStr = $"{createTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{meta.SenderId}|{meta.FunctionName}|{element.RichContent}";
dialogTexts.Add(metaStr);
var content = $" - {element.Content}";
@ -507,6 +570,18 @@ namespace BotSharp.Core.Repository
return states ?? new List<StateKeyValue>();
}
private List<ConversationBreakpoint> CollectConversationBreakpoints(string breakpointFile)
{
var breakpoints = new List<ConversationBreakpoint>();
if (!File.Exists(breakpointFile)) return breakpoints;
var content = File.ReadAllText(breakpointFile);
if (string.IsNullOrEmpty(content)) return breakpoints;
breakpoints = JsonSerializer.Deserialize<List<ConversationBreakpoint>>(content, _options);
return breakpoints ?? new List<ConversationBreakpoint>();
}
private bool HandleTruncatedDialogs(string convDir, string dialogDir, List<DialogElement> dialogs, int foundIdx)
{
var truncatedDialogs = dialogs.Where((x, idx) => idx < foundIdx).ToList();
@ -538,6 +613,16 @@ namespace BotSharp.Core.Repository
return isSaved;
}
private bool HandleTruncatedBreakpoints(string breakpointDir, List<ConversationBreakpoint> breakpoints, string refMessageId)
{
var targetIdx = breakpoints.FindIndex(x => x.MessageId == refMessageId);
var truncatedBreakpoints = breakpoints?.Where((x, idx) => idx < targetIdx)?
.ToList() ?? new List<ConversationBreakpoint>();
var isSaved = SaveTruncatedBreakpoints(breakpointDir, truncatedBreakpoints);
return isSaved;
}
private bool HandleTruncatedLogs(string convDir, DateTime refTime)
{
var contentLogDir = Path.Combine(convDir, "content_log");
@ -595,6 +680,16 @@ namespace BotSharp.Core.Repository
File.WriteAllText(stateDir, stateStr);
return true;
}
private bool SaveTruncatedBreakpoints(string breakpointDir, List<ConversationBreakpoint> breakpoints)
{
if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false;
if (!File.Exists(breakpointDir)) File.Create(breakpointDir);
var breakpointStr = JsonSerializer.Serialize(breakpoints, _options);
File.WriteAllText(breakpointDir, breakpointStr);
return true;
}
#endregion
}
}

1
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs
View file

@ -29,6 +29,7 @@ public partial class FileRepository : IBotSharpRepository
private const string STATS_FILE = "stats.json";
private const string DIALOG_FILE = "dialogs.txt";
private const string STATE_FILE = "state.json";
private const string BREAKPOINT_FILE = "breakpoint.json";
private const string EXECUTION_LOG_FILE = "execution.log";
private const string PLUGIN_CONFIG_FILE = "config.json";
private const string AGENT_TASK_PREFIX = "#metadata";

1
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs
View file

@ -11,5 +11,4 @@ public class ConversationDocument : MongoBase
public int DialogCount { get; set; }
public DateTime CreatedTime { get; set; }
public DateTime UpdatedTime { get; set; }
public DateTime Breakpoint { get; set; }
}

1
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs
View file

@ -6,4 +6,5 @@ public class ConversationStateDocument : MongoBase
{
public string ConversationId { get; set; }
public List<StateMongoElement> States { get; set; }
public List<BreakpointMongoElement> Breakpoints { get; set; }
}

8
src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs Normal file
View file

@ -0,0 +1,8 @@
namespace BotSharp.Plugin.MongoStorage.Models;
public class BreakpointMongoElement
{
public string? MessageId { get; set; }
public DateTime Breakpoint { get; set; }
public DateTime CreatedTime { get; set; }
}

91
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs
View file

@ -13,6 +13,7 @@ public partial class MongoRepository
{
if (conversation == null) return;
var utcNow = DateTime.UtcNow;
var convDoc = new ConversationDocument
{
Id = !string.IsNullOrEmpty(conversation.Id) ? conversation.Id : Guid.NewGuid().ToString(),
@ -22,9 +23,8 @@ public partial class MongoRepository
Channel = conversation.Channel,
TaskId = conversation.TaskId,
Status = conversation.Status,
CreatedTime = conversation.CreatedTime,
UpdatedTime = conversation.UpdatedTime,
Breakpoint = conversation.Breakpoint,
CreatedTime = utcNow,
UpdatedTime = utcNow
};
var dialogDoc = new ConversationDialogDocument
@ -44,11 +44,21 @@ public partial class MongoRepository
}
}).ToList();
var initialBreakpoints = new List<BreakpointMongoElement>()
{
new BreakpointMongoElement
{
Breakpoint = utcNow.AddMilliseconds(-100),
CreatedTime = utcNow
}
};
var stateDoc = new ConversationStateDocument
{
Id = Guid.NewGuid().ToString(),
ConversationId = convDoc.Id,
States = initialStates
States = initialStates,
Breakpoints = initialBreakpoints
};
_dc.Conversations.InsertOne(convDoc);
@ -142,16 +152,38 @@ public partial class MongoRepository
_dc.Conversations.UpdateOne(filterConv, updateConv);
}
public void UpdateConversationBreakpoint(string conversationId, DateTime breakpoint)
public void UpdateConversationBreakpoint(string conversationId, string messageId, DateTime breakpoint)
{
if (string.IsNullOrEmpty(conversationId)) return;
var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var updateConv = Builders<ConversationDocument>.Update
.Set(x => x.UpdatedTime, DateTime.UtcNow)
.Set(x => x.Breakpoint, breakpoint);
var newBreakpoint = new BreakpointMongoElement()
{
MessageId = messageId,
Breakpoint = breakpoint,
CreatedTime = DateTime.UtcNow
};
var filterState = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var updateState = Builders<ConversationStateDocument>.Update.Push(x => x.Breakpoints, newBreakpoint);
_dc.Conversations.UpdateOne(filterConv, updateConv);
_dc.ConversationStates.UpdateOne(filterState, updateState);
}
public DateTime GetConversationBreakpoint(string conversationId)
{
if (string.IsNullOrEmpty(conversationId))
{
return default;
}
var filter = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var state = _dc.ConversationStates.Find(filter).FirstOrDefault();
if (state == null || state.Breakpoints.IsNullOrEmpty())
{
return default;
}
return state.Breakpoints.LastOrDefault()?.Breakpoint ?? default;
}
public ConversationState GetConversationStates(string conversationId)
@ -226,8 +258,7 @@ public partial class MongoRepository
States = curStates,
DialogCount = conv.DialogCount,
CreatedTime = conv.CreatedTime,
UpdatedTime = conv.UpdatedTime,
Breakpoint = conv.Breakpoint
UpdatedTime = conv.UpdatedTime
};
}
@ -315,8 +346,7 @@ public partial class MongoRepository
Status = conv.Status,
DialogCount = conv.DialogCount,
CreatedTime = conv.CreatedTime,
UpdatedTime = conv.UpdatedTime,
Breakpoint = conv.Breakpoint
UpdatedTime = conv.UpdatedTime
});
}
@ -343,8 +373,7 @@ public partial class MongoRepository
Status = c.Status,
DialogCount = c.DialogCount,
CreatedTime = c.CreatedTime,
UpdatedTime = c.UpdatedTime,
Breakpoint = c.Breakpoint
UpdatedTime = c.UpdatedTime
}).ToList();
}
@ -416,19 +445,33 @@ public partial class MongoRepository
var stateFilter = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault();
if (foundStates != null && !foundStates.States.IsNullOrEmpty())
if (foundStates != null)
{
var truncatedStates = new List<StateMongoElement>();
foreach (var state in foundStates.States)
// Truncate states
if (!foundStates.States.IsNullOrEmpty())
{
var values = state.Values.Where(x => x.UpdateTime < refTime).ToList();
if (values.Count == 0) continue;
var truncatedStates = new List<StateMongoElement>();
foreach (var state in foundStates.States)
{
var values = state.Values.Where(x => x.UpdateTime < refTime).ToList();
if (values.Count == 0) continue;
state.Values = values;
truncatedStates.Add(state);
state.Values = values;
truncatedStates.Add(state);
}
foundStates.States = truncatedStates;
}
foundStates.States = truncatedStates;
// Truncate breakpoints
if (!foundStates.Breakpoints.IsNullOrEmpty())
{
var breakpoints = foundStates.Breakpoints ?? new List<BreakpointMongoElement>();
var targetIdx = breakpoints.FindIndex(x => x.MessageId == messageId);
var truncatedBreakpoints = breakpoints.Where((x, idx) => idx < targetIdx).ToList();
foundStates.Breakpoints = truncatedBreakpoints;
}
// Update
_dc.ConversationStates.ReplaceOne(stateFilter, foundStates);
}