SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #245 from iceljc/features/track-conversation-states

Browse source 
refine conversation states
This commit is contained in:
Haiping 2024-01-02 13:47:52 -06:00 committed by GitHub
parent 150699f605 9d47f41109
commit ea9ae4db58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 295 additions and 162 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationStateService.cs
View file

@ -8,11 +8,11 @@ namespace BotSharp.Abstraction.Conversations;
public interface IConversationStateService
{
string GetConversationId();
ConversationState Load(string conversationId);
Dictionary<string, string> Load(string conversationId);
string GetState(string name, string defaultValue = "");
bool ContainsState(string name);
ConversationState GetStates();
IConversationStateService SetState<T>(string name, T value);
Dictionary<string, string> GetStates();
IConversationStateService SetState<T>(string name, T value, bool isNeedVersion = true);
void SaveStateByArgs(JsonDocument args);
void CleanState();
void Save();

2
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs
View file

@ -13,7 +13,7 @@ public class Conversation
public List<DialogElement> Dialogs { get; set; } = new List<DialogElement>();
[JsonIgnore]
public ConversationState States { get; set; } = new ConversationState();
public Dictionary<string, string> States { get; set; } = new Dictionary<string, string>();
public string Status { get; set; } = ConversationStatus.Open;

11
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationState.cs
View file

@ -1,22 +1,17 @@
namespace BotSharp.Abstraction.Conversations.Models;
public class ConversationState : Dictionary<string, string>
public class ConversationState : Dictionary<string, List<StateValue>>
{
public ConversationState()
{
}
public ConversationState(List<StateKeyValue> pairs)
{
foreach (var pair in pairs)
{
this[pair.Key] = pair.Value;
this[pair.Key] = pair.Values;
}
}
public List<StateKeyValue> ToKeyValueList()
{
return this.Select(x => new StateKeyValue(x.Key, x.Value)).ToList();
}
}

17
src/Infrastructure/BotSharp.Abstraction/Conversations/Models/StateKeyValue.cs
View file

@ -3,16 +3,27 @@ namespace BotSharp.Abstraction.Conversations.Models;
public class StateKeyValue
{
public string Key { get; set; }
public string Value { get; set; }
public List<StateValue> Values { get; set; } = new List<StateValue>();
public StateKeyValue()
{
}
public StateKeyValue(string key, string value)
public StateKeyValue(string key, List<StateValue> values)
{
Key = key;
Value = value;
Values = values;
}
}
public class StateValue
{
public string Data { get; set; }
public DateTime UpdateTime { get; set; }
public StateValue()
{
}
}

2
src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructResult.cs
View file

@ -6,5 +6,5 @@ public class InstructResult : ITrackableMessage
public string MessageId { get; set; }
public string Text { get; set; }
public object Data { get; set; }
public ConversationState States { get; set; }
public Dictionary<string, string> States { get; set; }
}

2
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -33,7 +33,7 @@ public interface IBotSharpRepository
List<DialogElement> GetConversationDialogs(string conversationId);
void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements);
void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs);
List<StateKeyValue> GetConversationStates(string conversationId);
ConversationState GetConversationStates(string conversationId);
void UpdateConversationStates(string conversationId, List<StateKeyValue> states);
void UpdateConversationStatus(string conversationId, string status);
Conversation GetConversation(string conversationId);

5
src/Infrastructure/BotSharp.Abstraction/Repositories/Records/ConversationRecord.cs
View file

@ -1,5 +1,3 @@
using System.Text.Json.Serialization;
namespace BotSharp.Abstraction.Repositories.Records;
public class ConversationRecord : RecordBase
@ -18,9 +16,6 @@ public class ConversationRecord : RecordBase
[JsonIgnore]
public string Dialog { get; set; }
[JsonIgnore]
public List<StateKeyValue> States { get; set; }
[Required]
public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;

84
src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs
View file

@ -1,5 +1,5 @@
using BotSharp.Abstraction.Repositories;
using System.IO;
using System.Linq;
namespace BotSharp.Core.Conversations.Services;
@ -11,60 +11,87 @@ public class ConversationStateService : IConversationStateService, IDisposable
private readonly ILogger _logger;
private readonly IServiceProvider _services;
private ConversationState _states;
private BotSharpDatabaseSettings _dbSettings;
private string _conversationId;
private readonly IBotSharpRepository _db;
private List<StateKeyValue> _savedStates;
public ConversationStateService(ILogger<ConversationStateService> logger,
IServiceProvider services,
BotSharpDatabaseSettings dbSettings,
IServiceProvider services,
IBotSharpRepository db)
{
_logger = logger;
_services = services;
_dbSettings = dbSettings;
_db = db;
_states = new ConversationState();
}
public string GetConversationId() => _conversationId;
public IConversationStateService SetState<T>(string name, T value)
/// <summary>
/// Set conversation state
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="name"></param>
/// <param name="value"></param>
/// <param name="isNeedVersion">whether the state is related to message or not</param>
/// <returns></returns>
public IConversationStateService SetState<T>(string name, T value, bool isNeedVersion = true)
{
if (value == null)
{
return this;
}
var preValue = string.Empty;
var currentValue = value.ToString();
var hooks = _services.GetServices<IConversationHook>();
string preValue = _states.ContainsKey(name) ? _states[name] : "";
if (!_states.ContainsKey(name) || _states[name] != currentValue)
if (_states.TryGetValue(name, out var values))
{
preValue = values?.LastOrDefault()?.Data ?? string.Empty;
}
if (!_states.ContainsKey(name) || preValue != currentValue)
{
_states[name] = currentValue;
_logger.LogInformation($"[STATE] {name} = {value}");
foreach (var hook in hooks)
{
hook.OnStateChanged(name, preValue, currentValue).Wait();
}
var stateValue = new StateValue
{
Data = currentValue,
UpdateTime = DateTime.UtcNow
};
if (!_states.ContainsKey(name) || !isNeedVersion)
{
_states[name] = new List<StateValue> { stateValue };
}
else
{
_states[name].Add(stateValue);
}
}
return this;
}
public ConversationState Load(string conversationId)
public Dictionary<string, string> Load(string conversationId)
{
_conversationId = conversationId;
_savedStates = _db.GetConversationStates(_conversationId).ToList();
_states = _db.GetConversationStates(_conversationId);
var curStates = new Dictionary<string, string>();
if (!_savedStates.IsNullOrEmpty())
if (!_states.IsNullOrEmpty())
{
foreach (var data in _savedStates)
foreach (var state in _states)
{
_states[data.Key] = data.Value;
_logger.LogInformation($"[STATE] {data.Key} : {data.Value}");
var value = state.Value?.LastOrDefault()?.Data ?? string.Empty;
curStates[state.Key] = value;
_logger.LogInformation($"[STATE] {state.Key} : {value}");
}
}
@ -75,7 +102,7 @@ public class ConversationStateService : IConversationStateService, IDisposable
hook.OnStateLoaded(_states).Wait();
}
return _states;
return curStates;
}
public void Save()
@ -93,25 +120,32 @@ public class ConversationStateService : IConversationStateService, IDisposable
}
_db.UpdateConversationStates(_conversationId, states);
_logger.LogInformation($"Saved state {_conversationId}");
_logger.LogInformation($"Saved states of conversation {_conversationId}");
}
public void CleanState()
{
//File.Delete(_file);
_states.Clear();
}
public ConversationState GetStates()
=> _states;
public Dictionary<string, string> GetStates()
{
var curStates = new Dictionary<string, string>();
foreach (var state in _states)
{
curStates[state.Key] = state.Value?.LastOrDefault()?.Data ?? string.Empty;
}
return curStates;
}
public string GetState(string name, string defaultValue = "")
{
if (!_states.ContainsKey(name))
if (!_states.ContainsKey(name) || _states[name].IsNullOrEmpty())
{
return defaultValue;
}
return _states[name];
return _states[name].Last().Data;
}
public void Dispose()
@ -121,7 +155,9 @@ public class ConversationStateService : IConversationStateService, IDisposable
public bool ContainsState(string name)
{
return _states.ContainsKey(name) && !string.IsNullOrEmpty(_states[name]);
return _states.ContainsKey(name)
&& !_states[name].IsNullOrEmpty()
&& !string.IsNullOrEmpty(_states[name].Last().Data);
}
public void SaveStateByArgs(JsonDocument args)

6
src/Infrastructure/BotSharp.Core/Conversations/Services/TokenStatistics.cs
View file

@ -47,14 +47,14 @@ public class TokenStatistics : ITokenStatistics
// Accumulated Token
var stat = _services.GetRequiredService<IConversationStateService>();
var inputCount = int.Parse(stat.GetState("prompt_total", "0"));
stat.SetState("prompt_total", stats.PromptCount + inputCount);
stat.SetState("prompt_total", stats.PromptCount + inputCount, false);
var outputCount = int.Parse(stat.GetState("completion_total", "0"));
stat.SetState("completion_total", stats.CompletionCount + outputCount);
stat.SetState("completion_total", stats.CompletionCount + outputCount, false);
// Total cost
var total_cost = float.Parse(stat.GetState("llm_total_cost", "0"));
total_cost += Cost;
stat.SetState("llm_total_cost", total_cost);
stat.SetState("llm_total_cost", total_cost, false);
}
public void PrintStatistics()

2
src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file

@ -152,7 +152,7 @@ public class BotSharpDbContext : Database, IBotSharpRepository
throw new NotImplementedException();
}
public List<StateKeyValue> GetConversationStates(string conversationId)
public ConversationState GetConversationStates(string conversationId)
{
throw new NotImplementedException();
}

146
src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs
View file

@ -9,6 +9,7 @@ using BotSharp.Abstraction.Repositories.Filters;
using BotSharp.Abstraction.Repositories.Models;
using BotSharp.Abstraction.Routing.Settings;
using BotSharp.Abstraction.Evaluations.Settings;
using System.Text.Encodings.Web;
namespace BotSharp.Core.Repository;
@ -20,6 +21,17 @@ public class FileRepository : IBotSharpRepository
private readonly ConversationSetting _conversationSettings;
private JsonSerializerOptions _options;
private const string AGENT_FILE = "agent.json";
private const string AGENT_INSTRUCTION_FILE = "instruction";
private const string AGENT_FUNCTIONS_FILE = "functions.json";
private const string AGENT_SAMPLES_FILE = "samples.txt";
private const string USER_FILE = "user.json";
private const string USER_AGENT_FILE = "agents.json";
private const string CONVERSATION_FILE = "conversation.json";
private const string DIALOG_FILE = "dialogs.txt";
private const string STATE_FILE = "state.json";
private const string EXECUTION_LOG_FILE = "execution.log";
public FileRepository(
IServiceProvider services,
BotSharpDatabaseSettings dbSettings,
@ -35,7 +47,9 @@ public class FileRepository : IBotSharpRepository
{
PropertyNameCaseInsensitive = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = true
WriteIndented = true,
AllowTrailingCommas = true,
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
};
}
@ -59,7 +73,7 @@ public class FileRepository : IBotSharpRepository
{
foreach (var d in Directory.GetDirectories(dir))
{
var userFile = Path.Combine(d, "user.json");
var userFile = Path.Combine(d, USER_FILE);
if (!Directory.Exists(d) || !File.Exists(userFile))
continue;
@ -86,7 +100,7 @@ public class FileRepository : IBotSharpRepository
{
foreach (var d in Directory.GetDirectories(dir))
{
var file = Path.Combine(d, "agent.json");
var file = Path.Combine(d, AGENT_FILE);
if (!Directory.Exists(d) || !File.Exists(file))
continue;
@ -122,7 +136,7 @@ public class FileRepository : IBotSharpRepository
{
foreach (var d in Directory.GetDirectories(dir))
{
var file = Path.Combine(d, "agents.json");
var file = Path.Combine(d, USER_AGENT_FILE);
if (!Directory.Exists(d) || !File.Exists(file))
continue;
@ -171,7 +185,7 @@ public class FileRepository : IBotSharpRepository
{
Directory.CreateDirectory(dir);
}
var path = Path.Combine(dir, "agent.json");
var path = Path.Combine(dir, AGENT_FILE);
File.WriteAllText(path, JsonSerializer.Serialize(agent, _options));
}
}
@ -184,7 +198,7 @@ public class FileRepository : IBotSharpRepository
{
Directory.CreateDirectory(dir);
}
var path = Path.Combine(dir, "user.json");
var path = Path.Combine(dir, USER_FILE);
File.WriteAllText(path, JsonSerializer.Serialize(user, _options));
}
}
@ -198,7 +212,7 @@ public class FileRepository : IBotSharpRepository
if (agents.Any())
{
var dir = Path.Combine(_dbSettings.FileRepository, "users", uid);
var path = Path.Combine(dir, "agents.json");
var path = Path.Combine(dir, USER_AGENT_FILE);
File.WriteAllText(path, JsonSerializer.Serialize(agents, _options));
}
});
@ -357,7 +371,7 @@ public class FileRepository : IBotSharpRepository
if (agent == null) return;
var instructionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir,
agentId, $"instruction.{_agentSettings.TemplateFormat}");
agentId, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}");
File.WriteAllText(instructionFile, instruction);
}
@ -370,7 +384,7 @@ public class FileRepository : IBotSharpRepository
if (agent == null) return;
var functionFile = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir,
agentId, "functions.json");
agentId, AGENT_FUNCTIONS_FILE);
var functionText = JsonSerializer.Serialize(inputFunctions, _options);
File.WriteAllText(functionFile, functionText);
@ -436,7 +450,7 @@ public class FileRepository : IBotSharpRepository
var (agent, agentFile) = GetAgentFromFile(agentId);
if (agent == null) return;
var file = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, "samples.txt");
var file = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId, AGENT_SAMPLES_FILE);
File.WriteAllLines(file, samples);
}
@ -501,7 +515,7 @@ public class FileRepository : IBotSharpRepository
if (!string.IsNullOrEmpty(dir))
{
var json = File.ReadAllText(Path.Combine(dir, "agent.json"));
var json = File.ReadAllText(Path.Combine(dir, AGENT_FILE));
if (string.IsNullOrEmpty(json)) return null;
var record = JsonSerializer.Deserialize<Agent>(json, _options);
@ -629,22 +643,31 @@ public class FileRepository : IBotSharpRepository
Directory.CreateDirectory(dir);
}
var convDir = Path.Combine(dir, "conversation.json");
if (!File.Exists(convDir))
var convFile = Path.Combine(dir, CONVERSATION_FILE);
if (!File.Exists(convFile))
{
File.WriteAllText(convDir, JsonSerializer.Serialize(conversation, _options));
File.WriteAllText(convFile, JsonSerializer.Serialize(conversation, _options));
}
var dialogDir = Path.Combine(dir, "dialogs.txt");
if (!File.Exists(dialogDir))
var dialogFile = Path.Combine(dir, DIALOG_FILE);
if (!File.Exists(dialogFile))
{
File.WriteAllText(dialogDir, string.Empty);
File.WriteAllText(dialogFile, string.Empty);
}
var stateDir = Path.Combine(dir, "state.dict");
if (!File.Exists(stateDir))
var stateFile = Path.Combine(dir, STATE_FILE);
if (!File.Exists(stateFile))
{
File.WriteAllText(stateDir, string.Empty);
var states = conversation.States ?? new Dictionary<string, string>();
var initialStates = states.Select(x => new StateKeyValue
{
Key = x.Key,
Values = new List<StateValue>
{
new StateValue { Data = x.Value, UpdateTime = DateTime.UtcNow }
}
}).ToList();
File.WriteAllText(stateFile, JsonSerializer.Serialize(initialStates, _options));
}
}
@ -665,7 +688,7 @@ public class FileRepository : IBotSharpRepository
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var dialogDir = Path.Combine(convDir, "dialogs.txt");
var dialogDir = Path.Combine(convDir, DIALOG_FILE);
dialogs = CollectDialogElements(dialogDir);
}
@ -680,7 +703,7 @@ public class FileRepository : IBotSharpRepository
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var dialogDir = Path.Combine(convDir, "dialogs.txt");
var dialogDir = Path.Combine(convDir, DIALOG_FILE);
if (File.Exists(dialogDir))
{
var updated = dialogElements.Select((x, idx) =>
@ -704,22 +727,21 @@ public class FileRepository : IBotSharpRepository
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var dialogDir = Path.Combine(convDir, "dialogs.txt");
var dialogDir = Path.Combine(convDir, DIALOG_FILE);
if (File.Exists(dialogDir))
{
var texts = ParseDialogElements(dialogs);
File.AppendAllLines(dialogDir, texts);
}
}
return;
}
public void UpdateConversationTitle(string conversationId, string title)
{
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var convFile = Path.Combine(convDir, "conversation.json");
var convFile = Path.Combine(convDir, CONVERSATION_FILE);
var content = File.ReadAllText(convFile);
var record = JsonSerializer.Deserialize<Conversation>(content, _options);
if (record != null)
@ -730,33 +752,32 @@ public class FileRepository : IBotSharpRepository
}
}
}
public List<StateKeyValue> GetConversationStates(string conversationId)
public ConversationState GetConversationStates(string conversationId)
{
var curStates = new List<StateKeyValue>();
var states = new List<StateKeyValue>();
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var stateDir = Path.Combine(convDir, "state.dict");
curStates = CollectConversationStates(stateDir);
var stateFile = Path.Combine(convDir, STATE_FILE);
states = CollectConversationStates(stateFile);
}
return curStates;
return new ConversationState(states);
}
public void UpdateConversationStates(string conversationId, List<StateKeyValue> states)
{
var localStates = new List<string>();
if (states.IsNullOrEmpty()) return;
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var stateDir = Path.Combine(convDir, "state.dict");
if (File.Exists(stateDir))
var stateFile = Path.Combine(convDir, STATE_FILE);
if (File.Exists(stateFile))
{
foreach (var data in states)
{
localStates.Add($"{data.Key}={data.Value}");
}
File.WriteAllLines(stateDir, localStates);
var stateStr = JsonSerializer.Serialize(states, _options);
File.WriteAllText(stateFile, stateStr);
}
}
}
@ -766,7 +787,7 @@ public class FileRepository : IBotSharpRepository
var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var convFile = Path.Combine(convDir, "conversation.json");
var convFile = Path.Combine(convDir, CONVERSATION_FILE);
if (File.Exists(convFile))
{
var json = File.ReadAllText(convFile);
@ -783,21 +804,26 @@ public class FileRepository : IBotSharpRepository
var convDir = FindConversationDirectory(conversationId);
if (string.IsNullOrEmpty(convDir)) return null;
var convFile = Path.Combine(convDir, "conversation.json");
var convFile = Path.Combine(convDir, CONVERSATION_FILE);
var content = File.ReadAllText(convFile);
var record = JsonSerializer.Deserialize<Conversation>(content, _options);
var dialogFile = Path.Combine(convDir, "dialogs.txt");
var dialogFile = Path.Combine(convDir, DIALOG_FILE);
if (record != null)
{
record.Dialogs = CollectDialogElements(dialogFile);
}
var stateFile = Path.Combine(convDir, "state.dict");
var stateFile = Path.Combine(convDir, STATE_FILE);
if (record != null)
{
var states = CollectConversationStates(stateFile);
record.States = new ConversationState(states);
var curStates = new Dictionary<string, string>();
states.ForEach(x =>
{
curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty;
});
record.States = curStates;
}
return record;
@ -810,7 +836,7 @@ public class FileRepository : IBotSharpRepository
foreach (var d in Directory.GetDirectories(dir))
{
var path = Path.Combine(d, "conversation.json");
var path = Path.Combine(d, CONVERSATION_FILE);
if (!File.Exists(path)) continue;
var json = File.ReadAllText(path);
@ -838,7 +864,7 @@ public class FileRepository : IBotSharpRepository
foreach (var d in Directory.GetDirectories(dir))
{
var path = Path.Combine(d, "conversation.json");
var path = Path.Combine(d, CONVERSATION_FILE);
if (!File.Exists(path)) continue;
var json = File.ReadAllText(path);
@ -889,7 +915,7 @@ public class FileRepository : IBotSharpRepository
Directory.CreateDirectory(dir);
}
var file = Path.Combine(dir, "execution.log");
var file = Path.Combine(dir, EXECUTION_LOG_FILE);
File.AppendAllLines(file, logs);
}
@ -901,7 +927,7 @@ public class FileRepository : IBotSharpRepository
var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId);
if (!Directory.Exists(dir)) return logs;
var file = Path.Combine(dir, "execution.log");
var file = Path.Combine(dir, EXECUTION_LOG_FILE);
logs = File.ReadAllLines(file)?.ToList() ?? new List<string>();
return logs;
}
@ -949,7 +975,7 @@ public class FileRepository : IBotSharpRepository
private (Agent?, string) GetAgentFromFile(string agentId)
{
var dir = GetAgentDataDir(agentId);
var agentFile = Path.Combine(dir, "agent.json");
var agentFile = Path.Combine(dir, AGENT_FILE);
if (!File.Exists(agentFile)) return (null, string.Empty);
var json = File.ReadAllText(agentFile);
@ -959,7 +985,7 @@ public class FileRepository : IBotSharpRepository
private string FetchInstruction(string fileDir)
{
var file = Path.Combine(fileDir, $"instruction.{_agentSettings.TemplateFormat}");
var file = Path.Combine(fileDir, $"{AGENT_INSTRUCTION_FILE}.{_agentSettings.TemplateFormat}");
if (!File.Exists(file)) return string.Empty;
var instruction = File.ReadAllText(file);
@ -968,7 +994,7 @@ public class FileRepository : IBotSharpRepository
private List<FunctionDef> FetchFunctions(string fileDir)
{
var file = Path.Combine(fileDir, "functions.json");
var file = Path.Combine(fileDir, AGENT_FUNCTIONS_FILE);
if (!File.Exists(file)) return new List<FunctionDef>();
var functionsJson = File.ReadAllText(file);
@ -978,7 +1004,7 @@ public class FileRepository : IBotSharpRepository
private List<string> FetchSamples(string fileDir)
{
var file = Path.Combine(fileDir, "samples.txt");
var file = Path.Combine(fileDir, AGENT_SAMPLES_FILE);
if (!File.Exists(file)) return new List<string>();
return File.ReadAllLines(file)?.ToList() ?? new List<string>();
@ -1082,18 +1108,16 @@ public class FileRepository : IBotSharpRepository
return dialogTexts;
}
private List<StateKeyValue> CollectConversationStates(string stateDir)
private List<StateKeyValue> CollectConversationStates(string stateFile)
{
var states = new List<StateKeyValue>();
if (!File.Exists(stateDir)) return states;
if (!File.Exists(stateFile)) return states;
var dict = File.ReadAllLines(stateDir);
foreach (var line in dict)
{
var data = line.Split('=');
states.Add(new StateKeyValue(data[0], data[1]));
}
return states;
var stateStr = File.ReadAllText(stateFile);
if (string.IsNullOrEmpty(stateStr)) return states;
states = JsonSerializer.Deserialize<List<StateKeyValue>>(stateStr, _options);
return states ?? new List<StateKeyValue>();
}
private int GetNextLlmCompletionLogIndex(string logDir, string id)

2
src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/ConversationViewModel.cs
View file

@ -20,7 +20,7 @@ public class ConversationViewModel
public string Channel { get; set; } = ConversationChannel.OpenAPI;
public string Status { get; set; }
public ConversationState States { get; set; }
public Dictionary<string, string> States { get; set; }
[JsonPropertyName("updated_time")]
public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;

3
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Abstraction.Conversations.Models;
namespace BotSharp.Plugin.MongoStorage.Collections;
public class ConversationDocument : MongoBase
@ -9,7 +7,6 @@ public class ConversationDocument : MongoBase
public string Title { get; set; }
public string Channel { get; set; }
public string Status { get; set; }
public List<StateKeyValue> States { get; set; }
public DateTime CreatedTime { get; set; }
public DateTime UpdatedTime { get; set; }
}

9
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs Normal file
View file

@ -0,0 +1,9 @@
using BotSharp.Plugin.MongoStorage.Models;
namespace BotSharp.Plugin.MongoStorage.Collections;
public class ConversationStateDocument : MongoBase
{
public string ConversationId { get; set; }
public List<StateMongoElement> States { get; set; }
}

2
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogDocument.cs
View file

@ -5,5 +5,5 @@ namespace BotSharp.Plugin.MongoStorage.Collections;
public class LlmCompletionLogDocument : MongoBase
{
public string ConversationId { get; set; }
public List<PromptLogElement> Logs { get; set; }
public List<PromptLogMongoElement> Logs { get; set; }
}

2
src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogElement.cs → src/Plugins/BotSharp.Plugin.MongoStorage/Models/PromptLogMongoElement.cs
View file

@ -1,6 +1,6 @@
namespace BotSharp.Plugin.MongoStorage.Models;
public class PromptLogElement
public class PromptLogMongoElement
{
public string MessageId { get; set; }
public string AgentId { get; set; }

51
src/Plugins/BotSharp.Plugin.MongoStorage/Models/StateMongoElement.cs Normal file
View file

@ -0,0 +1,51 @@
using BotSharp.Abstraction.Conversations.Models;
namespace BotSharp.Plugin.MongoStorage.Models;
public class StateMongoElement
{
public string Key { get; set; }
public List<StateValueMongoElement> Values { get; set; }
public static StateMongoElement ToMongoElement(StateKeyValue state)
{
return new StateMongoElement
{
Key = state.Key,
Values = state.Values?.Select(x => StateValueMongoElement.ToMongoElement(x))?.ToList() ?? new List<StateValueMongoElement>()
};
}
public static StateKeyValue ToDomainElement(StateMongoElement state)
{
return new StateKeyValue
{
Key = state.Key,
Values = state.Values?.Select(x => StateValueMongoElement.ToDomainElement(x))?.ToList() ?? new List<StateValue>()
};
}
}
public class StateValueMongoElement
{
public string Data { get; set; }
public DateTime UpdateTime { get; set; }
public static StateValueMongoElement ToMongoElement(StateValue element)
{
return new StateValueMongoElement
{
Data = element.Data,
UpdateTime = element.UpdateTime
};
}
public static StateValue ToDomainElement(StateValueMongoElement element)
{
return new StateValue
{
Data = element.Data,
UpdateTime = element.UpdateTime
};
}
}

3
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -37,6 +37,9 @@ public class MongoDbContext
public IMongoCollection<ConversationDialogDocument> ConversationDialogs
=> Database.GetCollection<ConversationDialogDocument>($"{_collectionPrefix}_ConversationDialogs");
public IMongoCollection<ConversationStateDocument> ConversationStates
=> Database.GetCollection<ConversationStateDocument>($"{_collectionPrefix}_ConversationStates");
public IMongoCollection<ExecutionLogDocument> ExectionLogs
=> Database.GetCollection<ExecutionLogDocument>($"{_collectionPrefix}_ExecutionLogs");

102
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs
View file

@ -447,7 +447,7 @@ public class MongoRepository : IBotSharpRepository
{
query = query.Where(x => x.Name.ToLower() == filter.AgentName.ToLower());
}
if (filter.Disabled.HasValue)
{
query = query.Where(x => x.Disabled == filter.Disabled);
@ -514,9 +514,9 @@ public class MongoRepository : IBotSharpRepository
public List<Agent> GetAgentsByUser(string userId)
{
var agentIds = (from ua in _dc.UserAgents.AsQueryable()
join u in _dc.Users.AsQueryable() on ua.UserId equals u.Id
where ua.UserId == userId || u.ExternalId == userId
select ua.AgentId).ToList();
join u in _dc.Users.AsQueryable() on ua.UserId equals u.Id
where ua.UserId == userId || u.ExternalId == userId
select ua.AgentId).ToList();
var filter = new AgentFilter
{
@ -608,7 +608,7 @@ public class MongoRepository : IBotSharpRepository
{
return false;
}
}
#endregion
@ -617,7 +617,7 @@ public class MongoRepository : IBotSharpRepository
{
if (conversation == null) return;
var conv = new ConversationDocument
var convDoc = new ConversationDocument
{
Id = !string.IsNullOrEmpty(conversation.Id) ? conversation.Id : Guid.NewGuid().ToString(),
AgentId = conversation.AgentId,
@ -625,20 +625,37 @@ public class MongoRepository : IBotSharpRepository
Title = conversation.Title,
Channel = conversation.Channel,
Status = conversation.Status,
States = conversation.States?.ToKeyValueList() ?? new List<StateKeyValue>(),
CreatedTime = DateTime.UtcNow,
UpdatedTime = DateTime.UtcNow,
};
var dialog = new ConversationDialogDocument
var dialogDoc = new ConversationDialogDocument
{
Id = Guid.NewGuid().ToString(),
ConversationId = conv.Id,
ConversationId = convDoc.Id,
Dialogs = new List<DialogMongoElement>()
};
_dc.Conversations.InsertOne(conv);
_dc.ConversationDialogs.InsertOne(dialog);
var states = conversation.States ?? new Dictionary<string, string>();
var initialStates = states.Select(x => new StateMongoElement
{
Key = x.Key,
Values = new List<StateValueMongoElement>
{
new StateValueMongoElement { Data = x.Value, UpdateTime = DateTime.UtcNow }
}
}).ToList();
var stateDoc = new ConversationStateDocument
{
Id = Guid.NewGuid().ToString(),
ConversationId = convDoc.Id,
States = initialStates
};
_dc.Conversations.InsertOne(convDoc);
_dc.ConversationDialogs.InsertOne(dialogDoc);
_dc.ConversationStates.InsertOne(stateDoc);
}
public bool DeleteConversation(string conversationId)
@ -647,14 +664,16 @@ public class MongoRepository : IBotSharpRepository
var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var filterSates = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var filterExeLog = Builders<ExecutionLogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var filterPromptLog = Builders<LlmCompletionLogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var exeLogDeleted = _dc.ExectionLogs.DeleteMany(filterExeLog);
var promptLogDeleted = _dc.LlmCompletionLogs.DeleteMany(filterPromptLog);
var statesDeleted = _dc.ConversationStates.DeleteMany(filterSates);
var dialogDeleted = _dc.ConversationDialogs.DeleteMany(filterDialog);
var convDeleted = _dc.Conversations.DeleteMany(filterConv);
return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0
return convDeleted.DeletedCount > 0 || dialogDeleted.DeletedCount > 0 || statesDeleted.DeletedCount > 0
|| exeLogDeleted.DeletedCount > 0 || promptLogDeleted.DeletedCount > 0;
}
@ -688,7 +707,7 @@ public class MongoRepository : IBotSharpRepository
}
return x;
}).ToList();
_dc.ConversationDialogs.ReplaceOne(filterDialog, foundDialog);
}
@ -697,13 +716,7 @@ public class MongoRepository : IBotSharpRepository
if (string.IsNullOrEmpty(conversationId)) return;
var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var foundConv = _dc.Conversations.Find(filterConv).FirstOrDefault();
if (foundConv == null) return;
var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault();
if (foundDialog == null) return;
var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList();
var updateDialog = Builders<ConversationDialogDocument>.Update.PushEach(x => x.Dialogs, dialogElements);
var updateConv = Builders<ConversationDocument>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow);
@ -717,9 +730,6 @@ public class MongoRepository : IBotSharpRepository
if (string.IsNullOrEmpty(conversationId)) return;
var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var foundConv = _dc.Conversations.Find(filterConv).FirstOrDefault();
if (foundConv == null) return;
var updateConv = Builders<ConversationDocument>.Update
.Set(x => x.UpdatedTime, DateTime.UtcNow)
.Set(x => x.Title, title);
@ -727,30 +737,28 @@ public class MongoRepository : IBotSharpRepository
_dc.Conversations.UpdateOne(filterConv, updateConv);
}
public List<StateKeyValue> GetConversationStates(string conversationId)
public ConversationState GetConversationStates(string conversationId)
{
var states = new List<StateKeyValue>();
var states = new ConversationState();
if (string.IsNullOrEmpty(conversationId)) return states;
var filter = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var foundConversation = _dc.Conversations.Find(filter).FirstOrDefault();
var savedStates = foundConversation?.States ?? new List<StateKeyValue>();
return savedStates;
var filter = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var foundStates = _dc.ConversationStates.Find(filter).FirstOrDefault();
if (foundStates == null || foundStates.States.IsNullOrEmpty()) return states;
var savedStates = foundStates.States.Select(x => StateMongoElement.ToDomainElement(x)).ToList();
return new ConversationState(savedStates);
}
public void UpdateConversationStates(string conversationId, List<StateKeyValue> states)
{
if (string.IsNullOrEmpty(conversationId)) return;
if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return;
var filter = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var foundConv = _dc.Conversations.Find(filter).FirstOrDefault();
if (foundConv == null) return;
var filterStates = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList();
var updateStates = Builders<ConversationStateDocument>.Update.Set(x => x.States, saveStates);
var update = Builders<ConversationDocument>.Update
.Set(x => x.States, states)
.Set(x => x.UpdatedTime, DateTime.UtcNow);
_dc.Conversations.UpdateOne(filter, update);
_dc.ConversationStates.UpdateOne(filterStates, updateStates);
}
public void UpdateConversationStatus(string conversationId, string status)
@ -758,9 +766,6 @@ public class MongoRepository : IBotSharpRepository
if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(status)) return;
var filter = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var foundConv = _dc.Conversations.Find(filter).FirstOrDefault();
if (foundConv == null) return;
var update = Builders<ConversationDocument>.Update
.Set(x => x.Status, status)
.Set(x => x.UpdatedTime, DateTime.UtcNow);
@ -774,14 +779,21 @@ public class MongoRepository : IBotSharpRepository
var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var filterState = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var conv = _dc.Conversations.Find(filterConv).FirstOrDefault();
var dialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault();
var states = _dc.ConversationStates.Find(filterState).FirstOrDefault();
if (conv == null) return null;
var dialogElements = dialog?.Dialogs?.Select(x => DialogMongoElement.ToDomainElement(x))?.ToList() ?? new List<DialogElement>();
var curStates = new Dictionary<string, string>();
states.States.ForEach(x =>
{
curStates[x.Key] = x.Values?.LastOrDefault()?.Data ?? string.Empty;
});
return new Conversation
{
Id = conv.Id.ToString(),
@ -791,7 +803,7 @@ public class MongoRepository : IBotSharpRepository
Channel = conv.Channel,
Status = conv.Status,
Dialogs = dialogElements,
States = new ConversationState(conv.States ?? new List<StateKeyValue>()),
States = curStates,
CreatedTime = conv.CreatedTime,
UpdatedTime = conv.UpdatedTime
};
@ -911,8 +923,8 @@ public class MongoRepository : IBotSharpRepository
var filter = Builders<ExecutionLogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var update = Builders<ExecutionLogDocument>.Update
.SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
.PushEach(x => x.Logs, logs);
.SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
.PushEach(x => x.Logs, logs);
_dc.ExectionLogs.UpdateOne(filter, update, _options);
}
@ -938,7 +950,7 @@ public class MongoRepository : IBotSharpRepository
var conversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
var messageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
var logElement = new PromptLogElement
var logElement = new PromptLogMongoElement
{
MessageId = messageId,
AgentId = log.AgentId,