SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #786 from iceljc/features/add-crontab-storage

Browse source 
add crontab storage
This commit is contained in:
iceljc 2024-12-08 18:37:42 -06:00 committed by GitHub
parent 8d176b54f0 4fa714ff87
commit eac490e8ed
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 300 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
BotSharp.sln
View file

@ -121,6 +121,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Core.SideCar", "sr
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.VertexAI", "src\Plugins\BotSharp.Plugin.LangChain\BotSharp.Plugin.VertexAI.csproj", "{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Crontab", "src\Infrastructure\BotSharp.Core.Crontab\BotSharp.Core.Crontab.csproj", "{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -489,6 +491,14 @@ Global
{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B}.Release|Any CPU.Build.0 = Release|Any CPU
{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B}.Release|x64.ActiveCfg = Release|Any CPU
{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B}.Release|x64.Build.0 = Release|Any CPU
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Debug|x64.ActiveCfg = Debug|Any CPU
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Debug|x64.Build.0 = Debug|Any CPU
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Release|Any CPU.Build.0 = Release|Any CPU
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Release|x64.ActiveCfg = Release|Any CPU
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -547,6 +557,7 @@ Global
{F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
{6D3A54F9-4792-41DB-BE7D-4F7B1D918EAE} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
{7DA2DCD0-551B-432E-AA5C-22DDD3ED459B} = {D5293208-2BEF-42FC-A64C-5954F61720BA}
{F812BAAE-5A7D-4DF7-8E71-70696B51C61F} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

2
src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>

15
src/Infrastructure/BotSharp.Core.Crontab/Models/CrontabItem.cs → src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs
View file

@ -1,12 +1,25 @@
namespace BotSharp.Core.Crontab.Models;
namespace BotSharp.Abstraction.Crontab.Models;
public class CrontabItem : ScheduleTaskArgs
{
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("user_id")]
public string UserId { get; set; } = null!;
[JsonPropertyName("agent_id")]
public string AgentId { get; set; } = null!;
[JsonPropertyName("conversation_id")]
public string ConversationId { get; set; } = null!;
[JsonPropertyName("execution_result")]
public string ExecutionResult { get; set; } = null!;
[JsonPropertyName("created_time")]
public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
public override string ToString()
{
return $"{Topic}: {Description} [AgentId: {AgentId}, UserId: {UserId}]";

26
src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItemFilter.cs Normal file
View file

@ -0,0 +1,26 @@
namespace BotSharp.Abstraction.Crontab.Models;
public class CrontabItemFilter : Pagination
{
[JsonPropertyName("user_ids")]
public IEnumerable<string>? UserIds { get; set; }
[JsonPropertyName("agent_ids")]
public IEnumerable<string>? AgentIds { get; set; }
[JsonPropertyName("conversation_ids")]
public IEnumerable<string>? ConversationIds { get; set; }
[JsonPropertyName("topics")]
public IEnumerable<string>? Topics { get; set; }
public CrontabItemFilter()
{
}
public static CrontabItemFilter Empty()
{
return new CrontabItemFilter();
}
}

4
src/Infrastructure/BotSharp.Core.Crontab/Models/ScheduleTaskArgs.cs → src/Infrastructure/BotSharp.Abstraction/Crontab/Models/ScheduleTaskArgs.cs
View file

@ -1,6 +1,4 @@
using System.Text.Json.Serialization;
namespace BotSharp.Core.Crontab.Models;
namespace BotSharp.Abstraction.Crontab.Models;
public class ScheduleTaskArgs
{

5
src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file

@ -145,4 +145,9 @@ public interface IBotSharpRepository : IHaveServiceProvider
bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null);
PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter);
#endregion
#region Crontab
bool InsertCrontabItem(CrontabItem item) => throw new NotImplementedException();
PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter) => throw new NotImplementedException();
#endregion
}

3
src/Infrastructure/BotSharp.Abstraction/Using.cs
View file

@ -17,4 +17,5 @@ global using BotSharp.Abstraction.Translation.Attributes;
global using BotSharp.Abstraction.Messaging.Enums;
global using BotSharp.Abstraction.Files.Models;
global using BotSharp.Abstraction.Files.Enums;
global using BotSharp.Abstraction.Knowledges.Models;
global using BotSharp.Abstraction.Knowledges.Models;
global using BotSharp.Abstraction.Crontab.Models;

2
src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabHook.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Core.Crontab.Models;
namespace BotSharp.Core.Crontab.Abstraction;
public interface ICrontabHook

2
src/Infrastructure/BotSharp.Core.Crontab/Abstraction/ICrontabService.cs
View file

@ -1,5 +1,3 @@
using BotSharp.Core.Crontab.Models;
namespace BotSharp.Core.Crontab.Abstraction;
public interface ICrontabService

5
src/Infrastructure/BotSharp.Core.Crontab/BotSharp.Core.Crontab.csproj
View file

@ -21,7 +21,6 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
@ -29,4 +28,8 @@
<PackageReference Include="NCrontab" Version="3.3.3" />
</ItemGroup>
<ItemGroup>
<Folder Include="Models\" />
</ItemGroup>
</Project>

1
src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs
View file

@ -17,7 +17,6 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Crontab.Models;
using BotSharp.Core.Infrastructures;

4
src/Infrastructure/BotSharp.Core.Crontab/Using.cs
View file

@ -4,10 +4,10 @@ global using Microsoft.Extensions.Configuration;
global using Microsoft.Extensions.DependencyInjection;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Crontab.Models;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Conversations.Models;
global using BotSharp.Abstraction.Functions;
global using BotSharp.Core.Crontab.Services;
global using BotSharp.Core.Crontab.Abstraction;
global using BotSharp.Core.Crontab.Models;
global using BotSharp.Core.Crontab.Abstraction;

96
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Crontab.cs Normal file
View file

@ -0,0 +1,96 @@
using System.IO;
namespace BotSharp.Core.Repository;
public partial class FileRepository
{
public bool InsertCrontabItem(CrontabItem item)
{
if (item == null)
{
return false;
}
try
{
var baseDir = Path.Combine(_dbSettings.FileRepository, CRONTAB_FOLDER);
item.Id = Guid.NewGuid().ToString();
var dir = Path.Combine(baseDir, item.Id);
if (Directory.Exists(dir))
{
return false;
}
Directory.CreateDirectory(dir);
Thread.Sleep(50);
var itemFile = Path.Combine(dir, CRONTAB_FILE);
var json = JsonSerializer.Serialize(item, _options);
File.WriteAllText(itemFile, json);
return true;
}
catch (Exception ex)
{
_logger.LogError($"Error when saving crontab item: {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter)
{
if (filter == null)
{
filter = CrontabItemFilter.Empty();
}
var records = new List<CrontabItem>();
var dir = Path.Combine(_dbSettings.FileRepository, CRONTAB_FOLDER);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
var totalDirs = Directory.GetDirectories(dir);
foreach (var d in totalDirs)
{
var file = Path.Combine(d, CRONTAB_FILE);
if (!File.Exists(file)) continue;
var json = File.ReadAllText(file);
var record = JsonSerializer.Deserialize<CrontabItem>(json, _options);
if (record == null) continue;
var matched = true;
if (filter?.AgentIds != null)
{
matched = matched && filter.AgentIds.Contains(record.AgentId);
}
if (filter?.ConversationIds != null)
{
matched = matched && filter.ConversationIds.Contains(record.ConversationId);
}
if (filter?.UserIds != null)
{
matched = matched && filter.UserIds.Contains(record.UserId);
}
if (filter?.Topics != null)
{
matched = matched && filter.Topics.Contains(record.Topic);
}
if (!matched) continue;
records.Add(record);
}
return new PagedItems<CrontabItem>
{
Items = records.OrderByDescending(x => x.CreatedTime).Skip(filter.Offset).Take(filter.Size),
Count = records.Count(),
};
}
}

4
src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs
View file

@ -55,6 +55,9 @@ public partial class FileRepository : IBotSharpRepository
private const string PLUGIN_CONFIG_FILE = "config.json";
private const string STATS_FILE = "stats.json";
private const string CRONTAB_FOLDER = "crontabs";
private const string CRONTAB_FILE = "crontab.json";
public FileRepository(
IServiceProvider services,
BotSharpDatabaseSettings dbSettings,
@ -88,7 +91,6 @@ public partial class FileRepository : IBotSharpRepository
private List<Agent> _agents = new List<Agent>();
private List<RoleAgent> _roleAgents = new List<RoleAgent>();
private List<UserAgent> _userAgents = new List<UserAgent>();
private List<Conversation> _conversations = new List<Conversation>();
private PluginConfig? _pluginConfig = null;
private IQueryable<Role> Roles

2
src/Infrastructure/BotSharp.Core/Using.cs
View file

@ -14,7 +14,7 @@ global using BotSharp.Abstraction.Routing;
global using BotSharp.Abstraction.Plugins;
global using BotSharp.Abstraction.Agents;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Knowledges;
global using BotSharp.Abstraction.Crontab.Models;
global using BotSharp.Abstraction.Users;
global using BotSharp.Abstraction.Roles;
global using BotSharp.Abstraction.Roles.Models;

2
src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubCrontabHook.cs
View file

@ -1,5 +1,5 @@
using BotSharp.Abstraction.Crontab.Models;
using BotSharp.Core.Crontab.Abstraction;
using BotSharp.Core.Crontab.Models;
using Microsoft.AspNetCore.SignalR;
namespace BotSharp.Plugin.ChatHub.Hooks;

53
src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs Normal file
View file

@ -0,0 +1,53 @@
using BotSharp.Abstraction.Crontab.Models;
namespace BotSharp.Plugin.MongoStorage.Collections;
public class CrontabItemDocument : MongoBase
{
public string UserId { get; set; }
public string AgentId { get; set; }
public string ConversationId { get; set; }
public string ExecutionResult { get; set; }
public string Cron { get; set; }
public string Topic { get; set; }
public string Description { get; set; }
public string Script { get; set; }
public string Language { get; set; }
public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
public static CrontabItem ToDomainModel(CrontabItemDocument item)
{
return new CrontabItem
{
Id = item.Id,
UserId = item.UserId,
AgentId = item.AgentId,
ConversationId = item.ConversationId,
ExecutionResult = item.ExecutionResult,
Cron = item.Cron,
Topic = item.Topic,
Description = item.Description,
Script = item.Script,
Language = item.Language,
CreatedTime = item.CreatedTime
};
}
public static CrontabItemDocument ToMongoModel(CrontabItem item)
{
return new CrontabItemDocument
{
Id = item.Id,
UserId = item.UserId,
AgentId = item.AgentId,
ConversationId = item.ConversationId,
ExecutionResult = item.ExecutionResult,
Cron = item.Cron,
Topic = item.Topic,
Description = item.Description,
Script = item.Script,
Language = item.Language,
CreatedTime = item.CreatedTime
};
}
}

3
src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs
View file

@ -165,4 +165,7 @@ public class MongoDbContext
public IMongoCollection<RoleAgentDocument> RoleAgents
=> Database.GetCollection<RoleAgentDocument>($"{_collectionPrefix}_RoleAgents");
public IMongoCollection<CrontabItemDocument> CrontabItems
=> Database.GetCollection<CrontabItemDocument>($"{_collectionPrefix}_CronTabItems");
}

73
src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Crontab.cs Normal file
View file

@ -0,0 +1,73 @@
using BotSharp.Abstraction.Crontab.Models;
using Microsoft.Extensions.Logging;
namespace BotSharp.Plugin.MongoStorage.Repository;
public partial class MongoRepository
{
public bool InsertCrontabItem(CrontabItem item)
{
if (item == null)
{
return false;
}
try
{
item.Id = Guid.NewGuid().ToString();
var cronDoc = CrontabItemDocument.ToMongoModel(item);
_dc.CrontabItems.InsertOne(cronDoc);
return true;
}
catch (Exception ex)
{
_logger.LogError($"Error when saving crontab item: {ex.Message}\r\n{ex.InnerException}");
return false;
}
}
public PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter)
{
if (filter == null)
{
filter = CrontabItemFilter.Empty();
}
var cronBuilder = Builders<CrontabItemDocument>.Filter;
var cronFilters = new List<FilterDefinition<CrontabItemDocument>>() { cronBuilder.Empty };
// Filter conversations
if (filter?.AgentIds != null)
{
cronFilters.Add(cronBuilder.In(x => x.AgentId, filter.AgentIds));
}
if (filter?.ConversationIds != null)
{
cronFilters.Add(cronBuilder.In(x => x.ConversationId, filter.ConversationIds));
}
if (filter?.Topics != null)
{
cronFilters.Add(cronBuilder.In(x => x.Topic, filter.Topics));
}
if (filter?.UserIds != null)
{
cronFilters.Add(cronBuilder.In(x => x.UserId, filter.UserIds));
}
// Sort and paginate
var filterDef = cronBuilder.And(cronFilters);
var sortDef = Builders<CrontabItemDocument>.Sort.Descending(x => x.CreatedTime);
var cronDocs = _dc.CrontabItems.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList();
var count = _dc.CrontabItems.CountDocuments(filterDef);
var crontabItems = cronDocs.Select(x => CrontabItemDocument.ToDomainModel(x)).ToList();
return new PagedItems<CrontabItem>
{
Items = crontabItems,
Count = (int)count
};
}
}

3
src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverCrontabHook.cs
View file

@ -1,6 +1,5 @@
using BotSharp.Abstraction.Crontab.Models;
using BotSharp.Core.Crontab.Abstraction;
using BotSharp.Core.Crontab.Models;
using Microsoft.EntityFrameworkCore.Query;
namespace BotSharp.Plugin.SqlDriver.Hooks;

3
src/WebStarter/WebStarter.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>$(TargetFramework)</TargetFramework>
@ -29,6 +29,7 @@
<ItemGroup>
<ProjectReference Include="..\..\tests\BotSharp.Plugin.PizzaBot\BotSharp.Plugin.PizzaBot.csproj" />
<ProjectReference Include="..\BotSharp.ServiceDefaults\BotSharp.ServiceDefaults.csproj" />
<ProjectReference Include="..\Infrastructure\BotSharp.Core.Crontab\BotSharp.Core.Crontab.csproj" />
<ProjectReference Include="..\Infrastructure\BotSharp.Core.SideCar\BotSharp.Core.SideCar.csproj" />
</ItemGroup>

1
src/WebStarter/appsettings.json
View file

@ -328,6 +328,7 @@
"Assemblies": [
"BotSharp.Core",
"BotSharp.Core.SideCar",
"BotSharp.Core.Crontab",
"BotSharp.Logger",
"BotSharp.Plugin.MongoStorage",
"BotSharp.Plugin.Dashboard",