Merge pull request #549 from hchen2020/master

knowledge learner
This commit is contained in:
C. Oceania 2024-07-17 16:05:54 -05:00 committed by GitHub
commit dd36d0b58f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 335 additions and 32 deletions

View file

@ -2,9 +2,33 @@ namespace BotSharp.Abstraction.Agents.Enums;
public class BuiltInAgentId
{
/// <summary>
/// A routing agent can be used as a base router.
/// </summary>
public const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a";
/// <summary>
/// A demo agent used for open domain chatting
/// </summary>
public const string Chatbot = "01e2fc5c-2c89-4ec7-8470-7688608b496c";
/// <summary>
/// Human customer service
/// </summary>
public const string HumanSupport = "01dcc3e5-0af7-49e6-ad7a-a760bd12dc4b";
/// <summary>
/// Used as a container to host the shared tools/ utilities built in different plugins.
/// </summary>
public const string UtilityAssistant = "6745151e-6d46-4a02-8de4-1c4f21c7da95";
/// <summary>
/// Used when router can't route to any existing task agent
/// </summary>
public const string Fallback = "01fcc3e5-0af7-49e6-ad7a-a760bd12dc4d";
/// <summary>
/// Used by knowledgebase plugin to acquire domain knowledge
/// </summary>
public const string Learner = "01acc3e5-0af7-49e6-ad7a-a760bd12dc40";
}

View file

@ -7,6 +7,7 @@ public class StateConst
public const string NEXT_ACTION_AGENT = "next_action_agent";
public const string NEXT_ACTION_REASON = "next_action_reason";
public const string USER_GOAL_AGENT = "user_goal_agent";
public const string AGENT_REDIRECTION_REASON = "agent_redirection_reason";
public const string LANGUAGE = "language";
}

View file

@ -0,0 +1,10 @@
namespace BotSharp.Abstraction.Knowledges.Models;
public class ExtractedKnowledge
{
[JsonPropertyName("question")]
public string Question { get; set; } = string.Empty;
[JsonPropertyName("answer")]
public string Answer { get; set; } = string.Empty;
}

View file

@ -4,6 +4,6 @@ public interface IVectorDb
{
Task<List<string>> GetCollections();
Task CreateCollection(string collectionName, int dim);
Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<List<string>> Search(string collectionName, float[] vector, int limit = 5);
Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null);
Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f);
}

View file

@ -19,11 +19,7 @@ public partial class ConversationService
Agent agent = await agentService.LoadAgent(agentId);
var content = $"Received [{agent.Name}] {message.Role}: {message.Content}";
#if DEBUG
Console.WriteLine(content);
#else
_logger.LogInformation(content);
#endif
message.CurrentAgentId = agent.Id;
if (string.IsNullOrEmpty(message.SenderId))

View file

@ -89,6 +89,7 @@ public partial class RouteToAgentFn : IFunctionCallback
// Stack redirection agent
_context.Push(agentId, reason: $"REDIRECTION {reason}");
message.Content = reason;
states.SetState(StateConst.AGENT_REDIRECTION_REASON, reason, isNeedVersion: false);
}
}

View file

@ -1 +1,4 @@
You are a smart AI Assistant.
You are a smart AI Assistant.
{% if agent_redirection_reason %}
You've been reached out because: {{ agent_redirection_reason }}
{% endif %}

View file

@ -1 +1 @@
Break down the users most recent needs and figure out the instruction of next step.
Break down the users most recent needs and figure out the instruction of next step without explanation.

View file

@ -11,7 +11,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Anthropic.SDK" Version="3.2.1" />
<PackageReference Include="Anthropic.SDK" Version="3.2.3" />
</ItemGroup>
<ItemGroup>

View file

@ -1,4 +1,5 @@
using Anthropic.SDK.Common;
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.MLTasks.Settings;
using System.Text.Json;
using System.Text.Json.Nodes;
@ -160,13 +161,17 @@ public class ChatCompletionProvider : IChatCompletion
}
}
var state = _services.GetRequiredService<IConversationStateService>();
var temperature = decimal.Parse(state.GetState("temperature", "0.0"));
var maxToken = int.Parse(state.GetState("max_tokens", "512"));
var parameters = new MessageParameters()
{
Messages = messages,
MaxTokens = 256,
Model = settings.Version, // AnthropicModels.Claude3Haiku
MaxTokens = maxToken,
Model = settings.Name,
Stream = false,
Temperature = 0m,
Temperature = temperature,
SystemMessage = instruction,
Tools = new List<Function>() { }
};

View file

@ -16,6 +16,31 @@
<None Remove="agents\**" />
</ItemGroup>
<ItemGroup>
<None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\agent.json" />
<None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\functions\memorize_knowledge.json" />
<None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\instruction.liquid" />
<None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\knowledge_retrieval.fn.liquid" />
</ItemGroup>
<ItemGroup>
<Content Include="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\agent.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\functions\memorize_knowledge.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\instruction.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\knowledge_retrieval.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\knowledge_retrieval.fn.liquid">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<PackageReference Include="PdfPig" Version="0.1.8" />
<PackageReference Include="TensorFlow.Keras" Version="0.15.0" />
@ -23,6 +48,7 @@
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,6 @@
namespace BotSharp.Plugin.KnowledgeBase.Enum;
public class UtilityName
{
public const string KnowledgeRetrieval = "knowledge-retrieval";
}

View file

@ -0,0 +1,40 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.KnowledgeBase.Functions;
public class KnowledgeRetrievalFn : IFunctionCallback
{
public string Name => "knowledge_retrieval";
private readonly IServiceProvider _services;
private readonly KnowledgeBaseSettings _settings;
public KnowledgeRetrievalFn(IServiceProvider services, KnowledgeBaseSettings settings)
{
_services = services;
_settings = settings;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
var embedding = _services.GetServices<ITextEmbedding>()
.FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding));
var vector = await embedding.GetVectorsAsync(new List<string>
{
args.Question
});
var vectorDb = _services.GetRequiredService<IVectorDb>();
var id = Utilities.HashTextMd5(args.Question);
var knowledges = await vectorDb.Search("lessen", vector[0], "answer");
message.Content = string.Join("\r\n\r\n=====\r\n", knowledges);
return true;
}
}

View file

@ -0,0 +1,47 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.KnowledgeBase.Functions;
public class MemorizeKnowledgeFn : IFunctionCallback
{
public string Name => "memorize_knowledge";
private readonly IServiceProvider _services;
private readonly KnowledgeBaseSettings _settings;
public MemorizeKnowledgeFn(IServiceProvider services, KnowledgeBaseSettings settings)
{
_services = services;
_settings = settings;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
var embedding = _services.GetServices<ITextEmbedding>()
.First(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding));
var vector = await embedding.GetVectorsAsync(new List<string>
{
args.Question
});
var vectorDb = _services.GetRequiredService<IVectorDb>();
await vectorDb.CreateCollection("lessen", vector[0].Length);
var id = Utilities.HashTextMd5(args.Question);
var result = await vectorDb.Upsert("lessen", id, vector[0],
args.Question,
new Dictionary<string, string>
{
{ "answer", args.Answer }
});
message.Content = $"Save result: {(result ? "success" : "failed")}";
return true;
}
}

View file

@ -0,0 +1,66 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Models;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
using BotSharp.Plugin.KnowledgeBase.Enum;
namespace BotSharp.Plugin.KnowledgeBase.Hooks;
public class KnowledgeBaseAgentHook : AgentHookBase, IAgentHook
{
public override string SelfId => string.Empty;
public KnowledgeBaseAgentHook(IServiceProvider services, AgentSettings settings) : base(services, settings)
{
}
public override void OnAgentLoaded(Agent agent)
{
var conv = _services.GetRequiredService<IConversationService>();
var isConvMode = conv.IsConversationMode();
if (isConvMode)
{
AddUtility(agent, UtilityName.KnowledgeRetrieval, "knowledge_retrieval");
}
base.OnAgentLoaded(agent);
}
private void AddUtility(Agent agent, string utility, string functionName)
{
if (!IsEnableUtility(agent, utility)) return;
var (prompt, fn) = GetPromptAndFunction(functionName);
if (fn != null)
{
if (!string.IsNullOrWhiteSpace(prompt))
{
agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n";
}
if (agent.Functions == null)
{
agent.Functions = new List<FunctionDef> { fn };
}
else
{
agent.Functions.Add(fn);
}
}
}
private bool IsEnableUtility(Agent agent, string utility)
{
return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility);
}
private (string, FunctionDef?) GetPromptAndFunction(string functionName)
{
var db = _services.GetRequiredService<IBotSharpRepository>();
var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant);
var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty;
var fn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName));
return (prompt, fn);
}
}

View file

@ -0,0 +1,11 @@
using BotSharp.Plugin.KnowledgeBase.Enum;
namespace BotSharp.Plugin.KnowledgeBase.Hooks;
public class KnowledgeBaseUtilityHook : IAgentUtilityHook
{
public void AddUtilities(List<string> utilities)
{
utilities.Add(UtilityName.KnowledgeRetrieval);
}
}

View file

@ -1,5 +1,6 @@
using BotSharp.Abstraction.Plugins.Models;
using BotSharp.Abstraction.Settings;
using BotSharp.Plugin.KnowledgeBase.Hooks;
using Microsoft.Extensions.Configuration;
namespace BotSharp.Plugin.KnowledgeBase;
@ -24,6 +25,8 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
services.AddScoped<ITextChopper, TextChopperService>();
services.AddScoped<IKnowledgeService, KnowledgeService>();
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();
services.AddScoped<IAgentUtilityHook, KnowledgeBaseUtilityHook>();
services.AddScoped<IAgentHook, KnowledgeBaseAgentHook>();
}
public bool AttachMenu(List<PluginMenuDef> menu)

View file

@ -18,7 +18,7 @@ public class MemVectorDatabase : IVectorDb
return _collections.Select(x => x.Key).ToList();
}
public async Task<List<string>> Search(string collectionName, float[] vector, int limit = 5)
public async Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f)
{
if (!_vectors.ContainsKey(collectionName))
{
@ -37,7 +37,7 @@ public class MemVectorDatabase : IVectorDb
return texts;
}
public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
_vectors[collectionName].Add(new VecRecord
{
@ -45,6 +45,8 @@ public class MemVectorDatabase : IVectorDb
Vector = vector,
Text = text
});
return true;
}
private float[] CalEuclideanDistance(float[] vec, List<VecRecord> records)

View file

@ -68,7 +68,7 @@ public partial class KnowledgeService : IKnowledgeService
// Vector search
var db = GetVectorDb();
var result = await db.Search("shared", vector, limit: 10);
var result = await db.Search("shared", vector, "answer", limit: 10);
// Restore
return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}"));

View file

@ -0,0 +1,16 @@
{
"id": "01acc3e5-0af7-49e6-ad7a-a760bd12dc40",
"name": "Learner",
"description": "An avid learner who is eager to learn and acquire knowledge",
"iconUrl": "https://cdn-icons-png.flaticon.com/512/4578/4578413.png",
"type": "task",
"createdDateTime": "2024-07-17T00:00:00Z",
"updatedDateTime": "2024-07-17T00:00:00Z",
"disabled": false,
"isPublic": true,
"profiles": [ "tool" ],
"llmConfig": {
"provider": "anthropic",
"model": "claude-3-5-sonnet-20240620"
}
}

View file

@ -0,0 +1,18 @@
{
"name": "memorize_knowledge",
"description": "Retrieve related domain knowledge to handle user request",
"parameters": {
"type": "object",
"properties": {
"question": {
"type": "string",
"description": "User question with scenario detail"
},
"answer": {
"type": "string",
"description": "The answer to handle user question"
}
},
"required": [ "question", "answer" ]
}
}

View file

@ -0,0 +1,2 @@
You an avid learner who is eager to learn and acquire knowledge from the conversation.
You're good at learning useful domain knowledge and experience by interacting with user.

View file

@ -0,0 +1,14 @@
{
"name": "knowledge_retrieval",
"description": "Retrieve related domain knowledge to handle user request",
"parameters": {
"type": "object",
"properties": {
"question": {
"type": "string",
"description": "Knowledge topic with scenario detail"
}
},
"required": [ "question" ]
}
}

View file

@ -0,0 +1,3 @@
Call function knowledge_retrieval to retrieve related domain knowledge to handle user request.
You must retrieve existing KnowledgeBase to get prerequisite informations before you writing SQL query;
You must retrieve existing API specification from KnowledgeBase before calling a Web API;

View file

@ -17,12 +17,12 @@ public class FaissDb : IVectorDb
throw new NotImplementedException();
}
public Task<List<string>> Search(string collectionName, float[] vector, int limit = 10)
public Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 10, float confidence = 0.5f)
{
throw new NotImplementedException();
}
public Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
public Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
throw new NotImplementedException();
}

View file

@ -9,7 +9,7 @@ public class TextEmbeddingProvider : ITextEmbedding
protected readonly ILogger<TextEmbeddingProvider> _logger;
private const int DEFAULT_DIMENSION = 1536;
protected string _model;
protected string _model = "text-embedding-3-small";
public virtual string Provider => "openai";

View file

@ -67,7 +67,7 @@ public class QdrantDb : IVectorDb
}
}
public async Task Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
public async Task<bool> Upsert(string collectionName, string id, float[] vector, string text, Dictionary<string, string>? payload = null)
{
// Insert vectors
var point = new PointStruct()
@ -78,29 +78,37 @@ public class QdrantDb : IVectorDb
},
Vectors = vector,
Payload = { }
Payload =
{
{ "text", text }
}
};
foreach (var item in payload)
if (payload != null)
{
point.Payload.Add(item.Key, item.Value);
foreach (var item in payload)
{
point.Payload.Add(item.Key, item.Value);
}
}
var result = await GetClient().UpsertAsync(collectionName, points: new List<PointStruct>
var client = GetClient();
var result = await client.UpsertAsync(collectionName, points: new List<PointStruct>
{
point
});
return result.Status == UpdateStatus.Completed;
}
public async Task<List<string>> Search(string collectionName, float[] vector, int limit = 5)
public async Task<List<string>> Search(string collectionName, float[] vector, string returnFieldName, int limit = 5, float confidence = 0.5f)
{
var result = await GetClient().SearchAsync(collectionName, vector, limit: (ulong)limit);
var client = GetClient();
var points = await client.SearchAsync(collectionName, vector,
limit: (ulong)limit,
scoreThreshold: confidence);
var agentService = _services.GetRequiredService<IAgentService>();
var agentDataDir = agentService.GetAgentDataDir(collectionName);
var knowledgePath = Path.Combine(agentDataDir, "knowledge.txt");
var texts = File.ReadAllLines(knowledgePath);
return result.Select(x => texts[x.Id.Num]).ToList();
return points.Select(x => x.Payload[returnFieldName].StringValue).ToList();
}
}

View file

@ -2,4 +2,5 @@ You are connecting to {{ db_type }} database. Please generate SQL statements fol
Please call function sql_select if user wants to get or retrieve data from data tables.
If there are any parameters, please add them in the WHERE clause, each of which starts with "@".
For example, SELECT * FROM table WHERE Id=@Id AND Name=@Name
Avoid returning the entire record and only return the fields you need.
For example, SELECT Id FROM table WHERE Id=@Id AND Name=@Name