SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge branch 'SciSharp:master' into master

Browse source
This commit is contained in:
hchen2020 2024-10-01 09:03:08 -05:00 committed by GitHub
parent 06d0b398fc 62e1ce50e7
commit c511cb24cd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 87 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs
View file

@ -24,7 +24,7 @@ public class PrimaryStagePlanFn : IFunctionCallback
state.SetState("max_tokens", "4096");
var task = JsonSerializer.Deserialize<PrimaryRequirementRequest>(message.FunctionArgs);
var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
var collectionName = knowledgeSettings.Default.CollectionName;
// Get knowledge from vectordb
var hooks = _services.GetServices<IKnowledgeHook>();

10
src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs
View file

@ -17,16 +17,14 @@ public class SecondaryStagePlanFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var fn = _services.GetRequiredService<IRoutingService>();
var agentService = _services.GetRequiredService<IAgentService>();
var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
var states = _services.GetRequiredService<IConversationStateService>();
var msgSecondary = RoleDialogModel.From(message);
var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
var collectionName = knowledgeSettings.Default.CollectionName;
var planPrimary = states.GetState("planning_result");
var taskPrimary = states.GetState("requirement_detail");
var taskSecondary = JsonSerializer.Deserialize<SecondaryBreakdownTask>(msgSecondary.FunctionArgs);
@ -35,8 +33,8 @@ public class SecondaryStagePlanFn : IFunctionCallback
{
Confidence = 0.6f
});
var knowledgeResults = "";
knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));
// Get second stage planning prompt
var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
@ -45,7 +43,7 @@ public class SecondaryStagePlanFn : IFunctionCallback
var plannerAgent = new Agent
{
Id = string.Empty,
Id = BuiltInAgentId.Planner,
Name = "planning_2nd",
Instruction = secondPlanningPrompt,
TemplateDict = new Dictionary<string, object>(),

5
src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs
View file

@ -33,7 +33,7 @@ public class SummaryPlanFn : IFunctionCallback
var states = _services.GetRequiredService<IConversationStateService>();
var steps = states.GetState("planning_result").JsonArrayContent<SecondStagePlan>();
var allTables = new List<string>();
var ddlStatements = "";
var ddlStatements = string.Empty;
var relevantKnowledge = states.GetState("planning_result");
var dictionaryItems = states.GetState("dictionary_items");
@ -42,6 +42,7 @@ public class SummaryPlanFn : IFunctionCallback
allTables.AddRange(step.Tables);
}
var distinctTables = allTables.Distinct().ToList();
foreach (var table in distinctTables)
{
var msgCopy = RoleDialogModel.From(message);
@ -92,7 +93,7 @@ public class SummaryPlanFn : IFunctionCallback
return render.Render(template, new Dictionary<string, object>
{
{ "task_description", taskDescription },
{ "summary_requirements", string.Join("\r\n",additionalRequirements) },
{ "summary_requirements", string.Join("\r\n", additionalRequirements) },
{ "relevant_knowledges", relevantKnowledge },
{ "dictionary_items", dictionaryItems },
{ "table_structure", ddlStatement },

2
src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/agent.json
View file

@ -11,7 +11,7 @@
"profiles": [ "planning" ],
"utilities": [ "two-stage-planner" ],
"llmConfig": {
"provider": "azure-openai",
"provider": "openai",
"model": "gpt-4o",
"max_recursion_depth": 10
}

7
src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj
View file

@ -11,8 +11,11 @@
</PropertyGroup>
<ItemGroup>
<Compile Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\**" />
<Compile Remove="packages\**" />
<EmbeddedResource Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\**" />
<EmbeddedResource Remove="packages\**" />
<None Remove="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\**" />
<None Remove="packages\**" />
</ItemGroup>
@ -80,8 +83,4 @@
<ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="data\agents\beda4c12-e1ec-4b4b-b328-3df4a6687c4f\templates\" />
</ItemGroup>
</Project>

2
src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs
View file

@ -19,6 +19,6 @@ public class SqlDriverController : ControllerBase
public async Task<bool> ImportDbKnowledge(ImportDbKnowledgeRequest request)
{
var dbKnowledge = _services.GetRequiredService<DbKnowledgeService>();
return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema);
return await dbKnowledge.Import(request);
}
}

1
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs
View file

@ -44,7 +44,6 @@ public class ExecuteQueryFn : IFunctionCallback
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
using var connection = new SqlConnection(settings.SqlServerExecutionConnectionString ?? settings.SqlServerConnectionString);
var dictionary = new Dictionary<string, object>();
return connection.Query(string.Join("\r\n", sqlTexts));
}
}

45
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Plugin.SqlDriver.Models;
using Fluid.Ast.BinaryExpressions;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Logging;
using MySqlConnector;
@ -37,10 +36,6 @@ public class GetTableDefinitionFn : IFunctionCallback
};
message.Content = string.Join("\r\n\r\n", tableDdls);
//var states = _services.GetRequiredService<IConversationStateService>();
//states.SetState($"table_definition_{args.Table}", message.Content);
return true;
}
@ -48,7 +43,7 @@ public class GetTableDefinitionFn : IFunctionCallback
{
var settings = _services.GetRequiredService<SqlDriverSetting>();
var tableDdls = new List<string>();
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString);
using var connection = new MySqlConnection(settings.MySqlExecutionConnectionString ?? settings.MySqlConnectionString);
connection.Open();
foreach (var table in tables)
@ -76,7 +71,6 @@ public class GetTableDefinitionFn : IFunctionCallback
}
connection.Close();
return tableDdls;
}
@ -92,27 +86,27 @@ public class GetTableDefinitionFn : IFunctionCallback
try
{
var sql = @$"DECLARE @TableName NVARCHAR(128) = '{table}';
DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' (';
DECLARE @SQL NVARCHAR(MAX) = 'CREATE TABLE ' + @TableName + ' (';
SELECT @SQL = @SQL + '
' + COLUMN_NAME + ' ' +
DATA_TYPE +
CASE
WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%'
THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'
WHEN DATA_TYPE IN ('decimal', 'numeric')
THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'
ELSE ''
END + ' ' +
CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ','
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = @TableName
ORDER BY ORDINAL_POSITION;
SELECT @SQL = @SQL + '
' + COLUMN_NAME + ' ' +
DATA_TYPE +
CASE
WHEN CHARACTER_MAXIMUM_LENGTH IS NOT NULL AND DATA_TYPE LIKE '%char%'
THEN '(' + CAST(CHARACTER_MAXIMUM_LENGTH AS VARCHAR(10)) + ')'
WHEN DATA_TYPE IN ('decimal', 'numeric')
THEN '(' + CAST(NUMERIC_PRECISION AS VARCHAR(10)) + ',' + CAST(NUMERIC_SCALE AS VARCHAR(10)) + ')'
ELSE ''
END + ' ' +
CASE WHEN IS_NULLABLE = 'NO' THEN 'NOT NULL' ELSE 'NULL' END + ','
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = @TableName
ORDER BY ORDINAL_POSITION;
-- Remove the last comma and add closing parenthesis
SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');';
-- Remove the last comma and add closing parenthesis
SET @SQL = LEFT(@SQL, LEN(@SQL) - 1) + ');';
SELECT @SQL;";
SELECT @SQL;";
using var command = new SqlCommand(sql, connection);
using var reader = command.ExecuteReader();
@ -129,7 +123,6 @@ SELECT @SQL;";
}
connection.Close();
return tableDdls;
}
}

55
src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs
View file

@ -1,14 +1,9 @@
using Azure;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Agents.Services;
using BotSharp.Core.Infrastructures;
using BotSharp.Plugin.SqlDriver.Models;
using MySqlConnector;
using System.Text.RegularExpressions;
using static Dapper.SqlMapper;
using static System.Net.Mime.MediaTypeNames;
namespace BotSharp.Plugin.SqlDriver.Functions;
@ -35,15 +30,16 @@ public class LookupDictionaryFn : IFunctionCallback
var agentService = _services.GetRequiredService<IAgentService>();
var currentAgent = await agentService.LoadAgent(message.CurrentAgentId);
var dictionarySqlPrompt = await GetDictionarySQLPrompt(args.SqlStatement, msgCopy.Content);
var plannerAgent = new Agent
var agent = new Agent
{
Id = string.Empty,
Id = message.CurrentAgentId ?? string.Empty,
Name = "sqlDriver_DictionarySearch",
Instruction = dictionarySqlPrompt,
TemplateDict = new Dictionary<string, object>(),
LlmConfig = currentAgent.LlmConfig
};
var response = await GetAiResponse(plannerAgent);
var response = await GetAiResponse(agent);
args = JsonSerializer.Deserialize<LookupDictionary>(response.Content);
// check if need to instantely
@ -59,13 +55,16 @@ public class LookupDictionaryFn : IFunctionCallback
{
message.Content = JsonSerializer.Serialize(result);
}
var states = _services.GetRequiredService<IConversationStateService>();
var dictionaryItems = states.GetState("dictionary_items", "");
dictionaryItems += "\r\n\r\n" + args.Table + ":\r\n" + args.Reason + ":\r\n" + message.Content + "\r\n";
var newItem = BuildDictionaryItem(args.Table, args.Reason, message.Content);
dictionaryItems += !string.IsNullOrWhiteSpace(newItem) ? $"\r\n{newItem}\r\n" : string.Empty;
states.SetState("dictionary_items", dictionaryItems);
return true;
}
private async Task<string> GetDictionarySQLPrompt(string originalSql, string tableStructure)
{
var agentService = _services.GetRequiredService<IAgentService>();
@ -83,15 +82,45 @@ public class LookupDictionaryFn : IFunctionCallback
{ "response_format", responseFormat }
});
}
private async Task<RoleDialogModel> GetAiResponse(Agent plannerAgent)
private async Task<RoleDialogModel> GetAiResponse(Agent agent)
{
var text = "Check and correct the SQL statement.";
var message = new RoleDialogModel(AgentRole.User, text);
var completion = CompletionProvider.GetChatCompletion(_services,
provider: plannerAgent.LlmConfig.Provider,
model: plannerAgent.LlmConfig.Model);
provider: agent.LlmConfig.Provider,
model: agent.LlmConfig.Model);
return await completion.GetChatCompletions(plannerAgent, new List<RoleDialogModel> { message });
return await completion.GetChatCompletions(agent, new List<RoleDialogModel> { message });
}
private string BuildDictionaryItem(string? table, string? reason, string? result)
{
var res = string.Empty;
if (!string.IsNullOrWhiteSpace(table))
{
res += $"Table: {table}";
}
if (!string.IsNullOrWhiteSpace(reason))
{
if (!string.IsNullOrWhiteSpace(res))
{
res += "\r\n";
}
res += $"Reason: {reason}";
}
if (!string.IsNullOrWhiteSpace(result))
{
if (!string.IsNullOrWhiteSpace(res))
{
res += "\r\n";
}
res += $"Result: {result}";
}
return res;
}
}

1
src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs
View file

@ -2,7 +2,6 @@ using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Agents.Settings;
using BotSharp.Abstraction.Functions.Models;
using BotSharp.Abstraction.Repositories;
using System.Collections.Generic;
namespace BotSharp.Plugin.SqlDriver.Hooks;

1
src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs
View file

@ -1,7 +1,6 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Planning;
using BotSharp.Abstraction.Routing;
using BotSharp.Core.Agents.Services;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.SqlDriver.Hooks;

6
src/Plugins/BotSharp.Plugin.SqlDriver/Models/LookupDictionary.cs
View file

@ -5,11 +5,11 @@ namespace BotSharp.Plugin.SqlDriver.Models;
public class LookupDictionary
{
[JsonPropertyName("sql_statement")]
public string SqlStatement { get; set; }
public string? SqlStatement { get; set; }
[JsonPropertyName("reason")]
public string Reason { get; set; }
public string? Reason { get; set; }
[JsonPropertyName("table")]
public string Table { get; set; }
public string? Table { get; set; }
}

3
src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs
View file

@ -16,4 +16,7 @@ public class ImportDbKnowledgeRequest : RequestBase
{
[JsonPropertyName("schema")]
public string Schema { get; set; }
[JsonPropertyName("knowledgebase_collection")]
public string KnowledgebaseCollection { get; set; }
}

10
src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs
View file

@ -3,9 +3,9 @@ using Microsoft.Extensions.Logging;
using BotSharp.Core.Infrastructures;
using MySqlConnector;
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.Knowledges.Settings;
using BotSharp.Abstraction.Knowledges.Enums;
using BotSharp.Abstraction.VectorStorage.Models;
using BotSharp.Plugin.SqlDriver.Models;
namespace BotSharp.Plugin.SqlDriver.Services;
@ -22,12 +22,14 @@ public class DbKnowledgeService
_logger = logger;
}
public async Task<bool> Import(string provider, string model, string schema)
public async Task<bool> Import(ImportDbKnowledgeRequest request)
{
var sqlDriverSettings = _services.GetRequiredService<SqlDriverSetting>();
var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
var provider = request.Provider ?? "openai";
var model = request.Model ?? "gpt-4o";
var schema = request.Schema;
var collectionName = request.KnowledgebaseCollection;
var tables = new HashSet<string>();
using var connection = new MySqlConnection(sqlDriverSettings.MySqlConnectionString);

2
src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json
View file

@ -9,7 +9,7 @@
"isPublic": true,
"profiles": [ "database" ],
"llmConfig": {
"provider": "azure-openai",
"provider": "openai",
"model": "gpt-4o-mini"
},
"routingRules": [