Extract data in WebDriver.

This commit is contained in:
Haiping Chen 2024-01-05 21:24:13 -06:00
parent d19e39e9d7
commit 4299a67057
25 changed files with 353 additions and 127 deletions

View file

@ -0,0 +1,37 @@
using BotSharp.Abstraction.Functions;
using BotSharp.Abstraction.Routing.Models;
using BotSharp.Plugin.KnowledgeBase.LlmContexts;
namespace BotSharp.Plugin.KnowledgeBase.Functions;
public class SearchKnowledgesFn : IFunctionCallback
{
public string Name => "search_knowledges";
private readonly IServiceProvider _services;
public SearchKnowledgesFn(IServiceProvider services)
{
_services = services;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<KnowledgeContextIn>(message.FunctionArgs);
var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
var knowledge = await knowledgeService.GetKnowledges(new KnowledgeRetrievalModel
{
AgentId = message.CurrentAgentId,
Question = args.Question
});
if (string.IsNullOrEmpty(knowledge))
{
message.Content = "Can't find any relevant data in local knowledge base.";
var routingCtx = _services.GetRequiredService<RoutingContext>();
routingCtx.Pop();
}
return true;
}
}

View file

@ -1,26 +0,0 @@
using System;
namespace BotSharp.Plugin.KnowledgeBase;
public class KnowledgeBaseAgentHook : AgentHookBase
{
public KnowledgeBaseAgentHook(IServiceProvider services, AgentSettings settings)
: base(services, settings)
{
}
public override bool OnInstructionLoaded(string template, Dictionary<string, object> dict)
{
// Get relevant domain knowledge
/*if (_settings.EnableKnowledgeBase)
{
var knowledge = _services.GetRequiredService<IKnowledgeService>();
agent.Knowledges = await knowledge.GetKnowledges(new KnowledgeRetrievalModel
{
AgentId = agentId,
Question = string.Join("\n", wholeDialogs.Select(x => x.Content))
});
}*/
return base.OnInstructionLoaded(template, dict);
}
}

View file

@ -12,6 +12,8 @@ public class KnowledgeBasePlugin : IBotSharpPlugin
config.Bind("KnowledgeBase", settings);
services.AddSingleton(x => settings);
var a = config["KnowledgeBase"];
services.AddScoped<ITextChopper, TextChopperService>();
services.AddScoped<IKnowledgeService, KnowledgeService>();
services.AddSingleton<IPdf2TextConverter, PigPdf2TextConverter>();

View file

@ -0,0 +1,9 @@
using System.Text.Json.Serialization;
namespace BotSharp.Plugin.KnowledgeBase.LlmContexts;
public class KnowledgeContextIn
{
[JsonPropertyName("question")]
public string Question { get; set; }
}

View file

@ -7,20 +7,24 @@ public class MemVectorDatabase : IVectorDb
{
private readonly Dictionary<string, int> _collections = new Dictionary<string, int>();
private readonly Dictionary<string, List<VecRecord>> _vectors = new Dictionary<string, List<VecRecord>>();
public Task CreateCollection(string collectionName, int dim)
public async Task CreateCollection(string collectionName, int dim)
{
_collections[collectionName] = dim;
_vectors[collectionName] = new List<VecRecord>();
return Task.CompletedTask;
}
public Task<List<string>> GetCollections()
public async Task<List<string>> GetCollections()
{
return Task.FromResult(_collections.Select(x => x.Key).ToList());
return _collections.Select(x => x.Key).ToList();
}
public Task<List<string>> Search(string collectionName, float[] vector, int limit = 5)
public async Task<List<string>> Search(string collectionName, float[] vector, int limit = 5)
{
if (!_vectors.ContainsKey(collectionName))
{
return new List<string>();
}
var similarities = CalCosineSimilarity(vector, _vectors[collectionName]);
// var similarities2 = CalEuclideanDistance(vector, _vectors[collectionName]);
@ -30,10 +34,10 @@ public class MemVectorDatabase : IVectorDb
.Select(i => _vectors[collectionName][i].Text)
.ToList();
return Task.FromResult(texts);
return texts;
}
public Task Upsert(string collectionName, int id, float[] vector, string text)
public async Task Upsert(string collectionName, int id, float[] vector, string text)
{
_vectors[collectionName].Add(new VecRecord
{
@ -41,8 +45,6 @@ public class MemVectorDatabase : IVectorDb
Vector = vector,
Text = text
});
return Task.CompletedTask;
}
private float[] CalEuclideanDistance(float[] vec, List<VecRecord> records)

View file

@ -67,7 +67,8 @@ public class KnowledgeService : IKnowledgeService
var vector = await textEmbedding.GetVectorAsync(retrievalModel.Question);
// Vector search
var result = await GetVectorDb().Search(retrievalModel.AgentId, vector, limit: 10);
var db = GetVectorDb();
var result = await db.Search("shared", vector, limit: 10);
// Restore
return string.Join("\n\n", result.Select((x, i) => $"### Paragraph {i + 1} ###\n{x.Trim()}"));

View file

@ -13,7 +13,7 @@ namespace BotSharp.Plugin.Qdrant;
public class QdrantDb : IVectorDb
{
private readonly QdrantClient _client;
private QdrantClient _client;
private readonly QdrantSetting _setting;
private readonly IServiceProvider _services;
@ -22,11 +22,20 @@ public class QdrantDb : IVectorDb
{
_setting = setting;
_services = services;
_client = new QdrantClient
(
host: _setting.Url,
apiKey: _setting.ApiKey
);
}
private QdrantClient GetClient()
{
if (_client == null)
{
_client = new QdrantClient
(
host: _setting.Url,
apiKey: _setting.ApiKey
);
}
return _client;
}
public async Task<List<string>> GetCollections()
@ -42,7 +51,7 @@ public class QdrantDb : IVectorDb
if (!collections.Contains(collectionName))
{
// Create a new collection
await _client.CreateCollectionAsync(collectionName, new VectorParams()
await GetClient().CreateCollectionAsync(collectionName, new VectorParams()
{
Size = (ulong)dim,
Distance = Distance.Cosine
@ -65,7 +74,7 @@ public class QdrantDb : IVectorDb
public async Task Upsert(string collectionName, int id, float[] vector, string text)
{
// Insert vectors
await _client.UpsertAsync(collectionName, points: new List<PointStruct>
await GetClient().UpsertAsync(collectionName, points: new List<PointStruct>
{
new PointStruct()
{
@ -86,7 +95,7 @@ public class QdrantDb : IVectorDb
public async Task<List<string>> Search(string collectionName, float[] vector, int limit = 5)
{
var result = await _client.SearchAsync(collectionName, vector, limit: (ulong)limit);
var result = await GetClient().SearchAsync(collectionName, vector, limit: (ulong)limit);
var agentService = _services.GetRequiredService<IAgentService>();
var agentDataDir = agentService.GetAgentDataDir(collectionName);

View file

@ -6,12 +6,24 @@ public class PlaywrightInstance : IDisposable
IBrowser _browser;
IPage _page;
public IPlaywright Playwright => _playwright;
// public IPlaywright Playwright => _playwright;
public IBrowser Browser => _browser;
public IPage Page => _page;
public void SetPlaywright(IPlaywright playwright) { _playwright = playwright; }
public void SetBrowser(IBrowser browser) { _browser = browser; }
public async Task InitInstance()
{
if (_playwright == null)
{
_playwright = await Playwright.CreateAsync();
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = false,
Channel = "chrome",
});
}
}
public void SetPage(IPage page) { _page = page; }
public void Dispose()

View file

@ -10,24 +10,31 @@ public partial class PlaywrightWebDriver
var body = await _instance.Page.QuerySelectorAsync("body");
var str = new List<string>();
var anchors = await body.QuerySelectorAllAsync("a");
/*var anchors = await body.QuerySelectorAllAsync("a");
foreach (var a in anchors)
{
var text = await a.TextContentAsync();
str.Add($"<a>{(string.IsNullOrEmpty(text) ? "EMPTY" : text)}</a>");
}
}*/
var buttons = await body.QuerySelectorAllAsync("button");
foreach (var btn in buttons)
{
var text = await btn.TextContentAsync();
str.Add($"<button>{text}</button>");
var name = await btn.GetAttributeAsync("name");
var id = await btn.GetAttributeAsync("id");
str.Add($"<button name='{name}' id='{id}'>{text}</button>");
}
var driverService = _services.GetRequiredService<WebDriverService>();
var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId);
var htmlElementContextOut = await driverService.LocateElement(agent, string.Join("", str), context.ElementName, messageId);
var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
await element.ClickAsync();
var tags = await _instance.Page.QuerySelectorAllAsync(htmlElementContextOut.TagName);
var button = tags[htmlElementContextOut.Index];
if (button.AsElement() == null)
{
throw new Exception($"Can't find web element {context.ElementName}");
}
await button.ClickAsync();
}
}

View file

@ -0,0 +1,18 @@
using BotSharp.Plugin.WebDriver.Services;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task<string> ExtractData(Agent agent, BrowsingContextIn context, string messageId)
{
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var content = await body.InnerTextAsync();
var driverService = _services.GetRequiredService<WebDriverService>();
var answer = await driverService.ExtraData(agent, content, context.Question, messageId);
return answer;
}
}

View file

@ -0,0 +1,32 @@
using Microsoft.Extensions.Configuration;
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
public partial class PlaywrightWebDriver
{
public async Task InputUserPassword(Agent agent, BrowsingContextIn context, string messageId)
{
// Retrieve the page raw html and infer the element path
var body = await _instance.Page.QuerySelectorAsync("body");
var inputs = await body.QuerySelectorAllAsync("input");
var password = inputs.FirstOrDefault(x => x.GetAttributeAsync("type").Result == "password");
if (password == null)
{
throw new Exception($"Can't locate the web element {context.ElementName}.");
}
var config = _services.GetRequiredService<IConfiguration>();
try
{
var key = context.Password.Replace("@", "").Replace(".", ":");
var value = config.GetValue<string>(key);
await password.FillAsync(value);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
}
}

View file

@ -29,9 +29,21 @@ public partial class PlaywrightWebDriver
}
var driverService = _services.GetRequiredService<WebDriverService>();
var htmlElementContextOut = await driverService.FindElement(agent, string.Join("", str), context.ElementName, messageId);
var htmlElementContextOut = await driverService.LocateElement(agent, string.Join("", str), context.ElementName, messageId);
if (htmlElementContextOut.Index < 0)
{
throw new Exception($"Can't locate the web element {context.ElementName}.");
}
var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
await element.FillAsync(context.InputText);
try
{
await element.FillAsync(context.InputText);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
}
}

View file

@ -4,21 +4,7 @@ public partial class PlaywrightWebDriver
{
public async Task<IBrowser> LaunchBrowser(string? url)
{
if (_instance.Playwright == null)
{
var playwright = await Playwright.CreateAsync();
_instance.SetPlaywright(playwright);
}
if (_instance.Browser == null)
{
var browser = await _instance.Playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = false,
Channel = "chrome",
});
_instance.SetBrowser(browser);
}
await _instance.InitInstance();
if (!string.IsNullOrEmpty(url))
{

View file

@ -3,14 +3,14 @@ using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class ClickHtmlElementFn : IFunctionCallback
public class ClickButtonFn : IFunctionCallback
{
public string Name => "click_html_element";
public string Name => "click_button";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
public ClickHtmlElementFn(IServiceProvider services,
public ClickButtonFn(IServiceProvider services,
PlaywrightWebDriver driver)
{
_services = services;

View file

@ -0,0 +1,29 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class ExtractDataFn : IFunctionCallback
{
public string Name => "extract_data_from_page";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
public ExtractDataFn(IServiceProvider services,
PlaywrightWebDriver driver)
{
_services = services;
_driver = driver;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
message.Content = await _driver.ExtractData(agent, args, message.MessageId);
return true;
}
}

View file

@ -0,0 +1,31 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
namespace BotSharp.Plugin.WebDriver.Functions;
public class InputUserPasswordFn : IFunctionCallback
{
public string Name => "input_user_password";
private readonly IServiceProvider _services;
private readonly PlaywrightWebDriver _driver;
public InputUserPasswordFn(IServiceProvider services,
PlaywrightWebDriver driver)
{
_services = services;
_driver = driver;
}
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var agentService = _services.GetRequiredService<IAgentService>();
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.InputUserPassword(agent, args, message.MessageId);
message.Content = "Input password successfully";
return true;
}
}

View file

@ -1,4 +1,3 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
@ -26,7 +25,7 @@ public class InputUserTextFn : IFunctionCallback
var agent = await agentService.LoadAgent(message.CurrentAgentId);
await _driver.InputUserText(agent, args, message.MessageId);
message.Content = "Executed successfully.";
message.Content = "Input text successfully.";
return true;
}
}

View file

@ -19,10 +19,10 @@ public class OpenBrowserFn : IFunctionCallback
public async Task<bool> Execute(RoleDialogModel message)
{
var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
var browser = await _driver.LaunchBrowser(args.Url);
message.Content = "Executed successfully.";
message.Content = string.IsNullOrEmpty(args.Url) ? "Launch browser successfully." : $"Open website successfully.";
message.Content += "\r\nWhat would you like to do next?";
message.StopCompletion = true;
return true;
}
}

View file

@ -12,4 +12,10 @@ public class BrowsingContextIn
[JsonPropertyName("input_text")]
public string? InputText { get; set; }
[JsonPropertyName("password")]
public string? Password { get; set; }
[JsonPropertyName("question")]
public string? Question { get; set; }
}

View file

@ -0,0 +1,49 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.WebDriver.Services;
public partial class WebDriverService
{
public async Task<string> ExtraData(Agent agent, string html, string question, string messageId)
{
var parserInstruction = agent.Templates.First(x => x.Name == "extract_data").Content;
var render = _services.GetRequiredService<ITemplateRender>();
var prompt = render.Render(parserInstruction, new Dictionary<string, object>
{
{ "content", html },
{ "question", question }
});
var completer = CompletionProvider.GetCompletion(_services,
agentConfig: agent.LlmConfig);
if (completer is ITextCompletion textCompleter)
{
var result = await textCompleter.GetCompletion(prompt, agent.Id, messageId);
return result;
}
else if (completer is IChatCompletion chatCompleter)
{
var dialogs = new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, prompt)
{
CurrentAgentId = agent.Id,
MessageId = messageId
}
};
var result = chatCompleter.GetChatCompletions(new Agent
{
Id = agent.Id,
Name = agent.Name,
Instruction = "You're a Content Extrator."
}, dialogs);
return result.Content;
}
return null;
}
}

View file

@ -0,0 +1,49 @@
using BotSharp.Abstraction.Agents.Enums;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.WebDriver.Services;
public partial class WebDriverService
{
public async Task<HtmlElementContextOut> LocateElement(Agent agent, string html, string elementName, string messageId)
{
var parserInstruction = agent.Templates.First(x => x.Name == "html_parser").Content;
var render = _services.GetRequiredService<ITemplateRender>();
var prompt = render.Render(parserInstruction, new Dictionary<string, object>
{
{ "html_content", html },
{ "element_name", elementName }
});
var completer = CompletionProvider.GetCompletion(_services,
agentConfig: agent.LlmConfig);
if (completer is ITextCompletion textCompleter)
{
var result = await textCompleter.GetCompletion(prompt, agent.Id, messageId);
return result.JsonContent<HtmlElementContextOut>();
}
else if (completer is IChatCompletion chatCompleter)
{
var dialogs = new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, prompt)
{
CurrentAgentId = agent.Id,
MessageId = messageId
}
};
var result = chatCompleter.GetChatCompletions(new Agent
{
Id = agent.Id,
Name = agent.Name,
Instruction = "You're a HTML Parser."
}, dialogs);
return result.Content.JsonContent<HtmlElementContextOut>();
}
return null;
}
}

View file

@ -4,7 +4,7 @@ using BotSharp.Core.Infrastructures;
namespace BotSharp.Plugin.WebDriver.Services;
public class WebDriverService
public partial class WebDriverService
{
private readonly IServiceProvider _services;
@ -12,45 +12,4 @@ public class WebDriverService
{
_services = services;
}
public async Task<HtmlElementContextOut> FindElement(Agent agent, string html, string elementName, string messageId)
{
var parserInstruction = agent.Templates.First(x => x.Name == "html_parser").Content;
var render = _services.GetRequiredService<ITemplateRender>();
var prompt = render.Render(parserInstruction, new Dictionary<string, object>
{
{ "html_content", html },
{ "element_name", elementName }
});
var completer = CompletionProvider.GetCompletion(_services,
agentConfig: agent.LlmConfig);
if (completer is ITextCompletion textCompleter)
{
var result = await textCompleter.GetCompletion(prompt, agent.Id, messageId);
return result.JsonContent<HtmlElementContextOut>();
}
else if (completer is IChatCompletion chatCompleter)
{
var dialogs = new List<RoleDialogModel>
{
new RoleDialogModel(AgentRole.User, prompt)
{
CurrentAgentId = agent.Id,
MessageId = messageId
}
};
var result = chatCompleter.GetChatCompletions(new Agent
{
Id = agent.Id,
Name = agent.Name,
Instruction = "You're a HTML Parser."
}, dialogs);
return result.Content.JsonContent<HtmlElementContextOut>();
}
return null;
}
}

View file

@ -1,6 +1,6 @@
{
"name": "PizzaBot",
"description": "Pizza restaurant AI Bot",
"name": "AI Assistant",
"description": "AI assistant that can complete many different tasks",
"createdDateTime": "2023-08-18T10:39:32.2349685Z",
"updatedDateTime": "2023-08-18T14:39:32.2349686Z",
"id": "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a",

View file

@ -1,14 +1,14 @@
[
{
"text": "Hello, I'm an AI assistant that help you order a pizza."
"text": "Hello, I'm an AI assistant that help you do variety of tasks."
},
{
"rich_type": "quick_reply",
"text": "How can I help you today?",
"quick_replies": [
{
"title":"Order a pizza",
"payload":"order a pizza"
"title":"Access website",
"payload":"Launch Browser"
},
{
"title":"Who are you?",

View file

@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
@ -14,8 +14,11 @@
</ItemGroup>
<ItemGroup>
<Compile Remove="agents\**" />
<Compile Remove="documents\**" />
<EmbeddedResource Remove="agents\**" />
<EmbeddedResource Remove="documents\**" />
<None Remove="agents\**" />
<None Remove="documents\**" />
</ItemGroup>