SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add pipline pattern

Browse source
This commit is contained in:
Oceania2018 2018-06-14 10:03:01 -05:00
parent e1b585a578
commit 194f661545
11 changed files with 64 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
BotSharp.Core/Abstractions/INlpEntitizer.cs
View file

@ -1,15 +0,0 @@
using BotSharp.Core.Models;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
public interface INlpEntitizer
{
IConfiguration Configuration { get; set; }
List<NlpEntity> Entitize(string text);
}
}

14
BotSharp.Core/Abstractions/INlpFeaturizer.cs
View file

@ -1,14 +0,0 @@
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
public interface INlpFeaturizer
{
IConfiguration Configuration { get; set; }
List<decimal> Featurize(string text);
}
}

18
BotSharp.Core/Abstractions/INlpPipeline.cs Normal file
View file

@ -0,0 +1,18 @@
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Abstractions
{
/// <summary>
/// NLP process pipeline interface
/// </summary>
public interface INlpPipeline
{
IConfiguration Configuration { get; set; }
bool Process(String text, JObject data);
}
}

14
BotSharp.Core/Abstractions/INlpProvider.cs
View file

@ -1,14 +0,0 @@
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
public interface INlpProvider
{
IConfiguration Configuration { get; set; }
bool Load();
}
}

18
BotSharp.Core/Abstractions/INlpTokenizer.cs
View file

@ -1,18 +0,0 @@
using BotSharp.Core.Models;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
/// <summary>
/// Segment text into words, punctuations marks etc.
/// </summary>
public interface INlpTokenizer
{
IConfiguration Configuration { get; set; }
List<NlpToken> Tokenize(string text);
}
}

35
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -1,8 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using BotSharp.Core.Abstractions;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Newtonsoft.Json.Linq;
namespace BotSharp.Core.Engines
{
@ -23,32 +26,30 @@ namespace BotSharp.Core.Engines
public string Train()
{
var data = JObject.FromObject(new { });
// Get NLP Provider
string providerName = Database.Configuration.GetSection($"{config}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpProvider;
var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpPipeline;
provider.Configuration = Database.Configuration.GetSection("BotSharpAi");
provider.Process("How are you doing?", data);
provider.Load();
// pipe process
var pipelines = Database.Configuration.GetSection($"{config}:Pipe").Value
.Split(',')
.Select(x => x.Trim())
.ToList();
// Tokenize
var tokenizerName = provider.Configuration.GetSection("Pipe:Tokenizer").Value;
var tokenizer = TypeHelper.GetInstance(tokenizerName, Database.Assemblies) as INlpTokenizer;
tokenizer.Configuration = provider.Configuration;
var tokens = tokenizer.Tokenize("How are you doing?");
pipelines.ForEach(pipeName =>
{
var pipe = TypeHelper.GetInstance(pipeName, Database.Assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
var tokens = pipe.Process("How are you doing?", data);
// Featurize
var featurizerName = provider.Configuration.GetSection("Pipe:Featurizer").Value;
var featurizer = TypeHelper.GetInstance(featurizerName, Database.Assemblies) as INlpFeaturizer;
featurizer.Configuration = provider.Configuration;
var features = featurizer.Featurize("How are you doing?");
// Entitize
var entitizerName = provider.Configuration.GetSection("Pipe:Entitizer").Value;
var entitizer = TypeHelper.GetInstance(entitizerName, Database.Assemblies) as INlpEntitizer;
entitizer.Configuration = provider.Configuration;
var entities = entitizer.Entitize("Where are you going tomorrow ?");
});
return "";
}

10
BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs
View file

@ -1,24 +1,28 @@
using System;
using System.Collections.Generic;
using System.Text;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Models;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using RestSharp;
namespace BotSharp.Core.Engines.SpaCy
{
public class SpaCyEntitizer : INlpEntitizer
public class SpaCyEntitizer : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public List<NlpEntity> Entitize(string text)
public bool Process(string text, JObject data)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("entitize", Method.GET);
request.AddParameter("text", text);
var response = client.Execute<Result>(request);
return response.Data.Entities;
data.Add("Entities", JToken.FromObject(response.Data.Entities));
return response.IsSuccessful;
}
public class Result

8
BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs
View file

@ -1,4 +1,6 @@
using Microsoft.Extensions.Configuration;
using BotSharp.Core.Abstractions;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
@ -7,11 +9,11 @@ using System.Text;
namespace BotSharp.Core.Engines.SpaCy
{
public class SpaCyProvider : INlpProvider
public class SpaCyProvider : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public bool Load()
public bool Process(string text, JObject data)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("load", Method.GET);

12
BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs
View file

@ -1,5 +1,7 @@
using BotSharp.Core.Models;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Models;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
@ -7,18 +9,20 @@ using System.Text;
namespace BotSharp.Core.Engines.SpaCy
{
public class SpaCyTokenizer : INlpTokenizer
public class SpaCyTokenizer : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public List<NlpToken> Tokenize(string text)
public bool Process(string text, JObject data)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("tokenize", Method.GET);
request.AddParameter("text", text);
var response = client.Execute<Result>(request);
return response.Data.Tokens;
data.Add("tokens", JToken.FromObject(response.Data.Tokens));
return response.IsSuccessful;
}
public class Result

10
BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs
View file

@ -1,23 +1,27 @@
using System;
using System.Collections.Generic;
using System.Text;
using BotSharp.Core.Abstractions;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using RestSharp;
namespace BotSharp.Core.Engines.SpaCy
{
public class SpacyFeaturizer : INlpFeaturizer
public class SpacyFeaturizer : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public List<decimal> Featurize(string text)
public bool Process(string text, JObject data)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("featurize", Method.GET);
request.AddParameter("text", text);
var response = client.Execute<Result>(request);
return response.Data.Vectors;
data.Add("features", JToken.FromObject(response.Data.Vectors));
return response.IsSuccessful;
}
public class Result

6
BotSharp.UnitTest/Settings/settings.bot.json
View file

@ -9,10 +9,6 @@
"SpaCyProvider": {
"Url": "http://gtx.local:5005"
},
"Pipe": {
"Tokenizer": "SpaCyTokenizer",
"Featurizer": "SpacyFeaturizer",
"Entitizer": "SpaCyEntitizer"
}
"Pipe": "SpaCyTokenizer, SpacyFeaturizer, SpaCyEntitizer"
}
}