SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Separate training and predictive processing pipelines so that new processes can be added at any time during prediction without retraining the model.

Browse source
This commit is contained in:
Oceania2018 2018-08-12 23:14:02 -05:00
parent 3779704aac
commit a68e920ebf
15 changed files with 81 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BotSharp.Core/Abstractions/INlpNer.cs
View file

@ -5,7 +5,7 @@ using System.Text;
namespace BotSharp.Core.Abstractions
{
public interface INlpNer
public interface INlpNer : INlpPipeline
{
List<OntologyEnum> Ontologies { get; }
}

10
BotSharp.Core/Abstractions/INlpPipeline.cs
View file

@ -20,15 +20,5 @@ namespace BotSharp.Core.Abstractions
/// Common settings for Pipeline
/// </summary>
PipeSettings Settings { get; set; }
/// <summary>
/// Process
/// </summary>
/// <param name="agent"></param>
/// <param name="doc">Intermediate result</param>
/// <param name="meta">Meta data which is packed to model</param>
/// <returns></returns>
Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta);
Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta);
}
}

14
BotSharp.Core/Abstractions/INlpPredict.cs Normal file
View file

@ -0,0 +1,14 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Abstractions
{
public interface INlpPredict : INlpPipeline
{
Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta);
}
}

14
BotSharp.Core/Abstractions/INlpProvider.cs Normal file
View file

@ -0,0 +1,14 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Abstractions
{
public interface INlpProvider : INlpPipeline
{
Task<bool> Load(Agent agent, PipeModel meta);
}
}

21
BotSharp.Core/Abstractions/INlpTrain.cs Normal file
View file

@ -0,0 +1,21 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Abstractions
{
public interface INlpTrain : INlpPipeline
{
/// <summary>
/// Process
/// </summary>
/// <param name="agent"></param>
/// <param name="doc">Intermediate result</param>
/// <param name="meta">Meta data which is packed to model</param>
/// <returns></returns>
Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta);
}
}

2
BotSharp.Core/Engines/BotEngineBase.cs
View file

@ -37,7 +37,7 @@ namespace BotSharp.Core.Engines
public AIResponse TextRequest(AIRequest request)
{
var preditor = new BotPreditor();
var preditor = new BotPredictor();
var doc = preditor.Predict(agent, request).Result;
var parameters = new Dictionary<String, Object>();
doc.Sentences[0].Entities.ForEach(x => parameters.Add(x.Entity, x.Value));

19
BotSharp.Core/Engines/BotPreditor.cs → BotSharp.Core/Engines/BotPredictor.cs
View file

@ -15,7 +15,7 @@ using System.Threading.Tasks;
namespace BotSharp.Core.Engines
{
public class BotPreditor
public class BotPredictor
{
public async Task<NlpDoc> Predict(Agent agent, AIRequest request)
{
@ -30,7 +30,7 @@ namespace BotSharp.Core.Engines
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var providerPipe = meta.Pipeline.First();
var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpPipeline;
var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpProvider;
provider.Configuration = config.GetSection(meta.Platform);
var data = new NlpDoc
@ -44,7 +44,7 @@ namespace BotSharp.Core.Engines
}
};
await provider.Train(agent, data, providerPipe);
await provider.Load(agent, providerPipe);
meta.Pipeline.RemoveAt(0);
var settings = new PipeSettings
@ -58,14 +58,19 @@ namespace BotSharp.Core.Engines
{
Directory.CreateDirectory(settings.PredictDir);
}
// pipe process
meta.Pipeline.ForEach(async pipeMeta =>
var pipelines = provider.Configuration.GetValue<String>($"Pipe:predict")
.Split(',')
.Select(x => x.Trim())
.ToList();
pipelines.ForEach(async pipeName =>
{
var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline;
var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPredict;
pipe.Configuration = provider.Configuration;
pipe.Settings = settings;
await pipe.Predict(agent, data, pipeMeta);
await pipe.Predict(agent, data, meta.Pipeline.FirstOrDefault(x => x.Name == pipeName));
});
Console.WriteLine(JsonConvert.SerializeObject(data, new JsonSerializerSettings

6
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -47,7 +47,7 @@ namespace BotSharp.Core.Engines
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var platform = config.GetSection($"BotPlatform").Value;
string providerName = config.GetSection($"{platform}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpPipeline;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpTrain;
provider.Configuration = config.GetSection(platform);
var pipeModel = new PipeModel
@ -87,14 +87,14 @@ namespace BotSharp.Core.Engines
}
// pipe process
var pipelines = provider.Configuration.GetSection($"Pipe").Value
var pipelines = provider.Configuration.GetValue<String>($"Pipe:train")
.Split(',')
.Select(x => x.Trim())
.ToList();
pipelines.ForEach(async pipeName =>
{
var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline;
var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpTrain;
pipe.Configuration = provider.Configuration;
pipe.Settings = settings;
pipeModel = new PipeModel

2
BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs
View file

@ -13,7 +13,7 @@ using System.Threading.Tasks;
namespace BotSharp.Core.Engines.Classifiers
{
public class FasttextClassifier : INlpPipeline
public class FasttextClassifier : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }

2
BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs
View file

@ -18,7 +18,7 @@ using System.Threading.Tasks;
namespace BotSharp.Core.Engines.NERs
{
public class CRFsuiteEntityRecognizer : INlpPipeline, INlpNer
public class CRFsuiteEntityRecognizer : INlpTrain, INlpPredict, INlpNer
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }

4
BotSharp.Core/Engines/NERs/DucklingEntityRecognizer.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Core.Abstractions;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Text;
@ -8,5 +9,8 @@ namespace BotSharp.Core.Engines.NERs
public class DucklingEntityRecognizer : INlpNer
{
public List<OntologyEnum> Ontologies => throw new NotImplementedException();
public IConfiguration Configuration { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
public PipeSettings Settings { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
}
}

2
BotSharp.Core/Engines/NERs/WitAiEntityRecognizer.cs
View file

@ -15,7 +15,7 @@ using System.Threading.Tasks;
namespace BotSharp.Core.Engines.NERs
{
public class WitAiEntityRecognizer : INlpPipeline, INlpNer
public class WitAiEntityRecognizer : INlpPredict, INlpNer
{
public List<OntologyEnum> Ontologies
{

9
BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs
View file

@ -12,12 +12,12 @@ using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
public class SpaCyProvider : INlpPipeline
public class SpaCyProvider : INlpProvider
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
public async Task<bool> Load(Agent agent, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("load", Method.GET);
@ -30,11 +30,6 @@ namespace BotSharp.Core.Engines.SpaCy
return response.IsSuccessful;
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
return true;
}
private class Result
{
[JsonProperty("spaCy ver")]

2
BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs
View file

@ -14,7 +14,7 @@ using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
public class SpaCyTokenizer : INlpPipeline
public class SpaCyTokenizer : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }

7
BotSharp.WebHost/Settings/bot.json
View file

@ -10,9 +10,12 @@
"url": "http://localhost:5005"
},
"Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier",
"Pipe": {
"train": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier",
"predict": "SpaCyTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer"
},
"SpaCyTokenizer": {
"url": "http://localhost:5005"
},
"CRFsuiteEntityRecognizer": {
"fields": "y w pos chk",