SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add PipeSettings to hold common settings for training or predit.

Browse source
This commit is contained in:
haiping008@gmail.com 2018-08-09 16:54:52 -05:00
parent 2c35f6177a
commit efadfddf7b
14 changed files with 61 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
BotSharp.Core/Abstractions/INlpPipeline.cs
View file

@ -16,6 +16,11 @@ namespace BotSharp.Core.Abstractions
{
IConfiguration Configuration { get; set; }
/// <summary>
/// Common settings for Pipeline
/// </summary>
PipeSettings Settings { get; set; }
/// <summary>
/// Process
/// </summary>

2
BotSharp.Core/BotSharp.Core.csproj
View file

@ -35,7 +35,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="DotNetToolkit" Version="1.5.0" />
<PackageReference Include="DotNetToolkit" Version="1.5.1" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="1.8.0" />
<PackageReference Include="Microsoft.AspNetCore.Cryptography.KeyDerivation" Version="2.1.1" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />

8
BotSharp.Core/Engines/BotPreditor.cs
View file

@ -34,16 +34,24 @@ namespace BotSharp.Core.Engines
var data = JObject.FromObject(new
{
Text = request.Query.FirstOrDefault()
});
await provider.Train(agent, data, providerPipe);
meta.Pipeline.RemoveAt(0);
var settings = new PipeSettings
{
ModelDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id),
AlgorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
};
// pipe process
meta.Pipeline.ForEach(async pipeMeta =>
{
var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
pipe.Settings = settings;
await pipe.Predict(agent, data, pipeMeta);
});

18
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -71,16 +71,21 @@ namespace BotSharp.Core.Engines
Pipeline = new List<PipeModel>() { pipeModel }
};
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
if (!Directory.Exists(dirTrain))
var settings = new PipeSettings
{
Directory.CreateDirectory(dirTrain);
TrainDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id),
ModelDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id),
AlgorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
};
if (!Directory.Exists(settings.TrainDir))
{
Directory.CreateDirectory(settings.TrainDir);
}
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
if (!Directory.Exists(dirModel))
if (!Directory.Exists(settings.ModelDir))
{
Directory.CreateDirectory(dirModel);
Directory.CreateDirectory(settings.ModelDir);
}
// pipe process
@ -93,6 +98,7 @@ namespace BotSharp.Core.Engines
{
var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
pipe.Settings = settings;
pipeModel = new PipeModel
{
Name = pipeName,

13
BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs
View file

@ -20,6 +20,7 @@ namespace BotSharp.Core.Engines.CRFsuite
public class CRFsuiteEntityRecognizer : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
@ -32,11 +33,9 @@ namespace BotSharp.Core.Engines.CRFsuite
List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
List<List<TrainingData>> list = new List<List<TrainingData>>();
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
string rawTrainingDataFileName = Path.Join(dirTrain, "ner-crf.corpus.txt");
string parsedTrainingDataFileName = Path.Join(dirTrain, "ner-crf.parsed.txt");
string modelFileName = Path.Join(dirModel, meta.Model);
string rawTrainingDataFileName = Path.Join(Settings.TrainDir, "ner-crf.corpus.txt");
string parsedTrainingDataFileName = Path.Join(Settings.TrainDir, "ner-crf.parsed.txt");
string modelFileName = Path.Join(Settings.ModelDir, meta.Model);
using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create))
{
@ -65,9 +64,7 @@ namespace BotSharp.Core.Engines.CRFsuite
new MachineLearning.CRFsuite.Ner()
.NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, fields, uniFeatures.Split(" "), biFeatures.Split(" "));
var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x
CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();

17
BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs
View file

@ -14,21 +14,22 @@ namespace BotSharp.Core.Engines.Classifiers
public class FasttextClassifier : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
throw new NotImplementedException();
string modelFileName = Path.Join(Settings.ModelDir, meta.Model);
var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), "predict {modelFileName}.bin test.txt");
return true;
}
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
meta.Model = "classification-fasttext.model";
var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
string parsedTrainingDataFileName = Path.Join(dirTrain, $"classification-fasttext.parsed.txt");
string modelFileName = Path.Join(dirModel, meta.Model);
string parsedTrainingDataFileName = Path.Join(Settings.TrainDir, $"classification-fasttext.parsed.txt");
string modelFileName = Path.Join(Settings.ModelDir, meta.Model);
// assemble corpus
StringBuilder corpus = new StringBuilder();
@ -36,7 +37,7 @@ namespace BotSharp.Core.Engines.Classifiers
File.WriteAllText(parsedTrainingDataFileName, corpus.ToString());
var output = CmdHelper.Run(Path.Join(algorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}");
var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}");
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();

13
BotSharp.Core/Engines/PipeSettings.cs Normal file
View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
public class PipeSettings
{
public string TrainDir { get; set; }
public string ModelDir { get; set; }
public string AlgorithmDir { get; set; }
}
}

1
BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs
View file

@ -15,6 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy
public class SpaCyEntitizer : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{

1
BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs
View file

@ -17,6 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy
{
List<String> entitiesInTrainingSet = new List<string>();
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{

2
BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs
View file

@ -15,6 +15,8 @@ namespace BotSharp.Core.Engines.SpaCy
public class SpaCyProvider : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);

1
BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs
View file

@ -15,6 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy
public class SpaCyTagger : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{

1
BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs
View file

@ -16,6 +16,7 @@ namespace BotSharp.Core.Engines.SpaCy
public class SpaCyTextCategorizer : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{

1
BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs
View file

@ -17,6 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy
public class SpaCyTokenizer : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{

1
BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs
View file

@ -14,6 +14,7 @@ namespace BotSharp.Core.Engines.SpaCy
public class SpacyFeaturizer : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{