SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix training issue

Browse source
This commit is contained in:
Oceania2018 2018-10-03 12:21:19 -05:00
parent 22e6fa0ee1
commit a4be554d6c
25 changed files with 101 additions and 139 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
BotSharp.Core/Abstractions/INlpPredict.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using System;
using System.Collections.Generic;
using System.Text;

1
BotSharp.Core/Abstractions/INlpProvider.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using System;
using System.Collections.Generic;
using System.Text;

1
BotSharp.Core/Abstractions/INlpTrain.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using System;
using System.Collections.Generic;
using System.Text;

8
BotSharp.Core/AgentStorageInRedis.cs
View file

@ -69,7 +69,13 @@ namespace BotSharp.Core
agent.Id = Guid.NewGuid().ToString();
}
csredis.Set(agent.Id, JsonConvert.SerializeObject(agent));
var json = JsonConvert.SerializeObject(agent, new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
Formatting = Formatting.Indented,
});
csredis.Set(agent.Id, json);
return true;
}

1
BotSharp.Core/Engines/BotPredictor.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Core.Abstractions;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.AiRequest;
using BotSharp.Platform.Models.MachineLearning;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;

1
BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs
View file

@ -13,6 +13,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
namespace BotSharp.Core.Engines.BotSharp
{

1
BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs
View file

@ -3,6 +3,7 @@ using BotSharp.NLP;
using BotSharp.NLP.Classify;
using BotSharp.NLP.Txt2Vec;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;

2
BotSharp.Core/Engines/BotSharp/BotSharpAi.cs → BotSharp.Core/Engines/BotSharp/BotSharpNLU.cs
View file

@ -7,7 +7,7 @@ using BotSharp.Platform.Models;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpAi : BotEngineBase, IBotEngine
public class BotSharpNLU : BotEngineBase, IBotEngine
{
public override async Task Train(BotTrainOptions options)
{

1
BotSharp.Core/Engines/BotSharp/BotSharpProvider.cs
View file

@ -1,5 +1,6 @@
using BotSharp.Core.Abstractions;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;

1
BotSharp.Core/Engines/BotSharp/BotSharpTagger.cs
View file

@ -4,6 +4,7 @@ using BotSharp.NLP.Corpus;
using BotSharp.NLP.Tag;
using BotSharp.NLP.Tokenize;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;

1
BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs
View file

@ -2,6 +2,7 @@
using BotSharp.NLP;
using BotSharp.NLP.Tokenize;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;

7
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -6,6 +6,7 @@ using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
@ -39,7 +40,8 @@ namespace BotSharp.Core.Engines
// Get NLP Provider
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var engine = config.GetSection($"BotEngine").Value;
var platform = config.GetSection($"platform").Value;
var engine = config.GetSection($"{platform}:botEngine").Value;
string providerName = config.GetSection($"{engine}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpProvider;
provider.Configuration = config.GetSection(engine);
@ -73,7 +75,8 @@ namespace BotSharp.Core.Engines
var meta = new ModelMetaData
{
Platform = engine,
Platform = platform,
BotEngine = engine,
Language = agent.Language,
TrainingDate = DateTime.UtcNow,
Version = config.GetValue<String>($"Version"),

3
BotSharp.Platform.Abstraction/IPlatformBuilder.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.AiRequest;
using BotSharp.Platform.Models.AiResponse;
using BotSharp.Platform.Models.MachineLearning;
using System;
using System.Collections.Generic;
using System.Text;
@ -47,7 +48,7 @@ namespace BotSharp.Platform.Abstraction
/// <returns></returns>
bool SaveAgent(TAgent agent);
Task<bool> Train(TAgent agent, TrainingCorpus corpus);
Task<ModelMetaData> Train(TAgent agent, TrainingCorpus corpus);
AiResponse TextRequest(AiRequest request);
}

7
BotSharp.Core/Engines/ModelMetaData.cs → BotSharp.Platform.Models/MachineLearning/ModelMetaData.cs
View file

@ -1,13 +1,13 @@
using Newtonsoft.Json;
using System;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
namespace BotSharp.Platform.Models.MachineLearning
{
public class ModelMetaData
{
public string Platform { get; set; }
public string BotEngine { get; set; }
public string Language { get; set; }
public string Version { get; set; }
@ -16,7 +16,6 @@ namespace BotSharp.Core.Engines
/// <summary>
/// Model file fullpath
/// </summary>
[JsonIgnore]
public string Model { get; set; }
public List<PipeModel> Pipeline { get; set; }

7
BotSharp.Core/Engines/PipeModel.cs → BotSharp.Platform.Models/MachineLearning/PipeModel.cs
View file

@ -1,9 +1,8 @@
using Newtonsoft.Json.Linq;
using System;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
namespace BotSharp.Platform.Models.MachineLearning
{
public class PipeModel
{
@ -24,6 +23,6 @@ namespace BotSharp.Core.Engines
/// <summary>
/// Extra meta data according to pipe
/// </summary>
public JObject Meta { get; set; }
public Object Meta { get; set; }
}
}

3
BotSharp.RestApi/BotSharp.RestApi.csproj
View file

@ -20,8 +20,9 @@
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
<DocumentationFile>C:\Users\haipi\Documents\Projects\BotSharp\BotSharp.RestApi\BotSharp.RestApi.xml</DocumentationFile>
<DocumentationFile>bin\Debug\netstandard2.0\BotSharp.RestApi.xml</DocumentationFile>
<DefineConstants>TRACE;DEBUG</DefineConstants>
<OutputPath />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA|AnyCPU'">

2
BotSharp.WebHost/BotSharp.WebHost.csproj
View file

@ -91,7 +91,7 @@
<Content Update="Settings\auth.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Update="Settings\BotSharpAi.json">
<Content Update="Settings\BotSharpNLU.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Update="Settings\db.json">

6
BotSharp.WebHost/Program.cs
View file

@ -44,13 +44,7 @@ namespace BotSharp.WebHost
config.AddJsonFile(setting, optional: false, reloadOnChange: true);
});
})
#if ARTICULATE
.UseUrls("http://0.0.0.0:7500")
#elif RASA
.UseUrls("http://0.0.0.0:5000")
#else
.UseUrls("http://0.0.0.0:3112")
#endif
.UseStartup<Startup>()
.Build();
}

35
BotSharp.WebHost/Settings/ArticulateAi.json
View file

@ -1,36 +1,7 @@
{
"ArticulateAi": {
"Lang": "en",
"articulateAi": {
"botEngine": "BotSharpNLU",
"AgentStorage": "AgentStorageInRedis",
"Provider": "BotSharpProvider",
"BotSharpProvider": {
},
"Pipe": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier",
"BotSharpTokenizer": {
"tokenizer": "TreebankTokenizer"
},
"BotSharpIntentClassifier": {
"classifer": "NaiveBayesClassifier"
},
"BotSharpTagger": {
"tagger": "NGramTagger"
},
"BotSharpCRFNer": {
"template": "|App_Data|CRFLite/template.en"
},
"WitAiEntityRecognizer": {
"url": "https://api.wit.ai",
"resource": "message",
"serverAccessToken": "SERVER_ACCESS_TOKEN",
"version": "20180811"
}
"agentStorage": "AgentStorageInRedis"
}
}

34
BotSharp.WebHost/Settings/BotSharpAi.json
View file

@ -1,34 +0,0 @@
{
"BotSharpAi": {
"Lang": "en",
"Provider": "BotSharpProvider",
"BotSharpProvider": {
},
"Pipe": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier",
"BotSharpTokenizer": {
"tokenizer": "TreebankTokenizer"
},
"BotSharpIntentClassifier": {
"classifer": "SVMClassifier"
},
"BotSharpTagger": {
"tagger": "NGramTagger"
},
"BotSharpCRFNer": {
"template": "|App_Data|CRFLite/template.en"
},
"WitAiEntityRecognizer": {
"url": "https://api.wit.ai",
"resource": "message",
"serverAccessToken": "SERVER_ACCESS_TOKEN",
"version": "20180811"
}
}
}

35
BotSharp.WebHost/Settings/BotSharpNLU.json Normal file
View file

@ -0,0 +1,35 @@
{
"botSharpNLU": {
"lang": "en",
"provider": "BotSharpProvider",
"botSharpProvider": {
},
"pipe": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier",
"botSharpTokenizer": {
"tokenizer": "TreebankTokenizer"
},
"botSharpIntentClassifier": {
"classifer": "NaiveBayesClassifier"
},
"botSharpTagger": {
"tagger": "NGramTagger"
},
"botSharpCRFNer": {
"template": "|App_Data|CRFLite/template.en"
},
"witAiEntityRecognizer": {
"url": "https://api.wit.ai",
"resource": "message",
"serverAccessToken": "SERVER_ACCESS_TOKEN",
"version": "20180811"
}
}
}

35
BotSharp.WebHost/Settings/DialogflowAi.json
View file

@ -1,36 +1,7 @@
{
"DialogflowAi": {
"Lang": "en",
"dialogflowAi": {
"botEngine": "BotSharpNLU",
"AgentStorage": "AgentStorageInRedis",
"Provider": "BotSharpProvider",
"BotSharpProvider": {
},
"Pipe": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpIntentClassifier",
"BotSharpTokenizer": {
"tokenizer": "TreebankTokenizer"
},
"BotSharpIntentClassifier": {
"classifer": "NaiveBayesClassifier"
},
"BotSharpTagger": {
"tagger": "NGramTagger"
},
"BotSharpCRFNer": {
"template": "|App_Data|CRFLite/template.en"
},
"WitAiEntityRecognizer": {
"url": "https://api.wit.ai",
"resource": "message",
"serverAccessToken": "SERVER_ACCESS_TOKEN",
"version": "20180811"
}
"agentStorage": "AgentStorageInRedis"
}
}

10
BotSharp.WebHost/Settings/app.json
View file

@ -1,9 +1,11 @@
{
"Assemblies": "BotSharp.Core",
"BotEngine": "BotSharpAi",
"Version": "0.1.0",
"assemblies": "BotSharp.Core",
"MachineLearning": {
"platform": "DialogflowAi",
"version": "0.1.0",
"machineLearning": {
"dataDir": "D:\\Projects\\BotSharp\\Data"
}
}

32
BotSharp.WebHost/Startup.cs
View file

@ -1,8 +1,5 @@
using BotSharp.Core.Engines;
using BotSharp.Platform.Abstraction;
using DotNetToolkit;
using Colorful;
using DotNetToolkit.JwtHelper;
using EntityFrameworkCore.BootKit;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
@ -12,8 +9,10 @@ using Newtonsoft.Json.Serialization;
using Swashbuckle.AspNetCore.Swagger;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using Console = Colorful.Console;
namespace BotSharp.WebHost
{
@ -63,17 +62,10 @@ namespace BotSharp.WebHost
});
// register platform dependency
services.AddTransient<IBotEngine>((provider) =>
/*services.AddTransient<IBotEngine>((provider) =>
{
var assemblies = (String[])AppDomain.CurrentDomain.GetData("Assemblies");
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var implements = TypeHelper.GetClassesWithInterface<IBotEngine>(assemblies);
string platform = config.GetValue<String>("BotPlatform");
var implement = implements.FirstOrDefault(x => x.Name.Split('.').Last() == platform);
var instance = (IBotEngine)Activator.CreateInstance(implement);
return instance;
});
});*/
}
public void Configure(IApplicationBuilder app, IHostingEnvironment env)
@ -100,7 +92,9 @@ namespace BotSharp.WebHost
c.DocumentTitle = info.Title;
c.InjectStylesheet(Configuration.GetValue<String>("Swagger:Stylesheet"));
Console.WriteLine($"Current Mode: {info.Title}");
Console.WriteLine($"{info.Title} {info.Version} {info.License.Name}", Color.Gray);
Console.WriteLine($"{info.Description}", Color.Gray);
Console.WriteLine($"{info.Contact.Name}", Color.Gray);
});
app.Use(async (context, next) =>
@ -129,6 +123,16 @@ namespace BotSharp.WebHost
loader.Env = env;
loader.Config = Configuration;
loader.Load();*/
var platform = Configuration.GetValue<string>("Platform");
var engine = Configuration.GetValue<string>($"{platform}:BotEngine");
Formatter[] settings = new Formatter[]
{
new Formatter(platform, Color.Yellow),
new Formatter(engine, Color.Yellow),
};
Console.WriteLineFormatted("Platform Emulator: {0} powered by {1} NLU engine.", Color.White, settings);
}
}
}

5
Platform.Articulate/ArticulateAi.cs
View file

@ -5,6 +5,7 @@ using BotSharp.Platform.Abstraction;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.AiRequest;
using BotSharp.Platform.Models.AiResponse;
using BotSharp.Platform.Models.MachineLearning;
using DotNetToolkit;
using Platform.Articulate.Models;
using System;
@ -133,7 +134,7 @@ namespace Platform.Articulate
return base.SaveAgent(agent);
}
public async Task<bool> Train(TAgent agent, TrainingCorpus corpus)
public async Task<ModelMetaData> Train(TAgent agent, TrainingCorpus corpus)
{
string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id);
var model = "model_" + DateTime.UtcNow.ToString("yyyyMMdd");
@ -149,7 +150,7 @@ namespace Platform.Articulate
var info = await trainer.Train(agent, trainOptions);
return true;
return info;
}
public AiResponse TextRequest(AiRequest request)