SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add BotSharpTokenizer, BotSharpTagger for BotSharpAi pipeline.

Browse source
This commit is contained in:
Oceania2018 2018-08-22 16:36:39 -05:00
parent 0885ff37ec
commit f42a42d34e
11 changed files with 133 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
BotSharp.Core.UnitTest/AgentTest.cs
View file

@ -24,8 +24,7 @@ namespace BotSharp.Core.UnitTest
var agent = new Agent
{
Id = BOT_ID,
Language = "en",
UserId = Guid.NewGuid().ToString()
Language = "en"
};
var rasa = new RasaAi();
//int row = dc.DbTran(() => rasa.agent.SaveAgent(dc));

7
BotSharp.Core/Agents/Agent.cs
View file

@ -44,13 +44,6 @@ namespace BotSharp.Core.Agents
[StringLength(32)]
public String DeveloperAccessToken { get; set; }
/// <summary>
/// Who created this bot
/// </summary>
[Required]
[StringLength(36)]
public String UserId { get; set; }
[ForeignKey("AgentId")]
public List<Intent> Intents { get; set; }

24
BotSharp.Core/Engines/BotSharp/BotSharpProvider.cs Normal file
View file

@ -0,0 +1,24 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpProvider : INlpProvider
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Load(Agent agent, PipeModel meta)
{
meta.Meta = JObject.FromObject(new { version = "0.1.0" });
return true;
}
}
}

46
BotSharp.Core/Engines/BotSharp/BotSharpTagger.cs Normal file
View file

@ -0,0 +1,46 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Corpus;
using BotSharp.NLP.Tag;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpTagger : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
throw new NotImplementedException();
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
string dataDir = Path.Combine(Configuration.GetValue<String>("BotSharpTagger:dataDir"), "CoNLL");
var data = new CoNLLReader().Read(new ReaderOptions
{
DataDir = dataDir,
FileName = "conll2000_chunking_train.txt"
});
var tagger = new TaggerFactory<NGramTagger>(new TagOptions
{
NGram = 1,
Tag = "NN",
Corpus = data
}, SupportedLanguage.English);
doc.Sentences.ForEach(x => tagger.Tag(new Sentence { Words = x.Tokens }));
return true;
}
}
}

47
BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs Normal file
View file

@ -0,0 +1,47 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Tokenize;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpTokenizer : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
List<List<Token>> tokens = new List<List<Token>>();
var corpus = agent.Corpus;
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC
}, SupportedLanguage.English);
doc.Sentences = new List<NlpDocSentence>();
List<string> sentencesList = new List<string>();
corpus.UserSays.ForEach(say =>
{
doc.Sentences.Add(new NlpDocSentence
{
Tokens = tokenizer.Tokenize(say.Text),
Text = say.Text
});
});
return true;
}
}
}

4
BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs
View file

@ -38,10 +38,6 @@ namespace BotSharp.Core.Engines
var result = agent.ToObject<Agent>();
result.ClientAccessToken = agentHeader.ClientAccessToken;
result.DeveloperAccessToken = agentHeader.DeveloperAccessToken;
if(agentHeader.UserId != null)
{
result.UserId = agentHeader.UserId;
}
result.MlConfig = agent.ToObject<AgentMlConfig>();
result.MlConfig.MinConfidence = agent.MlMinConfidence;

4
BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs
View file

@ -39,10 +39,6 @@ namespace BotSharp.Core.Engines
var result = agent.ToObject<Agent>();
result.ClientAccessToken = agentHeader.ClientAccessToken;
result.DeveloperAccessToken = agentHeader.DeveloperAccessToken;
if(agentHeader.UserId != null)
{
result.UserId = agentHeader.UserId;
}
return result;
}

2
BotSharp.WebHost/BotSharp.WebHost.csproj
View file

@ -7,7 +7,7 @@
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
<DefineConstants>TRACE;DEBUG;MODE_RASA</DefineConstants>
<DefineConstants>TRACE;DEBUG;RASA_UI</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

5
BotSharp.WebHost/Program.cs
View file

@ -17,15 +17,14 @@ namespace BotSharp.WebHost
Microsoft.AspNetCore.WebHost.CreateDefaultBuilder(args)
.ConfigureAppConfiguration((hostingContext, config) =>
{
var env = hostingContext.HostingEnvironment;
var settings = Directory.GetFiles($"{env.ContentRootPath}{Path.DirectorySeparatorChar}Settings", "*.json");
var settings = Directory.GetFiles(Path.Combine(env.ContentRootPath, "Settings"), "*.json");
settings.ToList().ForEach(setting =>
{
config.AddJsonFile(setting, optional: false, reloadOnChange: true);
});
})
#if MODE_RASA
#if RASA_UI
.UseUrls("http://0.0.0.0:5000")
#else
.UseUrls("http://0.0.0.0:3112")

25
BotSharp.WebHost/Settings/bot.json
View file

@ -1,29 +1,26 @@
{
"RasaAi": {
"url": "http://10.2.21.200:5000"
},
"BotSharpAi": {
"Lang": "en",
"Provider": "SpaCyProvider",
"SpaCyProvider": {
"url": "http://localhost:5005"
},
"NltkProvider": {
"url": "http://localhost:5005"
},
"Pipe": {
"train": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier",
"predict": "SpaCyTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer"
"Provider": "BotSharpProvider",
"BotSharpProvider": {
},
"SpaCyTokenizer": {
"Pipe": {
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, FasttextClassifier",
"predict": "BotSharpTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer, FasttextClassifier"
},
"BotSharpTagger": {
"dataDir": "D:\\Projects\\BotSharp.NLP\\Data"
},
"CRFsuiteEntityRecognizer": {
"fields": "y w pos chk",
"uniFeatures": "w wl pos chk shape shaped type p1 p2 p3 p4 s1 s2 s3 s4 2d 4d d&a d&- d&/ d&, d&. up iu au al ad ao cu cl ca cd cs",
"biFeatures": "w pos chk shaped type"
},
"WitAiEntityRecognizer": {
"url": "https://api.wit.ai",
"resource": "message",

5
BotSharp.WebHost/Startup.cs
View file

@ -108,10 +108,7 @@ namespace BotSharp.WebHost
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
context.Request.Headers["ClientAccessToken"] = token;
var dc = new DefaultDataContextLoader().GetDefaultDc();
var userId = dc.Table<Agent>().FirstOrDefault(x => x.ClientAccessToken == token)?.UserId;
context.Request.Headers["Authorization"] = "Bearer " + JwtToken.GenerateToken(config, userId);
// context.Request.Headers["Authorization"] = "Bearer " + JwtToken.GenerateToken(config, userId);
}
await next.Invoke();