SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Fix parser api for RASA UI.

Browse source
This commit is contained in:
Oceania2018 2018-08-23 07:25:55 -05:00
parent 16ab106ce7
commit af2bd8d89f
11 changed files with 270114 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BotSharp.Core/Abstractions/IBotPlatform.cs
View file

@ -17,6 +17,8 @@ namespace BotSharp.Core.Engines
/// <returns></returns>
Agent LoadAgent(string id);
Agent LoadAgentFromFile<TAgentImporter>(string dataDir, AgentImportHeader agentHeader) where TAgentImporter : IAgentImporter, new();
AIResponse TextRequest(AIRequest request);
Task Train();

3
BotSharp.Core/Engines/BotPredictor.cs
View file

@ -20,7 +20,7 @@ namespace BotSharp.Core.Engines
public async Task<NlpDoc> Predict(Agent agent, AIRequest request)
{
// load model
var dir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
var dir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id, request.Model);
Console.WriteLine($"Load model from {dir}");
var metaJson = File.ReadAllText(Path.Combine(dir, "metadata.json"));
var meta = JsonConvert.DeserializeObject<ModelMetaData>(metaJson);
@ -49,6 +49,7 @@ namespace BotSharp.Core.Engines
var settings = new PipeSettings
{
ModelDir = dir,
ProjectDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id),
AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
};

23
BotSharp.Core/Engines/BotSharp/BotSharpTagger.cs
View file

@ -17,28 +17,35 @@ namespace BotSharp.Core.Engines.BotSharp
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
throw new NotImplementedException();
}
private TaggerFactory<NGramTagger> _tagger;
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
public BotSharpTagger()
{
string dataDir = Path.Combine(Configuration.GetValue<String>("BotSharpTagger:dataDir"), "CoNLL");
string dataDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Corpus", "CoNLL");
var data = new CoNLLReader().Read(new ReaderOptions
{
DataDir = dataDir,
FileName = "conll2000_chunking_train.txt"
});
var tagger = new TaggerFactory<NGramTagger>(new TagOptions
_tagger = new TaggerFactory<NGramTagger>(new TagOptions
{
NGram = 1,
Tag = "NN",
Corpus = data
}, SupportedLanguage.English);
}
doc.Sentences.ForEach(x => tagger.Tag(new Sentence { Words = x.Tokens }));
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
doc.Sentences.ForEach(x => _tagger.Tag(new Sentence { Words = x.Tokens }));
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
doc.Sentences.ForEach(x => _tagger.Tag(new Sentence { Words = x.Tokens }));
return true;
}

28
BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs
View file

@ -14,29 +14,35 @@ namespace BotSharp.Core.Engines.BotSharp
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
private TokenizerFactory<RegexTokenizer> _tokenizer;
public BotSharpTokenizer()
{
_tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC
}, SupportedLanguage.English);
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
// same as train
doc.Sentences.ForEach(snt =>
{
snt.Tokens = _tokenizer.Tokenize(snt.Text);
});
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
List<List<Token>> tokens = new List<List<Token>>();
var corpus = agent.Corpus;
var tokenizer = new TokenizerFactory<RegexTokenizer>(new TokenizationOptions
{
Pattern = RegexTokenizer.WORD_PUNC
}, SupportedLanguage.English);
doc.Sentences = new List<NlpDocSentence>();
List<string> sentencesList = new List<string>();
corpus.UserSays.ForEach(say =>
agent.Corpus.UserSays.ForEach(say =>
{
doc.Sentences.Add(new NlpDocSentence
{
Tokens = tokenizer.Tokenize(say.Text),
Tokens = _tokenizer.Tokenize(say.Text),
Text = say.Text
});
});

5
BotSharp.Core/Engines/Dialogflow/AIRequest.cs
View file

@ -17,6 +17,11 @@ namespace BotSharp.Core.Models
public OriginalRequest OriginalRequest { get; set; }
/// <summary>
/// What model is used to predict.
/// </summary>
public string Model { get; set; }
public AIRequest()
{
Contexts = new List<AIContext>();

14
BotSharp.RestApi/Rasa/ParseController.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Core.Engines;
using BotSharp.Core.Engines.Rasa;
using BotSharp.Core.Models;
using BotSharp.NLP;
using Microsoft.AspNetCore.Mvc;
@ -36,7 +37,6 @@ namespace BotSharp.RestApi.Rasa
[HttpPost, HttpGet]
public ActionResult<RasaResponse> Parse()
{
String clientAccessToken = Request.Headers["ClientAccessToken"];
var config = new AIConfiguration("", SupportedLanguage.English);
config.SessionId = "rasa nlu";
@ -47,10 +47,20 @@ namespace BotSharp.RestApi.Rasa
}
var request = JsonConvert.DeserializeObject<RasaRequestModel>(body);
//_platform.LoadAgent(clientAccessToken);
// Load agent
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", request.Project);
var modelPath = Path.Combine(projectPath, request.Model);
_platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath,
new AgentImportHeader
{
Id = request.Project,
Name = request.Project
});
var aIResponse = _platform.TextRequest(new AIRequest
{
Model = request.Model,
Query = new String[] { request.Text }
});

14
BotSharp.RestApi/Rasa/TrainController.cs
View file

@ -64,18 +64,17 @@ namespace BotSharp.RestApi.Rasa
}
// save corpus to agent dir
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects");
var dataPath = Path.Combine(projectPath, project);
var agentPath = Path.Combine(dataPath, "Temp");
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", project);
var modelPath = Path.Combine(projectPath, request.Model);
if (!Directory.Exists(agentPath))
if (!Directory.Exists(modelPath))
{
Directory.CreateDirectory(agentPath);
Directory.CreateDirectory(modelPath);
}
// Save raw data to file, then parse it to Agent instance.
// in order to unify the process.
var fileName = Path.Combine(agentPath, "corpus.json");
var fileName = Path.Combine(modelPath, "corpus.json");
System.IO.File.WriteAllText(fileName, JsonConvert.SerializeObject(request.Corpus, new JsonSerializerSettings
{
@ -84,8 +83,7 @@ namespace BotSharp.RestApi.Rasa
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
var bot = new RasaAi();
var agent = bot.LoadAgentFromFile<AgentImporterInRasa>(agentPath,
var agent = _platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath,
new AgentImportHeader
{
Id = request.Project,

2
BotSharp.WebHost/App_Data/Corpus/CoNLL/README.md Normal file
View file

@ -0,0 +1,2 @@
conll2000_chunking is downloaded from https://www.clips.uantwerpen.be/conll2000/chunking/
The train and test data consist of three columns separated by spaces. Each word has been put on a separate line and there is an empty line after each sentence. The first column contains the current word, the second its part-of-speech tag as derived by the Brill tagger and the third its chunk tag as derived from the WSJ corpus.

49389
BotSharp.WebHost/App_Data/Corpus/CoNLL/conll2000_chunking_test.txt Normal file
View file

File diff suppressed because it is too large Load diff

220663
BotSharp.WebHost/App_Data/Corpus/CoNLL/conll2000_chunking_train.txt Normal file
View file

File diff suppressed because it is too large Load diff

3
BotSharp.WebHost/Settings/bot.json
View file

@ -8,11 +8,10 @@
"Pipe": {
"train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, FasttextClassifier",
"predict": "BotSharpTokenizer, CRFsuiteEntityRecognizer, WitAiEntityRecognizer, FasttextClassifier"
"predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, FasttextClassifier"
},
"BotSharpTagger": {
"dataDir": "D:\\Projects\\BotSharp.NLP\\Data"
},
"CRFsuiteEntityRecognizer": {