SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Fix NER offset issue.

Browse source
This commit is contained in:
Oceania2018 2018-09-14 11:31:35 -05:00
parent 2c4872f068
commit 0c41b47057
12 changed files with 204 additions and 338 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs
View file

@ -58,9 +58,6 @@ namespace BotSharp.Core.Engines.BotSharp
var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input \"{parsedTrainingDataFileName}\" -output \"{modelFileName}\"", false);
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Aug 3, 2018";
return true;
}

188
BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs
View file

@ -1,8 +1,11 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Models.CRFLite;
using BotSharp.Models.CRFLite.Decoder;
using BotSharp.Models.CRFLite.Encoder;
using BotSharp.Models.NLP;
using BotSharp.NLP.Tokenize;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
@ -39,12 +42,12 @@ namespace BotSharp.Core.Engines.BotSharp
List<TrainingData> curLine = Merge(doc, doc.Sentences[i].Tokens, userSays[i].Entities);
curLine.ForEach(trainingData =>
{
string[] wordParams = { trainingData.Entity, trainingData.Token, trainingData.Pos, trainingData.Chunk };
string wordStr = string.Join(" ", wordParams);
sw.Write(wordStr + "\n");
string[] wordParams = { trainingData.Token, trainingData.Pos, trainingData.Entity };
string wordStr = string.Join("\t", wordParams);
sw.WriteLine(wordStr);
});
list.Add(curLine);
sw.Write("\n");
sw.WriteLine();
}
sw.Flush();
}
@ -52,7 +55,7 @@ namespace BotSharp.Core.Engines.BotSharp
string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString();
string template = Configuration.GetValue<String>($"BotSharpCRFNer:template");
template = template.Replace("|App_Data|", contentDir);
template = template.Replace("|App_Data|", contentDir + System.IO.Path.DirectorySeparatorChar);
var encoder = new CRFEncoder();
bool result = encoder.Learn(new EncoderOptions
@ -65,7 +68,7 @@ namespace BotSharp.Core.Engines.BotSharp
return result;
}
public List<TrainingData> Merge(NlpDoc doc, List<Token> tokens, List<TrainingIntentExpressionPart> entities)
private List<TrainingData> Merge(NlpDoc doc, List<Token> tokens, List<TrainingIntentExpressionPart> entities)
{
List<TrainingData> trainingTuple = new List<TrainingData>();
HashSet<String> entityWordBag = new HashSet<String>();
@ -74,47 +77,75 @@ namespace BotSharp.Core.Engines.BotSharp
for (int i = 0; i < tokens.Count; i++)
{
TrainingIntentExpressionPart curEntity = null;
if (entities != null)
{
bool entityFinded = false;
entities.ForEach(entity => {
if (!entityFinded)
{
var vDoc = new NlpDoc { Sentences = new List<NlpDocSentence> { new NlpDocSentence { Text = entity.Value } } };
doc.Tokenizer.Predict(null, vDoc, null);
string[] words = vDoc.Sentences[0].Tokens.Select(x => x.Text).ToArray();
if (entities == null) continue;
for (int j = 0; j < words.Length; j++)
bool entityFinded = false;
for (int entityIndex = 0; entityIndex < entities.Count; entityIndex++)
{
var entity = entities[entityIndex];
if (!entityFinded)
{
var vDoc = new NlpDoc { Sentences = new List<NlpDocSentence> { new NlpDocSentence { Text = entity.Value } } };
doc.Tokenizer.Predict(null, vDoc, null);
string[] words = vDoc.Sentences[0].Tokens.Select(x => x.Text).ToArray();
for (int j = 0; j < words.Length; j++)
{
if (tokens[i + j].Text == words[j])
{
if (tokens[i + j].Text == words[j])
wordCandidateCount++;
if (j == words.Length - 1)
{
wordCandidateCount++;
if (j == words.Length - 1)
curEntity = entity;
}
}
else
{
wordCandidateCount = 0;
break;
}
}
if (wordCandidateCount != 0) // && entity.Start == tokens[i].Offset)
{
String entityName = curEntity.Entity.Contains(":") ? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1) : curEntity.Entity;
for(int wordIndex = 0; wordIndex < words.Length; wordIndex++)
{
var tag = entityName;
if (wordIndex == 0)
{
if (words.Length == 1)
{
curEntity = entity;
tag = "S_" + entityName;
}
else
{
tag = "B_" + entityName;
}
}
else if (wordIndex == words.Length - 1)
{
tag = "E_" + entityName;
}
else
{
wordCandidateCount = 0;
break;
tag = "M_" + entityName;
}
var word = words[wordIndex];
trainingTuple.Add(new TrainingData(tag, word, tokens[i].Pos));
}
if (wordCandidateCount != 0) // && entity.Start == tokens[i].Offset)
{
String entityName = curEntity.Entity.Contains(":") ? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1) : curEntity.Entity;
foreach (string s in words)
{
trainingTuple.Add(new TrainingData(entityName, s, tokens[i].Pos, "I"));
}
entityFinded = true;
}
entityFinded = true;
}
});
}
}
if (wordCandidateCount == 0)
{
trainingTuple.Add(new TrainingData("O", tokens[i].Text, tokens[i].Pos, "O"));
trainingTuple.Add(new TrainingData("S", tokens[i].Text, tokens[i].Pos));
}
else
{
@ -127,7 +158,92 @@ namespace BotSharp.Core.Engines.BotSharp
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
throw new NotImplementedException();
var decoder = new CRFDecoder();
var options = new DecoderOptions
{
ModelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model)
};
//Load encoded model from file
decoder.LoadModel(options.ModelFileName);
//Create decoder tagger instance.
var tagger = decoder.CreateTagger(options.NBest, options.MaxWord);
tagger.set_vlevel(options.ProbLevel);
//Initialize result
var crf_out = new CRFSegOut[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
crf_out[i] = new CRFSegOut(options.MaxWord);
}
doc.Sentences.ForEach(sent =>
{
List<List<String>> dataset = new List<List<string>>();
dataset.AddRange(sent.Tokens.Select(token => new List<String> { token.Text, token.Pos }).ToList());
//predict given string's tags
decoder.Segment(crf_out, tagger, dataset);
var entities = new List<NlpEntity>();
for (int i = 0; i < sent.Tokens.Count; i++)
{
var entity = crf_out[0].result_;
entities.Add(new NlpEntity
{
Entity = entity[i],
Start = doc.Sentences[0].Tokens[i].Start,
Value = doc.Sentences[0].Tokens[i].Text,
Confidence = 0,
Extrator = "BotSharpCRFNer"
});
}
sent.Entities = MergeEntity(doc.Sentences[0].Text, entities);
});
return true;
}
private List<NlpEntity> MergeEntity(string sentence, List<NlpEntity> tokens)
{
List<NlpEntity> res = new List<NlpEntity>();
for(int i = 0; i < tokens.Count; i++)
{
var entity = tokens[i];
if (entity.Entity.StartsWith("S_"))
{
entity.Entity = entity.Entity.Split('_')[1];
res.Add(entity);
}
else if (entity.Entity.StartsWith("B_"))
{
entity.Entity = entity.Entity.Split('_')[1];
for(int j = i; j < tokens.Count; j++)
{
var token = tokens[j];
if (token.Entity.StartsWith("E_"))
{
res.Add(new NlpEntity
{
Value = sentence.Substring(entity.Start, token.End - entity.Start + 1),
Entity = entity.Entity,
Extrator = entity.Extrator,
Start = entity.Start,
Confidence = entity.Confidence
});
}
i++;
}
}
}
return res;
}
public class TrainingData
@ -135,14 +251,12 @@ namespace BotSharp.Core.Engines.BotSharp
public String Token { get; set; }
public String Entity { get; set; }
public String Pos { get; set; }
public String Chunk { get; set; }
public TrainingData(string entity, string token, string pos, string chunk)
public TrainingData(string entity, string token, string pos)
{
Token = token;
Entity = entity;
Pos = pos;
Chunk = chunk;
}
}
}

2
BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs
View file

@ -40,8 +40,6 @@ namespace BotSharp.Core.Engines.BotSharp
classifier.Train(sentences);
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Sep 12, 2018";
return true;
}

261
BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs
View file

@ -13,257 +13,48 @@ namespace BotSharp.NLP.UnitTest.CRFLite
[TestClass]
public class DecoderTest
{
object rdLocker = new object();
[TestMethod]
public void TestDecode()
{
var encoder = new CRFDecoder();
bool result = Decode(new DecoderOptions
var decoder = new CRFDecoder();
var options = new DecoderOptions
{
InputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\test\test.txt",
ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\model\ner_model_eng",
});
}
object rdLocker = new object();
bool Decode(DecoderOptions options)
{
var parallelOption = new ParallelOptions();
var watch = Stopwatch.StartNew();
var sr = new StreamReader(options.InputFileName);
StreamWriter sw = null, swSeg = null;
//Create CRFSharp wrapper instance. It's a global instance
var crfWrapper = new CRFDecoder();
ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model"
};
//Load encoded model from file
//Logger.WriteLine("Loading model from {0}", options.strModelFileName);
crfWrapper.LoadModel(options.ModelFileName);
decoder.LoadModel(options.ModelFileName);
var queueRecords = new ConcurrentQueue<List<List<string>>>();
var queueSegRecords = new ConcurrentQueue<List<List<string>>>();
//Create decoder tagger instance.
var tagger = decoder.CreateTagger(options.NBest, options.MaxWord);
tagger.set_vlevel(options.ProbLevel);
parallelOption.MaxDegreeOfParallelism = options.Thread;
Parallel.For(0, options.Thread, parallelOption, t =>
//Initialize result
var crf_out = new CRFSegOut[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
//Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance
var tagger = crfWrapper.CreateTagger(options.NBest, options.MaxWord);
tagger.set_vlevel(options.ProbLevel);
//Initialize result
var crf_out = new CRFSegOut[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
crf_out[i] = new CRFSegOut(tagger.crf_max_word_num);
}
var inbuf = new List<List<string>>();
while (true)
{
lock (rdLocker)
{
if (ReadRecord(inbuf, sr) == false)
{
break;
}
queueRecords.Enqueue(inbuf);
queueSegRecords.Enqueue(inbuf);
}
//Call CRFSharp wrapper to predict given string's tags
if (swSeg != null)
{
crfWrapper.Segment(crf_out, tagger, inbuf);
}
else
{
crfWrapper.Segment((CRFTermOut[])crf_out, (DecoderTagger)tagger, inbuf);
}
List<List<string>> peek = null;
//Save segmented tagged result into file
if (swSeg != null)
{
var rstList = ConvertCRFTermOutToStringList(inbuf, crf_out);
while (peek != inbuf)
{
queueSegRecords.TryPeek(out peek);
}
for (int index = 0; index < rstList.Count; index++)
{
var item = rstList[index];
swSeg.WriteLine(item);
}
queueSegRecords.TryDequeue(out peek);
peek = null;
}
//Save raw tagged result (with probability) into file
if (sw != null)
{
while (peek != inbuf)
{
queueRecords.TryPeek(out peek);
}
OutputRawResultToFile(inbuf, crf_out, tagger, sw);
queueRecords.TryDequeue(out peek);
}
}
});
sr.Close();
if (sw != null)
{
sw.Close();
crf_out[i] = new CRFSegOut(options.MaxWord);
}
if (swSeg != null)
{
swSeg.Close();
}
watch.Stop();
//Logger.WriteLine("Elapsed: {0} ms", watch.ElapsedMilliseconds);
return true;
var dataset = GetTestData();
//predict given string's tags
decoder.Segment(crf_out, tagger, dataset);
}
private bool ReadRecord(List<List<string>> inbuf, StreamReader sr)
private List<List<string>> GetTestData()
{
inbuf.Clear();
var dataset = new List<List<string>>();
while (true)
{
var strLine = sr.ReadLine();
if (strLine == null)
{
//At the end of current file
if (inbuf.Count == 0)
{
return false;
}
else
{
return true;
}
}
strLine = strLine.Trim();
if (strLine.Length == 0)
{
return true;
}
dataset.Add(new List<string> { "'", "PUN" });
dataset.Add(new List<string> { "'", "POS" });
dataset.Add(new List<string> { "Duchy", "NNP" });
dataset.Add(new List<string> { "of", "IN" });
dataset.Add(new List<string> { "Lithuania", "NNP" });
//Read feature set for each record
var items = strLine.Split(new char[] { '\t' });
inbuf.Add(new List<string>());
for (int index = 0; index < items.Length; index++)
{
var item = items[index];
inbuf[inbuf.Count - 1].Add(item);
}
}
}
//Output raw result with probability
private void OutputRawResultToFile(List<List<string>> inbuf, CRFTermOut[] crf_out, SegDecoderTagger tagger, StreamWriter sw)
{
for (var k = 0; k < crf_out.Length; k++)
{
if (crf_out[k] == null)
{
//No more result
break;
}
var sb = new StringBuilder();
var crf_seg_out = crf_out[k];
//Show the entire sequence probability
//For each token
for (var i = 0; i < inbuf.Count; i++)
{
//Show all features
for (var j = 0; j < inbuf[i].Count; j++)
{
sb.Append(inbuf[i][j]);
sb.Append("\t");
}
//Show the best result and its probability
sb.Append(crf_seg_out.result_[i]);
if (tagger.vlevel_ > 1)
{
sb.Append("\t");
sb.Append(crf_seg_out.weight_[i]);
//Show the probability of all tags
sb.Append("\t");
for (var j = 0; j < tagger.ysize_; j++)
{
sb.Append(tagger.yname(j));
sb.Append("/");
sb.Append(tagger.prob(i, j));
if (j < tagger.ysize_ - 1)
{
sb.Append("\t");
}
}
}
sb.AppendLine();
}
if (tagger.vlevel_ > 0)
{
sw.WriteLine("#{0}", crf_seg_out.prob);
}
sw.WriteLine(sb.ToString().Trim());
sw.WriteLine();
}
}
//Convert CRFSharp output format to string list
private List<string> ConvertCRFTermOutToStringList(List<List<string>> inbuf, CRFSegOut[] crf_out)
{
var sb = new StringBuilder();
for (var i = 0; i < inbuf.Count; i++)
{
sb.Append(inbuf[i][0]);
}
var strText = sb.ToString();
var rstList = new List<string>();
for (var i = 0; i < crf_out.Length; i++)
{
if (crf_out[i] == null)
{
//No more result
break;
}
sb.Clear();
var crf_term_out = crf_out[i];
for (var j = 0; j < crf_term_out.Count; j++)
{
var str = strText.Substring(crf_term_out.tokenList[j].Offset, crf_term_out.tokenList[j].Length);
var strNE = crf_term_out.tokenList[j].Tag;
sb.Append(str);
if (strNE.Length > 0)
{
sb.Append("[" + strNE + "]");
}
sb.Append(" ");
}
rstList.Add(sb.ToString().Trim());
}
return rstList;
return dataset;
}
}
}

44
BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs
View file

@ -43,49 +43,5 @@ namespace BotSharp.NLP.UnitTest.CRFLite
Assert.IsTrue(result);
}
object rdLocker = new object();
[TestMethod]
public void TestDecode()
{
var decoder = new CRFDecoder();
var options = new DecoderOptions
{
ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model"
};
//Load encoded model from file
decoder.LoadModel(options.ModelFileName);
//Create decoder tagger instance.
var tagger = decoder.CreateTagger(options.NBest, options.MaxWord);
tagger.set_vlevel(options.ProbLevel);
//Initialize result
var crf_out = new CRFSegOut[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
crf_out[i] = new CRFSegOut(options.MaxWord);
}
var dataset = GetTestData();
//predict given string's tags
decoder.Segment(crf_out, tagger, dataset);
}
private List<List<string>> GetTestData()
{
var dataset = new List<List<string>>();
dataset.Add(new List<string> { "'", "PUN" });
dataset.Add(new List<string> { "'", "POS" });
dataset.Add(new List<string> { "Duchy", "NNP" });
dataset.Add(new List<string> { "of", "IN" });
dataset.Add(new List<string> { "Lithuania", "NNP" });
return dataset;
}
}
}

14
BotSharp.NLP/Models/CRFLite/CRFEncoder.cs
View file

@ -61,24 +61,17 @@ namespace BotSharp.Models.CRFLite
var xList = modelWriter.ReadAllRecords();
modelWriter.Shrink(xList, args.MinFeatureFreq);
if (!modelWriter.SaveModelMetaData(args.ModelFileName))
{
return false;
}
else
{
}
if (!modelWriter.BuildFeatureSetIntoIndex(args.ModelFileName, args.SlotUsageRateThreshold, args.DebugLevel))
{
return false;
}
else
{
}
if (xList.Length == 0)
{
@ -101,6 +94,8 @@ namespace BotSharp.Models.CRFLite
bool runCRF(EncoderTagger[] x, ModelWriter modelWriter, bool orthant, EncoderOptions args)
{
Console.WriteLine("Running encoding process...");
var old_obj = double.MaxValue;
var converge = 0;
var lbfgs = new LBFGS(args.ThreadsNum);
@ -165,6 +160,9 @@ namespace BotSharp.Models.CRFLite
lbfgs.err += processList[i].err;
lbfgs.zeroone += processList[i].zeroone;
Console.WriteLine($"Thread: {i}, Iterating {itr} / {args.MaxIteration}");
Console.WriteLine($"{lbfgs.obj} {lbfgs.err} {lbfgs.zeroone}");
//Calculate error
for (var j = 0; j < modelWriter.y_.Count; j++)
{
@ -263,6 +261,8 @@ namespace BotSharp.Models.CRFLite
}
}
Console.WriteLine("Completed encoding process.");
return true;
}

2
BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs
View file

@ -40,7 +40,7 @@ namespace BotSharp.Models.CRFLite.Decoder
public DecoderOptions()
{
Thread = 1;
NBest = 1;
NBest = 2;
ProbLevel = 0;
MaxWord = 128;
}

4
BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs
View file

@ -14,7 +14,7 @@ namespace BotSharp.Models.CRFLite.Encoder
/// <summary>
/// Minimum feature frequency, if one feature's frequency is less than this value, the feature will be dropped.
/// </summary>
public int MinFeatureFreq = 2;
public int MinFeatureFreq = 1;
/// <summary>
/// Minimum diff value, when diff less than the value consecutive 3 times, the process will be ended.
@ -79,7 +79,7 @@ namespace BotSharp.Models.CRFLite.Encoder
public EncoderOptions()
{
MaxIteration = 100;
MinFeatureFreq = 2;
MinFeatureFreq = 1;
MinDifference = 0.0001;
SlotUsageRateThreshold = 0.95;
ThreadsNum = 1;

16
BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs
View file

@ -43,6 +43,8 @@ namespace BotSharp.Models.CRFLite.Encoder
//Regenerate feature id and shrink features with lower frequency
public void Shrink(EncoderTagger[] xList, int freq)
{
Console.WriteLine($"Shrink features lower than {freq} frequency");
var old2new = new CRFLite.Utils.BTreeDictionary<long, long>();
featureLexicalDict.Shrink(freq);
maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
@ -86,7 +88,7 @@ namespace BotSharp.Models.CRFLite.Encoder
var oldValue = Interlocked.Increment(ref arrayEncoderTaggerSize) - 1;
arrayEncoderTagger[oldValue] = _x;
if (oldValue % 10000 == 0)
if (oldValue % 100 == 0)
{
//Show current progress on console
Console.Write("{0}...", oldValue);
@ -94,10 +96,11 @@ namespace BotSharp.Models.CRFLite.Encoder
}
});
Console.WriteLine($"Read {trainCorpusList.Count} records");
trainCorpusList.Clear();
trainCorpusList = null;
Console.WriteLine();
return arrayEncoderTagger;
}
@ -136,6 +139,8 @@ namespace BotSharp.Models.CRFLite.Encoder
//Save indexed feature set into file
da.save(filename_featureset);
Console.WriteLine($"Saved featureset to {filename_featureset}");
if (string.IsNullOrWhiteSpace(modelFileName))
{
//Clean up all data
@ -220,6 +225,7 @@ namespace BotSharp.Models.CRFLite.Encoder
tofs.Close();
Console.WriteLine($"Saved meta data to {filename}");
return true;
}
@ -277,6 +283,8 @@ namespace BotSharp.Models.CRFLite.Encoder
bool OpenTemplateFile(string filename)
{
Console.WriteLine($"Open template: {filename}");
var ifs = new StreamReader(filename);
unigram_templs_ = new List<string>();
bigram_templs_ = new List<string>();
@ -305,6 +313,8 @@ namespace BotSharp.Models.CRFLite.Encoder
bool OpenTrainCorpusFile(string strTrainingCorpusFileName)
{
Console.WriteLine($"Open corpus: {strTrainingCorpusFileName}");
var ifs = new StreamReader(strTrainingCorpusFileName);
y_ = new List<string>();
trainCorpusList = new List<List<List<string>>>();

2
BotSharp.RestApi/BotSharp.RestApi.xml
View file

@ -120,7 +120,7 @@
Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project.
</summary>
<param name="model">Model name</param>
<param name="project"></param>
<param name="project">Agent name or agent id</param>
<returns></returns>
</member>
<member name="P:BotSharp.RestApi.Integrations.FacebookMessenger.WebhookMessageRecipient.Id">

4
BotSharp.RestApi/Rasa/TrainController.cs
View file

@ -38,10 +38,10 @@ namespace BotSharp.RestApi.Rasa
/// Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project.
/// </summary>
/// <param name="model">Model name</param>
/// <param name="project"></param>
/// <param name="project">Agent name or agent id</param>
/// <returns></returns>
[HttpPost]
public async Task<ActionResult<String>> Train([FromQuery] string model, [FromQuery] string project)
public async Task<ActionResult<String>> Train([FromQuery] string project, [FromQuery] string model)
{
string body = "";
using (var reader = new StreamReader(Request.Body))

2
Settings/bot.json
View file

@ -22,7 +22,7 @@
},
"BotSharpCRFNer": {
"template": "|App_Data|CRFLite\template.en"
"template": "|App_Data|CRFLite/template.en"
},
"CRFsuiteEntityRecognizer": {