From a26c4fa1daaad3447d43bf26cafa5da44ef3e748 Mon Sep 17 00:00:00 2001 From: botsharp2018 Date: Sat, 8 Sep 2018 22:59:01 -0500 Subject: [PATCH] Added Fasttext labeled data reader. --- BotSharp.Algorithm/Extensions/Split.cs | 24 ++++++++++++ .../NaiveBayesClassifierTest.cs | 38 ++++++++++++++++++- BotSharp.NLP/Classify/ClassifierFactory.cs | 2 +- BotSharp.NLP/Corpus/FasttextDataReader.cs | 21 ++++++++-- .../Corpus/LabeledPerFileNameReader.cs | 4 +- BotSharp.NLP/Corpus/ReaderOptions.cs | 2 + BotSharp.NLP/Sentence.cs | 5 +-- BotSharp.NLP/Tokenize/Token.cs | 2 +- BotSharp.NLP/Tokenize/TreebankTokenizer.cs | 2 +- 9 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 BotSharp.Algorithm/Extensions/Split.cs diff --git a/BotSharp.Algorithm/Extensions/Split.cs b/BotSharp.Algorithm/Extensions/Split.cs new file mode 100644 index 00000000..5f53058f --- /dev/null +++ b/BotSharp.Algorithm/Extensions/Split.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BotSharp.Algorithm.Extensions +{ + public static partial class IListExtensions + { + /// + /// Split dataset to training and test part. + /// + /// + /// + /// must between 0 and 1 + /// + public static Tuple, List> Split(this IList list, decimal percentage) + { + int boundary = int.Parse(Math.Floor(list.Count * percentage).ToString()); + + return new Tuple, List>(list.Take(boundary).ToList(), list.Skip(boundary).ToList()); + } + } +} diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index d4a037ca..44979a00 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -15,6 +15,42 @@ namespace BotSharp.NLP.UnitTest [TestClass] public class NaiveBayesClassifierTest : TestEssential { + [TestMethod] + public void CookingTest() + { + var reader = new FasttextDataReader(); + var sentences = reader.Read(new ReaderOptions + { + DataDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"), + FileName = "cooking.stackexchange.txt" + }); + + var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English); + sentences.ForEach(x => x.Words = tokenizer.Tokenize(x.Text)); + + sentences.Shuffle(); + + var options = new ClassifyOptions + { + TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + var dataset = sentences.Split(0.7M); + classifier.Train(dataset.Item1); + + int correct = 0; + dataset.Item2.ForEach(td => + { + var classes = classifier.Classify(td); + if (td.Label == classes[0].Item1) + { + correct++; + } + }); + + var accuracy = (float)correct / dataset.Item2.Count; + } + [TestMethod] public void GenderTest() { @@ -45,7 +81,7 @@ namespace BotSharp.NLP.UnitTest testData.ForEach(td => { var classes = classifier.Classify(td); - if(td.Labels[0] == classes[0].Item1) + if(td.Label == classes[0].Item1) { correct++; } diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index 3baba28a..24df05e2 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -38,7 +38,7 @@ namespace BotSharp.NLP.Classify { _classifier.Train(sentences.Select(x => new LabeledFeatureSet { - Label = x.Labels[0], + Label = x.Label, Features = GetFeatures(x.Words) }).ToList(), _options); } diff --git a/BotSharp.NLP/Corpus/FasttextDataReader.cs b/BotSharp.NLP/Corpus/FasttextDataReader.cs index 87a6e6f1..92306a24 100644 --- a/BotSharp.NLP/Corpus/FasttextDataReader.cs +++ b/BotSharp.NLP/Corpus/FasttextDataReader.cs @@ -14,6 +14,11 @@ namespace BotSharp.NLP.Corpus { public List Read(ReaderOptions options) { + if (String.IsNullOrEmpty(options.LabelPrefix)) + { + options.LabelPrefix = "__label__"; + } + var sentences = new List(); using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName))) { @@ -22,13 +27,21 @@ namespace BotSharp.NLP.Corpus string line = reader.ReadLine(); if (!String.IsNullOrEmpty(line)) { - var ms = Regex.Matches(line, @"__label__\w+\s").Cast().ToList(); + var ms = Regex.Matches(line, options.LabelPrefix + @"\S+") + .Cast() + .ToList(); - sentences.Add(new Sentence + var text = line.Substring(ms.Last().Index + ms.Last().Length + 1); + + ms.ForEach(m => { - // Label = lable, - Text = line + sentences.Add(new Sentence + { + Label = m.Value.Substring(options.LabelPrefix.Length), + Text = text + }); }); + } } } diff --git a/BotSharp.NLP/Corpus/LabeledPerFileNameReader.cs b/BotSharp.NLP/Corpus/LabeledPerFileNameReader.cs index 1a9eb3b7..17bf3cab 100644 --- a/BotSharp.NLP/Corpus/LabeledPerFileNameReader.cs +++ b/BotSharp.NLP/Corpus/LabeledPerFileNameReader.cs @@ -14,7 +14,7 @@ namespace BotSharp.NLP.Corpus { public List Read(ReaderOptions options) { - string lable = options.FileName.Split('.')[0]; + string label = options.FileName.Split('.')[0]; var sentences = new List(); using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName))) @@ -26,7 +26,7 @@ namespace BotSharp.NLP.Corpus { sentences.Add(new Sentence { - Labels = new List { lable }, + Label = label, Text = line }); } diff --git a/BotSharp.NLP/Corpus/ReaderOptions.cs b/BotSharp.NLP/Corpus/ReaderOptions.cs index 29229c7d..b60f6101 100644 --- a/BotSharp.NLP/Corpus/ReaderOptions.cs +++ b/BotSharp.NLP/Corpus/ReaderOptions.cs @@ -9,5 +9,7 @@ namespace BotSharp.NLP.Corpus public string DataDir { get; set; } public string FileName { get; set; } + + public string LabelPrefix { get; set; } } } diff --git a/BotSharp.NLP/Sentence.cs b/BotSharp.NLP/Sentence.cs index 50d075d6..e0914fa4 100644 --- a/BotSharp.NLP/Sentence.cs +++ b/BotSharp.NLP/Sentence.cs @@ -9,10 +9,7 @@ namespace BotSharp.NLP { public List Words { get; set; } - /// - /// Allow multiple classification - /// - public List Labels { get; set; } + public String Label { get; set; } public String Text { get; set; } } diff --git a/BotSharp.NLP/Tokenize/Token.cs b/BotSharp.NLP/Tokenize/Token.cs index 35f71d5e..c11cd3ed 100644 --- a/BotSharp.NLP/Tokenize/Token.cs +++ b/BotSharp.NLP/Tokenize/Token.cs @@ -52,7 +52,7 @@ namespace BotSharp.NLP.Tokenize { get { - return Start + Text.Length - 1; + return Start + Text.Length; } } diff --git a/BotSharp.NLP/Tokenize/TreebankTokenizer.cs b/BotSharp.NLP/Tokenize/TreebankTokenizer.cs index d820f215..25bebc89 100644 --- a/BotSharp.NLP/Tokenize/TreebankTokenizer.cs +++ b/BotSharp.NLP/Tokenize/TreebankTokenizer.cs @@ -128,7 +128,7 @@ namespace BotSharp.NLP.Tokenize var token = tokens[i]; token.Start = sentence.IndexOf(token.Text, startPos); - startPos = token.End + 1; + startPos = token.End; } }