diff --git a/BotSharp.Algorithm/Extensions/Split.cs b/BotSharp.Algorithm/Extensions/Split.cs
new file mode 100644
index 00000000..5f53058f
--- /dev/null
+++ b/BotSharp.Algorithm/Extensions/Split.cs
@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.Algorithm.Extensions
+{
+ public static partial class IListExtensions
+ {
+ ///
+ /// Split dataset to training and test part.
+ ///
+ ///
+ ///
+ /// must between 0 and 1
+ ///
+ public static Tuple, List> Split(this IList list, decimal percentage)
+ {
+ int boundary = int.Parse(Math.Floor(list.Count * percentage).ToString());
+
+ return new Tuple, List>(list.Take(boundary).ToList(), list.Skip(boundary).ToList());
+ }
+ }
+}
diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
index d4a037ca..44979a00 100644
--- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
+++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
@@ -15,6 +15,42 @@ namespace BotSharp.NLP.UnitTest
[TestClass]
public class NaiveBayesClassifierTest : TestEssential
{
+ [TestMethod]
+ public void CookingTest()
+ {
+ var reader = new FasttextDataReader();
+ var sentences = reader.Read(new ReaderOptions
+ {
+ DataDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
+ FileName = "cooking.stackexchange.txt"
+ });
+
+ var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English);
+ sentences.ForEach(x => x.Words = tokenizer.Tokenize(x.Text));
+
+ sentences.Shuffle();
+
+ var options = new ClassifyOptions
+ {
+ TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
+ };
+ var classifier = new ClassifierFactory(options, SupportedLanguage.English);
+ var dataset = sentences.Split(0.7M);
+ classifier.Train(dataset.Item1);
+
+ int correct = 0;
+ dataset.Item2.ForEach(td =>
+ {
+ var classes = classifier.Classify(td);
+ if (td.Label == classes[0].Item1)
+ {
+ correct++;
+ }
+ });
+
+ var accuracy = (float)correct / dataset.Item2.Count;
+ }
+
[TestMethod]
public void GenderTest()
{
@@ -45,7 +81,7 @@ namespace BotSharp.NLP.UnitTest
testData.ForEach(td =>
{
var classes = classifier.Classify(td);
- if(td.Labels[0] == classes[0].Item1)
+ if(td.Label == classes[0].Item1)
{
correct++;
}
diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs
index 3baba28a..24df05e2 100644
--- a/BotSharp.NLP/Classify/ClassifierFactory.cs
+++ b/BotSharp.NLP/Classify/ClassifierFactory.cs
@@ -38,7 +38,7 @@ namespace BotSharp.NLP.Classify
{
_classifier.Train(sentences.Select(x => new LabeledFeatureSet
{
- Label = x.Labels[0],
+ Label = x.Label,
Features = GetFeatures(x.Words)
}).ToList(), _options);
}
diff --git a/BotSharp.NLP/Corpus/FasttextDataReader.cs b/BotSharp.NLP/Corpus/FasttextDataReader.cs
index 87a6e6f1..92306a24 100644
--- a/BotSharp.NLP/Corpus/FasttextDataReader.cs
+++ b/BotSharp.NLP/Corpus/FasttextDataReader.cs
@@ -14,6 +14,11 @@ namespace BotSharp.NLP.Corpus
{
public List Read(ReaderOptions options)
{
+ if (String.IsNullOrEmpty(options.LabelPrefix))
+ {
+ options.LabelPrefix = "__label__";
+ }
+
var sentences = new List();
using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName)))
{
@@ -22,13 +27,21 @@ namespace BotSharp.NLP.Corpus
string line = reader.ReadLine();
if (!String.IsNullOrEmpty(line))
{
- var ms = Regex.Matches(line, @"__label__\w+\s").Cast().ToList();
+ var ms = Regex.Matches(line, options.LabelPrefix + @"\S+")
+ .Cast()
+ .ToList();
- sentences.Add(new Sentence
+ var text = line.Substring(ms.Last().Index + ms.Last().Length + 1);
+
+ ms.ForEach(m =>
{
- // Label = lable,
- Text = line
+ sentences.Add(new Sentence
+ {
+ Label = m.Value.Substring(options.LabelPrefix.Length),
+ Text = text
+ });
});
+
}
}
}
diff --git a/BotSharp.NLP/Corpus/LabeledPerFileNameReader.cs b/BotSharp.NLP/Corpus/LabeledPerFileNameReader.cs
index 1a9eb3b7..17bf3cab 100644
--- a/BotSharp.NLP/Corpus/LabeledPerFileNameReader.cs
+++ b/BotSharp.NLP/Corpus/LabeledPerFileNameReader.cs
@@ -14,7 +14,7 @@ namespace BotSharp.NLP.Corpus
{
public List Read(ReaderOptions options)
{
- string lable = options.FileName.Split('.')[0];
+ string label = options.FileName.Split('.')[0];
var sentences = new List();
using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName)))
@@ -26,7 +26,7 @@ namespace BotSharp.NLP.Corpus
{
sentences.Add(new Sentence
{
- Labels = new List { lable },
+ Label = label,
Text = line
});
}
diff --git a/BotSharp.NLP/Corpus/ReaderOptions.cs b/BotSharp.NLP/Corpus/ReaderOptions.cs
index 29229c7d..b60f6101 100644
--- a/BotSharp.NLP/Corpus/ReaderOptions.cs
+++ b/BotSharp.NLP/Corpus/ReaderOptions.cs
@@ -9,5 +9,7 @@ namespace BotSharp.NLP.Corpus
public string DataDir { get; set; }
public string FileName { get; set; }
+
+ public string LabelPrefix { get; set; }
}
}
diff --git a/BotSharp.NLP/Sentence.cs b/BotSharp.NLP/Sentence.cs
index 50d075d6..e0914fa4 100644
--- a/BotSharp.NLP/Sentence.cs
+++ b/BotSharp.NLP/Sentence.cs
@@ -9,10 +9,7 @@ namespace BotSharp.NLP
{
public List Words { get; set; }
- ///
- /// Allow multiple classification
- ///
- public List Labels { get; set; }
+ public String Label { get; set; }
public String Text { get; set; }
}
diff --git a/BotSharp.NLP/Tokenize/Token.cs b/BotSharp.NLP/Tokenize/Token.cs
index 35f71d5e..c11cd3ed 100644
--- a/BotSharp.NLP/Tokenize/Token.cs
+++ b/BotSharp.NLP/Tokenize/Token.cs
@@ -52,7 +52,7 @@ namespace BotSharp.NLP.Tokenize
{
get
{
- return Start + Text.Length - 1;
+ return Start + Text.Length;
}
}
diff --git a/BotSharp.NLP/Tokenize/TreebankTokenizer.cs b/BotSharp.NLP/Tokenize/TreebankTokenizer.cs
index d820f215..25bebc89 100644
--- a/BotSharp.NLP/Tokenize/TreebankTokenizer.cs
+++ b/BotSharp.NLP/Tokenize/TreebankTokenizer.cs
@@ -128,7 +128,7 @@ namespace BotSharp.NLP.Tokenize
var token = tokens[i];
token.Start = sentence.IndexOf(token.Text, startPos);
- startPos = token.End + 1;
+ startPos = token.End;
}
}