Added Fasttext labeled data reader.
This commit is contained in:
botsharp2018
parent
378bd43f09
commit
a26c4fa1da
24
BotSharp.Algorithm/Extensions/Split.cs
Normal file
24
BotSharp.Algorithm/Extensions/Split.cs
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace BotSharp.Algorithm.Extensions
|
||||
{
|
||||
public static partial class IListExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Split dataset to training and test part.
|
||||
/// </summary>
|
||||
/// <typeparam name="T"></typeparam>
|
||||
/// <param name="list"></param>
|
||||
/// <param name="percentage">must between 0 and 1</param>
|
||||
/// <returns></returns>
|
||||
public static Tuple<List<T>, List<T>> Split<T>(this IList<T> list, decimal percentage)
|
||||
{
|
||||
int boundary = int.Parse(Math.Floor(list.Count * percentage).ToString());
|
||||
|
||||
return new Tuple<List<T>, List<T>>(list.Take(boundary).ToList(), list.Skip(boundary).ToList());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -15,6 +15,42 @@ namespace BotSharp.NLP.UnitTest
|
|||
[TestClass]
|
||||
public class NaiveBayesClassifierTest : TestEssential
|
||||
{
|
||||
[TestMethod]
|
||||
public void CookingTest()
|
||||
{
|
||||
var reader = new FasttextDataReader();
|
||||
var sentences = reader.Read(new ReaderOptions
|
||||
{
|
||||
DataDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
|
||||
FileName = "cooking.stackexchange.txt"
|
||||
});
|
||||
|
||||
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
|
||||
sentences.ForEach(x => x.Words = tokenizer.Tokenize(x.Text));
|
||||
|
||||
sentences.Shuffle();
|
||||
|
||||
var options = new ClassifyOptions
|
||||
{
|
||||
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
|
||||
};
|
||||
var classifier = new ClassifierFactory<NaiveBayesClassifier>(options, SupportedLanguage.English);
|
||||
var dataset = sentences.Split(0.7M);
|
||||
classifier.Train(dataset.Item1);
|
||||
|
||||
int correct = 0;
|
||||
dataset.Item2.ForEach(td =>
|
||||
{
|
||||
var classes = classifier.Classify(td);
|
||||
if (td.Label == classes[0].Item1)
|
||||
{
|
||||
correct++;
|
||||
}
|
||||
});
|
||||
|
||||
var accuracy = (float)correct / dataset.Item2.Count;
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void GenderTest()
|
||||
{
|
||||
|
|
@ -45,7 +81,7 @@ namespace BotSharp.NLP.UnitTest
|
|||
testData.ForEach(td =>
|
||||
{
|
||||
var classes = classifier.Classify(td);
|
||||
if(td.Labels[0] == classes[0].Item1)
|
||||
if(td.Label == classes[0].Item1)
|
||||
{
|
||||
correct++;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ namespace BotSharp.NLP.Classify
|
|||
{
|
||||
_classifier.Train(sentences.Select(x => new LabeledFeatureSet
|
||||
{
|
||||
Label = x.Labels[0],
|
||||
Label = x.Label,
|
||||
Features = GetFeatures(x.Words)
|
||||
}).ToList(), _options);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,11 @@ namespace BotSharp.NLP.Corpus
|
|||
{
|
||||
public List<Sentence> Read(ReaderOptions options)
|
||||
{
|
||||
if (String.IsNullOrEmpty(options.LabelPrefix))
|
||||
{
|
||||
options.LabelPrefix = "__label__";
|
||||
}
|
||||
|
||||
var sentences = new List<Sentence>();
|
||||
using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName)))
|
||||
{
|
||||
|
|
@ -22,13 +27,21 @@ namespace BotSharp.NLP.Corpus
|
|||
string line = reader.ReadLine();
|
||||
if (!String.IsNullOrEmpty(line))
|
||||
{
|
||||
var ms = Regex.Matches(line, @"__label__\w+\s").Cast<Match>().ToList();
|
||||
var ms = Regex.Matches(line, options.LabelPrefix + @"\S+")
|
||||
.Cast<Match>()
|
||||
.ToList();
|
||||
|
||||
sentences.Add(new Sentence
|
||||
var text = line.Substring(ms.Last().Index + ms.Last().Length + 1);
|
||||
|
||||
ms.ForEach(m =>
|
||||
{
|
||||
// Label = lable,
|
||||
Text = line
|
||||
sentences.Add(new Sentence
|
||||
{
|
||||
Label = m.Value.Substring(options.LabelPrefix.Length),
|
||||
Text = text
|
||||
});
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ namespace BotSharp.NLP.Corpus
|
|||
{
|
||||
public List<Sentence> Read(ReaderOptions options)
|
||||
{
|
||||
string lable = options.FileName.Split('.')[0];
|
||||
string label = options.FileName.Split('.')[0];
|
||||
|
||||
var sentences = new List<Sentence>();
|
||||
using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName)))
|
||||
|
|
@ -26,7 +26,7 @@ namespace BotSharp.NLP.Corpus
|
|||
{
|
||||
sentences.Add(new Sentence
|
||||
{
|
||||
Labels = new List<string> { lable },
|
||||
Label = label,
|
||||
Text = line
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,5 +9,7 @@ namespace BotSharp.NLP.Corpus
|
|||
public string DataDir { get; set; }
|
||||
|
||||
public string FileName { get; set; }
|
||||
|
||||
public string LabelPrefix { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,10 +9,7 @@ namespace BotSharp.NLP
|
|||
{
|
||||
public List<Token> Words { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Allow multiple classification
|
||||
/// </summary>
|
||||
public List<String> Labels { get; set; }
|
||||
public String Label { get; set; }
|
||||
|
||||
public String Text { get; set; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ namespace BotSharp.NLP.Tokenize
|
|||
{
|
||||
get
|
||||
{
|
||||
return Start + Text.Length - 1;
|
||||
return Start + Text.Length;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ namespace BotSharp.NLP.Tokenize
|
|||
var token = tokens[i];
|
||||
token.Start = sentence.IndexOf(token.Text, startPos);
|
||||
|
||||
startPos = token.End + 1;
|
||||
startPos = token.End;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue