Shuffle and Reduce utility functions

BotSharp.Algorithm/Extensions/Reduce.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.Algorithm.Extensions
+{
+    public static partial class IListExtensions
+    {
+        /// <summary>
+        /// equivalent reduce function in Python
+        /// https://docs.python.org/3/library/functools.html?highlight=reduce#functools.reduce
+        /// </summary>
+        /// <typeparam name="TAccumulate"></typeparam>
+        /// <param name="source"></param>
+        /// <param name="func"></param>
+        /// <returns></returns>
+        public static TAccumulate Reduce<TAccumulate>(this IList<TAccumulate> source, Func<TAccumulate, TAccumulate, TAccumulate> func)
+        {
+            return source.Skip(1).Aggregate(source[0], func);
+        }
+    }
+}

BotSharp.Algorithm/Extensions/Shuffle.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace BotSharp.Algorithm.Extensions
+{
+    public static partial class IListExtensions
+    {
+        public static void Shuffle2<T>(this IList<T> list)
+        {
+            var provider = new RNGCryptoServiceProvider();
+            int count = list.Count;
+            while (count > 1)
+            {
+                var box = new byte[1];
+
+                do provider.GetBytes(box);
+                while (!(box[0] < count * (Byte.MaxValue / count)));
+
+                var k = (box[0] % count);
+                count--;
+
+                var value = list[k];
+                list[k] = list[count];
+                list[count] = value;
+            }
+        }
+
+        public static void Shuffle<T>(this IList<T> list)
+        {
+            var rng = new Random();
+            var count = list.Count;
+            while (count > 1)
+            {
+                count--;
+                var k = rng.Next(count + 1);
+                var value = list[k];
+                list[k] = list[count];
+                list[count] = value;
+            }
+        }
+    }
+}

BotSharp.Algorithm/Formulas/Lidstone.cs

@@ -50,7 +50,8 @@ namespace BotSharp.Algorithm.Formulas
         public double Prob(List<Probability> dist, string sample)
         {
             // observation x = (x1, ..., xd)
-            int x = dist.Find(f => f.Value == sample).Freq;
+            var p = dist.Find(f => f.Value == sample);
+            int x = p == null ? 0 : p.Freq;
 
             // N trials
             int _N = dist.Sum(f => f.Freq);

BotSharp.Core/BotSharp.Core.csproj

@@ -71,6 +71,7 @@ If you feel that this project is helpful to you, please Star on the project, we
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="BotSharp.NLP" Version="0.3.0" />
     <PackageReference Include="Colorful.Console" Version="1.2.9" />
     <PackageReference Include="DotNetToolkit" Version="1.6.0" />
     <PackageReference Include="EntityFrameworkCore.BootKit" Version="1.9.1" />

BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs

@@ -6,7 +6,9 @@ using Microsoft.VisualStudio.TestTools.UnitTesting;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text;
+using BotSharp.Algorithm.Extensions;
 
 namespace BotSharp.NLP.UnitTest
 {
@@ -31,10 +33,25 @@ namespace BotSharp.NLP.UnitTest
 
             corpus.ForEach(x => x.Words = tokenizer.Tokenize(x.Text));
 
-            classifier.Train(corpus);
+            // classifier.Train(corpus);
+            // string text = "Bridget";
+            // classifier.Classify(new Sentence { Text = text, Words = tokenizer.Tokenize(text) });
+            corpus.Shuffle();
+            var trainingData = corpus.Skip(2000).ToList();
+            classifier.Train(trainingData);
 
-            string text = "Aamir";
+            var testData = corpus.Take(2000).ToList();
-            classifier.Classify(new Sentence { Text = text, Words = tokenizer.Tokenize(text) });
+            int correct = 0;
+            testData.ForEach(td =>
+            {
+                var classes = classifier.Classify(td);
+                if(td.Label == classes[0].Item1)
+                {
+                    correct++;
+                }
+            });
+
+            var accuracy = (float)correct / testData.Count;
         }
 
         private List<Sentence> GetLabeledCorpus(ClassifyOptions options)

BotSharp.NLP/BotSharp.NLP.csproj

@@ -39,6 +39,7 @@ Naive Bayes Classifier</Description>
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="BotSharp.Algorithm" Version="0.1.0" />
     <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
   </ItemGroup>
 

BotSharp.NLP/Classify/ClassifierFactory.cs

@@ -22,14 +22,16 @@ namespace BotSharp.NLP.Classify
             _classifier = new IClassify();
         }
 
-        public void Classify(Sentence sentence)
+        public List<Tuple<string, double>> Classify(Sentence sentence)
         {
-            _classifier.Classify(new LabeledFeatureSet
+            var classes = _classifier.Classify(new LabeledFeatureSet
             {
                 Features = GetFeatures(sentence.Words)
             }, new ClassifyOptions
             {
             });
+
+            return classes.OrderByDescending(x => x.Item2).ToList();
         }
 
         public void Train(List<Sentence> sentences)

BotSharp.NLP/Classify/IClassifier.cs

@@ -8,6 +8,6 @@ namespace BotSharp.NLP.Classify
     {
         void Train(List<LabeledFeatureSet> featureSets, ClassifyOptions options);
 
-        void Classify(LabeledFeatureSet featureSet, ClassifyOptions options);
+        List<Tuple<string, double>> Classify(LabeledFeatureSet featureSet, ClassifyOptions options);
     }
 }

BotSharp.NLP/Classify/NaiveBayesClassifier.cs

@@ -17,6 +17,7 @@
  */
 
 using BotSharp.Algorithm;
+using BotSharp.Algorithm.Extensions;
 using BotSharp.Algorithm.Formulas;
 using System;
 using System.Collections.Generic;
@@ -32,6 +33,8 @@ namespace BotSharp.NLP.Classify
     /// This technique works well for topic classification; 
     /// say we have a set of academic papers, and we want to classify them into different topics (computer science, biology, mathematics).
     /// Naive Bayes is best for Less training data
+    /// P(X, Y) = P(Y|X)P(X) = P(X|Y)P(Y) => P(Y|X) = P(Y)P(X|Y)/P(X)
+    /// Y is label, X is features.
     /// </summary>
     public class NaiveBayesClassifier : IClassifier
     {
@@ -49,7 +52,9 @@ namespace BotSharp.NLP.Classify
                 })
                 .ToList();
 
-            var fNames = featureSets[0].Features.Select(x => x.Name).ToList();
+            var fNames = featureSets[0].Features.Select(x => x.Name)
+                .OrderBy(x => x)
+                .ToList();
 
             // combine all features.
             var allFeatureValues = new List<Feature>();
@@ -88,25 +93,40 @@ namespace BotSharp.NLP.Classify
             });
         }
 
-        public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
+        public List<Tuple<string, double>> Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
         {
             var estimator = new Lidstone();
 
             labelDist.ForEach(lf =>
             {
+                // prior probability
                 lf.Prob = estimator.Log2Prob(labelDist, lf.Value);
-            });
 
-            featureDist.ForEach(fd =>
+                // post probability P(X1,...,Xn|Y) = Sum(P(X1|Y) +...+ P(Xn|Y)
-            {
+                featureSet.Features.ForEach(f =>
-                fd.FeatureValues.ForEach(fv =>
                 {
-                    fv.Prob = estimator.Log2Prob(fd.FeatureValues, fv.Value);
+                    var fv = featureDist.Find(x => x.Label == lf.Value && x.FeatureName == f.Name).FeatureValues;
-
+                    lf.Prob += estimator.Log2Prob(fv, f.Value);
-                    var p = labelDist.Find(l => l.Value == fd.Label);
-                    p.Prob += fv.Prob;
                 });
             });
+
+            // add log
+            double[] logs = labelDist.Select(x => x.Prob).ToArray();
+
+            var sumLogs = logs.Reduce((log1, next) =>
+            {
+                double min = log1;
+                if (next < log1)
+                {
+                    min = next;
+                }
+
+                return min + Math.Log(Math.Pow(2, log1 - min) + Math.Pow(2, next - min), 2);
+            });
+
+            labelDist.ForEach(d => d.Prob -= sumLogs);
+
+            return labelDist.Select(x => new Tuple<string, double>(x.Value, x.Prob)).ToList();
         }
     }
 

BotSharp.NLP/Classify/SVMClassifier.cs

@@ -31,9 +31,9 @@ namespace BotSharp.NLP.Classify
     /// </summary>
     public class SVMClassifier : IClassifier
     {
-        public void Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
+        public List<Tuple<string, double>> Classify(LabeledFeatureSet featureSet, ClassifyOptions options)
         {
-            
+            return null;
         }
 
         public double[][] Predict(LabeledFeatureSet featureSet, ClassifyOptions options)

BotSharp.NLP/Corpus/FasttextDataReader.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace BotSharp.NLP.Corpus
+{
+    /// <summary>
+    /// Fasttext labeled data reader
+    /// </summary>
+    public class FasttextDataReader
+    {
+        public List<Sentence> Read(ReaderOptions options)
+        {
+            var sentences = new List<Sentence>();
+            using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName)))
+            {
+                while (!reader.EndOfStream)
+                {
+                    string line = reader.ReadLine();
+                    if (!String.IsNullOrEmpty(line))
+                    {
+                        var ms = Regex.Matches(line, @"__label__\w+\s").Cast<Match>().ToList();
+
+                        sentences.Add(new Sentence
+                        {
+                            // Label = lable,
+                            Text = line
+                        });
+                    }
+                }
+            }
+
+            return sentences;
+        }
+    }
+}