Added Lidstone smoothing estimator.

BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs

@@ -18,7 +18,7 @@ namespace BotSharp.NLP.UnitTest
         {
             var options = new ClassifyOptions
             {
-                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("BotSharp.NLP:dataDir"), "Gender")
+                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Gender")
             };
             var classifier = new ClassifierFactory<NaiveBayesClassifier>(options, SupportedLanguage.English);
 

BotSharp.NLP/Classify/ClassifierFactory.cs

@@ -38,10 +38,12 @@ namespace BotSharp.NLP.Classify
 
         private List<Feature> GetFeatures(List<Token> words)
         {
+            string text = words[0].Text;
             var features = new List<Feature>();
 
-            features.Add(new Feature("StartsWith(A)", words[0].Text.StartsWith("A").ToString()));
-            features.Add(new Feature("EndsWith(a)", words[0].Text.EndsWith("a").ToString()));
+            features.Add(new Feature("alwayson", "True"));
+            features.Add(new Feature("startswith", text[0].ToString().ToLower()));
+            features.Add(new Feature("endswith", text[text.Length - 1].ToString().ToLower()));
 
             return features;
         }

BotSharp.NLP/Classify/IEstimator.cs

@@ -0,0 +1,10 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Classify
+{
+    public interface IEstimator
+    {
+    }
+}

BotSharp.NLP/Classify/Lidstone.cs

@@ -0,0 +1,53 @@
+/*
+ * BotSharp.NLP Library
+ * Copyright (C) 2018 Haiping Chen
+ * 
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.Classify
+{
+    /// <summary>
+    /// Lidstone smoothing, is a technique used to smooth categorical data. 
+    /// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator:
+    /// Refer https://en.wikipedia.org/wiki/Additive_smoothing
+    /// </summary>
+    public class Lidstone : IEstimator
+    {
+        /// <summary>
+        /// x = (x1, …, xd) 
+        /// </summary>
+        private int _d;
+
+        /// <summary>
+        /// α > 0 is the smoothing parameter
+        /// </summary>
+        private float _a;
+
+        /// <summary>
+        /// N trials
+        /// </summary>
+        private int _N;
+
+        public Lidstone(float alpha, int bins)
+        {
+            _a = alpha;
+            _d = bins;
+        }
+    }
+}

BotSharp.NLP/Classify/NaiveBayesClassifier.cs

@@ -56,37 +56,33 @@ namespace BotSharp.NLP.Classify
                 Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList()
             }).ToList();
 
-            var allFeatureFreq = new List<FeatureFrequencyDistribution>();
-            featureSets.ForEach(fs =>
+            var featureFreqDist = new List<FeatureFrequencyDistribution>();
+
+            labelFreqDist.Select(x => x.Label).ToList().ForEach(label =>
             {
-                fs.Features.ForEach(f =>
+                var fSets = featureSets.Where(x => x.Label == label);
+                fNames.ForEach(fName =>
                 {
-                    allFeatureFreq.Add(new FeatureFrequencyDistribution
+                    var fsv = fSets.Select(fs => fs.Features.First(f => f.Name == fName))
+                        .GroupBy(f => f.Value)
+                        .Select(f => new Tuple<string, int>(f.Key, f.Count()))
+                        .OrderBy(f => f.Item1)
+                        .ToList();
+
+                    featureFreqDist.Add(new FeatureFrequencyDistribution
                     {
-                        Label = fs.Label,
-                        FeatureName = f.Name,
-                        FeatureValue = f.Value,
-                        Count = 1
+                        Label = label,
+                        FeatureName = fName,
+                        FeatureValues = fsv
                     });
                 });
             });
 
-            var featureFreqDist = allFeatureFreq.GroupBy(x => new { x.Label, x.FeatureName, x.FeatureValue })
-                .Select(x => new FeatureFrequencyDistribution
-                {
-                    Label = x.Key.Label,
-                    FeatureName = x.Key.FeatureName,
-                    FeatureValue = x.Key.FeatureValue,
-                    Count = x.Count()
-                }).ToList();
+            featureFreqDist.ForEach(ffd =>
+            {
 
-            var featureProbDist = featureFreqDist.GroupBy(x => new { x.Label, x.FeatureName })
-                .Select(x => new FeatureProbabilityDistribution
-                {
-                    Label = x.Key.Label,
-                    FeatureName = x.Key.FeatureName,
-                    Count = featureFreqDist.Where(ffd => ffd.Label == x.Key.Label && ffd.FeatureName == x.Key.FeatureName).Sum(ffd => ffd.Count)
-                }).ToList();
+
+            });
         }
     }
 
@@ -132,13 +128,11 @@ namespace BotSharp.NLP.Classify
 
         public string FeatureName { get; set; }
 
-        public string FeatureValue { get; set; }
-
-        public int Count { get; set; }
+        public List<Tuple<string, int>> FeatureValues { get; set; }
 
         public override string ToString()
         {
-            return $"{Label} {FeatureName} {FeatureValue} {Count}";
+            return $"{Label} {FeatureName} {FeatureValues.Count}";
         }
     }
 }

BotSharp.NLP/Tokenize/TokenizerFactory.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.NLP.Tokenize
 {
@@ -29,5 +31,27 @@ namespace BotSharp.NLP.Tokenize
         {
             return _tokenizer.Tokenize(sentence, _options);
         }
+
+        public List<List<Token>> Tokenize(List<String> sentences)
+        {
+            var sents = sentences.Select(s => new ParallelToken { Text = s }).ToList();
+
+            Parallel.ForEach(sents, (sentence) =>
+            {
+                sentence.Tokens = Tokenize(sentence.Text);
+            });
+
+            List<List<Token>> result = new List<List<Token>>();
+            sents.ForEach(x => result.Add(x.Tokens));
+
+            return result;
+        }
+
+        private class ParallelToken
+        {
+            public String Text { get; set; }
+
+            public List<Token> Tokens { get; set; }
+        }
     }
 }

BotSharp.WebHost/BotSharp.WebHost.csproj

@@ -12,7 +12,8 @@
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA|AnyCPU'">
-    <DefineConstants>TRACE;DEBUG</DefineConstants>
+    <DefineConstants>DEBUG;TRACE;RASA;NETCOREAPP;NETCOREAPP2_1;RASA;NETCOREAPP;NETCOREAPP2_1;RASA;NETCOREAPP;NETCOREAPP2_1;RASA;NETCOREAPP;NETCOREAPP2_1</DefineConstants>
+    <Optimize>false</Optimize>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='RASA NLU|AnyCPU'">

BotSharp.WebHost/Program.cs

@@ -18,7 +18,8 @@ namespace BotSharp.WebHost
                 .ConfigureAppConfiguration((hostingContext, config) =>
                 {
                     var env = hostingContext.HostingEnvironment;
-                    var settings = Directory.GetFiles(Path.Combine(env.ContentRootPath, "Settings"), "*.json");
+                    string dir = Path.GetFullPath(env.ContentRootPath + "/..");
+                    var settings = Directory.GetFiles(Path.Combine(dir, "Settings"), "*.json");
                     settings.ToList().ForEach(setting =>
                     {
                         config.AddJsonFile(setting, optional: false, reloadOnChange: true);

Settings/app.json

@@ -4,6 +4,6 @@
   "Version": "0.1.0",
 
   "MachineLearning": {
-    "dataDir": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data"
+    "dataDir": "D:\\Projects\\BotSharp\\Data"
   }
 }