From 567dcc335d84cdb35aacdea429d585011937d736 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Thu, 6 Sep 2018 17:32:51 -0500 Subject: [PATCH] Added Lidstone smoothing estimator. --- .../NaiveBayesClassifierTest.cs | 2 +- BotSharp.NLP/Classify/ClassifierFactory.cs | 6 ++- BotSharp.NLP/Classify/IEstimator.cs | 10 ++++ BotSharp.NLP/Classify/Lidstone.cs | 53 +++++++++++++++++++ BotSharp.NLP/Classify/NaiveBayesClassifier.cs | 48 ++++++++--------- BotSharp.NLP/Tokenize/TokenizerFactory.cs | 24 +++++++++ BotSharp.WebHost/BotSharp.WebHost.csproj | 3 +- BotSharp.WebHost/Program.cs | 3 +- .../Settings => Settings}/app.json | 2 +- .../Settings => Settings}/auth.json | 0 .../Settings => Settings}/bot.json | 0 .../Settings => Settings}/db.json | 0 .../Settings => Settings}/logging.json | 0 .../Settings => Settings}/swagger.json | 0 14 files changed, 118 insertions(+), 33 deletions(-) create mode 100644 BotSharp.NLP/Classify/IEstimator.cs create mode 100644 BotSharp.NLP/Classify/Lidstone.cs rename {BotSharp.WebHost/Settings => Settings}/app.json (63%) rename {BotSharp.WebHost/Settings => Settings}/auth.json (100%) rename {BotSharp.WebHost/Settings => Settings}/bot.json (100%) rename {BotSharp.WebHost/Settings => Settings}/db.json (100%) rename {BotSharp.WebHost/Settings => Settings}/logging.json (100%) rename {BotSharp.WebHost/Settings => Settings}/swagger.json (100%) diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index 9b66722b..d8070ebc 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -18,7 +18,7 @@ namespace BotSharp.NLP.UnitTest { var options = new ClassifyOptions { - TrainingCorpusDir = Path.Combine(Configuration.GetValue("BotSharp.NLP:dataDir"), "Gender") + TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Gender") }; var classifier = new ClassifierFactory(options, SupportedLanguage.English); diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index f012c9b0..c2823524 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -38,10 +38,12 @@ namespace BotSharp.NLP.Classify private List GetFeatures(List words) { + string text = words[0].Text; var features = new List(); - features.Add(new Feature("StartsWith(A)", words[0].Text.StartsWith("A").ToString())); - features.Add(new Feature("EndsWith(a)", words[0].Text.EndsWith("a").ToString())); + features.Add(new Feature("alwayson", "True")); + features.Add(new Feature("startswith", text[0].ToString().ToLower())); + features.Add(new Feature("endswith", text[text.Length - 1].ToString().ToLower())); return features; } diff --git a/BotSharp.NLP/Classify/IEstimator.cs b/BotSharp.NLP/Classify/IEstimator.cs new file mode 100644 index 00000000..305193e1 --- /dev/null +++ b/BotSharp.NLP/Classify/IEstimator.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Classify +{ + public interface IEstimator + { + } +} diff --git a/BotSharp.NLP/Classify/Lidstone.cs b/BotSharp.NLP/Classify/Lidstone.cs new file mode 100644 index 00000000..3602c61d --- /dev/null +++ b/BotSharp.NLP/Classify/Lidstone.cs @@ -0,0 +1,53 @@ +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Classify +{ + /// + /// Lidstone smoothing, is a technique used to smooth categorical data. + /// Given an observation x = (x1, …, xd) from a multinomial distribution with N trials, a "smoothed" version of the data gives the estimator: + /// Refer https://en.wikipedia.org/wiki/Additive_smoothing + /// + public class Lidstone : IEstimator + { + /// + /// x = (x1, …, xd) + /// + private int _d; + + /// + /// α > 0 is the smoothing parameter + /// + private float _a; + + /// + /// N trials + /// + private int _N; + + public Lidstone(float alpha, int bins) + { + _a = alpha; + _d = bins; + } + } +} diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index f6620b7d..6f99658d 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -56,37 +56,33 @@ namespace BotSharp.NLP.Classify Values = allFeatureValues.Where(x => x.Name == fn).Select(x => x.Value).Distinct().ToList() }).ToList(); - var allFeatureFreq = new List(); - featureSets.ForEach(fs => + var featureFreqDist = new List(); + + labelFreqDist.Select(x => x.Label).ToList().ForEach(label => { - fs.Features.ForEach(f => + var fSets = featureSets.Where(x => x.Label == label); + fNames.ForEach(fName => { - allFeatureFreq.Add(new FeatureFrequencyDistribution + var fsv = fSets.Select(fs => fs.Features.First(f => f.Name == fName)) + .GroupBy(f => f.Value) + .Select(f => new Tuple(f.Key, f.Count())) + .OrderBy(f => f.Item1) + .ToList(); + + featureFreqDist.Add(new FeatureFrequencyDistribution { - Label = fs.Label, - FeatureName = f.Name, - FeatureValue = f.Value, - Count = 1 + Label = label, + FeatureName = fName, + FeatureValues = fsv }); }); }); - var featureFreqDist = allFeatureFreq.GroupBy(x => new { x.Label, x.FeatureName, x.FeatureValue }) - .Select(x => new FeatureFrequencyDistribution - { - Label = x.Key.Label, - FeatureName = x.Key.FeatureName, - FeatureValue = x.Key.FeatureValue, - Count = x.Count() - }).ToList(); + featureFreqDist.ForEach(ffd => + { - var featureProbDist = featureFreqDist.GroupBy(x => new { x.Label, x.FeatureName }) - .Select(x => new FeatureProbabilityDistribution - { - Label = x.Key.Label, - FeatureName = x.Key.FeatureName, - Count = featureFreqDist.Where(ffd => ffd.Label == x.Key.Label && ffd.FeatureName == x.Key.FeatureName).Sum(ffd => ffd.Count) - }).ToList(); + + }); } } @@ -132,13 +128,11 @@ namespace BotSharp.NLP.Classify public string FeatureName { get; set; } - public string FeatureValue { get; set; } - - public int Count { get; set; } + public List> FeatureValues { get; set; } public override string ToString() { - return $"{Label} {FeatureName} {FeatureValue} {Count}"; + return $"{Label} {FeatureName} {FeatureValues.Count}"; } } } diff --git a/BotSharp.NLP/Tokenize/TokenizerFactory.cs b/BotSharp.NLP/Tokenize/TokenizerFactory.cs index 3d841ce0..32efc4e2 100644 --- a/BotSharp.NLP/Tokenize/TokenizerFactory.cs +++ b/BotSharp.NLP/Tokenize/TokenizerFactory.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; +using System.Threading.Tasks; namespace BotSharp.NLP.Tokenize { @@ -29,5 +31,27 @@ namespace BotSharp.NLP.Tokenize { return _tokenizer.Tokenize(sentence, _options); } + + public List> Tokenize(List sentences) + { + var sents = sentences.Select(s => new ParallelToken { Text = s }).ToList(); + + Parallel.ForEach(sents, (sentence) => + { + sentence.Tokens = Tokenize(sentence.Text); + }); + + List> result = new List>(); + sents.ForEach(x => result.Add(x.Tokens)); + + return result; + } + + private class ParallelToken + { + public String Text { get; set; } + + public List Tokens { get; set; } + } } } diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj b/BotSharp.WebHost/BotSharp.WebHost.csproj index 14882ccf..3c77fc2f 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj @@ -12,7 +12,8 @@ - TRACE;DEBUG + DEBUG;TRACE;RASA;NETCOREAPP;NETCOREAPP2_1;RASA;NETCOREAPP;NETCOREAPP2_1;RASA;NETCOREAPP;NETCOREAPP2_1;RASA;NETCOREAPP;NETCOREAPP2_1 + false diff --git a/BotSharp.WebHost/Program.cs b/BotSharp.WebHost/Program.cs index a9b88120..09362f52 100644 --- a/BotSharp.WebHost/Program.cs +++ b/BotSharp.WebHost/Program.cs @@ -18,7 +18,8 @@ namespace BotSharp.WebHost .ConfigureAppConfiguration((hostingContext, config) => { var env = hostingContext.HostingEnvironment; - var settings = Directory.GetFiles(Path.Combine(env.ContentRootPath, "Settings"), "*.json"); + string dir = Path.GetFullPath(env.ContentRootPath + "/.."); + var settings = Directory.GetFiles(Path.Combine(dir, "Settings"), "*.json"); settings.ToList().ForEach(setting => { config.AddJsonFile(setting, optional: false, reloadOnChange: true); diff --git a/BotSharp.WebHost/Settings/app.json b/Settings/app.json similarity index 63% rename from BotSharp.WebHost/Settings/app.json rename to Settings/app.json index e0736129..00c957a4 100644 --- a/BotSharp.WebHost/Settings/app.json +++ b/Settings/app.json @@ -4,6 +4,6 @@ "Version": "0.1.0", "MachineLearning": { - "dataDir": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data" + "dataDir": "D:\\Projects\\BotSharp\\Data" } } diff --git a/BotSharp.WebHost/Settings/auth.json b/Settings/auth.json similarity index 100% rename from BotSharp.WebHost/Settings/auth.json rename to Settings/auth.json diff --git a/BotSharp.WebHost/Settings/bot.json b/Settings/bot.json similarity index 100% rename from BotSharp.WebHost/Settings/bot.json rename to Settings/bot.json diff --git a/BotSharp.WebHost/Settings/db.json b/Settings/db.json similarity index 100% rename from BotSharp.WebHost/Settings/db.json rename to Settings/db.json diff --git a/BotSharp.WebHost/Settings/logging.json b/Settings/logging.json similarity index 100% rename from BotSharp.WebHost/Settings/logging.json rename to Settings/logging.json diff --git a/BotSharp.WebHost/Settings/swagger.json b/Settings/swagger.json similarity index 100% rename from BotSharp.WebHost/Settings/swagger.json rename to Settings/swagger.json