From f2005547680c1a0ebfe8e4180adeeca4868789da Mon Sep 17 00:00:00 2001
From: botsharp2018 <botsharp2018@gmail.com>
Date: Mon, 10 Sep 2018 22:35:33 -0500
Subject: [PATCH] One Hot Encoding

---
 .../NaiveBayesClassifierTest.cs               |  7 ++-
 .../Vector/OneHotEncodingTest.cs              | 40 +++++++++++++
 .../Classify/ITextFeatureExtractor.cs         |  1 +
 BotSharp.NLP/Sentence.cs                      |  2 +
 BotSharp.NLP/Txt2Vec/OneHotEncoder.cs         | 60 +++++++++++++++++++
 5 files changed, 109 insertions(+), 1 deletion(-)
 create mode 100644 BotSharp.NLP.UnitTest/Vector/OneHotEncodingTest.cs
 create mode 100644 BotSharp.NLP/Txt2Vec/OneHotEncoder.cs

diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
index 5a243fad..445bb12a 100644
--- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
+++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
@@ -9,6 +9,7 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using BotSharp.Algorithm.Extensions;
+using BotSharp.NLP.Txt2Vec;
 
 namespace BotSharp.NLP.UnitTest
 {
@@ -31,10 +32,14 @@ namespace BotSharp.NLP.UnitTest
             {
                 newSentences[i].Label = sentences[i].Label;
             }
-            sentences = newSentences.Take(10).ToList();
+            sentences = newSentences.ToList();
             
             sentences.Shuffle();
 
+            var encoder = new OneHotEncoder();
+            encoder.Sentences = sentences;
+            encoder.EncodeAll();
+
             var options = new ClassifyOptions
             {
                 TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
diff --git a/BotSharp.NLP.UnitTest/Vector/OneHotEncodingTest.cs b/BotSharp.NLP.UnitTest/Vector/OneHotEncodingTest.cs
new file mode 100644
index 00000000..5c39f326
--- /dev/null
+++ b/BotSharp.NLP.UnitTest/Vector/OneHotEncodingTest.cs
@@ -0,0 +1,40 @@
+using BotSharp.NLP.Corpus;
+using BotSharp.NLP.Tokenize;
+using BotSharp.NLP.Txt2Vec;
+using Microsoft.Extensions.Configuration;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.NLP.UnitTest.Vector
+{
+    [TestClass]
+    public class OneHotEncodingTest : TestEssential
+    {
+        [TestMethod]
+        public void OneHotTest()
+        {
+            var reader = new FasttextDataReader();
+            var sentences = reader.Read(new ReaderOptions
+            {
+                DataDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
+                FileName = "cooking.stackexchange.txt"
+            });
+
+            var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
+            var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
+            for (int i = 0; i < newSentences.Count; i++)
+            {
+                newSentences[i].Label = sentences[i].Label;
+            }
+            sentences = newSentences.ToList();
+
+            var encoder = new OneHotEncoder();
+            encoder.Sentences = sentences;
+            encoder.EncodeAll();
+        }
+    }
+}
diff --git a/BotSharp.NLP/Classify/ITextFeatureExtractor.cs b/BotSharp.NLP/Classify/ITextFeatureExtractor.cs
index f43006b0..d2aa67f1 100644
--- a/BotSharp.NLP/Classify/ITextFeatureExtractor.cs
+++ b/BotSharp.NLP/Classify/ITextFeatureExtractor.cs
@@ -12,5 +12,6 @@ namespace BotSharp.NLP.Classify
     public interface ITextFeatureExtractor
     {
         List<Feature> GetFeatures(List<Token> words);
+
     }
 }
diff --git a/BotSharp.NLP/Sentence.cs b/BotSharp.NLP/Sentence.cs
index e0914fa4..d7ac684e 100644
--- a/BotSharp.NLP/Sentence.cs
+++ b/BotSharp.NLP/Sentence.cs
@@ -12,5 +12,7 @@ namespace BotSharp.NLP
         public String Label { get; set; }
 
         public String Text { get; set; }
+
+        public double[] Vector { get; set; }
     }
 }
diff --git a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
new file mode 100644
index 00000000..7d630bac
--- /dev/null
+++ b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
@@ -0,0 +1,60 @@
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.NLP.Txt2Vec
+{
+    /// <summary>
+    /// A one hot encoding is a representation of categorical variables as binary vectors. 
+    /// Each integer value is represented as a binary vector that is all zero values except the index of the integer, which is marked with a 1.
+    /// </summary>
+    public class OneHotEncoder
+    {
+        public List<Sentence> Sentences { get; set; }
+
+        private List<string> words;
+
+        public void Encode(Sentence sentence)
+        {
+            InitDictionary();
+
+            var vector = words.Select(x => 0D).ToArray();
+
+            sentence.Words.ForEach(w =>
+            {
+                int index = words.IndexOf(w.Text.ToLower());
+                if(index > 0)
+                {
+                    vector[index] = 1;
+                }
+            });
+
+            sentence.Vector = vector;
+        }
+
+        public void EncodeAll()
+        {
+            InitDictionary();
+            Parallel.ForEach(Sentences, sent =>
+            {
+                Encode(sent);
+            });
+        }
+
+        private void InitDictionary()
+        {
+            if (words == null)
+            {
+                words = new List<string>();
+                Sentences.ForEach(x =>
+                {
+                    words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
+                });
+                words = words.Distinct().OrderBy(x => x).ToList();
+            }
+        }
+    }
+}