One Hot Encoding

BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs

@@ -9,6 +9,7 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using BotSharp.Algorithm.Extensions;
+using BotSharp.NLP.Txt2Vec;
 
 namespace BotSharp.NLP.UnitTest
 {
@@ -31,10 +32,14 @@ namespace BotSharp.NLP.UnitTest
             {
                 newSentences[i].Label = sentences[i].Label;
             }
-            sentences = newSentences.Take(10).ToList();
+            sentences = newSentences.ToList();
             
             sentences.Shuffle();
 
+            var encoder = new OneHotEncoder();
+            encoder.Sentences = sentences;
+            encoder.EncodeAll();
+
             var options = new ClassifyOptions
             {
                 TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")

BotSharp.NLP.UnitTest/Vector/OneHotEncodingTest.cs

@@ -0,0 +1,40 @@
+using BotSharp.NLP.Corpus;
+using BotSharp.NLP.Tokenize;
+using BotSharp.NLP.Txt2Vec;
+using Microsoft.Extensions.Configuration;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.NLP.UnitTest.Vector
+{
+    [TestClass]
+    public class OneHotEncodingTest : TestEssential
+    {
+        [TestMethod]
+        public void OneHotTest()
+        {
+            var reader = new FasttextDataReader();
+            var sentences = reader.Read(new ReaderOptions
+            {
+                DataDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
+                FileName = "cooking.stackexchange.txt"
+            });
+
+            var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
+            var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
+            for (int i = 0; i < newSentences.Count; i++)
+            {
+                newSentences[i].Label = sentences[i].Label;
+            }
+            sentences = newSentences.ToList();
+
+            var encoder = new OneHotEncoder();
+            encoder.Sentences = sentences;
+            encoder.EncodeAll();
+        }
+    }
+}

BotSharp.NLP/Classify/ITextFeatureExtractor.cs

@@ -12,5 +12,6 @@ namespace BotSharp.NLP.Classify
     public interface ITextFeatureExtractor
     {
         List<Feature> GetFeatures(List<Token> words);
+
     }
 }

BotSharp.NLP/Sentence.cs

@@ -12,5 +12,7 @@ namespace BotSharp.NLP
         public String Label { get; set; }
 
         public String Text { get; set; }
+
+        public double[] Vector { get; set; }
     }
 }

BotSharp.NLP/Txt2Vec/OneHotEncoder.cs

@@ -0,0 +1,60 @@
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.NLP.Txt2Vec
+{
+    /// <summary>
+    /// A one hot encoding is a representation of categorical variables as binary vectors. 
+    /// Each integer value is represented as a binary vector that is all zero values except the index of the integer, which is marked with a 1.
+    /// </summary>
+    public class OneHotEncoder
+    {
+        public List<Sentence> Sentences { get; set; }
+
+        private List<string> words;
+
+        public void Encode(Sentence sentence)
+        {
+            InitDictionary();
+
+            var vector = words.Select(x => 0D).ToArray();
+
+            sentence.Words.ForEach(w =>
+            {
+                int index = words.IndexOf(w.Text.ToLower());
+                if(index > 0)
+                {
+                    vector[index] = 1;
+                }
+            });
+
+            sentence.Vector = vector;
+        }
+
+        public void EncodeAll()
+        {
+            InitDictionary();
+            Parallel.ForEach(Sentences, sent =>
+            {
+                Encode(sent);
+            });
+        }
+
+        private void InitDictionary()
+        {
+            if (words == null)
+            {
+                words = new List<string>();
+                Sentences.ForEach(x =>
+                {
+                    words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Text.ToLower()));
+                });
+                words = words.Distinct().OrderBy(x => x).ToList();
+            }
+        }
+    }
+}