diff --git a/BotSharp.NLP.UnitTest/SVMClassifierTest.cs b/BotSharp.NLP.UnitTest/SVMClassifierTest.cs index 732af1f5..19e5a32b 100644 --- a/BotSharp.NLP.UnitTest/SVMClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/SVMClassifierTest.cs @@ -39,7 +39,7 @@ namespace BotSharp.NLP.UnitTest sentences.Add("The sun in the sky is bright."); sentences.Add("We can see the shining sun, the bright sun."); Args args = new Args(); - args.ModelFile = "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp.NLP\\BotSharp.NLP.UnitTest\\wordvec_enu.bin"; + args.ModelFile = "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\BotSharp.WebHost\\App_Data\\wordvec_enu.bin"; VectorGenerator vg = new VectorGenerator(args); var list = vg.Sentence2Vec(sentences); } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 4b28ee9e..124a49ea 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -124,7 +124,8 @@ namespace BotSharp.NLP.Classify public List GetData(List sentences) { - var extractor = new CountFeatureExtractor(); + // var extractor = new CountFeatureExtractor(); + var extractor = new Word2VecFeatureExtractor(); extractor.Sentences = sentences; if(features != null) { diff --git a/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs b/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs new file mode 100644 index 00000000..bf6e2101 --- /dev/null +++ b/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Bigtree.Algorithm.Matrix; +using Txt2Vec; + +namespace BotSharp.NLP.Featuring +{ + public class Word2VecFeatureExtractor : IFeatureExtractor + { + public int Dimension { get; set; } + public List Sentences { get; set; } + public List> Dictionary { get; set; } + public List Features { get; set; } + public Shape Shape { get; set; } + public VectorGenerator Vg { get; set; } + public int SentenceVectorSize { get; set; } + + public Word2VecFeatureExtractor() + { + Args args = new Args(); + args.ModelFile = "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\BotSharp.WebHost\\App_Data\\wordvec_enu.bin"; + this.Vg = new VectorGenerator(args); + this.SentenceVectorSize = this.Vg.Model.VectorSize * MaxSentenceTokenCounts(); + } + + public void Vectorize() + { + Sentences.ForEach(s => { + Vec sentenceVec = new Vec(); + s.Words.ForEach(word => { + Vec wordVec = Vg.Word2Vec(word.Text); + sentenceVec.VecNodes.AddRange(wordVec.VecNodes); + }); + while (sentenceVec.VecNodes.Count != SentenceVectorSize) + { + sentenceVec.VecNodes.Add(0); + } + s.Vector = sentenceVec.VecNodes.ToArray(); + }); + } + + private int MaxSentenceTokenCounts() + { + int maxCount = 0; + Sentences.ForEach(s=> { + if (s.Words.Count > maxCount) + { + maxCount = s.Words.Count; + } + }); + + return maxCount; + } + } +} diff --git a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs index 7c848031..fbf23aef 100644 --- a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs +++ b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs @@ -11,11 +11,13 @@ namespace Txt2Vec { public class VectorGenerator { - Txt2Vec.Model model = new Txt2Vec.Model(); + public Model Model { get; set; } + // Txt2Vec.Model model = new Txt2Vec.Model(); Dictionary dict = new Dictionary(); public VectorGenerator(Args args) { + this.Model = new Txt2Vec.Model(); bool bTxtFormat = false; string strModelFileName = args.ModelFile; @@ -30,7 +32,7 @@ namespace Txt2Vec throw new IOException(); } - model.LoadModel(strModelFileName, bTxtFormat); + this.Model.LoadModel(strModelFileName, bTxtFormat); } public List Sentence2Vec(List sentences, WeightingScheme weightingScheme = WeightingScheme.AVG) @@ -40,18 +42,20 @@ namespace Txt2Vec List> weights = null;// tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray()); List> matixList = new List>(); - + List sentenceVectorList = new List(); sentences.ForEach (sentence=>{ - List sentenceVectorList = new List(); - string[] words = sentence.Split(' '); - foreach (string word in words) - { - Vec vec = Word2Vec(word.ToLower()); - sentenceVectorList.Add(vec); - } - matixList.Add(sentenceVectorList); + //List sentenceVectorList = new List(); + //string[] words = sentence.Split(' '); + //foreach (string word in words) + //{ + // Vec vec = Word2Vec(word.ToLower()); + // sentenceVectorList.Add(vec); + //} + //matixList.Add(sentenceVectorList); }); + return sentenceVectorList; + /* List vectorList = new List(); // Traverse each sentence for (int i = 0; i < sentences.Count; i++) @@ -98,6 +102,7 @@ namespace Txt2Vec } return vectorList; + */ } public Vec SingleSentence2Vec(string sentence, WeightingScheme weightingScheme = WeightingScheme.AVG) @@ -156,7 +161,7 @@ namespace Txt2Vec { Vec vec= new Vec(); - Txt2Vec.Decoder decoder = new Txt2Vec.Decoder(model); + Txt2Vec.Decoder decoder = new Txt2Vec.Decoder(Model); string[] termList = new string[1]; termList[0] = word; vec.VecNodes = decoder.ToVector(termList).ToList(); @@ -167,7 +172,7 @@ namespace Txt2Vec public double Similarity(Vec vec1, Vec vec2) { double score = 0; - for (int i = 0; i < model.VectorSize; i++) + for (int i = 0; i < Model.VectorSize; i++) { score += vec1.VecNodes[i] * vec2.VecNodes[i]; }