SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

use distributed embedding instead of one hot encoding in SVMclassifier

Browse source
This commit is contained in:
PppBr 2018-10-03 12:15:46 -05:00
parent b929aa272e
commit 22e6fa0ee1
4 changed files with 77 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BotSharp.NLP.UnitTest/SVMClassifierTest.cs
View file

@ -39,7 +39,7 @@ namespace BotSharp.NLP.UnitTest
sentences.Add("The sun in the sky is bright.");
sentences.Add("We can see the shining sun, the bright sun.");
Args args = new Args();
args.ModelFile = "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp.NLP\\BotSharp.NLP.UnitTest\\wordvec_enu.bin";
args.ModelFile = "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\BotSharp.WebHost\\App_Data\\wordvec_enu.bin";
VectorGenerator vg = new VectorGenerator(args);
var list = vg.Sentence2Vec(sentences);
}

3
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -124,7 +124,8 @@ namespace BotSharp.NLP.Classify
public List<Node[]> GetData(List<Sentence> sentences)
{
var extractor = new CountFeatureExtractor();
// var extractor = new CountFeatureExtractor();
var extractor = new Word2VecFeatureExtractor();
extractor.Sentences = sentences;
if(features != null)
{

56
BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs Normal file
View file

@ -0,0 +1,56 @@
using System;
using System.Collections.Generic;
using System.Text;
using Bigtree.Algorithm.Matrix;
using Txt2Vec;
namespace BotSharp.NLP.Featuring
{
public class Word2VecFeatureExtractor : IFeatureExtractor
{
public int Dimension { get; set; }
public List<Sentence> Sentences { get; set; }
public List<Tuple<string, int>> Dictionary { get; set; }
public List<string> Features { get; set; }
public Shape Shape { get; set; }
public VectorGenerator Vg { get; set; }
public int SentenceVectorSize { get; set; }
public Word2VecFeatureExtractor()
{
Args args = new Args();
args.ModelFile = "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\BotSharp.WebHost\\App_Data\\wordvec_enu.bin";
this.Vg = new VectorGenerator(args);
this.SentenceVectorSize = this.Vg.Model.VectorSize * MaxSentenceTokenCounts();
}
public void Vectorize()
{
Sentences.ForEach(s => {
Vec sentenceVec = new Vec();
s.Words.ForEach(word => {
Vec wordVec = Vg.Word2Vec(word.Text);
sentenceVec.VecNodes.AddRange(wordVec.VecNodes);
});
while (sentenceVec.VecNodes.Count != SentenceVectorSize)
{
sentenceVec.VecNodes.Add(0);
}
s.Vector = sentenceVec.VecNodes.ToArray();
});
}
private int MaxSentenceTokenCounts()
{
int maxCount = 0;
Sentences.ForEach(s=> {
if (s.Words.Count > maxCount)
{
maxCount = s.Words.Count;
}
});
return maxCount;
}
}
}

31
BotSharp.NLP/Txt2Vec/VectorGenerator.cs
View file

@ -11,11 +11,13 @@ namespace Txt2Vec
{
public class VectorGenerator
{
Txt2Vec.Model model = new Txt2Vec.Model();
public Model Model { get; set; }
// Txt2Vec.Model model = new Txt2Vec.Model();
Dictionary<string, Vec> dict = new Dictionary<string, Vec>();
public VectorGenerator(Args args)
{
this.Model = new Txt2Vec.Model();
bool bTxtFormat = false;
string strModelFileName = args.ModelFile;
@ -30,7 +32,7 @@ namespace Txt2Vec
throw new IOException();
}
model.LoadModel(strModelFileName, bTxtFormat);
this.Model.LoadModel(strModelFileName, bTxtFormat);
}
public List<Vec> Sentence2Vec(List<string> sentences, WeightingScheme weightingScheme = WeightingScheme.AVG)
@ -40,18 +42,20 @@ namespace Txt2Vec
List<List<double>> weights = null;// tfidfGenerator.TFIDFWeightVectorsForSentences(sentences.ToArray());
List<List<Vec>> matixList = new List<List<Vec>>();
List<Vec> sentenceVectorList = new List<Vec>();
sentences.ForEach (sentence=>{
List<Vec> sentenceVectorList = new List<Vec>();
string[] words = sentence.Split(' ');
foreach (string word in words)
{
Vec vec = Word2Vec(word.ToLower());
sentenceVectorList.Add(vec);
}
matixList.Add(sentenceVectorList);
//List<Vec> sentenceVectorList = new List<Vec>();
//string[] words = sentence.Split(' ');
//foreach (string word in words)
//{
// Vec vec = Word2Vec(word.ToLower());
// sentenceVectorList.Add(vec);
//}
//matixList.Add(sentenceVectorList);
});
return sentenceVectorList;
/*
List<Vec> vectorList = new List<Vec>();
// Traverse each sentence
for (int i = 0; i < sentences.Count; i++)
@ -98,6 +102,7 @@ namespace Txt2Vec
}
return vectorList;
*/
}
public Vec SingleSentence2Vec(string sentence, WeightingScheme weightingScheme = WeightingScheme.AVG)
@ -156,7 +161,7 @@ namespace Txt2Vec
{
Vec vec= new Vec();
Txt2Vec.Decoder decoder = new Txt2Vec.Decoder(model);
Txt2Vec.Decoder decoder = new Txt2Vec.Decoder(Model);
string[] termList = new string[1];
termList[0] = word;
vec.VecNodes = decoder.ToVector(termList).ToList();
@ -167,7 +172,7 @@ namespace Txt2Vec
public double Similarity(Vec vec1, Vec vec2)
{
double score = 0;
for (int i = 0; i < model.VectorSize; i++)
for (int i = 0; i < Model.VectorSize; i++)
{
score += vec1.VecNodes[i] * vec2.VecNodes[i];
}