SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

added word embedding for svm classifer.

Browse source
This commit is contained in:
PppBr 2018-10-03 16:49:40 -05:00
parent 78fe7deb5f
commit 8c00131a69
8 changed files with 109 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
BotSharp.NLP.UnitTest/Vector/Word2VecTest.cs Normal file
View file

@ -0,0 +1,31 @@
using BotSharp.NLP.Txt2Vec;
using Microsoft.Extensions.Configuration;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Txt2Vec;
namespace BotSharp.NLP.UnitTest.Vector
{
public class Word2VecTest
{
[TestClass]
public class OneHotEncodingTest : TestEssential
{
[TestMethod]
public void Word2VecTest()
{
string sentence = "stop this song";
List<string> words = sentence.Split(' ').ToList();
Args args = new Args();
args.ModelFile = @"C:\Users\bpeng\Desktop\BoloReborn\Txt2VecDemo\wordvec_enu.bin";
VectorGenerator vg = new VectorGenerator(args);
vg.Distance(words);
}
}
}
}

1
BotSharp.NLP/Classify/ClassifyOptions.cs
View file

@ -13,6 +13,7 @@ namespace BotSharp.NLP.Classify
public string ModelName { get; set; }
public string FeaturesFileName { get; set; }
public string FeaturesInTfIdfFileName { get; set; }
public string DictionaryFileName { get; set; }
public string CategoriesFileName { get; set; }

24
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -40,6 +40,7 @@ namespace BotSharp.NLP.Classify
private List<string> categories;
private RangeTransform transform;
private Bigtree.Algorithm.SVM.Model model;
private List<string> featuresInTfIdf;
public void Train(List<Sentence> sentences, ClassifyOptions options)
{
@ -48,6 +49,12 @@ namespace BotSharp.NLP.Classify
public void SVMClassifierTrain(List<Sentence> sentences, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
{
var tfidf = new TfIdfFeatureExtractor();
tfidf.Dimension = options.Dimension;
tfidf.Sentences = sentences;
tfidf.CalBasedOnCategory();
featuresInTfIdf = tfidf.Keywords();
// copy test multiclass Model
Problem train = new Problem();
train.X = GetData(sentences).ToArray();
@ -124,7 +131,7 @@ namespace BotSharp.NLP.Classify
public List<Node[]> GetData(List<Sentence> sentences)
{
// var extractor = new CountFeatureExtractor();
//var extractor = new CountFeatureExtractor();
var extractor = new Word2VecFeatureExtractor();
extractor.Sentences = sentences;
if(features != null)
@ -137,7 +144,7 @@ namespace BotSharp.NLP.Classify
extractor.Dictionary = dictionary;
}
extractor.Vectorize();
extractor.Vectorize(featuresInTfIdf);
if(features == null)
{
@ -157,8 +164,9 @@ namespace BotSharp.NLP.Classify
for(int i = 0; i < extractor.Features.Count; i++)
{
int name = i;
var xx = sentence.Words.Find(x => x.Lemma == extractor.Features[i]);
/*var xx = sentence.Words.Find(x => x.Lemma == extractor.Features[i]);
if (xx == null)
{
@ -167,7 +175,9 @@ namespace BotSharp.NLP.Classify
else
{
curNodes.Add(new Node(name, xx.Vector));
}
}*/
curNodes.Add(new Node(i, sentence.Vector[i]));
}
datas.Add(curNodes.ToArray());
@ -181,9 +191,12 @@ namespace BotSharp.NLP.Classify
options.FeaturesFileName = Path.Combine(options.ModelDir, "features");
options.DictionaryFileName = Path.Combine(options.ModelDir, "dictionary");
options.CategoriesFileName = Path.Combine(options.ModelDir, "categories");
options.FeaturesInTfIdfFileName = Path.Combine(options.ModelDir, "featuresInTfIdf");
File.WriteAllText(options.FeaturesFileName, JsonConvert.SerializeObject(features));
File.WriteAllText(options.FeaturesInTfIdfFileName, JsonConvert.SerializeObject(featuresInTfIdf));
File.WriteAllText(options.DictionaryFileName, JsonConvert.SerializeObject(dictionary));
File.WriteAllText(options.CategoriesFileName, JsonConvert.SerializeObject(categories));
@ -201,9 +214,12 @@ namespace BotSharp.NLP.Classify
options.ModelFilePath = Path.Combine(options.ModelDir, options.ModelName);
options.TransformFilePath = Path.Combine(options.ModelDir, "transform");
options.CategoriesFileName = Path.Combine(options.ModelDir, "categories");
options.FeaturesInTfIdfFileName = Path.Combine(options.ModelDir, "featuresInTfIdf");
features = JsonConvert.DeserializeObject<List<String>>(File.ReadAllText(options.FeaturesFileName));
featuresInTfIdf = JsonConvert.DeserializeObject<List<String>>(File.ReadAllText(options.FeaturesInTfIdfFileName));
dictionary = JsonConvert.DeserializeObject<List<Tuple<string, int>>>(File.ReadAllText(options.DictionaryFileName));
categories = JsonConvert.DeserializeObject<List<String>>(File.ReadAllText(options.CategoriesFileName));

2
BotSharp.NLP/Featuring/CountFeatureExtractor.cs
View file

@ -37,7 +37,7 @@ namespace BotSharp.NLP.Featuring
public List<string> Features { get; set; }
public Shape Shape { get; set; }
public void Vectorize()
public void Vectorize(List<string> features)
{
CalculateDictionary();

2
BotSharp.NLP/Featuring/IFeatureExtractor.cs
View file

@ -30,7 +30,7 @@ namespace BotSharp.NLP.Featuring
/// <summary>
/// Vectorize sentence
/// </summary>
void Vectorize();
void Vectorize(List<string> features);
/// <summary>
/// Array shape

2
BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs
View file

@ -192,7 +192,7 @@ namespace BotSharp.NLP.Featuring
return result;
}
public void Vectorize()
public void Vectorize(List<string> features)
{
throw new NotImplementedException();
}

42
BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs
View file

@ -18,30 +18,48 @@ namespace BotSharp.NLP.Featuring
public Word2VecFeatureExtractor()
{
Args args = new Args();
args.ModelFile = "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\BotSharp.WebHost\\App_Data\\wordvec_enu.bin";
this.Vg = new VectorGenerator(args);
this.SentenceVectorSize = this.Vg.Model.VectorSize * MaxSentenceTokenCounts();
}
public void Vectorize()
public void Vectorize(List<string> features)
{
Init();
Sentences.ForEach(s => {
Vec sentenceVec = new Vec();
List<string> wordLemmas = new List<string>();
s.Words.ForEach(word => {
Vec wordVec = Vg.Word2Vec(word.Text);
sentenceVec.VecNodes.AddRange(wordVec.VecNodes);
if (features.Contains(word.Lemma))
{
wordLemmas.Add(word.Lemma);
}
});
while (sentenceVec.VecNodes.Count != SentenceVectorSize)
{
sentenceVec.VecNodes.Add(0);
}
Vec sentenceVec = Vg.Sent2Vec(wordLemmas);
s.Vector = sentenceVec.VecNodes.ToArray();
});
}
private void Init()
{
if(Vg == null)
{
Args args = new Args();
args.ModelFile = @"C:\Users\bpeng\Desktop\BoloReborn\Txt2VecDemo\wordvec_enu.bin";
Vg = new VectorGenerator(args);
SentenceVectorSize = this.Vg.Model.VectorSize * MaxSentenceTokenCounts();
Features = new List<string>();
for (int i = 0; i < SentenceVectorSize; i++)
{
Features.Add($"f-{i}");
}
}
}
private int MaxSentenceTokenCounts()
{
return 1;
int maxCount = 0;
Sentences.ForEach(s=> {
if (s.Words.Count > maxCount)

24
BotSharp.NLP/Txt2Vec/VectorGenerator.cs
View file

@ -169,6 +169,30 @@ namespace Txt2Vec
return vec;
}
public Vec Sent2Vec(List<string> words)
{
Vec vec = new Vec();
Txt2Vec.Decoder decoder = new Txt2Vec.Decoder(Model);
string[] termList = words.ToArray();
vec.VecNodes = decoder.ToVector(termList).ToList();
return vec;
}
public void Distance(List<string> words)
{
Txt2Vec.Decoder decoder = new Txt2Vec.Decoder(Model);
words.ForEach(word=> {
Console.WriteLine($"current word: {word}");
List<Result> sysnonyms = decoder.Distance(word);
sysnonyms.ForEach(s=> {
Console.WriteLine($"{s.strTerm}: {s.score}");
});
});
}
public double Similarity(Vec vec1, Vec vec2)
{
double score = 0;