diff --git a/BotSharp.NLP.UnitTest/Vector/Word2VecTest.cs b/BotSharp.NLP.UnitTest/Vector/Word2VecTest.cs new file mode 100644 index 00000000..994fa78f --- /dev/null +++ b/BotSharp.NLP.UnitTest/Vector/Word2VecTest.cs @@ -0,0 +1,31 @@ +using BotSharp.NLP.Txt2Vec; +using Microsoft.Extensions.Configuration; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Txt2Vec; + +namespace BotSharp.NLP.UnitTest.Vector +{ + public class Word2VecTest + { + [TestClass] + public class OneHotEncodingTest : TestEssential + { + [TestMethod] + public void Word2VecTest() + { + string sentence = "stop this song"; + List words = sentence.Split(' ').ToList(); + Args args = new Args(); + args.ModelFile = @"C:\Users\bpeng\Desktop\BoloReborn\Txt2VecDemo\wordvec_enu.bin"; + VectorGenerator vg = new VectorGenerator(args); + + vg.Distance(words); + } + } + } +} diff --git a/BotSharp.NLP/Classify/ClassifyOptions.cs b/BotSharp.NLP/Classify/ClassifyOptions.cs index 35671344..1e0b68f4 100644 --- a/BotSharp.NLP/Classify/ClassifyOptions.cs +++ b/BotSharp.NLP/Classify/ClassifyOptions.cs @@ -13,6 +13,7 @@ namespace BotSharp.NLP.Classify public string ModelName { get; set; } public string FeaturesFileName { get; set; } + public string FeaturesInTfIdfFileName { get; set; } public string DictionaryFileName { get; set; } public string CategoriesFileName { get; set; } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 124a49ea..9539622b 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -40,6 +40,7 @@ namespace BotSharp.NLP.Classify private List categories; private RangeTransform transform; private Bigtree.Algorithm.SVM.Model model; + private List featuresInTfIdf; public void Train(List sentences, ClassifyOptions options) { @@ -48,6 +49,12 @@ namespace BotSharp.NLP.Classify public void SVMClassifierTrain(List sentences, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) { + var tfidf = new TfIdfFeatureExtractor(); + tfidf.Dimension = options.Dimension; + tfidf.Sentences = sentences; + tfidf.CalBasedOnCategory(); + featuresInTfIdf = tfidf.Keywords(); + // copy test multiclass Model Problem train = new Problem(); train.X = GetData(sentences).ToArray(); @@ -124,7 +131,7 @@ namespace BotSharp.NLP.Classify public List GetData(List sentences) { - // var extractor = new CountFeatureExtractor(); + //var extractor = new CountFeatureExtractor(); var extractor = new Word2VecFeatureExtractor(); extractor.Sentences = sentences; if(features != null) @@ -137,7 +144,7 @@ namespace BotSharp.NLP.Classify extractor.Dictionary = dictionary; } - extractor.Vectorize(); + extractor.Vectorize(featuresInTfIdf); if(features == null) { @@ -157,8 +164,9 @@ namespace BotSharp.NLP.Classify for(int i = 0; i < extractor.Features.Count; i++) { + int name = i; - var xx = sentence.Words.Find(x => x.Lemma == extractor.Features[i]); + /*var xx = sentence.Words.Find(x => x.Lemma == extractor.Features[i]); if (xx == null) { @@ -167,7 +175,9 @@ namespace BotSharp.NLP.Classify else { curNodes.Add(new Node(name, xx.Vector)); - } + }*/ + + curNodes.Add(new Node(i, sentence.Vector[i])); } datas.Add(curNodes.ToArray()); @@ -181,9 +191,12 @@ namespace BotSharp.NLP.Classify options.FeaturesFileName = Path.Combine(options.ModelDir, "features"); options.DictionaryFileName = Path.Combine(options.ModelDir, "dictionary"); options.CategoriesFileName = Path.Combine(options.ModelDir, "categories"); + options.FeaturesInTfIdfFileName = Path.Combine(options.ModelDir, "featuresInTfIdf"); File.WriteAllText(options.FeaturesFileName, JsonConvert.SerializeObject(features)); + File.WriteAllText(options.FeaturesInTfIdfFileName, JsonConvert.SerializeObject(featuresInTfIdf)); + File.WriteAllText(options.DictionaryFileName, JsonConvert.SerializeObject(dictionary)); File.WriteAllText(options.CategoriesFileName, JsonConvert.SerializeObject(categories)); @@ -201,9 +214,12 @@ namespace BotSharp.NLP.Classify options.ModelFilePath = Path.Combine(options.ModelDir, options.ModelName); options.TransformFilePath = Path.Combine(options.ModelDir, "transform"); options.CategoriesFileName = Path.Combine(options.ModelDir, "categories"); + options.FeaturesInTfIdfFileName = Path.Combine(options.ModelDir, "featuresInTfIdf"); features = JsonConvert.DeserializeObject>(File.ReadAllText(options.FeaturesFileName)); + featuresInTfIdf = JsonConvert.DeserializeObject>(File.ReadAllText(options.FeaturesInTfIdfFileName)); + dictionary = JsonConvert.DeserializeObject>>(File.ReadAllText(options.DictionaryFileName)); categories = JsonConvert.DeserializeObject>(File.ReadAllText(options.CategoriesFileName)); diff --git a/BotSharp.NLP/Featuring/CountFeatureExtractor.cs b/BotSharp.NLP/Featuring/CountFeatureExtractor.cs index be8a0992..40e438f4 100644 --- a/BotSharp.NLP/Featuring/CountFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/CountFeatureExtractor.cs @@ -37,7 +37,7 @@ namespace BotSharp.NLP.Featuring public List Features { get; set; } public Shape Shape { get; set; } - public void Vectorize() + public void Vectorize(List features) { CalculateDictionary(); diff --git a/BotSharp.NLP/Featuring/IFeatureExtractor.cs b/BotSharp.NLP/Featuring/IFeatureExtractor.cs index a3d6559f..024edea5 100644 --- a/BotSharp.NLP/Featuring/IFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/IFeatureExtractor.cs @@ -30,7 +30,7 @@ namespace BotSharp.NLP.Featuring /// /// Vectorize sentence /// - void Vectorize(); + void Vectorize(List features); /// /// Array shape diff --git a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs index 9f9a01bf..66319597 100644 --- a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs @@ -192,7 +192,7 @@ namespace BotSharp.NLP.Featuring return result; } - public void Vectorize() + public void Vectorize(List features) { throw new NotImplementedException(); } diff --git a/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs b/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs index bf6e2101..c71ddbea 100644 --- a/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs @@ -18,30 +18,48 @@ namespace BotSharp.NLP.Featuring public Word2VecFeatureExtractor() { - Args args = new Args(); - args.ModelFile = "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\BotSharp.WebHost\\App_Data\\wordvec_enu.bin"; - this.Vg = new VectorGenerator(args); - this.SentenceVectorSize = this.Vg.Model.VectorSize * MaxSentenceTokenCounts(); + } - public void Vectorize() + public void Vectorize(List features) { + Init(); + Sentences.ForEach(s => { - Vec sentenceVec = new Vec(); + List wordLemmas = new List(); s.Words.ForEach(word => { - Vec wordVec = Vg.Word2Vec(word.Text); - sentenceVec.VecNodes.AddRange(wordVec.VecNodes); + if (features.Contains(word.Lemma)) + { + wordLemmas.Add(word.Lemma); + } }); - while (sentenceVec.VecNodes.Count != SentenceVectorSize) - { - sentenceVec.VecNodes.Add(0); - } + Vec sentenceVec = Vg.Sent2Vec(wordLemmas); + s.Vector = sentenceVec.VecNodes.ToArray(); }); + + + } + + private void Init() + { + if(Vg == null) + { + Args args = new Args(); + args.ModelFile = @"C:\Users\bpeng\Desktop\BoloReborn\Txt2VecDemo\wordvec_enu.bin"; + Vg = new VectorGenerator(args); + SentenceVectorSize = this.Vg.Model.VectorSize * MaxSentenceTokenCounts(); + Features = new List(); + for (int i = 0; i < SentenceVectorSize; i++) + { + Features.Add($"f-{i}"); + } + } } private int MaxSentenceTokenCounts() { + return 1; int maxCount = 0; Sentences.ForEach(s=> { if (s.Words.Count > maxCount) diff --git a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs index fbf23aef..b54bb642 100644 --- a/BotSharp.NLP/Txt2Vec/VectorGenerator.cs +++ b/BotSharp.NLP/Txt2Vec/VectorGenerator.cs @@ -169,6 +169,30 @@ namespace Txt2Vec return vec; } + public Vec Sent2Vec(List words) + { + Vec vec = new Vec(); + + Txt2Vec.Decoder decoder = new Txt2Vec.Decoder(Model); + string[] termList = words.ToArray(); + vec.VecNodes = decoder.ToVector(termList).ToList(); + + return vec; + } + + public void Distance(List words) + { + + Txt2Vec.Decoder decoder = new Txt2Vec.Decoder(Model); + words.ForEach(word=> { + Console.WriteLine($"current word: {word}"); + List sysnonyms = decoder.Distance(word); + sysnonyms.ForEach(s=> { + Console.WriteLine($"{s.strTerm}: {s.score}"); + }); + }); + } + public double Similarity(Vec vec1, Vec vec2) { double score = 0;