diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs index acdcc983..4c83f663 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs @@ -72,9 +72,16 @@ namespace BotSharp.Core.Engines.BotSharp { ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model), ModelDir = Settings.ModelDir, - ModelName = meta.Model + ModelName = meta.Model, + Word2VecFilePath = Configuration.GetValue("wordvecModel") }; + if (!String.IsNullOrEmpty(options.Word2VecFilePath)) + { + string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString(); + options.Word2VecFilePath = options.Word2VecFilePath.Replace("|App_Data|", contentDir + System.IO.Path.DirectorySeparatorChar); + } + _classifier = new ClassifierFactory(options, SupportedLanguage.English); string classifierName = Configuration.GetValue($"classifer"); diff --git a/BotSharp.NLP/Classify/ClassifyOptions.cs b/BotSharp.NLP/Classify/ClassifyOptions.cs index 1e0b68f4..f6ec2751 100644 --- a/BotSharp.NLP/Classify/ClassifyOptions.cs +++ b/BotSharp.NLP/Classify/ClassifyOptions.cs @@ -11,6 +11,7 @@ namespace BotSharp.NLP.Classify public string ModelFilePath { get; set; } public string ModelDir { get; set; } public string ModelName { get; set; } + public string Word2VecFilePath { get; set; } public string FeaturesFileName { get; set; } public string FeaturesInTfIdfFileName { get; set; } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 9539622b..9c5d8220 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -57,7 +57,7 @@ namespace BotSharp.NLP.Classify // copy test multiclass Model Problem train = new Problem(); - train.X = GetData(sentences).ToArray(); + train.X = GetData(sentences, options).ToArray(); train.Y = GetLabels(sentences).ToArray(); train.Count = train.X.Count(); train.MaxIndex = train.X[0].Count();//int.MaxValue; @@ -104,7 +104,7 @@ namespace BotSharp.NLP.Classify public double[][] Predict(Sentence sentence, ClassifyOptions options) { Problem predict = new Problem(); - predict.X = GetData(new List { sentence }).ToArray(); + predict.X = GetData(new List { sentence }, options).ToArray(); predict.Y = new double[1]; predict.Count = predict.X.Count(); predict.MaxIndex = features.Count; @@ -129,10 +129,11 @@ namespace BotSharp.NLP.Classify return labels; } - public List GetData(List sentences) + public List GetData(List sentences, ClassifyOptions options) { //var extractor = new CountFeatureExtractor(); var extractor = new Word2VecFeatureExtractor(); + extractor.ModelFile = options.Word2VecFilePath; extractor.Sentences = sentences; if(features != null) { diff --git a/BotSharp.NLP/Featuring/CountFeatureExtractor.cs b/BotSharp.NLP/Featuring/CountFeatureExtractor.cs index 40e438f4..62bb44c0 100644 --- a/BotSharp.NLP/Featuring/CountFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/CountFeatureExtractor.cs @@ -36,6 +36,7 @@ namespace BotSharp.NLP.Featuring public List> Dictionary { get; set; } public List Features { get; set; } public Shape Shape { get; set; } + public string ModelFile { get; set; } public void Vectorize(List features) { diff --git a/BotSharp.NLP/Featuring/IFeatureExtractor.cs b/BotSharp.NLP/Featuring/IFeatureExtractor.cs index 024edea5..04d25dc0 100644 --- a/BotSharp.NLP/Featuring/IFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/IFeatureExtractor.cs @@ -36,5 +36,10 @@ namespace BotSharp.NLP.Featuring /// Array shape /// Shape Shape { get; set; } + + /// + /// Pre-trained model file path + /// + string ModelFile { get; set; } } } diff --git a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs index 66319597..b48b36c9 100644 --- a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs @@ -40,6 +40,7 @@ namespace BotSharp.NLP.Featuring public List> Dictionary { get; set; } public List Features { get; set; } public Shape Shape { get; set; } + public string ModelFile { get; set; } public void Extract(Sentence sentence) { diff --git a/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs b/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs index c71ddbea..b331d523 100644 --- a/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs @@ -15,11 +15,7 @@ namespace BotSharp.NLP.Featuring public Shape Shape { get; set; } public VectorGenerator Vg { get; set; } public int SentenceVectorSize { get; set; } - - public Word2VecFeatureExtractor() - { - - } + public string ModelFile { get; set; } public void Vectorize(List features) { @@ -48,7 +44,7 @@ namespace BotSharp.NLP.Featuring Args args = new Args(); args.ModelFile = @"C:\Users\bpeng\Desktop\BoloReborn\Txt2VecDemo\wordvec_enu.bin"; Vg = new VectorGenerator(args); - SentenceVectorSize = this.Vg.Model.VectorSize * MaxSentenceTokenCounts(); + SentenceVectorSize = this.Vg.Model.VectorSize; Features = new List(); for (int i = 0; i < SentenceVectorSize; i++) { @@ -56,19 +52,5 @@ namespace BotSharp.NLP.Featuring } } } - - private int MaxSentenceTokenCounts() - { - return 1; - int maxCount = 0; - Sentences.ForEach(s=> { - if (s.Words.Count > maxCount) - { - maxCount = s.Words.Count; - } - }); - - return maxCount; - } } } diff --git a/BotSharp.WebHost/Settings/BotSharpNLU.json b/BotSharp.WebHost/Settings/BotSharpNLU.json index 58e79d4d..fd756c84 100644 --- a/BotSharp.WebHost/Settings/BotSharpNLU.json +++ b/BotSharp.WebHost/Settings/BotSharpNLU.json @@ -14,7 +14,8 @@ }, "botSharpIntentClassifier": { - "classifer": "NaiveBayesClassifier" + "classifer": "SVMClassifier", + "wordvecModel": "|App_Data|wordvec_enu.bin" }, "botSharpTagger": {