SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

make word2vec pre-traind model path is configurable.

Browse source
This commit is contained in:
Oceania2018 2018-10-03 17:28:47 -05:00
parent 52709ea4ac
commit d0e89f777d
8 changed files with 24 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs
View file

@ -72,9 +72,16 @@ namespace BotSharp.Core.Engines.BotSharp
{
ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model),
ModelDir = Settings.ModelDir,
ModelName = meta.Model
ModelName = meta.Model,
Word2VecFilePath = Configuration.GetValue<string>("wordvecModel")
};
if (!String.IsNullOrEmpty(options.Word2VecFilePath))
{
string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString();
options.Word2VecFilePath = options.Word2VecFilePath.Replace("|App_Data|", contentDir + System.IO.Path.DirectorySeparatorChar);
}
_classifier = new ClassifierFactory<SentenceFeatureExtractor>(options, SupportedLanguage.English);
string classifierName = Configuration.GetValue<String>($"classifer");

1
BotSharp.NLP/Classify/ClassifyOptions.cs
View file

@ -11,6 +11,7 @@ namespace BotSharp.NLP.Classify
public string ModelFilePath { get; set; }
public string ModelDir { get; set; }
public string ModelName { get; set; }
public string Word2VecFilePath { get; set; }
public string FeaturesFileName { get; set; }
public string FeaturesInTfIdfFileName { get; set; }

7
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -57,7 +57,7 @@ namespace BotSharp.NLP.Classify
// copy test multiclass Model
Problem train = new Problem();
train.X = GetData(sentences).ToArray();
train.X = GetData(sentences, options).ToArray();
train.Y = GetLabels(sentences).ToArray();
train.Count = train.X.Count();
train.MaxIndex = train.X[0].Count();//int.MaxValue;
@ -104,7 +104,7 @@ namespace BotSharp.NLP.Classify
public double[][] Predict(Sentence sentence, ClassifyOptions options)
{
Problem predict = new Problem();
predict.X = GetData(new List<Sentence> { sentence }).ToArray();
predict.X = GetData(new List<Sentence> { sentence }, options).ToArray();
predict.Y = new double[1];
predict.Count = predict.X.Count();
predict.MaxIndex = features.Count;
@ -129,10 +129,11 @@ namespace BotSharp.NLP.Classify
return labels;
}
public List<Node[]> GetData(List<Sentence> sentences)
public List<Node[]> GetData(List<Sentence> sentences, ClassifyOptions options)
{
//var extractor = new CountFeatureExtractor();
var extractor = new Word2VecFeatureExtractor();
extractor.ModelFile = options.Word2VecFilePath;
extractor.Sentences = sentences;
if(features != null)
{

1
BotSharp.NLP/Featuring/CountFeatureExtractor.cs
View file

@ -36,6 +36,7 @@ namespace BotSharp.NLP.Featuring
public List<Tuple<string, int>> Dictionary { get; set; }
public List<string> Features { get; set; }
public Shape Shape { get; set; }
public string ModelFile { get; set; }
public void Vectorize(List<string> features)
{

5
BotSharp.NLP/Featuring/IFeatureExtractor.cs
View file

@ -36,5 +36,10 @@ namespace BotSharp.NLP.Featuring
/// Array shape
/// </summary>
Shape Shape { get; set; }
/// <summary>
/// Pre-trained model file path
/// </summary>
string ModelFile { get; set; }
}
}

1
BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs
View file

@ -40,6 +40,7 @@ namespace BotSharp.NLP.Featuring
public List<Tuple<string, int>> Dictionary { get; set; }
public List<string> Features { get; set; }
public Shape Shape { get; set; }
public string ModelFile { get; set; }
public void Extract(Sentence sentence)
{

22
BotSharp.NLP/Featuring/Word2VecFeatureExtractor.cs
View file

@ -15,11 +15,7 @@ namespace BotSharp.NLP.Featuring
public Shape Shape { get; set; }
public VectorGenerator Vg { get; set; }
public int SentenceVectorSize { get; set; }
public Word2VecFeatureExtractor()
{
}
public string ModelFile { get; set; }
public void Vectorize(List<string> features)
{
@ -48,7 +44,7 @@ namespace BotSharp.NLP.Featuring
Args args = new Args();
args.ModelFile = @"C:\Users\bpeng\Desktop\BoloReborn\Txt2VecDemo\wordvec_enu.bin";
Vg = new VectorGenerator(args);
SentenceVectorSize = this.Vg.Model.VectorSize * MaxSentenceTokenCounts();
SentenceVectorSize = this.Vg.Model.VectorSize;
Features = new List<string>();
for (int i = 0; i < SentenceVectorSize; i++)
{
@ -56,19 +52,5 @@ namespace BotSharp.NLP.Featuring
}
}
}
private int MaxSentenceTokenCounts()
{
return 1;
int maxCount = 0;
Sentences.ForEach(s=> {
if (s.Words.Count > maxCount)
{
maxCount = s.Words.Count;
}
});
return maxCount;
}
}
}

3
BotSharp.WebHost/Settings/BotSharpNLU.json
View file

@ -14,7 +14,8 @@
},
"botSharpIntentClassifier": {
"classifer": "NaiveBayesClassifier"
"classifer": "SVMClassifier",
"wordvecModel": "|App_Data|wordvec_enu.bin"
},
"botSharpTagger": {