diff --git a/BotSharp.Algorithm/Matrix/Shape.cs b/BotSharp.Algorithm/Matrix/Shape.cs
new file mode 100644
index 00000000..abe56423
--- /dev/null
+++ b/BotSharp.Algorithm/Matrix/Shape.cs
@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.Algorithm.Matrix
+{
+ ///
+ /// Shape of the data arrays
+ ///
+ public class Shape
+ {
+ ///
+ /// Total number of samples
+ ///
+ public int Samples { get; set; }
+
+ ///
+ /// Total number of features
+ ///
+ public int Features { get; set; }
+ }
+}
diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs
index 1672aad3..a2cb6604 100644
--- a/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs
+++ b/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs
@@ -67,11 +67,11 @@ namespace BotSharp.Core.Engines.BotSharp
{
meta.Model = "intent.model";
- string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
-
var options = new ClassifyOptions
{
- ModelFilePath = modelFileName
+ ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model),
+ ModelDir = Settings.ModelDir,
+ ModelName = meta.Model
};
_classifier = new ClassifierFactory(options, SupportedLanguage.English);
diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
deleted file mode 100644
index c9b1d0a6..00000000
--- a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs
+++ /dev/null
@@ -1,148 +0,0 @@
-using BotSharp.Algorithm.Bayes;
-using BotSharp.Core.Abstractions;
-using BotSharp.Core.Agents;
-using BotSharp.NLP.Classify;
-using DotNetToolkit;
-using Microsoft.Extensions.Configuration;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using RestSharp;
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-using Txt2Vec;
-
-namespace BotSharp.Core.Engines.BotSharp
-{
- public class BotSharpSVMClassifier : INlpTrain, INlpPredict
- {
- public IConfiguration Configuration { get; set; }
- public PipeSettings Settings { get; set; }
-
- public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta)
- {
- string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
- string predictFileName = Path.Combine(Settings.TempDir, "svm-predict-tempfile.txt");
- File.WriteAllText(predictFileName, doc.Sentences[0].Text);
-
- var svmClassifier = new NLP.Classify.SVMClassifier();
- Args args = new Args();
- args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
- var featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), "");
- /*
- //
- var client = new RestClient("http://10.2.21.200:5005");
- var request = new RestRequest("doc2vec", Method.GET);
- request.AddParameter("text", doc.Sentences[0].Text);
- var response = client.Execute(request);
- PredResult pred = JsonConvert.DeserializeObject(response.Content);
-
- Vec vec = new Vec();
- vec.VecNodes = pred.Doc2Vec;
-
- LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(vec, "");
- //
- */
- ClassifyOptions classifyOptions = new ClassifyOptions();
- classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model"));
- classifyOptions.Transform = SVM.BotSharp.MachineLearning.RangeTransform.Read(Path.Combine(Settings.ModelDir, "transform_obj_data"));
- double[][] d = svmClassifier.Predict(featureSet, classifyOptions);
-
- string intent = null;
- decimal confidence = 0;
- double max = Double.MinValue;
- for (int i = 0; i < d[0].Count(); i++)
- {
- if (d[0][i] > max)
- {
- max = d[0][i];
- intent = agent.Intents[i].Name;
- confidence = (decimal)d[0][i];
- }
- }
-
- File.Delete(predictFileName);
-
- doc.Sentences[0].Intent = new TextClassificationResult
- {
- Classifier = "SVMClassifier",
- Label = intent,
- Confidence = confidence
- };
-
- return true;
- }
-
- public async Task Train(Agent agent, NlpDoc doc, PipeModel meta)
- {
- meta.Model = "classification-svm.model";
- string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-svm.parsed.txt");
- string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
-
- List labels = new List();
- List sentences = new List();
-
- agent.Corpus.UserSays.ForEach(x =>{
- agent.Intents.ForEach(intent => {
- if (intent.Name == x.Intent)
- {
- labels.Add(agent.Intents.IndexOf(intent).ToString());
- }
- });
- sentences.Add(x.Text);
- });
-
- NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier();
- Args args = new Args();
- args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin");
- var featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels);
-
- /*
- // try using spacy doc2vec
- var client = new RestClient("http://10.2.21.200:5005");
- var request = new RestRequest("batchdoc2vec", Method.POST);
- request.RequestFormat = DataFormat.Json;
-
- request.AddParameter("application/json", JsonConvert.SerializeObject(new {Sentences = sentences}), ParameterType.RequestBody);
-
- var response = client.Execute(request);
- Result res = JsonConvert.DeserializeObject(response.Content);
-
- List vecs = new List();
- foreach (List cur in res.Doc2vecList)
- {
- Vec vec = new Vec();
- vec.VecNodes = cur;
- vecs.Add(vec);
- }
- List featureSetList = svmClassifier.FeatureSetsGenerator(vecs, labels);
- //
- */
-
-
-
- ClassifyOptions classifyOptions = new ClassifyOptions();
- classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model");
- classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data");
- // svmClassifier.Train(featureSetList, classifyOptions);
-
- meta.Meta = new JObject();
- meta.Meta["compiled at"] = "Aug 31, 2018";
- return true;
- }
- }
- public class Result
- {
- public List> Doc2vecList { get; set; }
- }
-
- public class PredResult
- {
- public List Doc2Vec{ get; set; }
- }
-}
diff --git a/BotSharp.NLP.UnitTest/Featuring/CountFeatureExtractorTest.cs b/BotSharp.NLP.UnitTest/Featuring/CountFeatureExtractorTest.cs
new file mode 100644
index 00000000..740c4cd3
--- /dev/null
+++ b/BotSharp.NLP.UnitTest/Featuring/CountFeatureExtractorTest.cs
@@ -0,0 +1,63 @@
+using BotSharp.NLP.Featuring;
+using BotSharp.NLP.Tokenize;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace BotSharp.NLP.UnitTest.Featuring
+{
+ [TestClass]
+ public class CountFeatureExtractorTest : TestEssential
+ {
+ [TestMethod]
+ public void TestVectorizer()
+ {
+ var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English);
+ tokenizer.GetTokenizer();
+
+ var extractor = new CountFeatureExtractor();
+ extractor.Sentences = tokenizer.Tokenize(Corpus());
+ extractor.Vectorize();
+
+ var vectors = Vectors();
+
+ for (int i = 0; i < extractor.Sentences.Count; i++)
+ {
+ var sentence = extractor.Sentences[i];
+
+ for(int j = 0; j < extractor.Features.Count; j++)
+ {
+ var word = sentence.Words.Find(w => w.Lemma == extractor.Features[j]);
+
+ if(word != null)
+ {
+ Assert.IsTrue(word.Vector == vectors[i][j]);
+ }
+ }
+ }
+ }
+
+ public List Corpus()
+ {
+ return new List
+ {
+ "This is the first document.",
+ "This document is the second document.",
+ "And this is the third one.",
+ "Is this the first document?"
+ };
+ }
+
+ public int[][] Vectors()
+ {
+ return new int[4][]
+ {
+ new int []{ 0, 1, 1, 1, 0, 0, 1, 0, 1 },
+ new int []{ 0, 2, 0, 1, 0, 1, 1, 0, 1 },
+ new int []{ 1, 0, 0, 1, 1, 0, 1, 1, 1 },
+ new int []{ 0, 1, 1, 1, 0, 0, 1, 0, 1 }
+ };
+ }
+ }
+}
diff --git a/BotSharp.NLP.UnitTest/TestEssential.cs b/BotSharp.NLP.UnitTest/TestEssential.cs
index 4ae615a5..033a8735 100644
--- a/BotSharp.NLP.UnitTest/TestEssential.cs
+++ b/BotSharp.NLP.UnitTest/TestEssential.cs
@@ -12,7 +12,7 @@ namespace BotSharp.NLP.UnitTest
public TestEssential()
{
var rootDir = Path.GetFullPath($"{Directory.GetCurrentDirectory()}{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}");
- var settingsDir = Path.Combine(rootDir, "Settings");
+ var settingsDir = Path.Combine(rootDir, "BotSharp.WebHost", "Settings");
ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
var settings = Directory.GetFiles(settingsDir, "*.json");
diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs
index f1a9d479..14670f58 100644
--- a/BotSharp.NLP/Classify/ClassifierFactory.cs
+++ b/BotSharp.NLP/Classify/ClassifierFactory.cs
@@ -52,7 +52,9 @@ namespace BotSharp.NLP.Classify
{
var options = new ClassifyOptions
{
- ModelFilePath = _options.ModelFilePath
+ ModelFilePath = _options.ModelFilePath,
+ ModelDir = _options.ModelDir,
+ ModelName = _options.ModelName
};
_classifier.LoadModel(options);
diff --git a/BotSharp.NLP/Classify/ClassifyOptions.cs b/BotSharp.NLP/Classify/ClassifyOptions.cs
index 0c213fad..0b7ee12b 100644
--- a/BotSharp.NLP/Classify/ClassifyOptions.cs
+++ b/BotSharp.NLP/Classify/ClassifyOptions.cs
@@ -9,7 +9,13 @@ namespace BotSharp.NLP.Classify
{
public string TrainingCorpusDir { get; set; }
public string ModelFilePath { get; set; }
- public Model Model { get; set; }
+ public string ModelDir { get; set; }
+ public string ModelName { get; set; }
+
+ public string FeaturesFileName { get; set; }
+ public string DictionaryFileName { get; set; }
+ public string CategoriesFileName { get; set; }
+
public string PrediceOutputFile { get; set; }
public string TransformFilePath { get; set; }
public RangeTransform Transform { get; set; }
diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs
index 7bbf8429..fe95a4d5 100644
--- a/BotSharp.NLP/Classify/SVMClassifier.cs
+++ b/BotSharp.NLP/Classify/SVMClassifier.cs
@@ -24,6 +24,7 @@ using System.Text;
using BotSharp.Algorithm.Features;
using BotSharp.NLP.Featuring;
using BotSharp.NLP.Txt2Vec;
+using Newtonsoft.Json;
using SVM.BotSharp.MachineLearning;
using Txt2Vec;
@@ -34,71 +35,35 @@ namespace BotSharp.NLP.Classify
///
public class SVMClassifier : IClassifier
{
- private List words;
-
- public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options)
- {
- Problem predict = new Problem();
- List featureSets = new List();
- featureSets.Add(featureSet);
- predict.X = GetData(featureSets).ToArray();
- predict.Y = new double[1];
- predict.Count = predict.X.Count();
- predict.MaxIndex = 300;
-
- RangeTransform transform = options.Transform;
- Problem scaled = transform.Scale(predict);
-
- return Prediction.PredictLabelsProbability(options.Model, scaled);
- }
+ private List features;
+ private List> dictionary;
+ private List categories;
+ private RangeTransform transform;
+ private SVM.BotSharp.MachineLearning.Model model;
public void Train(List sentences, ClassifyOptions options)
{
- var tfidf = new TfIdfFeatureExtractor();
- tfidf.Dimension = options.Dimension;
- tfidf.Sentences = sentences;
- tfidf.CalBasedOnCategory();
-
- var encoder = new OneHotEncoder();
- encoder.Sentences = sentences;
- encoder.Words = tfidf.Keywords();
- words = encoder.EncodeAll();
-
- var featureSets = new List();
- sentences.ForEach(sent =>
- {
- var fl = new FeaturesWithLabel();
- fl.Label = sent.Label;
- fl.Features = sent.Words.Select(x => new Feature(words.IndexOf(x.Lemma).ToString(), words.Contains(x.Lemma) ? "1" : "0")).ToList();
- featureSets.Add(fl);
- });
-
- SVMClassifierTrain(featureSets, options);
+ SVMClassifierTrain(sentences, options);
}
- public List> Classify(Sentence sentence, ClassifyOptions options)
+ public void SVMClassifierTrain(List sentences, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
{
- throw new NotImplementedException();
- }
-
- public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null)
- {
// copy test multiclass Model
Problem train = new Problem();
- train.X = GetData(featureSets).ToArray();
- train.Y = GetLabels(featureSets).ToArray();
+ train.X = GetData(sentences).ToArray();
+ train.Y = GetLabels(sentences).ToArray();
train.Count = train.X.Count();
- train.MaxIndex = 300;//int.MaxValue;
+ train.MaxIndex = train.X[0].Count();//int.MaxValue;
Parameter param = new Parameter();
- RangeTransform transform = RangeTransform.Compute(train);
+ transform = RangeTransform.Compute(train);
Problem scaled = transform.Scale(train);
param.Gamma = 1.0 / 3;
param.SvmType = svm;
param.KernelType = kernel;
param.Probability = probability;
- int numberOfClasses = train.Y.Distinct().Count();
+ int numberOfClasses = train.Y.OrderBy(x => x).Distinct().Count();
if (numberOfClasses == 1)
{
throw new ArgumentException("Number of classes can't be one!");
@@ -108,85 +73,144 @@ namespace BotSharp.NLP.Classify
for (int i = 0; i < numberOfClasses; i++)
param.Weights[i] = 1;
}
- var model = Training.Train(scaled, param);
- RangeTransform.Write(options.TransformFilePath, transform);
- SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model);
+
+ model = Training.Train(scaled, param);
+
Console.Write("Training finished!");
}
- public List GetLabels(List featureSets)
+ public List> Classify(Sentence sentence, ClassifyOptions options)
{
- var categories = featureSets.Select(x => x.Label).Distinct().OrderBy(x => x).ToList();
- List labels = new List();
- foreach (var labelFeatureSet in featureSets)
+ var categoryList = new List>();
+
+ var result = Predict(sentence, options).FirstOrDefault();
+
+ for(int i = 0; i < result.Length; i++)
{
- labels.Add(double.Parse(categories.IndexOf(labelFeatureSet.Label).ToString()));
+ categoryList.Add(new Tuple(categories[i], result[i]));
+ }
+
+ return categoryList;
+ }
+
+ public double[][] Predict(Sentence sentence, ClassifyOptions options)
+ {
+ Problem predict = new Problem();
+ predict.X = GetData(new List { sentence }).ToArray();
+ predict.Y = new double[1];
+ predict.Count = predict.X.Count();
+ predict.MaxIndex = features.Count;
+
+ transform = options.Transform;
+ Problem scaled = transform.Scale(predict);
+
+ return Prediction.PredictLabelsProbability(model, scaled);
+ }
+
+ public List GetLabels(List sentences)
+ {
+ categories = sentences.Select(x => x.Label).Distinct().OrderBy(x => x).ToList();
+ List labels = new List();
+
+ foreach (var sentence in sentences)
+ {
+ var labelId = categories.IndexOf(sentence.Label).ToString();
+ labels.Add(double.Parse(labelId));
}
return labels;
}
- public List GetData(List featureSets)
+ public List GetData(List sentences)
{
+ var extractor = new CountFeatureExtractor();
+ extractor.Sentences = sentences;
+ if(features != null)
+ {
+ extractor.Features = features;
+ }
+
+ if(dictionary != null)
+ {
+ extractor.Dictionary = dictionary;
+ }
+
+ extractor.Vectorize();
+
+ if(features == null)
+ {
+ features = extractor.Features;
+ }
+
+ if(dictionary == null)
+ {
+ dictionary = extractor.Dictionary;
+ }
+
List datas = new List();
- foreach (var labelFeatureSet in featureSets)
+ foreach (var sentence in sentences)
{
List curNodes = new List();
- labelFeatureSet.Features.ForEach(features => {
- int name = Int32.Parse(features.Name);
- double value = double.Parse(features.Value);
- curNodes.Add(new Node(name, value));
- });
+
+ for(int i = 0; i < extractor.Features.Count; i++)
+ {
+ int name = i;
+ var xx = sentence.Words.Find(x => x.Lemma == extractor.Features[i]);
+
+ if (xx == null)
+ {
+ curNodes.Add(new Node(name, 0));
+ }
+ else
+ {
+ curNodes.Add(new Node(name, xx.Vector));
+ }
+ }
+
datas.Add(curNodes.ToArray());
}
return datas;
}
- public List FeatureSetsGenerator(List sentenceVectors, List labels)
- {
- var res = new List();
- int j;
- for (int i = 0; i < labels.Count; i++)
- {
- string curLabel = labels[i];
- Vec curVec = sentenceVectors[i];
- var labeledFeatureSet = new FeaturesWithLabel();
- j = 1;
- foreach (double node in curVec.VecNodes)
- {
- Feature feature = new Feature((j++).ToString(), node.ToString());
- labeledFeatureSet.Features.Add(feature);
- }
- labeledFeatureSet.Label = curLabel;
- res.Add(labeledFeatureSet);
- }
-
- return res;
- }
-
- public FeaturesWithLabel FeatureSetsGenerator(Vec sentenceVectors, String label)
- {
- var labeledFeatureSet = new FeaturesWithLabel();
- int j = 1;
- foreach (double node in sentenceVectors.VecNodes)
- {
- Feature feature = new Feature((j++).ToString(), node.ToString());
- labeledFeatureSet.Features.Add(feature);
- }
- labeledFeatureSet.Label = label;
-
- return labeledFeatureSet;
- }
-
public string SaveModel(ClassifyOptions options)
{
- throw new NotImplementedException();
+ options.TransformFilePath = Path.Combine(options.ModelDir, "transform");
+ options.FeaturesFileName = Path.Combine(options.ModelDir, "features");
+ options.DictionaryFileName = Path.Combine(options.ModelDir, "dictionary");
+ options.CategoriesFileName = Path.Combine(options.ModelDir, "categories");
+
+ File.WriteAllText(options.FeaturesFileName, JsonConvert.SerializeObject(features));
+
+ File.WriteAllText(options.DictionaryFileName, JsonConvert.SerializeObject(dictionary));
+
+ File.WriteAllText(options.CategoriesFileName, JsonConvert.SerializeObject(categories));
+
+ RangeTransform.Write(options.TransformFilePath, transform);
+ SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model);
+
+ return options.ModelFilePath;
}
object IClassifier.LoadModel(ClassifyOptions options)
{
- throw new NotImplementedException();
+ options.FeaturesFileName = Path.Combine(options.ModelDir, "features");
+ options.DictionaryFileName = Path.Combine(options.ModelDir, "dictionary");
+ options.ModelFilePath = Path.Combine(options.ModelDir, options.ModelName);
+ options.TransformFilePath = Path.Combine(options.ModelDir, "transform");
+ options.CategoriesFileName = Path.Combine(options.ModelDir, "categories");
+
+ features = JsonConvert.DeserializeObject>(File.ReadAllText(options.FeaturesFileName));
+
+ dictionary = JsonConvert.DeserializeObject>>(File.ReadAllText(options.DictionaryFileName));
+
+ categories = JsonConvert.DeserializeObject>(File.ReadAllText(options.CategoriesFileName));
+
+ model = SVM.BotSharp.MachineLearning.Model.Read(options.ModelFilePath);
+
+ options.Transform = RangeTransform.Read(options.TransformFilePath);
+
+ return model;
}
}
}
diff --git a/BotSharp.NLP/Featuring/CountFeatureExtractor.cs b/BotSharp.NLP/Featuring/CountFeatureExtractor.cs
new file mode 100644
index 00000000..37277ce7
--- /dev/null
+++ b/BotSharp.NLP/Featuring/CountFeatureExtractor.cs
@@ -0,0 +1,91 @@
+/*
+ * BotSharp.NLP Library
+ * Copyright (C) 2018 Haiping Chen
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+using BotSharp.Algorithm.Matrix;
+using BotSharp.NLP.Tokenize;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BotSharp.NLP.Featuring
+{
+ ///
+ /// Convert a collection of text documents to a matrix of token counts
+ ///
+ public class CountFeatureExtractor : IFeatureExtractor
+ {
+ public int Dimension { get; set; }
+ public List Sentences { get; set; }
+
+ public List> Dictionary { get; set; }
+ public List Features { get; set; }
+ public Shape Shape { get; set; }
+
+ public void Vectorize()
+ {
+ CalculateDictionary();
+
+ int[][] vec = new int[Sentences.Count][];
+
+ Sentences.ForEach(s =>
+ {
+ s.Vector = new double[Features.Count];
+ for (int i = 0; i < Features.Count; i++)
+ {
+ s.Vector[i] = s.Words.Count(w => w.Lemma == Features[i]);
+ }
+
+ for (int i = 0; i < s.Words.Count; i++)
+ {
+ var dic = Dictionary.Find(x => x.Item1 == s.Words[i].Lemma);
+ if(dic != null)
+ {
+ s.Words[i].Vector = s.Words.Count(w => w.Lemma == dic.Item1);
+ }
+ }
+ });
+ }
+
+ private void CalculateDictionary()
+ {
+ if (Dictionary == null)
+ {
+ List allWords = new List();
+
+ Sentences.ForEach(s =>
+ {
+ allWords.AddRange(s.Words);
+ });
+
+ Features = allWords.Where(w => w.IsAlpha).Select(x => x.Lemma).Distinct().OrderBy(x => x).ToList();
+
+ Dictionary = new List>();
+
+ allWords.Select(x => x.Lemma)
+ .Distinct()
+ .OrderBy(x => x)
+ .ToList()
+ .ForEach(word =>
+ {
+ Dictionary.Add(new Tuple(word, allWords.Count(x => x.Lemma == word)));
+ });
+ }
+ }
+ }
+}
diff --git a/BotSharp.NLP/Featuring/IFeatureExtractor.cs b/BotSharp.NLP/Featuring/IFeatureExtractor.cs
index 497e4a13..e0277286 100644
--- a/BotSharp.NLP/Featuring/IFeatureExtractor.cs
+++ b/BotSharp.NLP/Featuring/IFeatureExtractor.cs
@@ -1,4 +1,5 @@
-using System;
+using BotSharp.Algorithm.Matrix;
+using System;
using System.Collections.Generic;
using System.Text;
@@ -10,5 +11,30 @@ namespace BotSharp.NLP.Featuring
/// Feature dimension size
///
int Dimension { get; set; }
+
+ ///
+ /// The whole corpus
+ ///
+ List Sentences { get; set; }
+
+ ///
+ /// Feature names
+ ///
+ List Features { get; set; }
+
+ ///
+ /// All words and frequency
+ ///
+ List> Dictionary { get; set; }
+
+ ///
+ /// Vectorize sentence
+ ///
+ void Vectorize();
+
+ ///
+ /// Array shape
+ ///
+ Shape Shape { get; set; }
}
}
diff --git a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs
index e7132aa7..b88a45ac 100644
--- a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs
+++ b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs
@@ -16,6 +16,7 @@
* along with this program. If not, see .
*/
+using BotSharp.Algorithm.Matrix;
using BotSharp.NLP.Tokenize;
using System;
using System.Collections.Generic;
@@ -36,6 +37,10 @@ namespace BotSharp.NLP.Featuring
private List Categories { get; set; }
public int Dimension { get; set; }
+ public List> Dictionary { get; set; }
+ public List Features { get; set; }
+ public Shape Shape { get; set; }
+
public void Extract(Sentence sentence)
{
@@ -186,5 +191,10 @@ namespace BotSharp.NLP.Featuring
}
return result;
}
+
+ public void Vectorize()
+ {
+ throw new NotImplementedException();
+ }
}
}
diff --git a/BotSharp.NLP/Models/SVM/RangeTransform.cs b/BotSharp.NLP/Models/SVM/RangeTransform.cs
index 5e3a2e80..ec5a5aad 100644
--- a/BotSharp.NLP/Models/SVM/RangeTransform.cs
+++ b/BotSharp.NLP/Models/SVM/RangeTransform.cs
@@ -67,10 +67,18 @@ namespace SVM.BotSharp.MachineLearning
{
for (int j = 0; j < prob.X[i].Length; j++)
{
- int index = prob.X[i][j].Index - 1;
- double value = prob.X[i][j].Value;
- minVals[index] = Math.Min(minVals[index], value);
- maxVals[index] = Math.Max(maxVals[index], value);
+ try
+ {
+ int index = prob.X[i][j].Index;
+ double value = prob.X[i][j].Value;
+ minVals[index] = Math.Min(minVals[index], value);
+ maxVals[index] = Math.Max(maxVals[index], value);
+ }
+ catch (Exception ex)
+ {
+
+ }
+
}
}
for (int i = 0; i < prob.MaxIndex; i++)
@@ -145,7 +153,6 @@ namespace SVM.BotSharp.MachineLearning
/// The scaled value
public double Transform(double input, int index)
{
- index--;
double tmp = input - _inputStart[index];
if (_inputScale[index] == 0)
return 0;
diff --git a/BotSharp.NLP/Models/SVM/Solver.cs b/BotSharp.NLP/Models/SVM/Solver.cs
index 5faada66..7b7588bb 100644
--- a/BotSharp.NLP/Models/SVM/Solver.cs
+++ b/BotSharp.NLP/Models/SVM/Solver.cs
@@ -1514,6 +1514,8 @@ namespace SVM.BotSharp.MachineLearning
}
for (i = 0; i < nr_fold; i++)
{
+ Console.WriteLine($"Cross-validation decision values for probability estimates {i}");
+
int begin = i * prob.Count / nr_fold;
int end = (i + 1) * prob.Count / nr_fold;
int j, k;
@@ -1724,6 +1726,7 @@ namespace SVM.BotSharp.MachineLearning
model.SupportVectorIndices = new int[nSV];
int j = 0;
for (i = 0; i < prob.Count; i++)
+ {
if (Math.Abs(f.alpha[i]) > 0)
{
model.SupportVectors[j] = prob.X[i];
@@ -1731,6 +1734,8 @@ namespace SVM.BotSharp.MachineLearning
model.SupportVectorIndices[j] = i + 1;
++j;
}
+ }
+
}
else
{