diff --git a/BotSharp.Algorithm/Matrix/Shape.cs b/BotSharp.Algorithm/Matrix/Shape.cs new file mode 100644 index 00000000..abe56423 --- /dev/null +++ b/BotSharp.Algorithm/Matrix/Shape.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.Algorithm.Matrix +{ + /// + /// Shape of the data arrays + /// + public class Shape + { + /// + /// Total number of samples + /// + public int Samples { get; set; } + + /// + /// Total number of features + /// + public int Features { get; set; } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs index 1672aad3..a2cb6604 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs @@ -67,11 +67,11 @@ namespace BotSharp.Core.Engines.BotSharp { meta.Model = "intent.model"; - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - var options = new ClassifyOptions { - ModelFilePath = modelFileName + ModelFilePath = Path.Combine(Settings.ModelDir, meta.Model), + ModelDir = Settings.ModelDir, + ModelName = meta.Model }; _classifier = new ClassifierFactory(options, SupportedLanguage.English); diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs deleted file mode 100644 index c9b1d0a6..00000000 --- a/BotSharp.Core/Engines/BotSharp/BotSharpSVMClassifier.cs +++ /dev/null @@ -1,148 +0,0 @@ -using BotSharp.Algorithm.Bayes; -using BotSharp.Core.Abstractions; -using BotSharp.Core.Agents; -using BotSharp.NLP.Classify; -using DotNetToolkit; -using Microsoft.Extensions.Configuration; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using RestSharp; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Txt2Vec; - -namespace BotSharp.Core.Engines.BotSharp -{ - public class BotSharpSVMClassifier : INlpTrain, INlpPredict - { - public IConfiguration Configuration { get; set; } - public PipeSettings Settings { get; set; } - - public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) - { - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - string predictFileName = Path.Combine(Settings.TempDir, "svm-predict-tempfile.txt"); - File.WriteAllText(predictFileName, doc.Sentences[0].Text); - - var svmClassifier = new NLP.Classify.SVMClassifier(); - Args args = new Args(); - args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin"); - var featureSet = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).SingleSentence2Vec(doc.Sentences[0].Text), ""); - /* - // - var client = new RestClient("http://10.2.21.200:5005"); - var request = new RestRequest("doc2vec", Method.GET); - request.AddParameter("text", doc.Sentences[0].Text); - var response = client.Execute(request); - PredResult pred = JsonConvert.DeserializeObject(response.Content); - - Vec vec = new Vec(); - vec.VecNodes = pred.Doc2Vec; - - LabeledFeatureSet featureSet = svmClassifier.FeatureSetsGenerator(vec, ""); - // - */ - ClassifyOptions classifyOptions = new ClassifyOptions(); - classifyOptions.Model = SVM.BotSharp.MachineLearning.Model.Read(Path.Combine(Settings.ModelDir, "svm_classifier_model")); - classifyOptions.Transform = SVM.BotSharp.MachineLearning.RangeTransform.Read(Path.Combine(Settings.ModelDir, "transform_obj_data")); - double[][] d = svmClassifier.Predict(featureSet, classifyOptions); - - string intent = null; - decimal confidence = 0; - double max = Double.MinValue; - for (int i = 0; i < d[0].Count(); i++) - { - if (d[0][i] > max) - { - max = d[0][i]; - intent = agent.Intents[i].Name; - confidence = (decimal)d[0][i]; - } - } - - File.Delete(predictFileName); - - doc.Sentences[0].Intent = new TextClassificationResult - { - Classifier = "SVMClassifier", - Label = intent, - Confidence = confidence - }; - - return true; - } - - public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) - { - meta.Model = "classification-svm.model"; - string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-svm.parsed.txt"); - string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); - - List labels = new List(); - List sentences = new List(); - - agent.Corpus.UserSays.ForEach(x =>{ - agent.Intents.ForEach(intent => { - if (intent.Name == x.Intent) - { - labels.Add(agent.Intents.IndexOf(intent).ToString()); - } - }); - sentences.Add(x.Text); - }); - - NLP.Classify.SVMClassifier svmClassifier = new NLP.Classify.SVMClassifier(); - Args args = new Args(); - args.ModelFile = Path.Combine(Configuration.GetValue("BotSharpSVMClassifier:wordvec"), "wordvec_enu.bin"); - var featureSetList = svmClassifier.FeatureSetsGenerator(new VectorGenerator(args).Sentence2Vec(sentences), labels); - - /* - // try using spacy doc2vec - var client = new RestClient("http://10.2.21.200:5005"); - var request = new RestRequest("batchdoc2vec", Method.POST); - request.RequestFormat = DataFormat.Json; - - request.AddParameter("application/json", JsonConvert.SerializeObject(new {Sentences = sentences}), ParameterType.RequestBody); - - var response = client.Execute(request); - Result res = JsonConvert.DeserializeObject(response.Content); - - List vecs = new List(); - foreach (List cur in res.Doc2vecList) - { - Vec vec = new Vec(); - vec.VecNodes = cur; - vecs.Add(vec); - } - List featureSetList = svmClassifier.FeatureSetsGenerator(vecs, labels); - // - */ - - - - ClassifyOptions classifyOptions = new ClassifyOptions(); - classifyOptions.ModelFilePath = Path.Combine(Settings.ModelDir, "svm_classifier_model"); - classifyOptions.TransformFilePath = Path.Combine(Settings.ModelDir, "transform_obj_data"); - // svmClassifier.Train(featureSetList, classifyOptions); - - meta.Meta = new JObject(); - meta.Meta["compiled at"] = "Aug 31, 2018"; - return true; - } - } - public class Result - { - public List> Doc2vecList { get; set; } - } - - public class PredResult - { - public List Doc2Vec{ get; set; } - } -} diff --git a/BotSharp.NLP.UnitTest/Featuring/CountFeatureExtractorTest.cs b/BotSharp.NLP.UnitTest/Featuring/CountFeatureExtractorTest.cs new file mode 100644 index 00000000..740c4cd3 --- /dev/null +++ b/BotSharp.NLP.UnitTest/Featuring/CountFeatureExtractorTest.cs @@ -0,0 +1,63 @@ +using BotSharp.NLP.Featuring; +using BotSharp.NLP.Tokenize; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.UnitTest.Featuring +{ + [TestClass] + public class CountFeatureExtractorTest : TestEssential + { + [TestMethod] + public void TestVectorizer() + { + var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English); + tokenizer.GetTokenizer(); + + var extractor = new CountFeatureExtractor(); + extractor.Sentences = tokenizer.Tokenize(Corpus()); + extractor.Vectorize(); + + var vectors = Vectors(); + + for (int i = 0; i < extractor.Sentences.Count; i++) + { + var sentence = extractor.Sentences[i]; + + for(int j = 0; j < extractor.Features.Count; j++) + { + var word = sentence.Words.Find(w => w.Lemma == extractor.Features[j]); + + if(word != null) + { + Assert.IsTrue(word.Vector == vectors[i][j]); + } + } + } + } + + public List Corpus() + { + return new List + { + "This is the first document.", + "This document is the second document.", + "And this is the third one.", + "Is this the first document?" + }; + } + + public int[][] Vectors() + { + return new int[4][] + { + new int []{ 0, 1, 1, 1, 0, 0, 1, 0, 1 }, + new int []{ 0, 2, 0, 1, 0, 1, 1, 0, 1 }, + new int []{ 1, 0, 0, 1, 1, 0, 1, 1, 1 }, + new int []{ 0, 1, 1, 1, 0, 0, 1, 0, 1 } + }; + } + } +} diff --git a/BotSharp.NLP.UnitTest/TestEssential.cs b/BotSharp.NLP.UnitTest/TestEssential.cs index 4ae615a5..033a8735 100644 --- a/BotSharp.NLP.UnitTest/TestEssential.cs +++ b/BotSharp.NLP.UnitTest/TestEssential.cs @@ -12,7 +12,7 @@ namespace BotSharp.NLP.UnitTest public TestEssential() { var rootDir = Path.GetFullPath($"{Directory.GetCurrentDirectory()}{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}"); - var settingsDir = Path.Combine(rootDir, "Settings"); + var settingsDir = Path.Combine(rootDir, "BotSharp.WebHost", "Settings"); ConfigurationBuilder configurationBuilder = new ConfigurationBuilder(); var settings = Directory.GetFiles(settingsDir, "*.json"); diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index f1a9d479..14670f58 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -52,7 +52,9 @@ namespace BotSharp.NLP.Classify { var options = new ClassifyOptions { - ModelFilePath = _options.ModelFilePath + ModelFilePath = _options.ModelFilePath, + ModelDir = _options.ModelDir, + ModelName = _options.ModelName }; _classifier.LoadModel(options); diff --git a/BotSharp.NLP/Classify/ClassifyOptions.cs b/BotSharp.NLP/Classify/ClassifyOptions.cs index 0c213fad..0b7ee12b 100644 --- a/BotSharp.NLP/Classify/ClassifyOptions.cs +++ b/BotSharp.NLP/Classify/ClassifyOptions.cs @@ -9,7 +9,13 @@ namespace BotSharp.NLP.Classify { public string TrainingCorpusDir { get; set; } public string ModelFilePath { get; set; } - public Model Model { get; set; } + public string ModelDir { get; set; } + public string ModelName { get; set; } + + public string FeaturesFileName { get; set; } + public string DictionaryFileName { get; set; } + public string CategoriesFileName { get; set; } + public string PrediceOutputFile { get; set; } public string TransformFilePath { get; set; } public RangeTransform Transform { get; set; } diff --git a/BotSharp.NLP/Classify/SVMClassifier.cs b/BotSharp.NLP/Classify/SVMClassifier.cs index 7bbf8429..fe95a4d5 100644 --- a/BotSharp.NLP/Classify/SVMClassifier.cs +++ b/BotSharp.NLP/Classify/SVMClassifier.cs @@ -24,6 +24,7 @@ using System.Text; using BotSharp.Algorithm.Features; using BotSharp.NLP.Featuring; using BotSharp.NLP.Txt2Vec; +using Newtonsoft.Json; using SVM.BotSharp.MachineLearning; using Txt2Vec; @@ -34,71 +35,35 @@ namespace BotSharp.NLP.Classify /// public class SVMClassifier : IClassifier { - private List words; - - public double[][] Predict(FeaturesWithLabel featureSet, ClassifyOptions options) - { - Problem predict = new Problem(); - List featureSets = new List(); - featureSets.Add(featureSet); - predict.X = GetData(featureSets).ToArray(); - predict.Y = new double[1]; - predict.Count = predict.X.Count(); - predict.MaxIndex = 300; - - RangeTransform transform = options.Transform; - Problem scaled = transform.Scale(predict); - - return Prediction.PredictLabelsProbability(options.Model, scaled); - } + private List features; + private List> dictionary; + private List categories; + private RangeTransform transform; + private SVM.BotSharp.MachineLearning.Model model; public void Train(List sentences, ClassifyOptions options) { - var tfidf = new TfIdfFeatureExtractor(); - tfidf.Dimension = options.Dimension; - tfidf.Sentences = sentences; - tfidf.CalBasedOnCategory(); - - var encoder = new OneHotEncoder(); - encoder.Sentences = sentences; - encoder.Words = tfidf.Keywords(); - words = encoder.EncodeAll(); - - var featureSets = new List(); - sentences.ForEach(sent => - { - var fl = new FeaturesWithLabel(); - fl.Label = sent.Label; - fl.Features = sent.Words.Select(x => new Feature(words.IndexOf(x.Lemma).ToString(), words.Contains(x.Lemma) ? "1" : "0")).ToList(); - featureSets.Add(fl); - }); - - SVMClassifierTrain(featureSets, options); + SVMClassifierTrain(sentences, options); } - public List> Classify(Sentence sentence, ClassifyOptions options) + public void SVMClassifierTrain(List sentences, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) { - throw new NotImplementedException(); - } - - public void SVMClassifierTrain(List featureSets, ClassifyOptions options, SvmType svm = SvmType.C_SVC, KernelType kernel = KernelType.RBF, bool probability = true, string outputFile = null) - { // copy test multiclass Model Problem train = new Problem(); - train.X = GetData(featureSets).ToArray(); - train.Y = GetLabels(featureSets).ToArray(); + train.X = GetData(sentences).ToArray(); + train.Y = GetLabels(sentences).ToArray(); train.Count = train.X.Count(); - train.MaxIndex = 300;//int.MaxValue; + train.MaxIndex = train.X[0].Count();//int.MaxValue; Parameter param = new Parameter(); - RangeTransform transform = RangeTransform.Compute(train); + transform = RangeTransform.Compute(train); Problem scaled = transform.Scale(train); param.Gamma = 1.0 / 3; param.SvmType = svm; param.KernelType = kernel; param.Probability = probability; - int numberOfClasses = train.Y.Distinct().Count(); + int numberOfClasses = train.Y.OrderBy(x => x).Distinct().Count(); if (numberOfClasses == 1) { throw new ArgumentException("Number of classes can't be one!"); @@ -108,85 +73,144 @@ namespace BotSharp.NLP.Classify for (int i = 0; i < numberOfClasses; i++) param.Weights[i] = 1; } - var model = Training.Train(scaled, param); - RangeTransform.Write(options.TransformFilePath, transform); - SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model); + + model = Training.Train(scaled, param); + Console.Write("Training finished!"); } - public List GetLabels(List featureSets) + public List> Classify(Sentence sentence, ClassifyOptions options) { - var categories = featureSets.Select(x => x.Label).Distinct().OrderBy(x => x).ToList(); - List labels = new List(); - foreach (var labelFeatureSet in featureSets) + var categoryList = new List>(); + + var result = Predict(sentence, options).FirstOrDefault(); + + for(int i = 0; i < result.Length; i++) { - labels.Add(double.Parse(categories.IndexOf(labelFeatureSet.Label).ToString())); + categoryList.Add(new Tuple(categories[i], result[i])); + } + + return categoryList; + } + + public double[][] Predict(Sentence sentence, ClassifyOptions options) + { + Problem predict = new Problem(); + predict.X = GetData(new List { sentence }).ToArray(); + predict.Y = new double[1]; + predict.Count = predict.X.Count(); + predict.MaxIndex = features.Count; + + transform = options.Transform; + Problem scaled = transform.Scale(predict); + + return Prediction.PredictLabelsProbability(model, scaled); + } + + public List GetLabels(List sentences) + { + categories = sentences.Select(x => x.Label).Distinct().OrderBy(x => x).ToList(); + List labels = new List(); + + foreach (var sentence in sentences) + { + var labelId = categories.IndexOf(sentence.Label).ToString(); + labels.Add(double.Parse(labelId)); } return labels; } - public List GetData(List featureSets) + public List GetData(List sentences) { + var extractor = new CountFeatureExtractor(); + extractor.Sentences = sentences; + if(features != null) + { + extractor.Features = features; + } + + if(dictionary != null) + { + extractor.Dictionary = dictionary; + } + + extractor.Vectorize(); + + if(features == null) + { + features = extractor.Features; + } + + if(dictionary == null) + { + dictionary = extractor.Dictionary; + } + List datas = new List(); - foreach (var labelFeatureSet in featureSets) + foreach (var sentence in sentences) { List curNodes = new List(); - labelFeatureSet.Features.ForEach(features => { - int name = Int32.Parse(features.Name); - double value = double.Parse(features.Value); - curNodes.Add(new Node(name, value)); - }); + + for(int i = 0; i < extractor.Features.Count; i++) + { + int name = i; + var xx = sentence.Words.Find(x => x.Lemma == extractor.Features[i]); + + if (xx == null) + { + curNodes.Add(new Node(name, 0)); + } + else + { + curNodes.Add(new Node(name, xx.Vector)); + } + } + datas.Add(curNodes.ToArray()); } return datas; } - public List FeatureSetsGenerator(List sentenceVectors, List labels) - { - var res = new List(); - int j; - for (int i = 0; i < labels.Count; i++) - { - string curLabel = labels[i]; - Vec curVec = sentenceVectors[i]; - var labeledFeatureSet = new FeaturesWithLabel(); - j = 1; - foreach (double node in curVec.VecNodes) - { - Feature feature = new Feature((j++).ToString(), node.ToString()); - labeledFeatureSet.Features.Add(feature); - } - labeledFeatureSet.Label = curLabel; - res.Add(labeledFeatureSet); - } - - return res; - } - - public FeaturesWithLabel FeatureSetsGenerator(Vec sentenceVectors, String label) - { - var labeledFeatureSet = new FeaturesWithLabel(); - int j = 1; - foreach (double node in sentenceVectors.VecNodes) - { - Feature feature = new Feature((j++).ToString(), node.ToString()); - labeledFeatureSet.Features.Add(feature); - } - labeledFeatureSet.Label = label; - - return labeledFeatureSet; - } - public string SaveModel(ClassifyOptions options) { - throw new NotImplementedException(); + options.TransformFilePath = Path.Combine(options.ModelDir, "transform"); + options.FeaturesFileName = Path.Combine(options.ModelDir, "features"); + options.DictionaryFileName = Path.Combine(options.ModelDir, "dictionary"); + options.CategoriesFileName = Path.Combine(options.ModelDir, "categories"); + + File.WriteAllText(options.FeaturesFileName, JsonConvert.SerializeObject(features)); + + File.WriteAllText(options.DictionaryFileName, JsonConvert.SerializeObject(dictionary)); + + File.WriteAllText(options.CategoriesFileName, JsonConvert.SerializeObject(categories)); + + RangeTransform.Write(options.TransformFilePath, transform); + SVM.BotSharp.MachineLearning.Model.Write(options.ModelFilePath, model); + + return options.ModelFilePath; } object IClassifier.LoadModel(ClassifyOptions options) { - throw new NotImplementedException(); + options.FeaturesFileName = Path.Combine(options.ModelDir, "features"); + options.DictionaryFileName = Path.Combine(options.ModelDir, "dictionary"); + options.ModelFilePath = Path.Combine(options.ModelDir, options.ModelName); + options.TransformFilePath = Path.Combine(options.ModelDir, "transform"); + options.CategoriesFileName = Path.Combine(options.ModelDir, "categories"); + + features = JsonConvert.DeserializeObject>(File.ReadAllText(options.FeaturesFileName)); + + dictionary = JsonConvert.DeserializeObject>>(File.ReadAllText(options.DictionaryFileName)); + + categories = JsonConvert.DeserializeObject>(File.ReadAllText(options.CategoriesFileName)); + + model = SVM.BotSharp.MachineLearning.Model.Read(options.ModelFilePath); + + options.Transform = RangeTransform.Read(options.TransformFilePath); + + return model; } } } diff --git a/BotSharp.NLP/Featuring/CountFeatureExtractor.cs b/BotSharp.NLP/Featuring/CountFeatureExtractor.cs new file mode 100644 index 00000000..37277ce7 --- /dev/null +++ b/BotSharp.NLP/Featuring/CountFeatureExtractor.cs @@ -0,0 +1,91 @@ +/* + * BotSharp.NLP Library + * Copyright (C) 2018 Haiping Chen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +using BotSharp.Algorithm.Matrix; +using BotSharp.NLP.Tokenize; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BotSharp.NLP.Featuring +{ + /// + /// Convert a collection of text documents to a matrix of token counts + /// + public class CountFeatureExtractor : IFeatureExtractor + { + public int Dimension { get; set; } + public List Sentences { get; set; } + + public List> Dictionary { get; set; } + public List Features { get; set; } + public Shape Shape { get; set; } + + public void Vectorize() + { + CalculateDictionary(); + + int[][] vec = new int[Sentences.Count][]; + + Sentences.ForEach(s => + { + s.Vector = new double[Features.Count]; + for (int i = 0; i < Features.Count; i++) + { + s.Vector[i] = s.Words.Count(w => w.Lemma == Features[i]); + } + + for (int i = 0; i < s.Words.Count; i++) + { + var dic = Dictionary.Find(x => x.Item1 == s.Words[i].Lemma); + if(dic != null) + { + s.Words[i].Vector = s.Words.Count(w => w.Lemma == dic.Item1); + } + } + }); + } + + private void CalculateDictionary() + { + if (Dictionary == null) + { + List allWords = new List(); + + Sentences.ForEach(s => + { + allWords.AddRange(s.Words); + }); + + Features = allWords.Where(w => w.IsAlpha).Select(x => x.Lemma).Distinct().OrderBy(x => x).ToList(); + + Dictionary = new List>(); + + allWords.Select(x => x.Lemma) + .Distinct() + .OrderBy(x => x) + .ToList() + .ForEach(word => + { + Dictionary.Add(new Tuple(word, allWords.Count(x => x.Lemma == word))); + }); + } + } + } +} diff --git a/BotSharp.NLP/Featuring/IFeatureExtractor.cs b/BotSharp.NLP/Featuring/IFeatureExtractor.cs index 497e4a13..e0277286 100644 --- a/BotSharp.NLP/Featuring/IFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/IFeatureExtractor.cs @@ -1,4 +1,5 @@ -using System; +using BotSharp.Algorithm.Matrix; +using System; using System.Collections.Generic; using System.Text; @@ -10,5 +11,30 @@ namespace BotSharp.NLP.Featuring /// Feature dimension size /// int Dimension { get; set; } + + /// + /// The whole corpus + /// + List Sentences { get; set; } + + /// + /// Feature names + /// + List Features { get; set; } + + /// + /// All words and frequency + /// + List> Dictionary { get; set; } + + /// + /// Vectorize sentence + /// + void Vectorize(); + + /// + /// Array shape + /// + Shape Shape { get; set; } } } diff --git a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs index e7132aa7..b88a45ac 100644 --- a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs @@ -16,6 +16,7 @@ * along with this program. If not, see . */ +using BotSharp.Algorithm.Matrix; using BotSharp.NLP.Tokenize; using System; using System.Collections.Generic; @@ -36,6 +37,10 @@ namespace BotSharp.NLP.Featuring private List Categories { get; set; } public int Dimension { get; set; } + public List> Dictionary { get; set; } + public List Features { get; set; } + public Shape Shape { get; set; } + public void Extract(Sentence sentence) { @@ -186,5 +191,10 @@ namespace BotSharp.NLP.Featuring } return result; } + + public void Vectorize() + { + throw new NotImplementedException(); + } } } diff --git a/BotSharp.NLP/Models/SVM/RangeTransform.cs b/BotSharp.NLP/Models/SVM/RangeTransform.cs index 5e3a2e80..ec5a5aad 100644 --- a/BotSharp.NLP/Models/SVM/RangeTransform.cs +++ b/BotSharp.NLP/Models/SVM/RangeTransform.cs @@ -67,10 +67,18 @@ namespace SVM.BotSharp.MachineLearning { for (int j = 0; j < prob.X[i].Length; j++) { - int index = prob.X[i][j].Index - 1; - double value = prob.X[i][j].Value; - minVals[index] = Math.Min(minVals[index], value); - maxVals[index] = Math.Max(maxVals[index], value); + try + { + int index = prob.X[i][j].Index; + double value = prob.X[i][j].Value; + minVals[index] = Math.Min(minVals[index], value); + maxVals[index] = Math.Max(maxVals[index], value); + } + catch (Exception ex) + { + + } + } } for (int i = 0; i < prob.MaxIndex; i++) @@ -145,7 +153,6 @@ namespace SVM.BotSharp.MachineLearning /// The scaled value public double Transform(double input, int index) { - index--; double tmp = input - _inputStart[index]; if (_inputScale[index] == 0) return 0; diff --git a/BotSharp.NLP/Models/SVM/Solver.cs b/BotSharp.NLP/Models/SVM/Solver.cs index 5faada66..7b7588bb 100644 --- a/BotSharp.NLP/Models/SVM/Solver.cs +++ b/BotSharp.NLP/Models/SVM/Solver.cs @@ -1514,6 +1514,8 @@ namespace SVM.BotSharp.MachineLearning } for (i = 0; i < nr_fold; i++) { + Console.WriteLine($"Cross-validation decision values for probability estimates {i}"); + int begin = i * prob.Count / nr_fold; int end = (i + 1) * prob.Count / nr_fold; int j, k; @@ -1724,6 +1726,7 @@ namespace SVM.BotSharp.MachineLearning model.SupportVectorIndices = new int[nSV]; int j = 0; for (i = 0; i < prob.Count; i++) + { if (Math.Abs(f.alpha[i]) > 0) { model.SupportVectors[j] = prob.X[i]; @@ -1731,6 +1734,8 @@ namespace SVM.BotSharp.MachineLearning model.SupportVectorIndices[j] = i + 1; ++j; } + } + } else {