Improve intent accuracy greater than 0.7

BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs

@@ -88,13 +88,16 @@ namespace BotSharp.Algorithm.Bayes
         {
             int featureCount = features.Length;
             
-            double postProb = priorProb;
+            double postProb = Math.Log(priorProb);
 
             // loop features
             for (int x = 0; x < featureCount; x++)
             {
                 string key = $"{Y} f{x} {features[x]}";
-                postProb += condProbDictionary[key];
+                if(features[x] == 1)
+                {
+                    postProb += condProbDictionary[key];
+                }
             }
 
             return Math.Pow(2, postProb);

BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj

@@ -19,6 +19,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\BotSharp.Algorithm\BotSharp.Algorithm.csproj" />
     <ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 

BotSharp.Core.UnitTest/Performance/Spotify.cs

@@ -8,6 +8,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
+using BotSharp.Algorithm.Extensions;
 
 namespace BotSharp.Core.UnitTest.Performance
 {
@@ -21,26 +22,24 @@ namespace BotSharp.Core.UnitTest.Performance
         public void IntentAccuracy()
         {
             int correct = 0;
+            List<Tuple<string, string>> errors = new List<Tuple<string, string>>();
+
             var agent = LoadAgent();
 
-            for(int i = 0; i < Samples.Count; i++)
+            for (int i = 0; i < Samples.Count; i++)
             {
-                try
+                var aIResponse = _platform.TextRequest(Samples[i].Item1);
+                if (aIResponse.Result.Metadata.IntentName == Samples[i].Item2)
                 {
-                    var aIResponse = _platform.TextRequest(Samples[i].Item1);
+                    correct++;
-                    if (aIResponse.Result.Metadata.IntentName == Samples[i].Item2)
-                    {
-                        correct++;
-                    }
                 }
-                catch (Exception)
+                else
                 {
-
+                    errors.Add(new Tuple<string, string>(Samples[i].Item2, Samples[i].Item1.Query[0]));
                 }
-
             }
 
-            double accuracy = correct / (Samples.Count + 0.0);
+           double accuracy = correct / (Samples.Count + 0.0);
         }
 
         private Agent LoadAgent()
@@ -55,6 +54,12 @@ namespace BotSharp.Core.UnitTest.Performance
 
             // Init samples
             Samples = new List<Tuple<AIRequest, string>>();
+            /*agent.Corpus.UserSays = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>
+            {
+                new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music.play", Text = "play the 50 Great Beatles Songs playlist in Prime Music"},
+                new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music.play", Text = "reproduce a the track Monster by Rihanna ft Eminem"},
+                new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music_player_control.add_favorite", Text = "add this song to my favourites"}
+            };*/
             agent.Corpus.UserSays.ForEach(intent =>
             {
                 Samples.Add(new Tuple<AIRequest, string>(new AIRequest
@@ -68,8 +73,9 @@ namespace BotSharp.Core.UnitTest.Performance
                 }, intent.Intent));
             });
 
-            var samples = String.Join("\r\n", Samples.Select(x => $"__label__{x.Item2} {x.Item1.Query[0]}").ToList());
+            //Samples.Shuffle();
 
+            var samples = String.Join("\r\n", Samples.Select(x => $"__label__{x.Item2} {x.Item1.Query[0]}").ToList());
 
             return agent;
         }

BotSharp.Core/BotSharp.Core.csproj

@@ -66,8 +66,11 @@ If you feel that this project is helpful to you, please Star on the project, we
 
   <ItemGroup>
     <Compile Remove="Accounts\**" />
+    <Compile Remove="Engines\CoreNlp\**" />
     <EmbeddedResource Remove="Accounts\**" />
+    <EmbeddedResource Remove="Engines\CoreNlp\**" />
     <None Remove="Accounts\**" />
+    <None Remove="Engines\CoreNlp\**" />
   </ItemGroup>
 
   <ItemGroup>
@@ -80,10 +83,6 @@ If you feel that this project is helpful to you, please Star on the project, we
     <PackageReference Include="RestSharp" Version="106.3.1" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Engines\CoreNlp\" />
-  </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\BotSharp.NLP\BotSharp.NLP.csproj" />
   </ItemGroup>

BotSharp.Core/Engines/BotEngineBase.cs

@@ -46,7 +46,7 @@ namespace BotSharp.Core.Engines
             {
                 doc.Sentences[0].Entities = new List<NlpEntity>();
             }
-            doc.Sentences[0].Entities.ForEach(x => parameters.Add(x.Entity, x.Value));
+            doc.Sentences[0].Entities.ForEach(x => parameters[x.Entity] = x.Value);
 
             return new AIResponse
             {

BotSharp.Core/Engines/Nltk/NltkTokenizer.cs

@@ -1,108 +0,0 @@
-using BotSharp.Core.Abstractions;
-using BotSharp.Core.Agents;
-using BotSharp.Core.Models;
-using BotSharp.NLP.Tokenize;
-using EntityFrameworkCore.BootKit;
-using Microsoft.Extensions.Configuration;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using RestSharp;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace BotSharp.Core.Engines.SpaCy
-{
-    public class NltkTokenizer : INlpTrain, INlpPredict
-    {
-        public IConfiguration Configuration { get; set; }
-        public PipeSettings Settings { get; set; }
-
-        public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
-        {
-            var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
-            var request = new RestRequest("nltktokenizesentences", Method.POST);
-            List<List<Token>> tokens = new List<List<Token>>();
-            Boolean res = true;
-            var dc = new DefaultDataContextLoader().GetDefaultDc();
-            var corpus = agent.Corpus;
-
-            doc.Sentences = new List<NlpDocSentence>();
-            List<string> sentencesList = new List<string>();
-            corpus.UserSays.ForEach ( usersay => sentencesList.Add(usersay.Text));
-
-            request.RequestFormat = DataFormat.Json;
-
-            request.AddParameter("application/json", JsonConvert.SerializeObject(new Documents(sentencesList)), ParameterType.RequestBody);
-
-            var response = client.Execute<Result>(request);
-
-            tokens = response.Data.TokensList;
-
-            for (int i = 0; i < sentencesList.Count; i++)
-            {
-                doc.Sentences.Add(new NlpDocSentence
-                {
-                    Tokens = tokens[i],
-                    Text = sentencesList[i]
-                });
-            }
-            res = res && response.IsSuccessful;
-            return res;
-            /*
-             corpus.UserSays.ForEach(usersay => {
-                Console.WriteLine(usersay.Text);
-                request.AddParameter("text", usersay.Text);
-                var response = client.Execute<Result>(request);
-                
-                tokens.Add(response.Data.Tokens);
-
-                doc.Sentences.Add(new NlpDocSentence
-                {
-                    Tokens = response.Data.Tokens,
-                    Text = usersay.Text
-                });
-
-                res = res && response.IsSuccessful;
-            });
-             */
-        }
-
-        public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
-        {
-            var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
-            var request = new RestRequest("nltktokenizesentences", Method.POST);
-            List<List<Token>> tokens = new List<List<Token>>();
-            Boolean res = true;
-            var corpus = agent.Corpus;
-
-            request.AddParameter("sentences", doc.Sentences[0].Text);
-            var response = client.Execute<Result>(request);
-            
-            //tokens.Add(response.Data.Tokens);
-
-            res = res && response.IsSuccessful;
-
-            doc.Sentences[0].Tokens = tokens[0];
-
-            return true;
-        }
-
-        private class Result
-        {
-            public List<List<Token>> TokensList { get; set; }
-        }
-
-        private class Documents
-        {
-            public List<string> Sentences { get; set; }
-
-            public Documents(List<string> sentences)
-            {
-                this.Sentences = sentences;
-            }
-        }
-    }
-}

BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs

@@ -88,12 +88,12 @@ namespace BotSharp.Core.Engines.Rasa
         public void LoadIntents(Agent agent)
         {
             string data = File.ReadAllText(Path.Combine(AgentDir, "corpus.json"));
-            var rasa = JsonConvert.DeserializeObject<RasaAgent>(data);
+            var rasa = JsonConvert.DeserializeObject<RasaAgentImportModel>(data);
 
-            agent.Intents = rasa.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList();
+            agent.Intents = rasa.Data.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList();
 
             agent.Intents.ForEach(intent => {
-                ImportIntentUserSays(intent, rasa.UserSays);
+                ImportIntentUserSays(intent, rasa.Data.UserSays);
             });
 
         }

BotSharp.Core/Engines/Rasa/RasaAgent.cs

@@ -21,4 +21,10 @@ namespace BotSharp.Core.Engines.Rasa
         [JsonProperty("regex_features")]
         public List<RasaTrainingRegex> Regex { get; set; }
     }
+
+    public class RasaAgentImportModel
+    {
+        [JsonProperty("rasa_nlu_data")]
+        public RasaAgent Data { get; set; }
+    }
 }

BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs

@@ -39,16 +39,17 @@ namespace BotSharp.NLP.UnitTest
             var options = new ClassifyOptions
             {
                 ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"),
-                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
+                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
+                Dimension = 100
             };
             var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
             
-            var dataset = sentences.Split(1M);
+            var dataset = sentences.Split(0.7M);
             classifier.Train(dataset.Item1);
 
             int correct = 0;
             int total = 0;
-            dataset.Item1.ForEach(td =>
+            dataset.Item2.ForEach(td =>
             {
                 var classes = classifier.Classify(td);
                 if (td.Label == classes[0].Item1)
@@ -127,5 +128,53 @@ namespace BotSharp.NLP.UnitTest
 
             return genders;
         }
+
+        [TestMethod]
+        public void SpotifyTest()
+        {
+            var reader = new FasttextDataReader();
+            var sentences = reader.Read(new ReaderOptions
+            {
+                DataDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify"),
+                FileName = "spotify.txt"
+            });
+
+            var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
+            var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
+            for (int i = 0; i < newSentences.Count; i++)
+            {
+                newSentences[i].Label = sentences[i].Label;
+            }
+            sentences = newSentences.ToList();
+
+            sentences.Shuffle();
+
+            var options = new ClassifyOptions
+            {
+                ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify", "nb.model"),
+                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify")
+            };
+            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
+
+            var dataset = sentences.Split(0.7M);
+            classifier.Train(dataset.Item1);
+
+            int correct = 0;
+            int total = 0;
+            dataset.Item2.ForEach(td =>
+            {
+                var classes = classifier.Classify(td);
+                if (td.Label == classes[0].Item1)
+                {
+                    correct++;
+                }
+                total++;
+            });
+
+            var accuracy = (float)correct / total;
+
+            Assert.IsTrue(accuracy > 0.6);
+        }
+
     }
 }

BotSharp.NLP/Classify/ClassifierFactory.cs

@@ -44,7 +44,9 @@ namespace BotSharp.NLP.Classify
 
             var classes = _classifier.Classify(sentence, options);
 
-            return classes.OrderByDescending(x => x.Item2).ToList();
+            classes = classes.OrderByDescending(x => x.Item2).ToList();
+
+            return classes;
         }
     }
 }

BotSharp.NLP/Classify/ClassifyOptions.cs

@@ -13,5 +13,10 @@ namespace BotSharp.NLP.Classify
         public string PrediceOutputFile { get; set; }
         public string TransformFilePath { get; set; }
         public RangeTransform Transform { get; set; }
+
+        /// <summary>
+        /// Feature dimension
+        /// </summary>
+        public int Dimension { get; set; }
     }
 }

BotSharp.NLP/Classify/NaiveBayesClassifier.cs

@@ -55,12 +55,13 @@ namespace BotSharp.NLP.Classify
         public void Train(List<Sentence> sentences, ClassifyOptions options)
         {
             var tfidf = new TfIdfFeatureExtractor();
+            tfidf.Dimension = options.Dimension;
             tfidf.Sentences = sentences;
             tfidf.CalBasedOnCategory();
-            var keyWords = tfidf.Keywords();
+
-            string keywords2 = String.Join(",", keyWords.ToArray());
             var encoder = new OneHotEncoder();
             encoder.Sentences = sentences;
+            encoder.Words = tfidf.Keywords();
             words = encoder.EncodeAll();
 
             var featureSets = sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
@@ -118,7 +119,8 @@ namespace BotSharp.NLP.Classify
                 lf.Prob = nb.PosteriorProb();
             });*/
 
-            return results;
+            double total = results.Select(x => x.Item2).Sum();
+            return results.Select(x => new Tuple<string, double>(x.Item1, x.Item2 / total)).ToList();
         }
 
         public string SaveModel(ClassifyOptions options)

BotSharp.NLP/Featuring/IFeatureExtractor.cs

@@ -6,5 +6,9 @@ namespace BotSharp.NLP.Featuring
 {
     public interface IFeatureExtractor
     {
+        /// <summary>
+        /// Feature dimension size
+        /// </summary>
+        int Dimension { get; set; }
     }
 }

BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs

@@ -34,6 +34,7 @@ namespace BotSharp.NLP.Featuring
         private List<Tuple<String, double>> tfs;
 
         private List<string> Categories { get; set; }
+        public int Dimension { get; set; }
 
         public void Extract(Sentence sentence)
         {
@@ -42,10 +43,26 @@ namespace BotSharp.NLP.Featuring
 
         public List<string> Keywords()
         {
+            if(Dimension == 0)
+            {
+                Dimension = Categories.Count * 3;
+
+                if(Dimension > 300)
+                {
+                    Dimension = 300;
+                }
+
+                if(Dimension < 30)
+                {
+                    Dimension = 30;
+                }
+            }
+
             var tfs2 = tfs.OrderByDescending(x => x.Item2)
                 .Select(x => x.Item1)
                 .Distinct()
-                .Take((int)Math.Floor(Sentences.Count / Categories.Count * 1.5))
+                .Take(Dimension)
+                .OrderBy(x => x)
                 .ToList();
 
             return tfs2;
@@ -59,7 +76,7 @@ namespace BotSharp.NLP.Featuring
 
             Sentences.ForEach(sent =>
             {
-                sent.Words.ForEach(word =>
+                sent.Words.Where(x => x.IsAlpha).ToList().ForEach(word =>
                 {
                     // TF
                     int c1 = sent.Words.Count(x => x.Lemma == word.Lemma);
@@ -82,6 +99,17 @@ namespace BotSharp.NLP.Featuring
 
             Categories = Sentences.Select(x => x.Label).Distinct().ToList();
 
+            List<Tuple<string, string>> allTextByCategory = new List<Tuple<string, string>>();
+
+            Categories.ForEach(label =>
+            {
+                var allTokens = new List<Token>();
+                Sentences.Where(x => x.Label == label)
+                    .ToList()
+                    .ForEach(s => allTokens.AddRange(s.Words));
+                allTextByCategory.Add(new Tuple<string, string>(label, String.Join(" ", allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma))));
+            });
+
             Categories.ForEach(label =>
             {
                 var allTokens = new List<Token>();
@@ -89,7 +117,7 @@ namespace BotSharp.NLP.Featuring
                     .ToList()
                     .ForEach(s => allTokens.AddRange(s.Words));
 
-                allTokens.Select(x => x.Lemma).Distinct()
+                allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma).Distinct()
                     .ToList()
                     .ForEach(word =>
                     {
@@ -98,8 +126,15 @@ namespace BotSharp.NLP.Featuring
                         double tf = (c1 + 1.0) / allTokens.Count();
 
                         // IDF
-                        var c2 = Sentences.Where(s => s.Words.Select(x => x.Lemma).Contains(word))
+                        var c2 = 0;
-                            .GroupBy(x => x.Label).Count();
+                        allTextByCategory.ForEach(all =>
+                        {
+                            if(Regex.IsMatch(all.Item2, word))
+                            {
+                                c2++;
+                            }
+                        });
+
                         double idf = Math.Log(Categories.Count / (c2 + 1.0));
 
                         tfs.Add(new Tuple<string, double>(word, tf * idf));

BotSharp.NLP/Txt2Vec/OneHotEncoder.cs

@@ -25,7 +25,7 @@ namespace BotSharp.NLP.Txt2Vec
 
             sentence.Words.ForEach(w =>
             {
-                int index = Words.IndexOf(w.Lemma.ToLower());
+                int index = Words.IndexOf(w.Lemma);
                 if(index > 0)
                 {
                     vector[index] = 1;
@@ -49,7 +49,12 @@ namespace BotSharp.NLP.Txt2Vec
         {
             if (Words == null)
             {
-                Words = "shuffle,pause,resume,next,stop,previous,continue,mode,repeat,back,music,play,enough,off,them,playlist,skip,restart,favourites,on,add,go,again,turn,save,my,station,favourite,start,by,playing,please,now,running,move,gym,yoga,backward,one,favorites,mark,as,remember,fave,what,forward,me,and,could,once,more,can".Split(',').ToList();
+                Words = new List<string>();
+                Sentences.ForEach(x =>
+                {
+                    Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Lemma));
+                });
+                Words = Words.Distinct().OrderBy(x => x).ToList();
             }
 
             return Words;

BotSharp.RestApi/AgentController.cs

@@ -38,9 +38,22 @@ namespace BotSharp.RestApi
         [HttpGet]
         public ActionResult<List<Agent>> AllAgents()
         {
-            var dc = new DefaultDataContextLoader().GetDefaultDc();
+            List<Agent> agents = new List<Agent>();
 
-            return dc.Table<Agent>().ToList();
+            string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects");
+
+            var names = Directory.EnumerateDirectories(agentDir).Select(x => x.Split(Path.DirectorySeparatorChar).Last()).ToList();
+
+            names.ForEach(name =>
+            {
+                agents.Add(new Agent
+                {
+                    Name = name
+                });
+
+            });
+
+            return agents;
         }
 
         /// <summary>