diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs
index 977f207f..05a3e2f3 100644
--- a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs
+++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs
@@ -88,13 +88,16 @@ namespace BotSharp.Algorithm.Bayes
{
int featureCount = features.Length;
- double postProb = priorProb;
+ double postProb = Math.Log(priorProb);
// loop features
for (int x = 0; x < featureCount; x++)
{
string key = $"{Y} f{x} {features[x]}";
- postProb += condProbDictionary[key];
+ if(features[x] == 1)
+ {
+ postProb += condProbDictionary[key];
+ }
}
return Math.Pow(2, postProb);
diff --git a/BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj b/BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj
index 1874b1a8..86788b78 100644
--- a/BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj
+++ b/BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj
@@ -19,6 +19,7 @@
+
diff --git a/BotSharp.Core.UnitTest/Performance/Spotify.cs b/BotSharp.Core.UnitTest/Performance/Spotify.cs
index 236d0c3f..e52a8f59 100644
--- a/BotSharp.Core.UnitTest/Performance/Spotify.cs
+++ b/BotSharp.Core.UnitTest/Performance/Spotify.cs
@@ -8,6 +8,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
+using BotSharp.Algorithm.Extensions;
namespace BotSharp.Core.UnitTest.Performance
{
@@ -21,26 +22,24 @@ namespace BotSharp.Core.UnitTest.Performance
public void IntentAccuracy()
{
int correct = 0;
+ List> errors = new List>();
+
var agent = LoadAgent();
- for(int i = 0; i < Samples.Count; i++)
+ for (int i = 0; i < Samples.Count; i++)
{
- try
+ var aIResponse = _platform.TextRequest(Samples[i].Item1);
+ if (aIResponse.Result.Metadata.IntentName == Samples[i].Item2)
{
- var aIResponse = _platform.TextRequest(Samples[i].Item1);
- if (aIResponse.Result.Metadata.IntentName == Samples[i].Item2)
- {
- correct++;
- }
+ correct++;
}
- catch (Exception)
+ else
{
-
+ errors.Add(new Tuple(Samples[i].Item2, Samples[i].Item1.Query[0]));
}
-
}
- double accuracy = correct / (Samples.Count + 0.0);
+ double accuracy = correct / (Samples.Count + 0.0);
}
private Agent LoadAgent()
@@ -55,6 +54,12 @@ namespace BotSharp.Core.UnitTest.Performance
// Init samples
Samples = new List>();
+ /*agent.Corpus.UserSays = new List>
+ {
+ new TrainingIntentExpression{ Intent = "music.play", Text = "play the 50 Great Beatles Songs playlist in Prime Music"},
+ new TrainingIntentExpression{ Intent = "music.play", Text = "reproduce a the track Monster by Rihanna ft Eminem"},
+ new TrainingIntentExpression{ Intent = "music_player_control.add_favorite", Text = "add this song to my favourites"}
+ };*/
agent.Corpus.UserSays.ForEach(intent =>
{
Samples.Add(new Tuple(new AIRequest
@@ -68,8 +73,9 @@ namespace BotSharp.Core.UnitTest.Performance
}, intent.Intent));
});
- var samples = String.Join("\r\n", Samples.Select(x => $"__label__{x.Item2} {x.Item1.Query[0]}").ToList());
+ //Samples.Shuffle();
+ var samples = String.Join("\r\n", Samples.Select(x => $"__label__{x.Item2} {x.Item1.Query[0]}").ToList());
return agent;
}
diff --git a/BotSharp.Core/BotSharp.Core.csproj b/BotSharp.Core/BotSharp.Core.csproj
index a7a25a63..409bb78b 100644
--- a/BotSharp.Core/BotSharp.Core.csproj
+++ b/BotSharp.Core/BotSharp.Core.csproj
@@ -66,8 +66,11 @@ If you feel that this project is helpful to you, please Star on the project, we
+
+
+
@@ -80,10 +83,6 @@ If you feel that this project is helpful to you, please Star on the project, we
-
-
-
-
diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs
index 710d02ee..bba696db 100644
--- a/BotSharp.Core/Engines/BotEngineBase.cs
+++ b/BotSharp.Core/Engines/BotEngineBase.cs
@@ -46,7 +46,7 @@ namespace BotSharp.Core.Engines
{
doc.Sentences[0].Entities = new List();
}
- doc.Sentences[0].Entities.ForEach(x => parameters.Add(x.Entity, x.Value));
+ doc.Sentences[0].Entities.ForEach(x => parameters[x.Entity] = x.Value);
return new AIResponse
{
diff --git a/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs b/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs
deleted file mode 100644
index b1a505b7..00000000
--- a/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs
+++ /dev/null
@@ -1,108 +0,0 @@
-using BotSharp.Core.Abstractions;
-using BotSharp.Core.Agents;
-using BotSharp.Core.Models;
-using BotSharp.NLP.Tokenize;
-using EntityFrameworkCore.BootKit;
-using Microsoft.Extensions.Configuration;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using RestSharp;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace BotSharp.Core.Engines.SpaCy
-{
- public class NltkTokenizer : INlpTrain, INlpPredict
- {
- public IConfiguration Configuration { get; set; }
- public PipeSettings Settings { get; set; }
-
- public async Task Train(Agent agent, NlpDoc doc, PipeModel meta)
- {
- var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
- var request = new RestRequest("nltktokenizesentences", Method.POST);
- List> tokens = new List>();
- Boolean res = true;
- var dc = new DefaultDataContextLoader().GetDefaultDc();
- var corpus = agent.Corpus;
-
- doc.Sentences = new List();
- List sentencesList = new List();
- corpus.UserSays.ForEach ( usersay => sentencesList.Add(usersay.Text));
-
- request.RequestFormat = DataFormat.Json;
-
- request.AddParameter("application/json", JsonConvert.SerializeObject(new Documents(sentencesList)), ParameterType.RequestBody);
-
- var response = client.Execute(request);
-
- tokens = response.Data.TokensList;
-
- for (int i = 0; i < sentencesList.Count; i++)
- {
- doc.Sentences.Add(new NlpDocSentence
- {
- Tokens = tokens[i],
- Text = sentencesList[i]
- });
- }
- res = res && response.IsSuccessful;
- return res;
- /*
- corpus.UserSays.ForEach(usersay => {
- Console.WriteLine(usersay.Text);
- request.AddParameter("text", usersay.Text);
- var response = client.Execute(request);
-
- tokens.Add(response.Data.Tokens);
-
- doc.Sentences.Add(new NlpDocSentence
- {
- Tokens = response.Data.Tokens,
- Text = usersay.Text
- });
-
- res = res && response.IsSuccessful;
- });
- */
- }
-
- public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta)
- {
- var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
- var request = new RestRequest("nltktokenizesentences", Method.POST);
- List> tokens = new List>();
- Boolean res = true;
- var corpus = agent.Corpus;
-
- request.AddParameter("sentences", doc.Sentences[0].Text);
- var response = client.Execute(request);
-
- //tokens.Add(response.Data.Tokens);
-
- res = res && response.IsSuccessful;
-
- doc.Sentences[0].Tokens = tokens[0];
-
- return true;
- }
-
- private class Result
- {
- public List> TokensList { get; set; }
- }
-
- private class Documents
- {
- public List Sentences { get; set; }
-
- public Documents(List sentences)
- {
- this.Sentences = sentences;
- }
- }
- }
-}
diff --git a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs
index f3811c40..7975823f 100644
--- a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs
+++ b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs
@@ -88,12 +88,12 @@ namespace BotSharp.Core.Engines.Rasa
public void LoadIntents(Agent agent)
{
string data = File.ReadAllText(Path.Combine(AgentDir, "corpus.json"));
- var rasa = JsonConvert.DeserializeObject(data);
+ var rasa = JsonConvert.DeserializeObject(data);
- agent.Intents = rasa.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList();
+ agent.Intents = rasa.Data.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList();
agent.Intents.ForEach(intent => {
- ImportIntentUserSays(intent, rasa.UserSays);
+ ImportIntentUserSays(intent, rasa.Data.UserSays);
});
}
diff --git a/BotSharp.Core/Engines/Rasa/RasaAgent.cs b/BotSharp.Core/Engines/Rasa/RasaAgent.cs
index fc9c7748..1851aa3c 100644
--- a/BotSharp.Core/Engines/Rasa/RasaAgent.cs
+++ b/BotSharp.Core/Engines/Rasa/RasaAgent.cs
@@ -21,4 +21,10 @@ namespace BotSharp.Core.Engines.Rasa
[JsonProperty("regex_features")]
public List Regex { get; set; }
}
+
+ public class RasaAgentImportModel
+ {
+ [JsonProperty("rasa_nlu_data")]
+ public RasaAgent Data { get; set; }
+ }
}
diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
index 2b8c2e0b..331caed3 100644
--- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
+++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
@@ -39,16 +39,17 @@ namespace BotSharp.NLP.UnitTest
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"),
- TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
+ TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
+ Dimension = 100
};
var classifier = new ClassifierFactory(options, SupportedLanguage.English);
- var dataset = sentences.Split(1M);
+ var dataset = sentences.Split(0.7M);
classifier.Train(dataset.Item1);
int correct = 0;
int total = 0;
- dataset.Item1.ForEach(td =>
+ dataset.Item2.ForEach(td =>
{
var classes = classifier.Classify(td);
if (td.Label == classes[0].Item1)
@@ -127,5 +128,53 @@ namespace BotSharp.NLP.UnitTest
return genders;
}
+
+ [TestMethod]
+ public void SpotifyTest()
+ {
+ var reader = new FasttextDataReader();
+ var sentences = reader.Read(new ReaderOptions
+ {
+ DataDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "spotify"),
+ FileName = "spotify.txt"
+ });
+
+ var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English);
+ var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
+ for (int i = 0; i < newSentences.Count; i++)
+ {
+ newSentences[i].Label = sentences[i].Label;
+ }
+ sentences = newSentences.ToList();
+
+ sentences.Shuffle();
+
+ var options = new ClassifyOptions
+ {
+ ModelFilePath = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "spotify", "nb.model"),
+ TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "spotify")
+ };
+ var classifier = new ClassifierFactory(options, SupportedLanguage.English);
+
+ var dataset = sentences.Split(0.7M);
+ classifier.Train(dataset.Item1);
+
+ int correct = 0;
+ int total = 0;
+ dataset.Item2.ForEach(td =>
+ {
+ var classes = classifier.Classify(td);
+ if (td.Label == classes[0].Item1)
+ {
+ correct++;
+ }
+ total++;
+ });
+
+ var accuracy = (float)correct / total;
+
+ Assert.IsTrue(accuracy > 0.6);
+ }
+
}
}
diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs
index 5f91b735..cc754488 100644
--- a/BotSharp.NLP/Classify/ClassifierFactory.cs
+++ b/BotSharp.NLP/Classify/ClassifierFactory.cs
@@ -44,7 +44,9 @@ namespace BotSharp.NLP.Classify
var classes = _classifier.Classify(sentence, options);
- return classes.OrderByDescending(x => x.Item2).ToList();
+ classes = classes.OrderByDescending(x => x.Item2).ToList();
+
+ return classes;
}
}
}
diff --git a/BotSharp.NLP/Classify/ClassifyOptions.cs b/BotSharp.NLP/Classify/ClassifyOptions.cs
index fce017fd..0c213fad 100644
--- a/BotSharp.NLP/Classify/ClassifyOptions.cs
+++ b/BotSharp.NLP/Classify/ClassifyOptions.cs
@@ -13,5 +13,10 @@ namespace BotSharp.NLP.Classify
public string PrediceOutputFile { get; set; }
public string TransformFilePath { get; set; }
public RangeTransform Transform { get; set; }
+
+ ///
+ /// Feature dimension
+ ///
+ public int Dimension { get; set; }
}
}
diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs
index 610628e4..8288dd28 100644
--- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs
+++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs
@@ -55,12 +55,13 @@ namespace BotSharp.NLP.Classify
public void Train(List sentences, ClassifyOptions options)
{
var tfidf = new TfIdfFeatureExtractor();
+ tfidf.Dimension = options.Dimension;
tfidf.Sentences = sentences;
tfidf.CalBasedOnCategory();
- var keyWords = tfidf.Keywords();
- string keywords2 = String.Join(",", keyWords.ToArray());
+
var encoder = new OneHotEncoder();
encoder.Sentences = sentences;
+ encoder.Words = tfidf.Keywords();
words = encoder.EncodeAll();
var featureSets = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList();
@@ -118,7 +119,8 @@ namespace BotSharp.NLP.Classify
lf.Prob = nb.PosteriorProb();
});*/
- return results;
+ double total = results.Select(x => x.Item2).Sum();
+ return results.Select(x => new Tuple(x.Item1, x.Item2 / total)).ToList();
}
public string SaveModel(ClassifyOptions options)
diff --git a/BotSharp.NLP/Featuring/IFeatureExtractor.cs b/BotSharp.NLP/Featuring/IFeatureExtractor.cs
index 785f1d30..497e4a13 100644
--- a/BotSharp.NLP/Featuring/IFeatureExtractor.cs
+++ b/BotSharp.NLP/Featuring/IFeatureExtractor.cs
@@ -6,5 +6,9 @@ namespace BotSharp.NLP.Featuring
{
public interface IFeatureExtractor
{
+ ///
+ /// Feature dimension size
+ ///
+ int Dimension { get; set; }
}
}
diff --git a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs
index ce709549..e7132aa7 100644
--- a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs
+++ b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs
@@ -34,6 +34,7 @@ namespace BotSharp.NLP.Featuring
private List> tfs;
private List Categories { get; set; }
+ public int Dimension { get; set; }
public void Extract(Sentence sentence)
{
@@ -42,10 +43,26 @@ namespace BotSharp.NLP.Featuring
public List Keywords()
{
+ if(Dimension == 0)
+ {
+ Dimension = Categories.Count * 3;
+
+ if(Dimension > 300)
+ {
+ Dimension = 300;
+ }
+
+ if(Dimension < 30)
+ {
+ Dimension = 30;
+ }
+ }
+
var tfs2 = tfs.OrderByDescending(x => x.Item2)
.Select(x => x.Item1)
.Distinct()
- .Take((int)Math.Floor(Sentences.Count / Categories.Count * 1.5))
+ .Take(Dimension)
+ .OrderBy(x => x)
.ToList();
return tfs2;
@@ -59,7 +76,7 @@ namespace BotSharp.NLP.Featuring
Sentences.ForEach(sent =>
{
- sent.Words.ForEach(word =>
+ sent.Words.Where(x => x.IsAlpha).ToList().ForEach(word =>
{
// TF
int c1 = sent.Words.Count(x => x.Lemma == word.Lemma);
@@ -82,6 +99,17 @@ namespace BotSharp.NLP.Featuring
Categories = Sentences.Select(x => x.Label).Distinct().ToList();
+ List> allTextByCategory = new List>();
+
+ Categories.ForEach(label =>
+ {
+ var allTokens = new List();
+ Sentences.Where(x => x.Label == label)
+ .ToList()
+ .ForEach(s => allTokens.AddRange(s.Words));
+ allTextByCategory.Add(new Tuple(label, String.Join(" ", allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma))));
+ });
+
Categories.ForEach(label =>
{
var allTokens = new List();
@@ -89,7 +117,7 @@ namespace BotSharp.NLP.Featuring
.ToList()
.ForEach(s => allTokens.AddRange(s.Words));
- allTokens.Select(x => x.Lemma).Distinct()
+ allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma).Distinct()
.ToList()
.ForEach(word =>
{
@@ -98,8 +126,15 @@ namespace BotSharp.NLP.Featuring
double tf = (c1 + 1.0) / allTokens.Count();
// IDF
- var c2 = Sentences.Where(s => s.Words.Select(x => x.Lemma).Contains(word))
- .GroupBy(x => x.Label).Count();
+ var c2 = 0;
+ allTextByCategory.ForEach(all =>
+ {
+ if(Regex.IsMatch(all.Item2, word))
+ {
+ c2++;
+ }
+ });
+
double idf = Math.Log(Categories.Count / (c2 + 1.0));
tfs.Add(new Tuple(word, tf * idf));
diff --git a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
index 5cf83cb5..d220c426 100644
--- a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
+++ b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
@@ -25,7 +25,7 @@ namespace BotSharp.NLP.Txt2Vec
sentence.Words.ForEach(w =>
{
- int index = Words.IndexOf(w.Lemma.ToLower());
+ int index = Words.IndexOf(w.Lemma);
if(index > 0)
{
vector[index] = 1;
@@ -49,7 +49,12 @@ namespace BotSharp.NLP.Txt2Vec
{
if (Words == null)
{
- Words = "shuffle,pause,resume,next,stop,previous,continue,mode,repeat,back,music,play,enough,off,them,playlist,skip,restart,favourites,on,add,go,again,turn,save,my,station,favourite,start,by,playing,please,now,running,move,gym,yoga,backward,one,favorites,mark,as,remember,fave,what,forward,me,and,could,once,more,can".Split(',').ToList();
+ Words = new List();
+ Sentences.ForEach(x =>
+ {
+ Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Lemma));
+ });
+ Words = Words.Distinct().OrderBy(x => x).ToList();
}
return Words;
diff --git a/BotSharp.RestApi/AgentController.cs b/BotSharp.RestApi/AgentController.cs
index bc3a4455..aee08112 100644
--- a/BotSharp.RestApi/AgentController.cs
+++ b/BotSharp.RestApi/AgentController.cs
@@ -38,9 +38,22 @@ namespace BotSharp.RestApi
[HttpGet]
public ActionResult> AllAgents()
{
- var dc = new DefaultDataContextLoader().GetDefaultDc();
+ List agents = new List();
- return dc.Table().ToList();
+ string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects");
+
+ var names = Directory.EnumerateDirectories(agentDir).Select(x => x.Split(Path.DirectorySeparatorChar).Last()).ToList();
+
+ names.ForEach(name =>
+ {
+ agents.Add(new Agent
+ {
+ Name = name
+ });
+
+ });
+
+ return agents;
}
///