SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Improve intent accuracy greater than 0.7

Browse source
This commit is contained in:
Oceania2018 2018-09-13 15:01:40 -05:00
parent a173569681
commit 0b09b20103
16 changed files with 168 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs
View file

@ -88,13 +88,16 @@ namespace BotSharp.Algorithm.Bayes
{
int featureCount = features.Length;
double postProb = priorProb;
double postProb = Math.Log(priorProb);
// loop features
for (int x = 0; x < featureCount; x++)
{
string key = $"{Y} f{x} {features[x]}";
postProb += condProbDictionary[key];
if(features[x] == 1)
{
postProb += condProbDictionary[key];
}
}
return Math.Pow(2, postProb);

1
BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj
View file

@ -19,6 +19,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BotSharp.Algorithm\BotSharp.Algorithm.csproj" />
<ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
</ItemGroup>

30
BotSharp.Core.UnitTest/Performance/Spotify.cs
View file

@ -8,6 +8,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using BotSharp.Algorithm.Extensions;
namespace BotSharp.Core.UnitTest.Performance
{
@ -21,26 +22,24 @@ namespace BotSharp.Core.UnitTest.Performance
public void IntentAccuracy()
{
int correct = 0;
List<Tuple<string, string>> errors = new List<Tuple<string, string>>();
var agent = LoadAgent();
for(int i = 0; i < Samples.Count; i++)
for (int i = 0; i < Samples.Count; i++)
{
try
var aIResponse = _platform.TextRequest(Samples[i].Item1);
if (aIResponse.Result.Metadata.IntentName == Samples[i].Item2)
{
var aIResponse = _platform.TextRequest(Samples[i].Item1);
if (aIResponse.Result.Metadata.IntentName == Samples[i].Item2)
{
correct++;
}
correct++;
}
catch (Exception)
else
{
errors.Add(new Tuple<string, string>(Samples[i].Item2, Samples[i].Item1.Query[0]));
}
}
double accuracy = correct / (Samples.Count + 0.0);
double accuracy = correct / (Samples.Count + 0.0);
}
private Agent LoadAgent()
@ -55,6 +54,12 @@ namespace BotSharp.Core.UnitTest.Performance
// Init samples
Samples = new List<Tuple<AIRequest, string>>();
/*agent.Corpus.UserSays = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>
{
new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music.play", Text = "play the 50 Great Beatles Songs playlist in Prime Music"},
new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music.play", Text = "reproduce a the track Monster by Rihanna ft Eminem"},
new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music_player_control.add_favorite", Text = "add this song to my favourites"}
};*/
agent.Corpus.UserSays.ForEach(intent =>
{
Samples.Add(new Tuple<AIRequest, string>(new AIRequest
@ -68,8 +73,9 @@ namespace BotSharp.Core.UnitTest.Performance
}, intent.Intent));
});
var samples = String.Join("\r\n", Samples.Select(x => $"__label__{x.Item2} {x.Item1.Query[0]}").ToList());
//Samples.Shuffle();
var samples = String.Join("\r\n", Samples.Select(x => $"__label__{x.Item2} {x.Item1.Query[0]}").ToList());
return agent;
}

7
BotSharp.Core/BotSharp.Core.csproj
View file

@ -66,8 +66,11 @@ If you feel that this project is helpful to you, please Star on the project, we
<ItemGroup>
<Compile Remove="Accounts\**" />
<Compile Remove="Engines\CoreNlp\**" />
<EmbeddedResource Remove="Accounts\**" />
<EmbeddedResource Remove="Engines\CoreNlp\**" />
<None Remove="Accounts\**" />
<None Remove="Engines\CoreNlp\**" />
</ItemGroup>
<ItemGroup>
@ -80,10 +83,6 @@ If you feel that this project is helpful to you, please Star on the project, we
<PackageReference Include="RestSharp" Version="106.3.1" />
</ItemGroup>
<ItemGroup>
<Folder Include="Engines\CoreNlp\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\BotSharp.NLP\BotSharp.NLP.csproj" />
</ItemGroup>

2
BotSharp.Core/Engines/BotEngineBase.cs
View file

@ -46,7 +46,7 @@ namespace BotSharp.Core.Engines
{
doc.Sentences[0].Entities = new List<NlpEntity>();
}
doc.Sentences[0].Entities.ForEach(x => parameters.Add(x.Entity, x.Value));
doc.Sentences[0].Entities.ForEach(x => parameters[x.Entity] = x.Value);
return new AIResponse
{

108
BotSharp.Core/Engines/Nltk/NltkTokenizer.cs
View file

@ -1,108 +0,0 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Models;
using BotSharp.NLP.Tokenize;
using EntityFrameworkCore.BootKit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
public class NltkTokenizer : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
var request = new RestRequest("nltktokenizesentences", Method.POST);
List<List<Token>> tokens = new List<List<Token>>();
Boolean res = true;
var dc = new DefaultDataContextLoader().GetDefaultDc();
var corpus = agent.Corpus;
doc.Sentences = new List<NlpDocSentence>();
List<string> sentencesList = new List<string>();
corpus.UserSays.ForEach ( usersay => sentencesList.Add(usersay.Text));
request.RequestFormat = DataFormat.Json;
request.AddParameter("application/json", JsonConvert.SerializeObject(new Documents(sentencesList)), ParameterType.RequestBody);
var response = client.Execute<Result>(request);
tokens = response.Data.TokensList;
for (int i = 0; i < sentencesList.Count; i++)
{
doc.Sentences.Add(new NlpDocSentence
{
Tokens = tokens[i],
Text = sentencesList[i]
});
}
res = res && response.IsSuccessful;
return res;
/*
corpus.UserSays.ForEach(usersay => {
Console.WriteLine(usersay.Text);
request.AddParameter("text", usersay.Text);
var response = client.Execute<Result>(request);
tokens.Add(response.Data.Tokens);
doc.Sentences.Add(new NlpDocSentence
{
Tokens = response.Data.Tokens,
Text = usersay.Text
});
res = res && response.IsSuccessful;
});
*/
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
var request = new RestRequest("nltktokenizesentences", Method.POST);
List<List<Token>> tokens = new List<List<Token>>();
Boolean res = true;
var corpus = agent.Corpus;
request.AddParameter("sentences", doc.Sentences[0].Text);
var response = client.Execute<Result>(request);
//tokens.Add(response.Data.Tokens);
res = res && response.IsSuccessful;
doc.Sentences[0].Tokens = tokens[0];
return true;
}
private class Result
{
public List<List<Token>> TokensList { get; set; }
}
private class Documents
{
public List<string> Sentences { get; set; }
public Documents(List<string> sentences)
{
this.Sentences = sentences;
}
}
}
}

6
BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs
View file

@ -88,12 +88,12 @@ namespace BotSharp.Core.Engines.Rasa
public void LoadIntents(Agent agent)
{
string data = File.ReadAllText(Path.Combine(AgentDir, "corpus.json"));
var rasa = JsonConvert.DeserializeObject<RasaAgent>(data);
var rasa = JsonConvert.DeserializeObject<RasaAgentImportModel>(data);
agent.Intents = rasa.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList();
agent.Intents = rasa.Data.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList();
agent.Intents.ForEach(intent => {
ImportIntentUserSays(intent, rasa.UserSays);
ImportIntentUserSays(intent, rasa.Data.UserSays);
});
}

6
BotSharp.Core/Engines/Rasa/RasaAgent.cs
View file

@ -21,4 +21,10 @@ namespace BotSharp.Core.Engines.Rasa
[JsonProperty("regex_features")]
public List<RasaTrainingRegex> Regex { get; set; }
}
public class RasaAgentImportModel
{
[JsonProperty("rasa_nlu_data")]
public RasaAgent Data { get; set; }
}
}

55
BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs
View file

@ -39,16 +39,17 @@ namespace BotSharp.NLP.UnitTest
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"),
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
Dimension = 100
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var dataset = sentences.Split(1M);
var dataset = sentences.Split(0.7M);
classifier.Train(dataset.Item1);
int correct = 0;
int total = 0;
dataset.Item1.ForEach(td =>
dataset.Item2.ForEach(td =>
{
var classes = classifier.Classify(td);
if (td.Label == classes[0].Item1)
@ -127,5 +128,53 @@ namespace BotSharp.NLP.UnitTest
return genders;
}
[TestMethod]
public void SpotifyTest()
{
var reader = new FasttextDataReader();
var sentences = reader.Read(new ReaderOptions
{
DataDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify"),
FileName = "spotify.txt"
});
var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
for (int i = 0; i < newSentences.Count; i++)
{
newSentences[i].Label = sentences[i].Label;
}
sentences = newSentences.ToList();
sentences.Shuffle();
var options = new ClassifyOptions
{
ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify", "nb.model"),
TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify")
};
var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
var dataset = sentences.Split(0.7M);
classifier.Train(dataset.Item1);
int correct = 0;
int total = 0;
dataset.Item2.ForEach(td =>
{
var classes = classifier.Classify(td);
if (td.Label == classes[0].Item1)
{
correct++;
}
total++;
});
var accuracy = (float)correct / total;
Assert.IsTrue(accuracy > 0.6);
}
}
}

4
BotSharp.NLP/Classify/ClassifierFactory.cs
View file

@ -44,7 +44,9 @@ namespace BotSharp.NLP.Classify
var classes = _classifier.Classify(sentence, options);
return classes.OrderByDescending(x => x.Item2).ToList();
classes = classes.OrderByDescending(x => x.Item2).ToList();
return classes;
}
}
}

5
BotSharp.NLP/Classify/ClassifyOptions.cs
View file

@ -13,5 +13,10 @@ namespace BotSharp.NLP.Classify
public string PrediceOutputFile { get; set; }
public string TransformFilePath { get; set; }
public RangeTransform Transform { get; set; }
/// <summary>
/// Feature dimension
/// </summary>
public int Dimension { get; set; }
}
}

8
BotSharp.NLP/Classify/NaiveBayesClassifier.cs
View file

@ -55,12 +55,13 @@ namespace BotSharp.NLP.Classify
public void Train(List<Sentence> sentences, ClassifyOptions options)
{
var tfidf = new TfIdfFeatureExtractor();
tfidf.Dimension = options.Dimension;
tfidf.Sentences = sentences;
tfidf.CalBasedOnCategory();
var keyWords = tfidf.Keywords();
string keywords2 = String.Join(",", keyWords.ToArray());
var encoder = new OneHotEncoder();
encoder.Sentences = sentences;
encoder.Words = tfidf.Keywords();
words = encoder.EncodeAll();
var featureSets = sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
@ -118,7 +119,8 @@ namespace BotSharp.NLP.Classify
lf.Prob = nb.PosteriorProb();
});*/
return results;
double total = results.Select(x => x.Item2).Sum();
return results.Select(x => new Tuple<string, double>(x.Item1, x.Item2 / total)).ToList();
}
public string SaveModel(ClassifyOptions options)

4
BotSharp.NLP/Featuring/IFeatureExtractor.cs
View file

@ -6,5 +6,9 @@ namespace BotSharp.NLP.Featuring
{
public interface IFeatureExtractor
{
/// <summary>
/// Feature dimension size
/// </summary>
int Dimension { get; set; }
}
}

45
BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs
View file

@ -34,6 +34,7 @@ namespace BotSharp.NLP.Featuring
private List<Tuple<String, double>> tfs;
private List<string> Categories { get; set; }
public int Dimension { get; set; }
public void Extract(Sentence sentence)
{
@ -42,10 +43,26 @@ namespace BotSharp.NLP.Featuring
public List<string> Keywords()
{
if(Dimension == 0)
{
Dimension = Categories.Count * 3;
if(Dimension > 300)
{
Dimension = 300;
}
if(Dimension < 30)
{
Dimension = 30;
}
}
var tfs2 = tfs.OrderByDescending(x => x.Item2)
.Select(x => x.Item1)
.Distinct()
.Take((int)Math.Floor(Sentences.Count / Categories.Count * 1.5))
.Take(Dimension)
.OrderBy(x => x)
.ToList();
return tfs2;
@ -59,7 +76,7 @@ namespace BotSharp.NLP.Featuring
Sentences.ForEach(sent =>
{
sent.Words.ForEach(word =>
sent.Words.Where(x => x.IsAlpha).ToList().ForEach(word =>
{
// TF
int c1 = sent.Words.Count(x => x.Lemma == word.Lemma);
@ -82,6 +99,17 @@ namespace BotSharp.NLP.Featuring
Categories = Sentences.Select(x => x.Label).Distinct().ToList();
List<Tuple<string, string>> allTextByCategory = new List<Tuple<string, string>>();
Categories.ForEach(label =>
{
var allTokens = new List<Token>();
Sentences.Where(x => x.Label == label)
.ToList()
.ForEach(s => allTokens.AddRange(s.Words));
allTextByCategory.Add(new Tuple<string, string>(label, String.Join(" ", allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma))));
});
Categories.ForEach(label =>
{
var allTokens = new List<Token>();
@ -89,7 +117,7 @@ namespace BotSharp.NLP.Featuring
.ToList()
.ForEach(s => allTokens.AddRange(s.Words));
allTokens.Select(x => x.Lemma).Distinct()
allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma).Distinct()
.ToList()
.ForEach(word =>
{
@ -98,8 +126,15 @@ namespace BotSharp.NLP.Featuring
double tf = (c1 + 1.0) / allTokens.Count();
// IDF
var c2 = Sentences.Where(s => s.Words.Select(x => x.Lemma).Contains(word))
.GroupBy(x => x.Label).Count();
var c2 = 0;
allTextByCategory.ForEach(all =>
{
if(Regex.IsMatch(all.Item2, word))
{
c2++;
}
});
double idf = Math.Log(Categories.Count / (c2 + 1.0));
tfs.Add(new Tuple<string, double>(word, tf * idf));

9
BotSharp.NLP/Txt2Vec/OneHotEncoder.cs
View file

@ -25,7 +25,7 @@ namespace BotSharp.NLP.Txt2Vec
sentence.Words.ForEach(w =>
{
int index = Words.IndexOf(w.Lemma.ToLower());
int index = Words.IndexOf(w.Lemma);
if(index > 0)
{
vector[index] = 1;
@ -49,7 +49,12 @@ namespace BotSharp.NLP.Txt2Vec
{
if (Words == null)
{
Words = "shuffle,pause,resume,next,stop,previous,continue,mode,repeat,back,music,play,enough,off,them,playlist,skip,restart,favourites,on,add,go,again,turn,save,my,station,favourite,start,by,playing,please,now,running,move,gym,yoga,backward,one,favorites,mark,as,remember,fave,what,forward,me,and,could,once,more,can".Split(',').ToList();
Words = new List<string>();
Sentences.ForEach(x =>
{
Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Lemma));
});
Words = Words.Distinct().OrderBy(x => x).ToList();
}
return Words;

17
BotSharp.RestApi/AgentController.cs
View file

@ -38,9 +38,22 @@ namespace BotSharp.RestApi
[HttpGet]
public ActionResult<List<Agent>> AllAgents()
{
var dc = new DefaultDataContextLoader().GetDefaultDc();
List<Agent> agents = new List<Agent>();
return dc.Table<Agent>().ToList();
string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects");
var names = Directory.EnumerateDirectories(agentDir).Select(x => x.Split(Path.DirectorySeparatorChar).Last()).ToList();
names.ForEach(name =>
{
agents.Add(new Agent
{
Name = name
});
});
return agents;
}
/// <summary>