diff --git a/.gitignore b/.gitignore index 8d5aa23c..d61438e2 100644 --- a/.gitignore +++ b/.gitignore @@ -286,21 +286,9 @@ __pycache__/ *.btm.cs *.odx.cs *.xsd.cs -/App_Data -/Bot.WebStarter/App_Data/bot-rasa.db -/Bot.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant -/BotSharp.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant -/BotSharp.UnitTest/App_Data/DbInitializer/Agents -/BotSharp.UnitTest/App_Data/BotSharp.db /BotSharp.WebHost/App_Data/BotSharp.db /BotSharp.UI -/BotSharp.WebHost/App_Data/ModelFiles -/BotSharp.WebHost/App_Data/TrainingFiles -/BotSharp.WebHost/App_Data/PredictFiles /BotSharp.WebHost/App_Data/Projects /BotSharp.WebHost/PublishOutput /Data -/docs/build /docs/_build -/BotSharp.WebHost/App_Data/AgentArchive/Smart Niraj.zip -/BotSharp.NLP.UnitTest/wordvec_enu.bin diff --git a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs index 977f207f..05a3e2f3 100644 --- a/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs +++ b/BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs @@ -88,13 +88,16 @@ namespace BotSharp.Algorithm.Bayes { int featureCount = features.Length; - double postProb = priorProb; + double postProb = Math.Log(priorProb); // loop features for (int x = 0; x < featureCount; x++) { string key = $"{Y} f{x} {features[x]}"; - postProb += condProbDictionary[key]; + if(features[x] == 1) + { + postProb += condProbDictionary[key]; + } } return Math.Pow(2, postProb); diff --git a/BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj b/BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj index 1874b1a8..86788b78 100644 --- a/BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj +++ b/BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj @@ -19,6 +19,7 @@ + diff --git a/BotSharp.Core.UnitTest/Performance/Spotify.cs b/BotSharp.Core.UnitTest/Performance/Spotify.cs new file mode 100644 index 00000000..e52a8f59 --- /dev/null +++ b/BotSharp.Core.UnitTest/Performance/Spotify.cs @@ -0,0 +1,83 @@ +using BotSharp.Core.Agents; +using BotSharp.Core.Engines; +using BotSharp.Core.Engines.BotSharp; +using BotSharp.Core.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using BotSharp.Algorithm.Extensions; + +namespace BotSharp.Core.UnitTest.Performance +{ + [TestClass] + public class Spotify : TestEssential + { + private List> Samples; + private IBotPlatform _platform; + + [TestMethod] + public void IntentAccuracy() + { + int correct = 0; + List> errors = new List>(); + + var agent = LoadAgent(); + + for (int i = 0; i < Samples.Count; i++) + { + var aIResponse = _platform.TextRequest(Samples[i].Item1); + if (aIResponse.Result.Metadata.IntentName == Samples[i].Item2) + { + correct++; + } + else + { + errors.Add(new Tuple(Samples[i].Item2, Samples[i].Item1.Query[0])); + } + } + + double accuracy = correct / (Samples.Count + 0.0); + } + + private Agent LoadAgent() + { + _platform = new BotSharpAi(); + + // Load agent + var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", "Spotify"); + string model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last().Split(Path.DirectorySeparatorChar).Last(); + var modelPath = Path.Combine(projectPath, model); + var agent = _platform.LoadAgentFromFile(modelPath); + + // Init samples + Samples = new List>(); + /*agent.Corpus.UserSays = new List> + { + new TrainingIntentExpression{ Intent = "music.play", Text = "play the 50 Great Beatles Songs playlist in Prime Music"}, + new TrainingIntentExpression{ Intent = "music.play", Text = "reproduce a the track Monster by Rihanna ft Eminem"}, + new TrainingIntentExpression{ Intent = "music_player_control.add_favorite", Text = "add this song to my favourites"} + };*/ + agent.Corpus.UserSays.ForEach(intent => + { + Samples.Add(new Tuple(new AIRequest + { + AgentDir = projectPath, + Model = model, + Query = new String[] + { + intent.Text + } + }, intent.Intent)); + }); + + //Samples.Shuffle(); + + var samples = String.Join("\r\n", Samples.Select(x => $"__label__{x.Item2} {x.Item1.Query[0]}").ToList()); + + return agent; + } + } +} diff --git a/BotSharp.Core.UnitTest/TestEssential.cs b/BotSharp.Core.UnitTest/TestEssential.cs index 2c42fdf8..f6257b45 100644 --- a/BotSharp.Core.UnitTest/TestEssential.cs +++ b/BotSharp.Core.UnitTest/TestEssential.cs @@ -18,9 +18,9 @@ namespace BotSharp.Core.UnitTest public TestEssential() { contentRoot = $"{Directory.GetCurrentDirectory()}{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}BotSharp.WebHost{Path.DirectorySeparatorChar}"; - + contentRoot = Path.GetFullPath(contentRoot); ConfigurationBuilder configurationBuilder = new ConfigurationBuilder(); - var settings = Directory.GetFiles(contentRoot + $"Settings{Path.DirectorySeparatorChar}", "*.json"); + var settings = Directory.GetFiles(contentRoot + $"..{Path.DirectorySeparatorChar}Settings{Path.DirectorySeparatorChar}", "*.json"); settings.ToList().ForEach(setting => { configurationBuilder.AddJsonFile(setting, optional: false, reloadOnChange: true); diff --git a/BotSharp.Core/BotSharp.Core.csproj b/BotSharp.Core/BotSharp.Core.csproj index a7a25a63..409bb78b 100644 --- a/BotSharp.Core/BotSharp.Core.csproj +++ b/BotSharp.Core/BotSharp.Core.csproj @@ -66,8 +66,11 @@ If you feel that this project is helpful to you, please Star on the project, we + + + @@ -80,10 +83,6 @@ If you feel that this project is helpful to you, please Star on the project, we - - - - diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs index 710d02ee..bba696db 100644 --- a/BotSharp.Core/Engines/BotEngineBase.cs +++ b/BotSharp.Core/Engines/BotEngineBase.cs @@ -46,7 +46,7 @@ namespace BotSharp.Core.Engines { doc.Sentences[0].Entities = new List(); } - doc.Sentences[0].Entities.ForEach(x => parameters.Add(x.Entity, x.Value)); + doc.Sentences[0].Entities.ForEach(x => parameters[x.Entity] = x.Value); return new AIResponse { diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs index fa276a49..b3db17a7 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs @@ -58,9 +58,6 @@ namespace BotSharp.Core.Engines.BotSharp var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input \"{parsedTrainingDataFileName}\" -output \"{modelFileName}\"", false); Console.WriteLine($"Saved model to {modelFileName}"); - meta.Meta = new JObject(); - meta.Meta["compiled at"] = "Aug 3, 2018"; - return true; } diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs new file mode 100644 index 00000000..42b6160b --- /dev/null +++ b/BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs @@ -0,0 +1,263 @@ +using BotSharp.Core.Abstractions; +using BotSharp.Core.Agents; +using BotSharp.Models.CRFLite; +using BotSharp.Models.CRFLite.Decoder; +using BotSharp.Models.CRFLite.Encoder; +using BotSharp.Models.NLP; +using BotSharp.NLP.Tokenize; +using DotNetToolkit; +using Microsoft.Extensions.Configuration; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Engines.BotSharp +{ + public class BotSharpCRFNer : INlpTrain, INlpPredict + { + public IConfiguration Configuration { get; set; } + public PipeSettings Settings { get; set; } + + public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) + { + var corpus = agent.Corpus; + + meta.Model = "ner-crf.model"; + + List> userSays = corpus.UserSays; + List> list = new List>(); + + string rawTrainingDataFileName = System.IO.Path.Combine(Settings.TempDir, "ner-crf.corpus.txt"); + string modelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model); + + using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create)) + { + using (StreamWriter sw = new StreamWriter(fs)) + { + for (int i = 0; i < doc.Sentences.Count; i++) + { + List curLine = Merge(doc, doc.Sentences[i].Tokens, userSays[i].Entities); + curLine.ForEach(trainingData => + { + string[] wordParams = { trainingData.Token, trainingData.Pos, trainingData.Entity }; + string wordStr = string.Join("\t", wordParams); + sw.WriteLine(wordStr); + }); + list.Add(curLine); + sw.WriteLine(); + } + sw.Flush(); + } + } + + string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString(); + string template = Configuration.GetValue($"BotSharpCRFNer:template"); + template = template.Replace("|App_Data|", contentDir + System.IO.Path.DirectorySeparatorChar); + + var encoder = new CRFEncoder(); + bool result = encoder.Learn(new EncoderOptions + { + TrainingCorpusFileName = rawTrainingDataFileName, + TemplateFileName = template, + ModelFileName = modelFileName, + }); + + return result; + } + + private List Merge(NlpDoc doc, List tokens, List entities) + { + List trainingTuple = new List(); + HashSet entityWordBag = new HashSet(); + int wordCandidateCount = 0; + + for (int i = 0; i < tokens.Count; i++) + { + TrainingIntentExpressionPart curEntity = null; + if (entities == null) continue; + + bool entityFinded = false; + for (int entityIndex = 0; entityIndex < entities.Count; entityIndex++) + { + var entity = entities[entityIndex]; + + if (!entityFinded) + { + var vDoc = new NlpDoc { Sentences = new List { new NlpDocSentence { Text = entity.Value } } }; + doc.Tokenizer.Predict(null, vDoc, null); + string[] words = vDoc.Sentences[0].Tokens.Select(x => x.Text).ToArray(); + + for (int j = 0; j < words.Length; j++) + { + if (tokens[i + j].Text == words[j]) + { + wordCandidateCount++; + if (j == words.Length - 1) + { + curEntity = entity; + } + } + else + { + wordCandidateCount = 0; + break; + } + } + if (wordCandidateCount != 0) // && entity.Start == tokens[i].Offset) + { + String entityName = curEntity.Entity.Contains(":") ? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1) : curEntity.Entity; + + for(int wordIndex = 0; wordIndex < words.Length; wordIndex++) + { + var tag = entityName; + + if (wordIndex == 0) + { + if (words.Length == 1) + { + tag = "S_" + entityName; + } + else + { + tag = "B_" + entityName; + } + } + else if (wordIndex == words.Length - 1) + { + tag = "E_" + entityName; + } + else + { + tag = "M_" + entityName; + } + + var word = words[wordIndex]; + trainingTuple.Add(new TrainingData(tag, word, tokens[i].Pos)); + } + + entityFinded = true; + } + } + } + + if (wordCandidateCount == 0) + { + trainingTuple.Add(new TrainingData("S", tokens[i].Text, tokens[i].Pos)); + } + else + { + i = i + wordCandidateCount - 1; + } + } + + return trainingTuple; + } + + public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) + { + var decoder = new CRFDecoder(); + var options = new DecoderOptions + { + ModelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model) + }; + + //Load encoded model from file + decoder.LoadModel(options.ModelFileName); + + //Create decoder tagger instance. + var tagger = decoder.CreateTagger(options.NBest, options.MaxWord); + tagger.set_vlevel(options.ProbLevel); + + //Initialize result + var crf_out = new CRFSegOut[options.NBest]; + for (var i = 0; i < options.NBest; i++) + { + crf_out[i] = new CRFSegOut(options.MaxWord); + } + + doc.Sentences.ForEach(sent => + { + List> dataset = new List>(); + dataset.AddRange(sent.Tokens.Select(token => new List { token.Text, token.Pos }).ToList()); + //predict given string's tags + decoder.Segment(crf_out, tagger, dataset); + + var entities = new List(); + + for (int i = 0; i < sent.Tokens.Count; i++) + { + var entity = crf_out[0].result_; + entities.Add(new NlpEntity + { + Entity = entity[i], + Start = doc.Sentences[0].Tokens[i].Start, + Value = doc.Sentences[0].Tokens[i].Text, + Confidence = 0, + Extrator = "BotSharpCRFNer" + }); + } + + sent.Entities = MergeEntity(doc.Sentences[0].Text, entities); + }); + + return true; + } + + private List MergeEntity(string sentence, List tokens) + { + List res = new List(); + + for(int i = 0; i < tokens.Count; i++) + { + var entity = tokens[i]; + + if (entity.Entity.StartsWith("S_")) + { + entity.Entity = entity.Entity.Split('_')[1]; + res.Add(entity); + } + else if (entity.Entity.StartsWith("B_")) + { + entity.Entity = entity.Entity.Split('_')[1]; + + for(int j = i; j < tokens.Count; j++) + { + var token = tokens[j]; + if (token.Entity.StartsWith("E_")) + { + res.Add(new NlpEntity + { + Value = sentence.Substring(entity.Start, token.End - entity.Start + 1), + Entity = entity.Entity, + Extrator = entity.Extrator, + Start = entity.Start, + Confidence = entity.Confidence + }); + } + + i++; + } + } + } + + return res; + } + + public class TrainingData + { + public String Token { get; set; } + public String Entity { get; set; } + public String Pos { get; set; } + + public TrainingData(string entity, string token, string pos) + { + Token = token; + Entity = entity; + Pos = pos; + } + } + } +} diff --git a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs index bc1380e5..11845549 100644 --- a/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs +++ b/BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs @@ -40,8 +40,6 @@ namespace BotSharp.Core.Engines.BotSharp classifier.Train(sentences); Console.WriteLine($"Saved model to {modelFileName}"); - meta.Meta = new JObject(); - meta.Meta["compiled at"] = "Sep 12, 2018"; return true; } diff --git a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs index b1d85122..d67c3739 100644 --- a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs @@ -38,7 +38,6 @@ namespace BotSharp.Core.Engines.NERs public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) { - var dc = new DefaultDataContextLoader().GetDefaultDc(); var corpus = agent.Corpus; meta.Model = "ner-crf.model"; diff --git a/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs b/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs deleted file mode 100644 index b1a505b7..00000000 --- a/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs +++ /dev/null @@ -1,108 +0,0 @@ -using BotSharp.Core.Abstractions; -using BotSharp.Core.Agents; -using BotSharp.Core.Models; -using BotSharp.NLP.Tokenize; -using EntityFrameworkCore.BootKit; -using Microsoft.Extensions.Configuration; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using RestSharp; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace BotSharp.Core.Engines.SpaCy -{ - public class NltkTokenizer : INlpTrain, INlpPredict - { - public IConfiguration Configuration { get; set; } - public PipeSettings Settings { get; set; } - - public async Task Train(Agent agent, NlpDoc doc, PipeModel meta) - { - var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value); - var request = new RestRequest("nltktokenizesentences", Method.POST); - List> tokens = new List>(); - Boolean res = true; - var dc = new DefaultDataContextLoader().GetDefaultDc(); - var corpus = agent.Corpus; - - doc.Sentences = new List(); - List sentencesList = new List(); - corpus.UserSays.ForEach ( usersay => sentencesList.Add(usersay.Text)); - - request.RequestFormat = DataFormat.Json; - - request.AddParameter("application/json", JsonConvert.SerializeObject(new Documents(sentencesList)), ParameterType.RequestBody); - - var response = client.Execute(request); - - tokens = response.Data.TokensList; - - for (int i = 0; i < sentencesList.Count; i++) - { - doc.Sentences.Add(new NlpDocSentence - { - Tokens = tokens[i], - Text = sentencesList[i] - }); - } - res = res && response.IsSuccessful; - return res; - /* - corpus.UserSays.ForEach(usersay => { - Console.WriteLine(usersay.Text); - request.AddParameter("text", usersay.Text); - var response = client.Execute(request); - - tokens.Add(response.Data.Tokens); - - doc.Sentences.Add(new NlpDocSentence - { - Tokens = response.Data.Tokens, - Text = usersay.Text - }); - - res = res && response.IsSuccessful; - }); - */ - } - - public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) - { - var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value); - var request = new RestRequest("nltktokenizesentences", Method.POST); - List> tokens = new List>(); - Boolean res = true; - var corpus = agent.Corpus; - - request.AddParameter("sentences", doc.Sentences[0].Text); - var response = client.Execute(request); - - //tokens.Add(response.Data.Tokens); - - res = res && response.IsSuccessful; - - doc.Sentences[0].Tokens = tokens[0]; - - return true; - } - - private class Result - { - public List> TokensList { get; set; } - } - - private class Documents - { - public List Sentences { get; set; } - - public Documents(List sentences) - { - this.Sentences = sentences; - } - } - } -} diff --git a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs index f3811c40..7975823f 100644 --- a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs +++ b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs @@ -88,12 +88,12 @@ namespace BotSharp.Core.Engines.Rasa public void LoadIntents(Agent agent) { string data = File.ReadAllText(Path.Combine(AgentDir, "corpus.json")); - var rasa = JsonConvert.DeserializeObject(data); + var rasa = JsonConvert.DeserializeObject(data); - agent.Intents = rasa.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList(); + agent.Intents = rasa.Data.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList(); agent.Intents.ForEach(intent => { - ImportIntentUserSays(intent, rasa.UserSays); + ImportIntentUserSays(intent, rasa.Data.UserSays); }); } diff --git a/BotSharp.Core/Engines/Rasa/RasaAgent.cs b/BotSharp.Core/Engines/Rasa/RasaAgent.cs index fc9c7748..1851aa3c 100644 --- a/BotSharp.Core/Engines/Rasa/RasaAgent.cs +++ b/BotSharp.Core/Engines/Rasa/RasaAgent.cs @@ -21,4 +21,10 @@ namespace BotSharp.Core.Engines.Rasa [JsonProperty("regex_features")] public List Regex { get; set; } } + + public class RasaAgentImportModel + { + [JsonProperty("rasa_nlu_data")] + public RasaAgent Data { get; set; } + } } diff --git a/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs b/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs index fc247354..2642c4b8 100644 --- a/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs +++ b/BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs @@ -13,266 +13,48 @@ namespace BotSharp.NLP.UnitTest.CRFLite [TestClass] public class DecoderTest { + object rdLocker = new object(); + [TestMethod] public void TestDecode() { - var encoder = new CRFDecoder(); - bool result = Decode(new DecoderOptions + var decoder = new CRFDecoder(); + var options = new DecoderOptions { - InputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\test\test.txt", - ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\model\ner_model_eng", - OutputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\test\output.txt" - }); - } - - object rdLocker = new object(); - - bool Decode(DecoderOptions options) - { - var parallelOption = new ParallelOptions(); - var watch = Stopwatch.StartNew(); - - var sr = new StreamReader(options.InputFileName); - StreamWriter sw = null, swSeg = null; - - if (options.OutputFileName != null && options.OutputFileName.Length > 0) - { - sw = new StreamWriter(options.OutputFileName); - } - if (options.OutputSegFileName != null && options.OutputSegFileName.Length > 0) - { - swSeg = new StreamWriter(options.OutputSegFileName); - } - - //Create CRFSharp wrapper instance. It's a global instance - var crfWrapper = new CRFDecoder(); + ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model" + }; //Load encoded model from file - //Logger.WriteLine("Loading model from {0}", options.strModelFileName); - crfWrapper.LoadModel(options.ModelFileName); + decoder.LoadModel(options.ModelFileName); - var queueRecords = new ConcurrentQueue>>(); - var queueSegRecords = new ConcurrentQueue>>(); + //Create decoder tagger instance. + var tagger = decoder.CreateTagger(options.NBest, options.MaxWord); + tagger.set_vlevel(options.ProbLevel); - parallelOption.MaxDegreeOfParallelism = options.Thread; - Parallel.For(0, options.Thread, parallelOption, t => + //Initialize result + var crf_out = new CRFSegOut[options.NBest]; + for (var i = 0; i < options.NBest; i++) { - - //Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance - var tagger = crfWrapper.CreateTagger(options.NBest, options.MaxWord); - tagger.set_vlevel(options.ProbLevel); - - //Initialize result - var crf_out = new crf_seg_out[options.NBest]; - for (var i = 0; i < options.NBest; i++) - { - crf_out[i] = new crf_seg_out(tagger.crf_max_word_num); - } - - var inbuf = new List>(); - while (true) - { - lock (rdLocker) - { - if (ReadRecord(inbuf, sr) == false) - { - break; - } - - queueRecords.Enqueue(inbuf); - queueSegRecords.Enqueue(inbuf); - } - - //Call CRFSharp wrapper to predict given string's tags - if (swSeg != null) - { - crfWrapper.Segment(crf_out, tagger, inbuf); - } - else - { - crfWrapper.Segment((CRFTermOut[])crf_out, (DecoderTagger)tagger, inbuf); - } - - List> peek = null; - //Save segmented tagged result into file - if (swSeg != null) - { - var rstList = ConvertCRFTermOutToStringList(inbuf, crf_out); - while (peek != inbuf) - { - queueSegRecords.TryPeek(out peek); - } - for (int index = 0; index < rstList.Count; index++) - { - var item = rstList[index]; - swSeg.WriteLine(item); - } - queueSegRecords.TryDequeue(out peek); - peek = null; - } - - //Save raw tagged result (with probability) into file - if (sw != null) - { - while (peek != inbuf) - { - queueRecords.TryPeek(out peek); - } - OutputRawResultToFile(inbuf, crf_out, tagger, sw); - queueRecords.TryDequeue(out peek); - - } - } - }); - - - sr.Close(); - - if (sw != null) - { - sw.Close(); + crf_out[i] = new CRFSegOut(options.MaxWord); } - if (swSeg != null) - { - swSeg.Close(); - } - watch.Stop(); - //Logger.WriteLine("Elapsed: {0} ms", watch.ElapsedMilliseconds); - return true; + var dataset = GetTestData(); + + //predict given string's tags + decoder.Segment(crf_out, tagger, dataset); } - private bool ReadRecord(List> inbuf, StreamReader sr) + private List> GetTestData() { - inbuf.Clear(); + var dataset = new List>(); - while (true) - { - var strLine = sr.ReadLine(); - if (strLine == null) - { - //At the end of current file - if (inbuf.Count == 0) - { - return false; - } - else - { - return true; - } - } - strLine = strLine.Trim(); - if (strLine.Length == 0) - { - return true; - } + dataset.Add(new List { "'", "PUN" }); + dataset.Add(new List { "'", "POS" }); + dataset.Add(new List { "Duchy", "NNP" }); + dataset.Add(new List { "of", "IN" }); + dataset.Add(new List { "Lithuania", "NNP" }); - //Read feature set for each record - var items = strLine.Split(new char[] { '\t' }); - inbuf.Add(new List()); - for (int index = 0; index < items.Length; index++) - { - var item = items[index]; - inbuf[inbuf.Count - 1].Add(item); - } - } - } - - //Output raw result with probability - private void OutputRawResultToFile(List> inbuf, CRFTermOut[] crf_out, SegDecoderTagger tagger, StreamWriter sw) - { - for (var k = 0; k < crf_out.Length; k++) - { - if (crf_out[k] == null) - { - //No more result - break; - } - - var sb = new StringBuilder(); - - var crf_seg_out = crf_out[k]; - //Show the entire sequence probability - //For each token - for (var i = 0; i < inbuf.Count; i++) - { - //Show all features - for (var j = 0; j < inbuf[i].Count; j++) - { - sb.Append(inbuf[i][j]); - sb.Append("\t"); - } - - //Show the best result and its probability - sb.Append(crf_seg_out.result_[i]); - - if (tagger.vlevel_ > 1) - { - sb.Append("\t"); - sb.Append(crf_seg_out.weight_[i]); - - //Show the probability of all tags - sb.Append("\t"); - for (var j = 0; j < tagger.ysize_; j++) - { - sb.Append(tagger.yname(j)); - sb.Append("/"); - sb.Append(tagger.prob(i, j)); - - if (j < tagger.ysize_ - 1) - { - sb.Append("\t"); - } - } - } - sb.AppendLine(); - } - if (tagger.vlevel_ > 0) - { - sw.WriteLine("#{0}", crf_seg_out.prob); - } - sw.WriteLine(sb.ToString().Trim()); - sw.WriteLine(); - } - } - - //Convert CRFSharp output format to string list - private List ConvertCRFTermOutToStringList(List> inbuf, crf_seg_out[] crf_out) - { - var sb = new StringBuilder(); - for (var i = 0; i < inbuf.Count; i++) - { - sb.Append(inbuf[i][0]); - } - - var strText = sb.ToString(); - var rstList = new List(); - for (var i = 0; i < crf_out.Length; i++) - { - if (crf_out[i] == null) - { - //No more result - break; - } - - sb.Clear(); - var crf_term_out = crf_out[i]; - for (var j = 0; j < crf_term_out.Count; j++) - { - var str = strText.Substring(crf_term_out.tokenList[j].offset, crf_term_out.tokenList[j].length); - var strNE = crf_term_out.tokenList[j].strTag; - - sb.Append(str); - if (strNE.Length > 0) - { - sb.Append("[" + strNE + "]"); - } - sb.Append(" "); - } - rstList.Add(sb.ToString().Trim()); - } - - return rstList; + return dataset; } } } diff --git a/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs b/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs index 36bd1dd5..5f4d82ce 100644 --- a/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs +++ b/BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs @@ -1,22 +1,47 @@ using BotSharp.Models.CRFLite; +using BotSharp.Models.CRFLite.Decoder; using BotSharp.Models.CRFLite.Encoder; using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; namespace BotSharp.NLP.UnitTest.CRFLite { [TestClass] public class EncoderTest { + /// + /// + /// [TestMethod] public void TestEncode() { var encoder = new CRFEncoder(); bool result = encoder.Learn(new EncoderOptions { - TrainingCorpusFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\corpus\eng.1K.training", - TemplateFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\template.NE", - ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\model\ner_model_eng" + /* + * traing corups format, split by tab, sentences is seperated by blank row + * + ! PUN S + Tokyo NNP S_LOCATION + and CC S + New NNP B_LOCATION + York NNP E_LOCATION + are VBP S + major JJ S + financial JJ S + centers NNS S + . PUN S + */ + TrainingCorpusFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\eng.1k.training", + TemplateFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\template.en", + ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model" }); + + Assert.IsTrue(result); } } } diff --git a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs index 2b8c2e0b..331caed3 100644 --- a/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs +++ b/BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs @@ -39,16 +39,17 @@ namespace BotSharp.NLP.UnitTest var options = new ClassifyOptions { ModelFilePath = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"), - TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange") + TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"), + Dimension = 100 }; var classifier = new ClassifierFactory(options, SupportedLanguage.English); - var dataset = sentences.Split(1M); + var dataset = sentences.Split(0.7M); classifier.Train(dataset.Item1); int correct = 0; int total = 0; - dataset.Item1.ForEach(td => + dataset.Item2.ForEach(td => { var classes = classifier.Classify(td); if (td.Label == classes[0].Item1) @@ -127,5 +128,53 @@ namespace BotSharp.NLP.UnitTest return genders; } + + [TestMethod] + public void SpotifyTest() + { + var reader = new FasttextDataReader(); + var sentences = reader.Read(new ReaderOptions + { + DataDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "spotify"), + FileName = "spotify.txt" + }); + + var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English); + var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList()); + for (int i = 0; i < newSentences.Count; i++) + { + newSentences[i].Label = sentences[i].Label; + } + sentences = newSentences.ToList(); + + sentences.Shuffle(); + + var options = new ClassifyOptions + { + ModelFilePath = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "spotify", "nb.model"), + TrainingCorpusDir = Path.Combine(Configuration.GetValue("MachineLearning:dataDir"), "Text Classification", "spotify") + }; + var classifier = new ClassifierFactory(options, SupportedLanguage.English); + + var dataset = sentences.Split(0.7M); + classifier.Train(dataset.Item1); + + int correct = 0; + int total = 0; + dataset.Item2.ForEach(td => + { + var classes = classifier.Classify(td); + if (td.Label == classes[0].Item1) + { + correct++; + } + total++; + }); + + var accuracy = (float)correct / total; + + Assert.IsTrue(accuracy > 0.6); + } + } } diff --git a/BotSharp.NLP/Classify/ClassifierFactory.cs b/BotSharp.NLP/Classify/ClassifierFactory.cs index 5f91b735..cc754488 100644 --- a/BotSharp.NLP/Classify/ClassifierFactory.cs +++ b/BotSharp.NLP/Classify/ClassifierFactory.cs @@ -44,7 +44,9 @@ namespace BotSharp.NLP.Classify var classes = _classifier.Classify(sentence, options); - return classes.OrderByDescending(x => x.Item2).ToList(); + classes = classes.OrderByDescending(x => x.Item2).ToList(); + + return classes; } } } diff --git a/BotSharp.NLP/Classify/ClassifyOptions.cs b/BotSharp.NLP/Classify/ClassifyOptions.cs index fce017fd..0c213fad 100644 --- a/BotSharp.NLP/Classify/ClassifyOptions.cs +++ b/BotSharp.NLP/Classify/ClassifyOptions.cs @@ -13,5 +13,10 @@ namespace BotSharp.NLP.Classify public string PrediceOutputFile { get; set; } public string TransformFilePath { get; set; } public RangeTransform Transform { get; set; } + + /// + /// Feature dimension + /// + public int Dimension { get; set; } } } diff --git a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs index 38f28d7b..8288dd28 100644 --- a/BotSharp.NLP/Classify/NaiveBayesClassifier.cs +++ b/BotSharp.NLP/Classify/NaiveBayesClassifier.cs @@ -55,12 +55,13 @@ namespace BotSharp.NLP.Classify public void Train(List sentences, ClassifyOptions options) { var tfidf = new TfIdfFeatureExtractor(); + tfidf.Dimension = options.Dimension; tfidf.Sentences = sentences; tfidf.CalBasedOnCategory(); - var keyWords = tfidf.Features(); - string keywords2 = String.Join(",", keyWords.ToArray()); + var encoder = new OneHotEncoder(); encoder.Sentences = sentences; + encoder.Words = tfidf.Keywords(); words = encoder.EncodeAll(); var featureSets = sentences.Select(x => new Tuple(x.Label, x.Vector)).ToList(); @@ -118,7 +119,8 @@ namespace BotSharp.NLP.Classify lf.Prob = nb.PosteriorProb(); });*/ - return results; + double total = results.Select(x => x.Item2).Sum(); + return results.Select(x => new Tuple(x.Item1, x.Item2 / total)).ToList(); } public string SaveModel(ClassifyOptions options) diff --git a/BotSharp.NLP/Featuring/IFeatureExtractor.cs b/BotSharp.NLP/Featuring/IFeatureExtractor.cs index 785f1d30..497e4a13 100644 --- a/BotSharp.NLP/Featuring/IFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/IFeatureExtractor.cs @@ -6,5 +6,9 @@ namespace BotSharp.NLP.Featuring { public interface IFeatureExtractor { + /// + /// Feature dimension size + /// + int Dimension { get; set; } } } diff --git a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs index 8f3ccf77..e7132aa7 100644 --- a/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs +++ b/BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs @@ -34,18 +34,35 @@ namespace BotSharp.NLP.Featuring private List> tfs; private List Categories { get; set; } + public int Dimension { get; set; } public void Extract(Sentence sentence) { } - public List Features() + public List Keywords() { + if(Dimension == 0) + { + Dimension = Categories.Count * 3; + + if(Dimension > 300) + { + Dimension = 300; + } + + if(Dimension < 30) + { + Dimension = 30; + } + } + var tfs2 = tfs.OrderByDescending(x => x.Item2) .Select(x => x.Item1) .Distinct() - .Take(Sentences.Count / Categories.Count) + .Take(Dimension) + .OrderBy(x => x) .ToList(); return tfs2; @@ -59,7 +76,7 @@ namespace BotSharp.NLP.Featuring Sentences.ForEach(sent => { - sent.Words.ForEach(word => + sent.Words.Where(x => x.IsAlpha).ToList().ForEach(word => { // TF int c1 = sent.Words.Count(x => x.Lemma == word.Lemma); @@ -82,6 +99,17 @@ namespace BotSharp.NLP.Featuring Categories = Sentences.Select(x => x.Label).Distinct().ToList(); + List> allTextByCategory = new List>(); + + Categories.ForEach(label => + { + var allTokens = new List(); + Sentences.Where(x => x.Label == label) + .ToList() + .ForEach(s => allTokens.AddRange(s.Words)); + allTextByCategory.Add(new Tuple(label, String.Join(" ", allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma)))); + }); + Categories.ForEach(label => { var allTokens = new List(); @@ -89,7 +117,7 @@ namespace BotSharp.NLP.Featuring .ToList() .ForEach(s => allTokens.AddRange(s.Words)); - allTokens.Select(x => x.Lemma).Distinct() + allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma).Distinct() .ToList() .ForEach(word => { @@ -98,8 +126,15 @@ namespace BotSharp.NLP.Featuring double tf = (c1 + 1.0) / allTokens.Count(); // IDF - var c2 = Sentences.Where(s => s.Words.Select(x => x.Lemma).Contains(word)) - .GroupBy(x => x.Label).Count(); + var c2 = 0; + allTextByCategory.ForEach(all => + { + if(Regex.IsMatch(all.Item2, word)) + { + c2++; + } + }); + double idf = Math.Log(Categories.Count / (c2 + 1.0)); tfs.Add(new Tuple(word, tf * idf)); diff --git a/BotSharp.NLP/Models/CRFLite/CRFDecoder.cs b/BotSharp.NLP/Models/CRFLite/CRFDecoder.cs index 38f5fb9e..2686a4c3 100644 --- a/BotSharp.NLP/Models/CRFLite/CRFDecoder.cs +++ b/BotSharp.NLP/Models/CRFLite/CRFDecoder.cs @@ -74,7 +74,7 @@ namespace BotSharp.Models.CRFLite } //Segment given text - public int Segment(crf_seg_out[] pout, //segment result + public int Segment(CRFSegOut[] pout, //segment result SegDecoderTagger tagger, //Tagger per thread List> inbuf //feature set for segment ) diff --git a/BotSharp.NLP/Models/CRFLite/CRFEncoder.cs b/BotSharp.NLP/Models/CRFLite/CRFEncoder.cs index cc85b626..886e6f71 100644 --- a/BotSharp.NLP/Models/CRFLite/CRFEncoder.cs +++ b/BotSharp.NLP/Models/CRFLite/CRFEncoder.cs @@ -61,24 +61,17 @@ namespace BotSharp.Models.CRFLite var xList = modelWriter.ReadAllRecords(); - modelWriter.Shrink(xList, args.MinFeatureFreq); if (!modelWriter.SaveModelMetaData(args.ModelFileName)) { return false; } - else - { - } if (!modelWriter.BuildFeatureSetIntoIndex(args.ModelFileName, args.SlotUsageRateThreshold, args.DebugLevel)) { return false; } - else - { - } if (xList.Length == 0) { @@ -101,6 +94,8 @@ namespace BotSharp.Models.CRFLite bool runCRF(EncoderTagger[] x, ModelWriter modelWriter, bool orthant, EncoderOptions args) { + Console.WriteLine("Running encoding process..."); + var old_obj = double.MaxValue; var converge = 0; var lbfgs = new LBFGS(args.ThreadsNum); @@ -165,6 +160,9 @@ namespace BotSharp.Models.CRFLite lbfgs.err += processList[i].err; lbfgs.zeroone += processList[i].zeroone; + Console.WriteLine($"Thread: {i}, Iterating {itr} / {args.MaxIteration}"); + Console.WriteLine($"{lbfgs.obj} {lbfgs.err} {lbfgs.zeroone}"); + //Calculate error for (var j = 0; j < modelWriter.y_.Count; j++) { @@ -263,6 +261,8 @@ namespace BotSharp.Models.CRFLite } } + Console.WriteLine("Completed encoding process."); + return true; } diff --git a/BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs b/BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs index ed8e234c..c5949ccd 100644 --- a/BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs +++ b/BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs @@ -7,13 +7,13 @@ namespace BotSharp.Models.CRFLite { public class SegToken { - public int offset; - public int length; - public string strTag; //CRF对应于term组合后的Tag字符串 - public double fWeight; //对应属性id的概率值，或者得分 + public int Offset; + public int Length; + public string Tag; + public double Weight; }; - public class crf_seg_out : CRFTermOut + public class CRFSegOut : CRFTermOut { //Segmented token by merging raw CRF model output public int termTotalLength; // the total term length in character @@ -30,7 +30,7 @@ namespace BotSharp.Models.CRFLite tokenList.Clear(); } - public crf_seg_out(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM): + public CRFSegOut(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM): base(max_word_num) { termTotalLength = 0; diff --git a/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs b/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs index f7834fd0..b7b19d65 100644 --- a/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs +++ b/BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs @@ -8,7 +8,7 @@ namespace BotSharp.Models.CRFLite.Decoder public class DecoderOptions { /// - /// + /// Model file path /// public string ModelFileName; @@ -17,16 +17,6 @@ namespace BotSharp.Models.CRFLite.Decoder /// public string InputFileName; - /// - /// - /// - public string OutputFileName; - - /// - /// - /// - public string OutputSegFileName; - /// /// /// @@ -43,16 +33,16 @@ namespace BotSharp.Models.CRFLite.Decoder public int ProbLevel; /// - /// + /// Max words length in one sentence /// public int MaxWord; public DecoderOptions() { Thread = 1; - NBest = 1; + NBest = 2; ProbLevel = 0; - MaxWord = 100; + MaxWord = 128; } } } diff --git a/BotSharp.NLP/Models/CRFLite/Decoder/ModelReader.cs b/BotSharp.NLP/Models/CRFLite/Decoder/ModelReader.cs index 880a06a4..60f8dc63 100644 --- a/BotSharp.NLP/Models/CRFLite/Decoder/ModelReader.cs +++ b/BotSharp.NLP/Models/CRFLite/Decoder/ModelReader.cs @@ -68,7 +68,7 @@ namespace BotSharp.Models.CRFLite.Decoder LoadFeatureWeights(); } - //获取key对应的特征id + //get key feature id public virtual int get_id(string str) { return da.SearchByPerfectMatch(str); @@ -143,26 +143,20 @@ namespace BotSharp.Models.CRFLite.Decoder var sr = new StreamReader(metadataStream); string strLine; - //读入版本号 strLine = sr.ReadLine(); version = uint.Parse(strLine.Split(':')[1].Trim()); - //读入cost_factor strLine = sr.ReadLine(); cost_factor_ = double.Parse(strLine.Split(':')[1].Trim()); - //读入maxid strLine = sr.ReadLine(); maxid_ = long.Parse(strLine.Split(':')[1].Trim()); - //读入xsize strLine = sr.ReadLine(); xsize_ = uint.Parse(strLine.Split(':')[1].Trim()); - //读入空行 strLine = sr.ReadLine(); - //读入待标注的标签 y_ = new List(); while (true) { @@ -174,7 +168,7 @@ namespace BotSharp.Models.CRFLite.Decoder y_.Add(strLine); } - //读入unigram和bigram模板 + // load unigram and bigram template unigram_templs_ = new List(); bigram_templs_ = new List(); while (sr.EndOfStream == false) diff --git a/BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs b/BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs index 410862cb..7ea76303 100644 --- a/BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs +++ b/BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs @@ -14,7 +14,7 @@ namespace BotSharp.Models.CRFLite.Encoder /// /// Minimum feature frequency, if one feature's frequency is less than this value, the feature will be dropped. /// - public int MinFeatureFreq = 2; + public int MinFeatureFreq = 1; /// /// Minimum diff value, when diff less than the value consecutive 3 times, the process will be ended. @@ -79,7 +79,7 @@ namespace BotSharp.Models.CRFLite.Encoder public EncoderOptions() { MaxIteration = 100; - MinFeatureFreq = 2; + MinFeatureFreq = 1; MinDifference = 0.0001; SlotUsageRateThreshold = 0.95; ThreadsNum = 1; diff --git a/BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs b/BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs index b440bd8b..c711de3f 100644 --- a/BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs +++ b/BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs @@ -43,6 +43,8 @@ namespace BotSharp.Models.CRFLite.Encoder //Regenerate feature id and shrink features with lower frequency public void Shrink(EncoderTagger[] xList, int freq) { + Console.WriteLine($"Shrink features lower than {freq} frequency"); + var old2new = new CRFLite.Utils.BTreeDictionary(); featureLexicalDict.Shrink(freq); maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count); @@ -86,7 +88,7 @@ namespace BotSharp.Models.CRFLite.Encoder var oldValue = Interlocked.Increment(ref arrayEncoderTaggerSize) - 1; arrayEncoderTagger[oldValue] = _x; - if (oldValue % 10000 == 0) + if (oldValue % 100 == 0) { //Show current progress on console Console.Write("{0}...", oldValue); @@ -94,10 +96,11 @@ namespace BotSharp.Models.CRFLite.Encoder } }); + Console.WriteLine($"Read {trainCorpusList.Count} records"); + trainCorpusList.Clear(); trainCorpusList = null; - - Console.WriteLine(); + return arrayEncoderTagger; } @@ -136,6 +139,8 @@ namespace BotSharp.Models.CRFLite.Encoder //Save indexed feature set into file da.save(filename_featureset); + Console.WriteLine($"Saved featureset to {filename_featureset}"); + if (string.IsNullOrWhiteSpace(modelFileName)) { //Clean up all data @@ -220,6 +225,7 @@ namespace BotSharp.Models.CRFLite.Encoder tofs.Close(); + Console.WriteLine($"Saved meta data to {filename}"); return true; } @@ -277,6 +283,8 @@ namespace BotSharp.Models.CRFLite.Encoder bool OpenTemplateFile(string filename) { + Console.WriteLine($"Open template: {filename}"); + var ifs = new StreamReader(filename); unigram_templs_ = new List(); bigram_templs_ = new List(); @@ -305,6 +313,8 @@ namespace BotSharp.Models.CRFLite.Encoder bool OpenTrainCorpusFile(string strTrainingCorpusFileName) { + Console.WriteLine($"Open corpus: {strTrainingCorpusFileName}"); + var ifs = new StreamReader(strTrainingCorpusFileName); y_ = new List(); trainCorpusList = new List>>(); diff --git a/BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs b/BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs index 6904b6c0..8a1a9a98 100644 --- a/BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs +++ b/BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs @@ -13,7 +13,7 @@ namespace BotSharp.Models.CRFLite crf_max_word_num = this_crf_max_word_num; } - int seg_termbuf_build(crf_seg_out term_buf) + int seg_termbuf_build(CRFSegOut term_buf) { term_buf.Clear(); @@ -42,24 +42,24 @@ namespace BotSharp.Models.CRFLite i == x_.Count - 1) { var tkn = new SegToken(); - tkn.length = term_len; - tkn.offset = term_buf.termTotalLength; + tkn.Length = term_len; + tkn.Offset = term_buf.termTotalLength; var spos = strTag.IndexOf('_'); if (spos < 0) { if (strTag == "NOR") { - tkn.strTag = ""; + tkn.Tag = ""; } else { - tkn.strTag = strTag; + tkn.Tag = strTag; } } else { - tkn.strTag = strTag.Substring(spos + 1); + tkn.Tag = strTag.Substring(spos + 1); } term_buf.termTotalLength += term_len; @@ -67,10 +67,10 @@ namespace BotSharp.Models.CRFLite switch (vlevel_) { case 0: - tkn.fWeight = 0.0; + tkn.Weight = 0.0; break; case 2: - tkn.fWeight = weight / num; + tkn.Weight = weight / num; weight = 0.0; num = 0; break; @@ -86,7 +86,7 @@ namespace BotSharp.Models.CRFLite } - public int output(crf_seg_out[] pout) + public int output(CRFSegOut[] pout) { var n = 0; var ret = 0; diff --git a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs index 510a37d9..d220c426 100644 --- a/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs +++ b/BotSharp.NLP/Txt2Vec/OneHotEncoder.cs @@ -25,7 +25,7 @@ namespace BotSharp.NLP.Txt2Vec sentence.Words.ForEach(w => { - int index = Words.IndexOf(w.Lemma.ToLower()); + int index = Words.IndexOf(w.Lemma); if(index > 0) { vector[index] = 1; @@ -49,7 +49,12 @@ namespace BotSharp.NLP.Txt2Vec { if (Words == null) { - // Words = "shuffle,pause,resume,next,stop,previous,continue,mode,repeat,back,music,play,enough,off,them,playlist,skip,restart,favourites,on,add,go,again,turn,save,my,station,favourite,start,by,playing,please,now,running,move".Split(',').ToList(); + Words = new List(); + Sentences.ForEach(x => + { + Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Lemma)); + }); + Words = Words.Distinct().OrderBy(x => x).ToList(); } return Words; diff --git a/BotSharp.RestApi/AgentController.cs b/BotSharp.RestApi/AgentController.cs index bc3a4455..aee08112 100644 --- a/BotSharp.RestApi/AgentController.cs +++ b/BotSharp.RestApi/AgentController.cs @@ -38,9 +38,22 @@ namespace BotSharp.RestApi [HttpGet] public ActionResult> AllAgents() { - var dc = new DefaultDataContextLoader().GetDefaultDc(); + List agents = new List(); - return dc.Table().ToList(); + string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects"); + + var names = Directory.EnumerateDirectories(agentDir).Select(x => x.Split(Path.DirectorySeparatorChar).Last()).ToList(); + + names.ForEach(name => + { + agents.Add(new Agent + { + Name = name + }); + + }); + + return agents; } /// diff --git a/BotSharp.RestApi/BotSharp.RestApi.xml b/BotSharp.RestApi/BotSharp.RestApi.xml index 9ec6d8f0..2231b601 100644 --- a/BotSharp.RestApi/BotSharp.RestApi.xml +++ b/BotSharp.RestApi/BotSharp.RestApi.xml @@ -120,7 +120,7 @@ Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project. Model name - + Agent name or agent id diff --git a/BotSharp.RestApi/Rasa/TrainController.cs b/BotSharp.RestApi/Rasa/TrainController.cs index ae2eb3cb..a0d2785e 100644 --- a/BotSharp.RestApi/Rasa/TrainController.cs +++ b/BotSharp.RestApi/Rasa/TrainController.cs @@ -38,10 +38,10 @@ namespace BotSharp.RestApi.Rasa /// Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project. /// /// Model name - /// + /// Agent name or agent id /// [HttpPost] - public async Task> Train([FromQuery] string model, [FromQuery] string project) + public async Task> Train([FromQuery] string project, [FromQuery] string model) { string body = ""; using (var reader = new StreamReader(Request.Body)) diff --git a/BotSharp.UI/package-lock.json b/BotSharp.UI/package-lock.json index 180561de..0d286e99 100644 --- a/BotSharp.UI/package-lock.json +++ b/BotSharp.UI/package-lock.json @@ -3487,7 +3487,7 @@ "dev": true, "requires": { "boom": "2.x.x", - "cryptiles": "2.x.x", + "cryptiles": "~>4.1.2", "hoek": "2.x.x", "sntp": "1.x.x" } @@ -8726,7 +8726,7 @@ "integrity": "sha1-iSIN32t1GuUrX3JISGNShZa7hME=", "dev": true, "requires": { - "macaddress": "^0.2.8" + "macaddress": "~>0.2.9" } }, "uniqs": { diff --git a/BotSharp.WebHost/Algorithms/crfsuite b/BotSharp.WebHost/Algorithms/crfsuite deleted file mode 100755 index 3fab0c91..00000000 Binary files a/BotSharp.WebHost/Algorithms/crfsuite and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/crfsuite.exe b/BotSharp.WebHost/Algorithms/crfsuite.exe deleted file mode 100644 index 74c7ad44..00000000 Binary files a/BotSharp.WebHost/Algorithms/crfsuite.exe and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/cyggcc_s-seh-1.dll b/BotSharp.WebHost/Algorithms/cyggcc_s-seh-1.dll deleted file mode 100644 index e5d3d8c8..00000000 Binary files a/BotSharp.WebHost/Algorithms/cyggcc_s-seh-1.dll and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/cygstdc++-6.dll b/BotSharp.WebHost/Algorithms/cygstdc++-6.dll deleted file mode 100644 index 03606607..00000000 Binary files a/BotSharp.WebHost/Algorithms/cygstdc++-6.dll and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/cygwin1.dll b/BotSharp.WebHost/Algorithms/cygwin1.dll deleted file mode 100644 index e2a5919a..00000000 Binary files a/BotSharp.WebHost/Algorithms/cygwin1.dll and /dev/null differ diff --git a/BotSharp.WebHost/Algorithms/fasttext.exe b/BotSharp.WebHost/Algorithms/fasttext.exe deleted file mode 100644 index 0ffcdd4c..00000000 Binary files a/BotSharp.WebHost/Algorithms/fasttext.exe and /dev/null differ diff --git a/BotSharp.WebHost/App_Data/CRFLite/template.en b/BotSharp.WebHost/App_Data/CRFLite/template.en new file mode 100644 index 00000000..5b009054 --- /dev/null +++ b/BotSharp.WebHost/App_Data/CRFLite/template.en @@ -0,0 +1,18 @@ +# Unigram +U01:%x[-1,0] +U02:%x[0,0] +U03:%x[1,0] +U05:%x[-1,0]/%x[0,0] +U06:%x[0,0]/%x[1,0] + +U11:%x[-1,1] +U12:%x[0,1] +U13:%x[1,1] +U16:%x[-1,1]/%x[0,1] +U17:%x[0,1]/%x[1,1] + +U20:%x[-1,0]/%x[0,0]/%x[1,0] +U21:%x[-1,1]/%x[0,1]/%x[1,1] + +# Bigram +B diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj b/BotSharp.WebHost/BotSharp.WebHost.csproj index 3c77fc2f..e5032769 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj @@ -37,7 +37,6 @@ - @@ -45,7 +44,6 @@ - @@ -53,7 +51,6 @@ - @@ -61,7 +58,6 @@ - diff --git a/BotSharp.WebHost/Program.cs b/BotSharp.WebHost/Program.cs index 09362f52..23601a8c 100644 --- a/BotSharp.WebHost/Program.cs +++ b/BotSharp.WebHost/Program.cs @@ -18,10 +18,20 @@ namespace BotSharp.WebHost .ConfigureAppConfiguration((hostingContext, config) => { var env = hostingContext.HostingEnvironment; - string dir = Path.GetFullPath(env.ContentRootPath + "/.."); - var settings = Directory.GetFiles(Path.Combine(dir, "Settings"), "*.json"); + string dir = Path.GetFullPath(env.ContentRootPath); + string settingsFolder = Path.Combine(dir, "Settings"); + + if (!Directory.Exists(settingsFolder)) + { + dir = Path.GetFullPath(env.ContentRootPath + "/.."); + } + + settingsFolder = Path.Combine(dir, "Settings"); + Console.WriteLine($"Settings folder: {settingsFolder}"); + var settings = Directory.GetFiles(settingsFolder, "*.json"); settings.ToList().ForEach(setting => { + Console.WriteLine($"Read {setting}"); config.AddJsonFile(setting, optional: false, reloadOnChange: true); }); }) diff --git a/Dockerfile b/Dockerfile index 93fae0a5..9da2a29c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,15 +5,13 @@ WORKDIR /source # copies the rest of your code COPY . . RUN dotnet build -RUN dotnet publish --output /app/ --configuration Debug +RUN dotnet publish --output /app --configuration RASA -# install facebookresearch fasttext -RUN wget https://github.com/facebookresearch/fastText/archive/v0.1.0.zip -RUN apt-get update -RUN apt-get install -y unzip make g++ -RUN unzip v0.1.0.zip -WORKDIR /source/fastText-0.1.0/ -RUN make +# copy Settings folder +COPY Settings /app/Settings + +# App_Data +COPY BotSharp.WebHost/App_Data /app/App_Data # stage 2: run WORKDIR /app diff --git a/README.rst b/README.rst index a6c0f753..30f78333 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,14 @@ -The Open Source AI Bot Platform Builder for Enterprise +The Open Source AI Bot Platform Builder ====================================================== -*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to our enterprise developers using the BotSharp Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your enterprise robots and precisely control every step of the AI processing pipeline."* +.. image:: https://img.shields.io/badge/gitter-join%20chat-brightgreen.svg + :target: `gitter`_ -**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in enterprise information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. +*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to our .NET developers using the BotSharp AI BOT Platform Builder to build a CaaP. It opens up as much learning power as possible for your own robots and precisely control every step of the AI processing pipeline."* -It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. More friendly to corporate developers. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope. +**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. + +It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. More friendly to corporate developers. BotSharp adopts machine learning algrithm in C# directly. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope. Why we do this? because we all know python is not friendly programming language for enterprise developers, it's not only because it's low performance but also it's a type weak language, it will be a disater if you use python to build your bussiness system. @@ -45,3 +48,4 @@ If you feel that this project is helpful to you, please Star on the project, we .. _Docker: https://www.docker.com .. _Rasa UI: https://github.com/paschmann/rasa-ui .. _Articulate UI: https://spg.ai/projects/articulate +.. _gitter: https://gitter.im/botsharpcore/Lobby diff --git a/Settings/bot.json b/Settings/bot.json index 34e48d70..b41b7b92 100644 --- a/Settings/bot.json +++ b/Settings/bot.json @@ -7,19 +7,23 @@ }, "Pipe": { - "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier", - "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier" + "train": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier", + "predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier" }, "BotSharpNBayesClassifier": { }, "BotSharpSVMClassifier": { - "wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data" + "wordvec": "" }, "BotSharpTagger": { }, + + "BotSharpCRFNer": { + "template": "|App_Data|CRFLite/template.en" + }, "CRFsuiteEntityRecognizer": { "fields": "y w pos chk", diff --git a/Settings/swagger.json b/Settings/swagger.json index 8d3b771c..18f714be 100644 --- a/Settings/swagger.json +++ b/Settings/swagger.json @@ -2,7 +2,7 @@ "Swagger": { "Contact": { "Email": "haiping008@gmail.com", - "Name": "Haiping Chen", + "Name": "BotSharp AI", "Url": "https://github.com/Oceania2018" }, "Description": "BotSharp is a chatbot platform written in C# (.net core), and it's developed for enterprise usage.", diff --git a/docker-compose-articulateui.yml b/docker-compose-articulateui.yml index 3241d002..550b28b4 100644 --- a/docker-compose-articulateui.yml +++ b/docker-compose-articulateui.yml @@ -2,7 +2,7 @@ version: '3.0' services: api: - image: samtecspg/articulate-api:repo-head + image: samtecspg/articulate-api:0.12.1 ports: ['0.0.0.0:7500:7500'] networks: ['botsharp-network'] entrypoint: ['node', 'start.js'] @@ -10,7 +10,7 @@ services: - SWAGGER_BASE_PATH ui: - image: samtecspg/articulate-ui:repo-head + image: samtecspg/articulate-ui:0.12.1 ports: ['0.0.0.0:3000:3000'] networks: ['botsharp-network'] environment: @@ -21,10 +21,12 @@ services: image: botsharpdocker/botsharp:latest ports: ['0.0.0.0:5000:5000'] networks: ['botsharp-network'] + duckling: image: samtecspg/duckling:0.1.6.0 ports: ['0.0.0.0:8000:8000'] networks: ['botsharp-network'] + redis: image: redis:4.0.6-alpine ports: ['0.0.0.0:6379:6379'] diff --git a/docs/agent/import-agent.rst b/docs/agent/import-agent.rst new file mode 100644 index 00000000..98c5c599 --- /dev/null +++ b/docs/agent/import-agent.rst @@ -0,0 +1,45 @@ +Import Agent +============ +Designed as a multi-platform framework, BotSharp allows developers to create their own Bot platforms and support multiple Bot platform services. It supports multiple Bot import, export and message reply formats such as Dialogflow and Rasa. +Support for importing and exporting between platforms. + +**First, export agent from other chatbot platform.** + +In general, the platform provides the ability to export to a compressed file. Different platform has different export method. + +**Second, add meta.json to zip file.** + +meta.json is used to tell BotSharp where the agent is exported from. It should looks like below: + +.. code-block:: json + + { + "Id": "YOURS", + "Name": "YOURS", + "Platform": "Dialogflow", + "ClientAccessToken": "YOURS", + "DeveloperAccessToken": "YOURS", + "Integrations": [] + } + +Extract zip file and add the meta.json to the zip file. + +1. Google Dialogflow +:: + +"Platform": "Dialogflow" + +2. RASA +:: + +"Platform": "Rasa" + +3. Microsoft LUIS + +**Last, upload updated zip file.** + +Upload zip file in REST API. + +|RestoreAgentFromZipScreenshot| + +.. |RestoreAgentFromZipScreenshot| image:: /static/screenshots/RestoreAgentFromZip.png \ No newline at end of file diff --git a/docs/agent/optimize-agent.rst b/docs/agent/optimize-agent.rst new file mode 100644 index 00000000..16ebbf11 --- /dev/null +++ b/docs/agent/optimize-agent.rst @@ -0,0 +1,4 @@ +Optimized Robot +=============== + +The semantic understanding ability of the robot can be improved by modifying the parameters of the training and the selection of the hyperparameters. \ No newline at end of file diff --git a/docs/agent/test-agent.rst b/docs/agent/test-agent.rst new file mode 100644 index 00000000..b478ceab --- /dev/null +++ b/docs/agent/test-agent.rst @@ -0,0 +1,13 @@ +Test Agent +=========== + +After the training is complete, you can start testing the Agent. Enter the robot name and the statement you want to test. + +|APITestInputScreenshot| + +After clicking the execution, you will get the result returned by the server, which contains the user intent and the entity value. + +|APITestResultScreenshot| + +.. |APITestInputScreenshot| image:: /static/screenshots/APITestInput.png +.. |APITestResultScreenshot| image:: /static/screenshots/APITestResult.png \ No newline at end of file diff --git a/docs/agent/train-agent.rst b/docs/agent/train-agent.rst new file mode 100644 index 00000000..f0b0a302 --- /dev/null +++ b/docs/agent/train-agent.rst @@ -0,0 +1,20 @@ +Train Agent +=========== + +When you successfully import the Agent, the next step is to train your Agent and let it run according to your pre-designed process. + +Fill in your Agent name and click the "Train" button for a while (depending on the size of the data). + +|APITrainStartScreenshot| + +During the training, the console will enter the training status immediately. + +|APITrainInProgressScreenshot| + +After the training is completed, you will get the details of a model. + +|APITrainCompletedScreenshot| + +.. |APITrainStartScreenshot| image:: /static/screenshots/APITrainStart.png +.. |APITrainInProgressScreenshot| image:: /static/screenshots/APITrainInProgress.png +.. |APITrainCompletedScreenshot| image:: /static/screenshots/APITrainCompleted.png \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index b77b0a23..3c645669 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -101,7 +101,7 @@ html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ['static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. diff --git a/docs/index.rst b/docs/index.rst index 5bdb3395..5eef073b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,12 +6,15 @@ .. image:: https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png :height: 30px -The Open Source AI Bot Platform Builder for Enterprise +.. image:: https://img.shields.io/badge/gitter-join%20chat-brightgreen.svg + :target: `gitter`_ + +The Open Source AI Bot Platform Builder ====================================================== -*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to our enterprise developers using the BotSharp Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your enterprise robots and precisely control every step of the AI processing pipeline."* +*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to .NET developers using the BotSharp Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your robots and precisely control every step of the AI processing pipeline."* -**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in enterprise information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. +**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. More friendly to corporate developers. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope. @@ -22,13 +25,16 @@ BotSharp is in accordance with components princple strictly, decouples every par Some Features ------------- +* Integrated debugging is easier without relying on any other machine learning algorithm libraries. * Built-in multi-Agents management, easy to build Bot as a Service platform. * Context In/ Out with lifespan to make conversion flow be controllable. -* Use the natural language processing pipeline mechanism and the popular NLP algorithm library to build your own unique robot processing flow. +* Use the natural language processing pipeline mechanism to work with extensions easily, and build your own unique robot processing flows. +* Rewrote NLP algorithm from ground without historical issues. * Support export/ import agent from other bot platforms directly. * Support different UI providers like `Rasa UI`_ and `Articulate UI`_. * Support for multiple data request and response formats such as Rasa NLU and Dialogflow. * Integrate with popular social platforms like Facebook Messenger, Slack and Telegram. +* Multi-core parallel computing optimization, High-Performance C# on GPUs in Hybridizer. Indices and tables ================== @@ -36,6 +42,7 @@ The main documentation for the site is organized into a couple sections: * :ref:`User Documentation ` * :ref:`Integration Documentation ` +* :ref:`NLP Documentation ` * :ref:`search` .. _user-docs: @@ -45,16 +52,37 @@ The main documentation for the site is organized into a couple sections: :caption: User Documentation: installation + agent/import-agent + agent/train-agent + agent/test-agent + agent/optimize-agent .. _integration-docs: .. toctree:: - :maxdepth: 2 - :caption: Integration Documentation: + :maxdepth: 3 + :caption: Channels Integration Documentation: integrations/facebook-messenger + integrations/slack + integrations/telegram + integrations/skype -If you feel that this project is helpful to you, please Star on the project, we will be very grateful. +.. _nlp-docs: + +.. toctree:: + :maxdepth: 2 + :caption: NLP Documentation: + + models/crf + models/nb + models/ngram + models/svm + models/tfidf + models/penntreebank + +If you feel that this project is helpful to you, please Star us on the project, we will be very grateful. .. _Rasa UI: https://github.com/paschmann/rasa-ui .. _Articulate UI: https://spg.ai/projects/articulate +.. _gitter: https://gitter.im/botsharpcore/Lobby \ No newline at end of file diff --git a/docs/installation.rst b/docs/installation.rst index d214fa3b..b604f964 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,7 +2,7 @@ Installation ============ BotSharp strictly follows the modular design principle and adopts a structure in which views and logic are separated. So you can choose the front-end Bot design and management interface. -If you want to use the `RASA UI`_ as a front end, you can use the rasaui-specific compose file to quickly experience BotSharp. +If you want to use the `Articulate UI`_ as a front end, you can use the articulateui-specific compose file to quickly experience BotSharp. Docker Composer ^^^^^^^^^^^^^^^ @@ -20,6 +20,7 @@ You can use docker compose to run, make sure you've got `Docker`_ installed. PS D:\BotSharp\> docker-compose -f docker-compose-articulateui.yml up Point your web browser at http://localhost:3000 and enjoy Articulate-UI with BotSharp. +|ArticulateHomeScreenshot| 2. Integrate with `Rasa UI`_, you can use docker compose to run. @@ -29,8 +30,14 @@ Point your web browser at http://localhost:3000 and enjoy Articulate-UI with Bot Point your web browser at http://localhost:5001 and enjoy Rasa-UI with BotSharp. +|RasaUIHomeScreenshot| + +3. Integrate with `Rasa Talk`_ + + Building BotSharp ^^^^^^^^^^^^^^^^^ +If you are a .NET developer and want to develop extensions or fix bug for BotSharp, you would CTRL + F5 to run it locally in debug mode. Make sure the `Microsoft .NET Core`_ build environment is installed. Building solution using dotnet CLI (preferred). @@ -43,6 +50,8 @@ Building solution using dotnet CLI (preferred). Install in docker container ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +If you just want to run BotSharp as a backend NLU engine, you can run it standalone in docker. + :: PS D:\> git clone https://github.com/Oceania2018/BotSharp @@ -61,7 +70,9 @@ Start a container: PS D:\BotSharp\> docker run -it -p 5000:5000 botsharp +Access restful APIs: http://localhost:5000 if you are using RASA response format. +|APIHomeScreenshot| Install in NuGet @@ -81,5 +92,10 @@ Use BotSharp.NLP as a natural language processing toolkit alone. .. _Rasa UI: https://github.com/paschmann/rasa-ui .. _Articulate UI: https://spg.ai/projects/articulate +.. _Rasa Talk: https://github.com/jackdh/RasaTalk .. _Microsoft .NET Core: https://www.microsoft.com/net/download -.. _Docker: https://www.docker.com \ No newline at end of file +.. _Docker: https://www.docker.com + +.. |APIHomeScreenshot| image:: /static/screenshots/APIHome.png +.. |ArticulateHomeScreenshot| image:: /static/screenshots/ArticulateHome.png +.. |RasaUIHomeScreenshot| image:: /static/screenshots/RasaUIHome.png \ No newline at end of file diff --git a/docs/integrations/skype.rst b/docs/integrations/skype.rst new file mode 100644 index 00000000..b374a8b4 --- /dev/null +++ b/docs/integrations/skype.rst @@ -0,0 +1,2 @@ +Skype Chabot +============ diff --git a/docs/integrations/slack.rst b/docs/integrations/slack.rst new file mode 100644 index 00000000..dcb6ef83 --- /dev/null +++ b/docs/integrations/slack.rst @@ -0,0 +1,6 @@ +Slack App +========= + +A bot is a type of Slack App designed to interact with users via conversation. + +A bot is the same as a regular app: it can access the same range of APIs and do all of the magical things that a Slack App can do. \ No newline at end of file diff --git a/docs/integrations/telegram.rst b/docs/integrations/telegram.rst new file mode 100644 index 00000000..a84945ed --- /dev/null +++ b/docs/integrations/telegram.rst @@ -0,0 +1,4 @@ +Telegram Bot Platform +===================== + +Bots are simply Telegram accounts operated by software – not people – and they'll often have AI features. They can do anything – teach, play, search, broadcast, remind, connect, integrate with other services, or even pass commands to the Internet of Things. \ No newline at end of file diff --git a/docs/models/penntreebank.rst b/docs/models/penntreebank.rst new file mode 100644 index 00000000..b18671d9 --- /dev/null +++ b/docs/models/penntreebank.rst @@ -0,0 +1,2 @@ +Penn Treebank +============= diff --git a/docs/models/tfidf.rst b/docs/models/tfidf.rst new file mode 100644 index 00000000..6daa95f6 --- /dev/null +++ b/docs/models/tfidf.rst @@ -0,0 +1,2 @@ +TF-IDF +====== diff --git a/docs/static/screenshots/APIHome.png b/docs/static/screenshots/APIHome.png new file mode 100644 index 00000000..d75dc395 Binary files /dev/null and b/docs/static/screenshots/APIHome.png differ diff --git a/docs/static/screenshots/APITestInput.png b/docs/static/screenshots/APITestInput.png new file mode 100644 index 00000000..41b4fab0 Binary files /dev/null and b/docs/static/screenshots/APITestInput.png differ diff --git a/docs/static/screenshots/APITestResult.png b/docs/static/screenshots/APITestResult.png new file mode 100644 index 00000000..f1a4a07d Binary files /dev/null and b/docs/static/screenshots/APITestResult.png differ diff --git a/docs/static/screenshots/APITrainCompleted.png b/docs/static/screenshots/APITrainCompleted.png new file mode 100644 index 00000000..b5f3d682 Binary files /dev/null and b/docs/static/screenshots/APITrainCompleted.png differ diff --git a/docs/static/screenshots/APITrainInProgress.png b/docs/static/screenshots/APITrainInProgress.png new file mode 100644 index 00000000..270e89db Binary files /dev/null and b/docs/static/screenshots/APITrainInProgress.png differ diff --git a/docs/static/screenshots/APITrainStart.png b/docs/static/screenshots/APITrainStart.png new file mode 100644 index 00000000..611c05da Binary files /dev/null and b/docs/static/screenshots/APITrainStart.png differ diff --git a/docs/static/screenshots/ArticulateHome.png b/docs/static/screenshots/ArticulateHome.png new file mode 100644 index 00000000..ef5d837a Binary files /dev/null and b/docs/static/screenshots/ArticulateHome.png differ diff --git a/docs/static/screenshots/RasaUIHome.png b/docs/static/screenshots/RasaUIHome.png new file mode 100644 index 00000000..298f5142 Binary files /dev/null and b/docs/static/screenshots/RasaUIHome.png differ diff --git a/docs/static/screenshots/RestoreAgentFromZip.png b/docs/static/screenshots/RestoreAgentFromZip.png new file mode 100644 index 00000000..445cc8a3 Binary files /dev/null and b/docs/static/screenshots/RestoreAgentFromZip.png differ