Merge pull request #3 from Oceania2018/master

merge newest code

.gitignore

@@ -286,21 +286,9 @@ __pycache__/
 *.btm.cs
 *.odx.cs
 *.xsd.cs
-/App_Data
-/Bot.WebStarter/App_Data/bot-rasa.db
-/Bot.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant
-/BotSharp.WebStarter/App_Data/DbInitializer/Agents/Dialogflow/VirtualAssistant
-/BotSharp.UnitTest/App_Data/DbInitializer/Agents
-/BotSharp.UnitTest/App_Data/BotSharp.db
 /BotSharp.WebHost/App_Data/BotSharp.db
 /BotSharp.UI
-/BotSharp.WebHost/App_Data/ModelFiles
-/BotSharp.WebHost/App_Data/TrainingFiles
-/BotSharp.WebHost/App_Data/PredictFiles
 /BotSharp.WebHost/App_Data/Projects
 /BotSharp.WebHost/PublishOutput
 /Data
-/docs/build
 /docs/_build
-/BotSharp.WebHost/App_Data/AgentArchive/Smart Niraj.zip
-/BotSharp.NLP.UnitTest/wordvec_enu.bin

BotSharp.Algorithm/Bayes/MultinomiaNaiveBayes.cs

@@ -88,13 +88,16 @@ namespace BotSharp.Algorithm.Bayes
         {
             int featureCount = features.Length;
             
-            double postProb = priorProb;
+            double postProb = Math.Log(priorProb);
 
             // loop features
             for (int x = 0; x < featureCount; x++)
             {
                 string key = $"{Y} f{x} {features[x]}";
-                postProb += condProbDictionary[key];
+                if(features[x] == 1)
+                {
+                    postProb += condProbDictionary[key];
+                }
             }
 
             return Math.Pow(2, postProb);

BotSharp.Core.UnitTest/BotSharp.Core.UnitTest.csproj

@@ -19,6 +19,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\BotSharp.Algorithm\BotSharp.Algorithm.csproj" />
     <ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 

BotSharp.Core.UnitTest/Performance/Spotify.cs

@@ -0,0 +1,83 @@
+using BotSharp.Core.Agents;
+using BotSharp.Core.Engines;
+using BotSharp.Core.Engines.BotSharp;
+using BotSharp.Core.Models;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using BotSharp.Algorithm.Extensions;
+
+namespace BotSharp.Core.UnitTest.Performance
+{
+    [TestClass]
+    public class Spotify : TestEssential
+    {
+        private List<Tuple<AIRequest, string>> Samples;
+        private IBotPlatform _platform;
+
+        [TestMethod]
+        public void IntentAccuracy()
+        {
+            int correct = 0;
+            List<Tuple<string, string>> errors = new List<Tuple<string, string>>();
+
+            var agent = LoadAgent();
+
+            for (int i = 0; i < Samples.Count; i++)
+            {
+                var aIResponse = _platform.TextRequest(Samples[i].Item1);
+                if (aIResponse.Result.Metadata.IntentName == Samples[i].Item2)
+                {
+                    correct++;
+                }
+                else
+                {
+                    errors.Add(new Tuple<string, string>(Samples[i].Item2, Samples[i].Item1.Query[0]));
+                }
+            }
+
+           double accuracy = correct / (Samples.Count + 0.0);
+        }
+
+        private Agent LoadAgent()
+        {
+            _platform = new BotSharpAi();
+
+            // Load agent
+            var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", "Spotify");
+            string model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last().Split(Path.DirectorySeparatorChar).Last();
+            var modelPath = Path.Combine(projectPath, model);
+            var agent = _platform.LoadAgentFromFile(modelPath);
+
+            // Init samples
+            Samples = new List<Tuple<AIRequest, string>>();
+            /*agent.Corpus.UserSays = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>
+            {
+                new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music.play", Text = "play the 50 Great Beatles Songs playlist in Prime Music"},
+                new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music.play", Text = "reproduce a the track Monster by Rihanna ft Eminem"},
+                new TrainingIntentExpression<TrainingIntentExpressionPart>{ Intent = "music_player_control.add_favorite", Text = "add this song to my favourites"}
+            };*/
+            agent.Corpus.UserSays.ForEach(intent =>
+            {
+                Samples.Add(new Tuple<AIRequest, string>(new AIRequest
+                {
+                    AgentDir = projectPath,
+                    Model = model,
+                    Query = new String[]
+                    {
+                        intent.Text
+                    }
+                }, intent.Intent));
+            });
+
+            //Samples.Shuffle();
+
+            var samples = String.Join("\r\n", Samples.Select(x => $"__label__{x.Item2} {x.Item1.Query[0]}").ToList());
+
+            return agent;
+        }
+    }
+}

BotSharp.Core.UnitTest/TestEssential.cs

@@ -18,9 +18,9 @@ namespace BotSharp.Core.UnitTest
         public TestEssential()
         {
             contentRoot = $"{Directory.GetCurrentDirectory()}{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}..{Path.DirectorySeparatorChar}BotSharp.WebHost{Path.DirectorySeparatorChar}";
-
+            contentRoot = Path.GetFullPath(contentRoot);
             ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
-            var settings = Directory.GetFiles(contentRoot + $"Settings{Path.DirectorySeparatorChar}", "*.json");
+            var settings = Directory.GetFiles(contentRoot + $"..{Path.DirectorySeparatorChar}Settings{Path.DirectorySeparatorChar}", "*.json");
             settings.ToList().ForEach(setting =>
             {
                 configurationBuilder.AddJsonFile(setting, optional: false, reloadOnChange: true);

BotSharp.Core/BotSharp.Core.csproj

@@ -66,8 +66,11 @@ If you feel that this project is helpful to you, please Star on the project, we
 
   <ItemGroup>
     <Compile Remove="Accounts\**" />
+    <Compile Remove="Engines\CoreNlp\**" />
     <EmbeddedResource Remove="Accounts\**" />
+    <EmbeddedResource Remove="Engines\CoreNlp\**" />
     <None Remove="Accounts\**" />
+    <None Remove="Engines\CoreNlp\**" />
   </ItemGroup>
 
   <ItemGroup>
@@ -80,10 +83,6 @@ If you feel that this project is helpful to you, please Star on the project, we
     <PackageReference Include="RestSharp" Version="106.3.1" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Engines\CoreNlp\" />
-  </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\BotSharp.NLP\BotSharp.NLP.csproj" />
   </ItemGroup>

BotSharp.Core/Engines/BotEngineBase.cs

@@ -46,7 +46,7 @@ namespace BotSharp.Core.Engines
             {
                 doc.Sentences[0].Entities = new List<NlpEntity>();
             }
-            doc.Sentences[0].Entities.ForEach(x => parameters.Add(x.Entity, x.Value));
+            doc.Sentences[0].Entities.ForEach(x => parameters[x.Entity] = x.Value);
 
             return new AIResponse
             {

BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs

@@ -58,9 +58,6 @@ namespace BotSharp.Core.Engines.BotSharp
             var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input \"{parsedTrainingDataFileName}\" -output \"{modelFileName}\"", false);
 
             Console.WriteLine($"Saved model to {modelFileName}");
-            meta.Meta = new JObject();
-            meta.Meta["compiled at"] = "Aug 3, 2018";
-
 
             return true;
         }

BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs

@@ -0,0 +1,263 @@
+using BotSharp.Core.Abstractions;
+using BotSharp.Core.Agents;
+using BotSharp.Models.CRFLite;
+using BotSharp.Models.CRFLite.Decoder;
+using BotSharp.Models.CRFLite.Encoder;
+using BotSharp.Models.NLP;
+using BotSharp.NLP.Tokenize;
+using DotNetToolkit;
+using Microsoft.Extensions.Configuration;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BotSharp.Core.Engines.BotSharp
+{
+    public class BotSharpCRFNer : INlpTrain, INlpPredict
+    {
+        public IConfiguration Configuration { get; set; }
+        public PipeSettings Settings { get; set; }
+
+        public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
+        {
+            var corpus = agent.Corpus;
+
+            meta.Model = "ner-crf.model";
+
+            List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
+            List<List<TrainingData>> list = new List<List<TrainingData>>();
+
+            string rawTrainingDataFileName = System.IO.Path.Combine(Settings.TempDir, "ner-crf.corpus.txt");
+            string modelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model);
+
+            using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create))
+            {
+                using (StreamWriter sw = new StreamWriter(fs))
+                {
+                    for (int i = 0; i < doc.Sentences.Count; i++)
+                    {
+                        List<TrainingData> curLine = Merge(doc, doc.Sentences[i].Tokens, userSays[i].Entities);
+                        curLine.ForEach(trainingData =>
+                        {
+                            string[] wordParams = { trainingData.Token, trainingData.Pos, trainingData.Entity };
+                            string wordStr = string.Join("\t", wordParams);
+                            sw.WriteLine(wordStr);
+                        });
+                        list.Add(curLine);
+                        sw.WriteLine();
+                    }
+                    sw.Flush();
+                }
+            }
+
+            string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString();
+            string template = Configuration.GetValue<String>($"BotSharpCRFNer:template");
+            template = template.Replace("|App_Data|", contentDir + System.IO.Path.DirectorySeparatorChar);
+
+            var encoder = new CRFEncoder();
+            bool result = encoder.Learn(new EncoderOptions
+            {
+                TrainingCorpusFileName = rawTrainingDataFileName,
+                TemplateFileName = template,
+                ModelFileName = modelFileName,
+            });
+
+            return result;
+        }
+
+        private List<TrainingData> Merge(NlpDoc doc, List<Token> tokens, List<TrainingIntentExpressionPart> entities)
+        {
+            List<TrainingData> trainingTuple = new List<TrainingData>();
+            HashSet<String> entityWordBag = new HashSet<String>();
+            int wordCandidateCount = 0;
+
+            for (int i = 0; i < tokens.Count; i++)
+            {
+                TrainingIntentExpressionPart curEntity = null;
+                if (entities == null) continue;
+
+                bool entityFinded = false;
+                for (int entityIndex = 0; entityIndex < entities.Count; entityIndex++)
+                {
+                    var entity = entities[entityIndex];
+
+                    if (!entityFinded)
+                    {
+                        var vDoc = new NlpDoc { Sentences = new List<NlpDocSentence> { new NlpDocSentence { Text = entity.Value } } };
+                        doc.Tokenizer.Predict(null, vDoc, null);
+                        string[] words = vDoc.Sentences[0].Tokens.Select(x => x.Text).ToArray();
+
+                        for (int j = 0; j < words.Length; j++)
+                        {
+                            if (tokens[i + j].Text == words[j])
+                            {
+                                wordCandidateCount++;
+                                if (j == words.Length - 1)
+                                {
+                                    curEntity = entity;
+                                }
+                            }
+                            else
+                            {
+                                wordCandidateCount = 0;
+                                break;
+                            }
+                        }
+                        if (wordCandidateCount != 0) // && entity.Start == tokens[i].Offset)
+                        {
+                            String entityName = curEntity.Entity.Contains(":") ? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1) : curEntity.Entity;
+
+                            for(int wordIndex = 0; wordIndex < words.Length; wordIndex++)
+                            {
+                                var tag = entityName;
+
+                                if (wordIndex == 0)
+                                {
+                                    if (words.Length == 1)
+                                    {
+                                        tag = "S_" + entityName;
+                                    }
+                                    else
+                                    {
+                                        tag = "B_" + entityName;
+                                    }
+                                }
+                                else if (wordIndex == words.Length - 1)
+                                {
+                                    tag = "E_" + entityName;
+                                }
+                                else
+                                {
+                                    tag = "M_" + entityName;
+                                }
+
+                                var word = words[wordIndex];
+                                trainingTuple.Add(new TrainingData(tag, word, tokens[i].Pos));
+                            }
+
+                            entityFinded = true;
+                        }
+                    }
+                }
+
+                if (wordCandidateCount == 0)
+                {
+                    trainingTuple.Add(new TrainingData("S", tokens[i].Text, tokens[i].Pos));
+                }
+                else
+                {
+                    i = i + wordCandidateCount - 1;
+                }
+            }
+
+            return trainingTuple;
+        }
+
+        public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
+        {
+            var decoder = new CRFDecoder();
+            var options = new DecoderOptions
+            {
+                ModelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model)
+            };
+
+            //Load encoded model from file
+            decoder.LoadModel(options.ModelFileName);
+
+            //Create decoder tagger instance.
+            var tagger = decoder.CreateTagger(options.NBest, options.MaxWord);
+            tagger.set_vlevel(options.ProbLevel);
+
+            //Initialize result
+            var crf_out = new CRFSegOut[options.NBest];
+            for (var i = 0; i < options.NBest; i++)
+            {
+                crf_out[i] = new CRFSegOut(options.MaxWord);
+            }
+
+            doc.Sentences.ForEach(sent =>
+            {
+                List<List<String>> dataset = new List<List<string>>();
+                dataset.AddRange(sent.Tokens.Select(token => new List<String> { token.Text, token.Pos }).ToList());
+                //predict given string's tags
+                decoder.Segment(crf_out, tagger, dataset);
+
+                var entities = new List<NlpEntity>();
+
+                for (int i = 0; i < sent.Tokens.Count; i++)
+                {
+                    var entity = crf_out[0].result_;
+                    entities.Add(new NlpEntity
+                    {
+                        Entity = entity[i],
+                        Start = doc.Sentences[0].Tokens[i].Start,
+                        Value = doc.Sentences[0].Tokens[i].Text,
+                        Confidence = 0,
+                        Extrator = "BotSharpCRFNer"
+                    });
+                }
+
+                sent.Entities = MergeEntity(doc.Sentences[0].Text, entities);
+            });
+
+            return true;
+        }
+
+        private List<NlpEntity> MergeEntity(string sentence, List<NlpEntity> tokens)
+        {
+            List<NlpEntity> res = new List<NlpEntity>();
+
+            for(int i = 0; i < tokens.Count; i++)
+            {
+                var entity = tokens[i];
+
+                if (entity.Entity.StartsWith("S_"))
+                {
+                    entity.Entity = entity.Entity.Split('_')[1];
+                    res.Add(entity);
+                }
+                else if (entity.Entity.StartsWith("B_"))
+                {
+                    entity.Entity = entity.Entity.Split('_')[1];
+
+                    for(int j = i; j < tokens.Count; j++)
+                    {
+                        var token = tokens[j];
+                        if (token.Entity.StartsWith("E_"))
+                        {
+                            res.Add(new NlpEntity
+                            {
+                                Value = sentence.Substring(entity.Start, token.End - entity.Start + 1),
+                                Entity = entity.Entity,
+                                Extrator = entity.Extrator,
+                                Start = entity.Start,
+                                Confidence = entity.Confidence
+                            });
+                        }
+
+                        i++;
+                    }
+                }
+            }
+
+            return res;
+        }
+
+        public class TrainingData
+        {
+            public String Token { get; set; }
+            public String Entity { get; set; }
+            public String Pos { get; set; }
+
+            public TrainingData(string entity, string token, string pos)
+            {
+                Token = token;
+                Entity = entity;
+                Pos = pos;
+            }
+        }
+    }
+}

BotSharp.Core/Engines/BotSharp/BotSharpNBayesClassifier.cs

@@ -40,8 +40,6 @@ namespace BotSharp.Core.Engines.BotSharp
             classifier.Train(sentences);
 
             Console.WriteLine($"Saved model to {modelFileName}");
-            meta.Meta = new JObject();
-            meta.Meta["compiled at"] = "Sep 12, 2018";
 
             return true;
         }

BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs

@@ -38,7 +38,6 @@ namespace BotSharp.Core.Engines.NERs
 
         public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
         {
-            var dc = new DefaultDataContextLoader().GetDefaultDc();
             var corpus = agent.Corpus;
 
             meta.Model = "ner-crf.model";

BotSharp.Core/Engines/Nltk/NltkTokenizer.cs

@@ -1,108 +0,0 @@
-using BotSharp.Core.Abstractions;
-using BotSharp.Core.Agents;
-using BotSharp.Core.Models;
-using BotSharp.NLP.Tokenize;
-using EntityFrameworkCore.BootKit;
-using Microsoft.Extensions.Configuration;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using RestSharp;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace BotSharp.Core.Engines.SpaCy
-{
-    public class NltkTokenizer : INlpTrain, INlpPredict
-    {
-        public IConfiguration Configuration { get; set; }
-        public PipeSettings Settings { get; set; }
-
-        public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
-        {
-            var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
-            var request = new RestRequest("nltktokenizesentences", Method.POST);
-            List<List<Token>> tokens = new List<List<Token>>();
-            Boolean res = true;
-            var dc = new DefaultDataContextLoader().GetDefaultDc();
-            var corpus = agent.Corpus;
-
-            doc.Sentences = new List<NlpDocSentence>();
-            List<string> sentencesList = new List<string>();
-            corpus.UserSays.ForEach ( usersay => sentencesList.Add(usersay.Text));
-
-            request.RequestFormat = DataFormat.Json;
-
-            request.AddParameter("application/json", JsonConvert.SerializeObject(new Documents(sentencesList)), ParameterType.RequestBody);
-
-            var response = client.Execute<Result>(request);
-
-            tokens = response.Data.TokensList;
-
-            for (int i = 0; i < sentencesList.Count; i++)
-            {
-                doc.Sentences.Add(new NlpDocSentence
-                {
-                    Tokens = tokens[i],
-                    Text = sentencesList[i]
-                });
-            }
-            res = res && response.IsSuccessful;
-            return res;
-            /*
-             corpus.UserSays.ForEach(usersay => {
-                Console.WriteLine(usersay.Text);
-                request.AddParameter("text", usersay.Text);
-                var response = client.Execute<Result>(request);
-                
-                tokens.Add(response.Data.Tokens);
-
-                doc.Sentences.Add(new NlpDocSentence
-                {
-                    Tokens = response.Data.Tokens,
-                    Text = usersay.Text
-                });
-
-                res = res && response.IsSuccessful;
-            });
-             */
-        }
-
-        public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
-        {
-            var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value);
-            var request = new RestRequest("nltktokenizesentences", Method.POST);
-            List<List<Token>> tokens = new List<List<Token>>();
-            Boolean res = true;
-            var corpus = agent.Corpus;
-
-            request.AddParameter("sentences", doc.Sentences[0].Text);
-            var response = client.Execute<Result>(request);
-            
-            //tokens.Add(response.Data.Tokens);
-
-            res = res && response.IsSuccessful;
-
-            doc.Sentences[0].Tokens = tokens[0];
-
-            return true;
-        }
-
-        private class Result
-        {
-            public List<List<Token>> TokensList { get; set; }
-        }
-
-        private class Documents
-        {
-            public List<string> Sentences { get; set; }
-
-            public Documents(List<string> sentences)
-            {
-                this.Sentences = sentences;
-            }
-        }
-    }
-}

BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs

@@ -88,12 +88,12 @@ namespace BotSharp.Core.Engines.Rasa
         public void LoadIntents(Agent agent)
         {
             string data = File.ReadAllText(Path.Combine(AgentDir, "corpus.json"));
-            var rasa = JsonConvert.DeserializeObject<RasaAgent>(data);
+            var rasa = JsonConvert.DeserializeObject<RasaAgentImportModel>(data);
 
-            agent.Intents = rasa.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList();
+            agent.Intents = rasa.Data.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList();
 
             agent.Intents.ForEach(intent => {
-                ImportIntentUserSays(intent, rasa.UserSays);
+                ImportIntentUserSays(intent, rasa.Data.UserSays);
             });
 
         }

BotSharp.Core/Engines/Rasa/RasaAgent.cs

@@ -21,4 +21,10 @@ namespace BotSharp.Core.Engines.Rasa
         [JsonProperty("regex_features")]
         public List<RasaTrainingRegex> Regex { get; set; }
     }
+
+    public class RasaAgentImportModel
+    {
+        [JsonProperty("rasa_nlu_data")]
+        public RasaAgent Data { get; set; }
+    }
 }

BotSharp.NLP.UnitTest/CRFLite/DecoderTest.cs

@@ -13,266 +13,48 @@ namespace BotSharp.NLP.UnitTest.CRFLite
     [TestClass]
     public class DecoderTest
     {
+        object rdLocker = new object();
+
         [TestMethod]
         public void TestDecode()
         {
-            var encoder = new CRFDecoder();
-            bool result = Decode(new DecoderOptions
+            var decoder = new CRFDecoder();
+            var options = new DecoderOptions
             {
-                InputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\test\test.txt",
-                ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\model\ner_model_eng",
-                OutputFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\test\output.txt"
-            });
-        }
-
-        object rdLocker = new object();
-
-        bool Decode(DecoderOptions options)
-        {
-            var parallelOption = new ParallelOptions();
-            var watch = Stopwatch.StartNew();
-
-            var sr = new StreamReader(options.InputFileName);
-            StreamWriter sw = null, swSeg = null;
-
-            if (options.OutputFileName != null && options.OutputFileName.Length > 0)
-            {
-                sw = new StreamWriter(options.OutputFileName);
-            }
-            if (options.OutputSegFileName != null && options.OutputSegFileName.Length > 0)
-            {
-                swSeg = new StreamWriter(options.OutputSegFileName);
-            }
-
-            //Create CRFSharp wrapper instance. It's a global instance
-            var crfWrapper = new CRFDecoder();
+                ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model"
+            };
 
             //Load encoded model from file
-            //Logger.WriteLine("Loading model from {0}", options.strModelFileName);
-            crfWrapper.LoadModel(options.ModelFileName);
+            decoder.LoadModel(options.ModelFileName);
 
-            var queueRecords = new ConcurrentQueue<List<List<string>>>();
-            var queueSegRecords = new ConcurrentQueue<List<List<string>>>();
+            //Create decoder tagger instance.
+            var tagger = decoder.CreateTagger(options.NBest, options.MaxWord);
+            tagger.set_vlevel(options.ProbLevel);
 
-            parallelOption.MaxDegreeOfParallelism = options.Thread;
-            Parallel.For(0, options.Thread, parallelOption, t =>
+            //Initialize result
+            var crf_out = new CRFSegOut[options.NBest];
+            for (var i = 0; i < options.NBest; i++)
             {
-
-                //Create decoder tagger instance. If the running environment is multi-threads, each thread needs a separated instance
-                var tagger = crfWrapper.CreateTagger(options.NBest, options.MaxWord);
-                tagger.set_vlevel(options.ProbLevel);
-
-                //Initialize result
-                var crf_out = new crf_seg_out[options.NBest];
-                for (var i = 0; i < options.NBest; i++)
-                {
-                    crf_out[i] = new crf_seg_out(tagger.crf_max_word_num);
-                }
-
-                var inbuf = new List<List<string>>();
-                while (true)
-                {
-                    lock (rdLocker)
-                    {
-                        if (ReadRecord(inbuf, sr) == false)
-                        {
-                            break;
-                        }
-
-                        queueRecords.Enqueue(inbuf);
-                        queueSegRecords.Enqueue(inbuf);
-                    }
-
-                    //Call CRFSharp wrapper to predict given string's tags
-                    if (swSeg != null)
-                    {
-                        crfWrapper.Segment(crf_out, tagger, inbuf);
-                    }
-                    else
-                    {
-                        crfWrapper.Segment((CRFTermOut[])crf_out, (DecoderTagger)tagger, inbuf);
-                    }
-
-                    List<List<string>> peek = null;
-                    //Save segmented tagged result into file
-                    if (swSeg != null)
-                    {
-                        var rstList = ConvertCRFTermOutToStringList(inbuf, crf_out);
-                        while (peek != inbuf)
-                        {
-                            queueSegRecords.TryPeek(out peek);
-                        }
-                        for (int index = 0; index < rstList.Count; index++)
-                        {
-                            var item = rstList[index];
-                            swSeg.WriteLine(item);
-                        }
-                        queueSegRecords.TryDequeue(out peek);
-                        peek = null;
-                    }
-
-                    //Save raw tagged result (with probability) into file
-                    if (sw != null)
-                    {
-                        while (peek != inbuf)
-                        {
-                            queueRecords.TryPeek(out peek);
-                        }
-                        OutputRawResultToFile(inbuf, crf_out, tagger, sw);
-                        queueRecords.TryDequeue(out peek);
-
-                    }
-                }
-            });
-
-
-            sr.Close();
-
-            if (sw != null)
-            {
-                sw.Close();
+                crf_out[i] = new CRFSegOut(options.MaxWord);
             }
-            if (swSeg != null)
-            {
-                swSeg.Close();
-            }
-            watch.Stop();
-            //Logger.WriteLine("Elapsed: {0} ms", watch.ElapsedMilliseconds);
 
-            return true;
+            var dataset = GetTestData();
+
+            //predict given string's tags
+            decoder.Segment(crf_out, tagger, dataset);
         }
 
-        private bool ReadRecord(List<List<string>> inbuf, StreamReader sr)
+        private List<List<string>> GetTestData()
         {
-            inbuf.Clear();
+            var dataset = new List<List<string>>();
 
-            while (true)
-            {
-                var strLine = sr.ReadLine();
-                if (strLine == null)
-                {
-                    //At the end of current file
-                    if (inbuf.Count == 0)
-                    {
-                        return false;
-                    }
-                    else
-                    {
-                        return true;
-                    }
-                }
-                strLine = strLine.Trim();
-                if (strLine.Length == 0)
-                {
-                    return true;
-                }
+            dataset.Add(new List<string> { "'", "PUN" });
+            dataset.Add(new List<string> { "'", "POS" });
+            dataset.Add(new List<string> { "Duchy", "NNP" });
+            dataset.Add(new List<string> { "of", "IN" });
+            dataset.Add(new List<string> { "Lithuania", "NNP" });
 
-                //Read feature set for each record
-                var items = strLine.Split(new char[] { '\t' });
-                inbuf.Add(new List<string>());
-                for (int index = 0; index < items.Length; index++)
-                {
-                    var item = items[index];
-                    inbuf[inbuf.Count - 1].Add(item);
-                }
-            }
-        }
-
-        //Output raw result with probability
-        private void OutputRawResultToFile(List<List<string>> inbuf, CRFTermOut[] crf_out, SegDecoderTagger tagger, StreamWriter sw)
-        {
-            for (var k = 0; k < crf_out.Length; k++)
-            {
-                if (crf_out[k] == null)
-                {
-                    //No more result
-                    break;
-                }
-
-                var sb = new StringBuilder();
-
-                var crf_seg_out = crf_out[k];
-                //Show the entire sequence probability
-                //For each token
-                for (var i = 0; i < inbuf.Count; i++)
-                {
-                    //Show all features
-                    for (var j = 0; j < inbuf[i].Count; j++)
-                    {
-                        sb.Append(inbuf[i][j]);
-                        sb.Append("\t");
-                    }
-
-                    //Show the best result and its probability
-                    sb.Append(crf_seg_out.result_[i]);
-
-                    if (tagger.vlevel_ > 1)
-                    {
-                        sb.Append("\t");
-                        sb.Append(crf_seg_out.weight_[i]);
-
-                        //Show the probability of all tags
-                        sb.Append("\t");
-                        for (var j = 0; j < tagger.ysize_; j++)
-                        {
-                            sb.Append(tagger.yname(j));
-                            sb.Append("/");
-                            sb.Append(tagger.prob(i, j));
-
-                            if (j < tagger.ysize_ - 1)
-                            {
-                                sb.Append("\t");
-                            }
-                        }
-                    }
-                    sb.AppendLine();
-                }
-                if (tagger.vlevel_ > 0)
-                {
-                    sw.WriteLine("#{0}", crf_seg_out.prob);
-                }
-                sw.WriteLine(sb.ToString().Trim());
-                sw.WriteLine();
-            }
-        }
-
-        //Convert CRFSharp output format to string list
-        private List<string> ConvertCRFTermOutToStringList(List<List<string>> inbuf, crf_seg_out[] crf_out)
-        {
-            var sb = new StringBuilder();
-            for (var i = 0; i < inbuf.Count; i++)
-            {
-                sb.Append(inbuf[i][0]);
-            }
-
-            var strText = sb.ToString();
-            var rstList = new List<string>();
-            for (var i = 0; i < crf_out.Length; i++)
-            {
-                if (crf_out[i] == null)
-                {
-                    //No more result
-                    break;
-                }
-
-                sb.Clear();
-                var crf_term_out = crf_out[i];
-                for (var j = 0; j < crf_term_out.Count; j++)
-                {
-                    var str = strText.Substring(crf_term_out.tokenList[j].offset, crf_term_out.tokenList[j].length);
-                    var strNE = crf_term_out.tokenList[j].strTag;
-
-                    sb.Append(str);
-                    if (strNE.Length > 0)
-                    {
-                        sb.Append("[" + strNE + "]");
-                    }
-                    sb.Append(" ");
-                }
-                rstList.Add(sb.ToString().Trim());
-            }
-
-            return rstList;
+            return dataset;
         }
     }
 }

BotSharp.NLP.UnitTest/CRFLite/EncoderTest.cs

@@ -1,22 +1,47 @@
 using BotSharp.Models.CRFLite;
+using BotSharp.Models.CRFLite.Decoder;
 using BotSharp.Models.CRFLite.Encoder;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
 
 namespace BotSharp.NLP.UnitTest.CRFLite
 {
     [TestClass]
     public class EncoderTest
     {
+        /// <summary>
+        /// 
+        /// </summary>
         [TestMethod]
         public void TestEncode()
         {
             var encoder = new CRFEncoder();
             bool result = encoder.Learn(new EncoderOptions
             {
-                TrainingCorpusFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\corpus\eng.1K.training",
-                TemplateFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\template.NE",
-                ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\English\model\ner_model_eng"
+                /*
+                 * traing corups format, split by tab, sentences is seperated by blank row
+                 * 
+                    ! PUN S
+                    Tokyo NNP S_LOCATION
+                    and	CC S
+                    New	NNP	B_LOCATION
+                    York NNP	E_LOCATION
+                    are	VBP	S
+                    major JJ S
+                    financial JJ S
+                    centers	NNS	S
+                    . PUN S
+                 */
+                TrainingCorpusFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\eng.1k.training",
+                TemplateFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\template.en",
+                ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model"
             });
+
+            Assert.IsTrue(result);
         }
     }
 }

BotSharp.NLP.UnitTest/NaiveBayesClassifierTest.cs

@@ -39,16 +39,17 @@ namespace BotSharp.NLP.UnitTest
             var options = new ClassifyOptions
             {
                 ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange", "nb.model"),
-                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange")
+                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
+                Dimension = 100
             };
             var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
             
-            var dataset = sentences.Split(1M);
+            var dataset = sentences.Split(0.7M);
             classifier.Train(dataset.Item1);
 
             int correct = 0;
             int total = 0;
-            dataset.Item1.ForEach(td =>
+            dataset.Item2.ForEach(td =>
             {
                 var classes = classifier.Classify(td);
                 if (td.Label == classes[0].Item1)
@@ -127,5 +128,53 @@ namespace BotSharp.NLP.UnitTest
 
             return genders;
         }
+
+        [TestMethod]
+        public void SpotifyTest()
+        {
+            var reader = new FasttextDataReader();
+            var sentences = reader.Read(new ReaderOptions
+            {
+                DataDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify"),
+                FileName = "spotify.txt"
+            });
+
+            var tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions { }, SupportedLanguage.English);
+            var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
+            for (int i = 0; i < newSentences.Count; i++)
+            {
+                newSentences[i].Label = sentences[i].Label;
+            }
+            sentences = newSentences.ToList();
+
+            sentences.Shuffle();
+
+            var options = new ClassifyOptions
+            {
+                ModelFilePath = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify", "nb.model"),
+                TrainingCorpusDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "spotify")
+            };
+            var classifier = new ClassifierFactory<NaiveBayesClassifier, SentenceFeatureExtractor>(options, SupportedLanguage.English);
+
+            var dataset = sentences.Split(0.7M);
+            classifier.Train(dataset.Item1);
+
+            int correct = 0;
+            int total = 0;
+            dataset.Item2.ForEach(td =>
+            {
+                var classes = classifier.Classify(td);
+                if (td.Label == classes[0].Item1)
+                {
+                    correct++;
+                }
+                total++;
+            });
+
+            var accuracy = (float)correct / total;
+
+            Assert.IsTrue(accuracy > 0.6);
+        }
+
     }
 }

BotSharp.NLP/Classify/ClassifierFactory.cs

@@ -44,7 +44,9 @@ namespace BotSharp.NLP.Classify
 
             var classes = _classifier.Classify(sentence, options);
 
-            return classes.OrderByDescending(x => x.Item2).ToList();
+            classes = classes.OrderByDescending(x => x.Item2).ToList();
+
+            return classes;
         }
     }
 }

BotSharp.NLP/Classify/ClassifyOptions.cs

@@ -13,5 +13,10 @@ namespace BotSharp.NLP.Classify
         public string PrediceOutputFile { get; set; }
         public string TransformFilePath { get; set; }
         public RangeTransform Transform { get; set; }
+
+        /// <summary>
+        /// Feature dimension
+        /// </summary>
+        public int Dimension { get; set; }
     }
 }

BotSharp.NLP/Classify/NaiveBayesClassifier.cs

@@ -55,12 +55,13 @@ namespace BotSharp.NLP.Classify
         public void Train(List<Sentence> sentences, ClassifyOptions options)
         {
             var tfidf = new TfIdfFeatureExtractor();
+            tfidf.Dimension = options.Dimension;
             tfidf.Sentences = sentences;
             tfidf.CalBasedOnCategory();
-            var keyWords = tfidf.Features();
-            string keywords2 = String.Join(",", keyWords.ToArray());
+
             var encoder = new OneHotEncoder();
             encoder.Sentences = sentences;
+            encoder.Words = tfidf.Keywords();
             words = encoder.EncodeAll();
 
             var featureSets = sentences.Select(x => new Tuple<string, double[]>(x.Label, x.Vector)).ToList();
@@ -118,7 +119,8 @@ namespace BotSharp.NLP.Classify
                 lf.Prob = nb.PosteriorProb();
             });*/
 
-            return results;
+            double total = results.Select(x => x.Item2).Sum();
+            return results.Select(x => new Tuple<string, double>(x.Item1, x.Item2 / total)).ToList();
         }
 
         public string SaveModel(ClassifyOptions options)

BotSharp.NLP/Featuring/IFeatureExtractor.cs

@@ -6,5 +6,9 @@ namespace BotSharp.NLP.Featuring
 {
     public interface IFeatureExtractor
     {
+        /// <summary>
+        /// Feature dimension size
+        /// </summary>
+        int Dimension { get; set; }
     }
 }

BotSharp.NLP/Featuring/TfIdfFeatureExtractor.cs

@@ -34,18 +34,35 @@ namespace BotSharp.NLP.Featuring
         private List<Tuple<String, double>> tfs;
 
         private List<string> Categories { get; set; }
+        public int Dimension { get; set; }
 
         public void Extract(Sentence sentence)
         {
 
         }
 
-        public List<string> Features()
+        public List<string> Keywords()
         {
+            if(Dimension == 0)
+            {
+                Dimension = Categories.Count * 3;
+
+                if(Dimension > 300)
+                {
+                    Dimension = 300;
+                }
+
+                if(Dimension < 30)
+                {
+                    Dimension = 30;
+                }
+            }
+
             var tfs2 = tfs.OrderByDescending(x => x.Item2)
                 .Select(x => x.Item1)
                 .Distinct()
-                .Take(Sentences.Count / Categories.Count)
+                .Take(Dimension)
+                .OrderBy(x => x)
                 .ToList();
 
             return tfs2;
@@ -59,7 +76,7 @@ namespace BotSharp.NLP.Featuring
 
             Sentences.ForEach(sent =>
             {
-                sent.Words.ForEach(word =>
+                sent.Words.Where(x => x.IsAlpha).ToList().ForEach(word =>
                 {
                     // TF
                     int c1 = sent.Words.Count(x => x.Lemma == word.Lemma);
@@ -82,6 +99,17 @@ namespace BotSharp.NLP.Featuring
 
             Categories = Sentences.Select(x => x.Label).Distinct().ToList();
 
+            List<Tuple<string, string>> allTextByCategory = new List<Tuple<string, string>>();
+
+            Categories.ForEach(label =>
+            {
+                var allTokens = new List<Token>();
+                Sentences.Where(x => x.Label == label)
+                    .ToList()
+                    .ForEach(s => allTokens.AddRange(s.Words));
+                allTextByCategory.Add(new Tuple<string, string>(label, String.Join(" ", allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma))));
+            });
+
             Categories.ForEach(label =>
             {
                 var allTokens = new List<Token>();
@@ -89,7 +117,7 @@ namespace BotSharp.NLP.Featuring
                     .ToList()
                     .ForEach(s => allTokens.AddRange(s.Words));
 
-                allTokens.Select(x => x.Lemma).Distinct()
+                allTokens.Where(x => x.IsAlpha).Select(x => x.Lemma).Distinct()
                     .ToList()
                     .ForEach(word =>
                     {
@@ -98,8 +126,15 @@ namespace BotSharp.NLP.Featuring
                         double tf = (c1 + 1.0) / allTokens.Count();
 
                         // IDF
-                        var c2 = Sentences.Where(s => s.Words.Select(x => x.Lemma).Contains(word))
-                            .GroupBy(x => x.Label).Count();
+                        var c2 = 0;
+                        allTextByCategory.ForEach(all =>
+                        {
+                            if(Regex.IsMatch(all.Item2, word))
+                            {
+                                c2++;
+                            }
+                        });
+
                         double idf = Math.Log(Categories.Count / (c2 + 1.0));
 
                         tfs.Add(new Tuple<string, double>(word, tf * idf));

BotSharp.NLP/Models/CRFLite/CRFDecoder.cs

@@ -74,7 +74,7 @@ namespace BotSharp.Models.CRFLite
         }
 
         //Segment given text
-        public int Segment(crf_seg_out[] pout, //segment result
+        public int Segment(CRFSegOut[] pout, //segment result
             SegDecoderTagger tagger, //Tagger per thread
             List<List<string>> inbuf //feature set for segment
             )

BotSharp.NLP/Models/CRFLite/CRFEncoder.cs

@@ -61,24 +61,17 @@ namespace BotSharp.Models.CRFLite
 
             var xList = modelWriter.ReadAllRecords();
 
-
             modelWriter.Shrink(xList, args.MinFeatureFreq);
 
             if (!modelWriter.SaveModelMetaData(args.ModelFileName))
             {
                 return false;
             }
-            else
-            {
-            }
 
             if (!modelWriter.BuildFeatureSetIntoIndex(args.ModelFileName, args.SlotUsageRateThreshold, args.DebugLevel))
             {
                 return false;
             }
-            else
-            {
-            }
 
             if (xList.Length == 0)
             {
@@ -101,6 +94,8 @@ namespace BotSharp.Models.CRFLite
 
         bool runCRF(EncoderTagger[] x, ModelWriter modelWriter, bool orthant, EncoderOptions args)
         {
+            Console.WriteLine("Running encoding process...");
+
             var old_obj = double.MaxValue;
             var converge = 0;
             var lbfgs = new LBFGS(args.ThreadsNum);
@@ -165,6 +160,9 @@ namespace BotSharp.Models.CRFLite
                     lbfgs.err += processList[i].err;
                     lbfgs.zeroone += processList[i].zeroone;
 
+                    Console.WriteLine($"Thread: {i}, Iterating {itr} / {args.MaxIteration}");
+                    Console.WriteLine($"{lbfgs.obj} {lbfgs.err} {lbfgs.zeroone}");
+
                     //Calculate error
                     for (var j = 0; j < modelWriter.y_.Count; j++)
                     {
@@ -263,6 +261,8 @@ namespace BotSharp.Models.CRFLite
                 }
             }
 
+            Console.WriteLine("Completed encoding process.");
+
             return true;
         }
 

BotSharp.NLP/Models/CRFLite/CRFSharpHelper.cs

@@ -7,13 +7,13 @@ namespace BotSharp.Models.CRFLite
 {
     public class SegToken
     {
-        public int offset;
-        public int length;
-        public string strTag; //CRF对应于term组合后的Tag字符串
-        public double fWeight;   //对应属性id的概率值，或者得分
+        public int Offset;
+        public int Length;
+        public string Tag;
+        public double Weight;
     };
 
-    public class crf_seg_out : CRFTermOut
+    public class CRFSegOut : CRFTermOut
     {
         //Segmented token by merging raw CRF model output
         public int termTotalLength;          // the total term length in character
@@ -30,7 +30,7 @@ namespace BotSharp.Models.CRFLite
             tokenList.Clear();
         }
 
-        public crf_seg_out(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM):
+        public CRFSegOut(int max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM):
             base(max_word_num)
         {
             termTotalLength = 0;

BotSharp.NLP/Models/CRFLite/Decoder/DecoderOptions.cs

@@ -8,7 +8,7 @@ namespace BotSharp.Models.CRFLite.Decoder
     public class DecoderOptions
     {
         /// <summary>
-        /// 
+        /// Model file path
         /// </summary>
         public string ModelFileName;
 
@@ -17,16 +17,6 @@ namespace BotSharp.Models.CRFLite.Decoder
         /// </summary>
         public string InputFileName;
 
-        /// <summary>
-        /// 
-        /// </summary>
-        public string OutputFileName;
-
-        /// <summary>
-        /// 
-        /// </summary>
-        public string OutputSegFileName;
-
         /// <summary>
         /// 
         /// </summary>
@@ -43,16 +33,16 @@ namespace BotSharp.Models.CRFLite.Decoder
         public int ProbLevel;
 
         /// <summary>
-        /// 
+        /// Max words length in one sentence
         /// </summary>
         public int MaxWord;
 
         public DecoderOptions()
         {
             Thread = 1;
-            NBest = 1;
+            NBest = 2;
             ProbLevel = 0;
-            MaxWord = 100;
+            MaxWord = 128;
         }
     }
 }

BotSharp.NLP/Models/CRFLite/Decoder/ModelReader.cs

@@ -68,7 +68,7 @@ namespace BotSharp.Models.CRFLite.Decoder
             LoadFeatureWeights();
         }
 
-        //获取key对应的特征id
+        //get key feature id
         public virtual int get_id(string str)
         {
             return da.SearchByPerfectMatch(str);
@@ -143,26 +143,20 @@ namespace BotSharp.Models.CRFLite.Decoder
                 var sr = new StreamReader(metadataStream);
                 string strLine;
 
-                //读入版本号
                 strLine = sr.ReadLine();
                 version = uint.Parse(strLine.Split(':')[1].Trim());
 
-                //读入cost_factor
                 strLine = sr.ReadLine();
                 cost_factor_ = double.Parse(strLine.Split(':')[1].Trim());
 
-                //读入maxid
                 strLine = sr.ReadLine();
                 maxid_ = long.Parse(strLine.Split(':')[1].Trim());
 
-                //读入xsize
                 strLine = sr.ReadLine();
                 xsize_ = uint.Parse(strLine.Split(':')[1].Trim());
 
-                //读入空行
                 strLine = sr.ReadLine();
 
-                //读入待标注的标签
                 y_ = new List<string>();
                 while (true)
                 {
@@ -174,7 +168,7 @@ namespace BotSharp.Models.CRFLite.Decoder
                     y_.Add(strLine);
                 }
 
-                //读入unigram和bigram模板
+                // load unigram and bigram template
                 unigram_templs_ = new List<string>();
                 bigram_templs_ = new List<string>();
                 while (sr.EndOfStream == false)

BotSharp.NLP/Models/CRFLite/Encoder/EncoderOptions.cs

@@ -14,7 +14,7 @@ namespace BotSharp.Models.CRFLite.Encoder
         /// <summary>
         /// Minimum feature frequency, if one feature's frequency is less than this value, the feature will be dropped.
         /// </summary>
-        public int MinFeatureFreq = 2;
+        public int MinFeatureFreq = 1;
 
         /// <summary>
         /// Minimum diff value, when diff less than the value consecutive 3 times, the process will be ended.
@@ -79,7 +79,7 @@ namespace BotSharp.Models.CRFLite.Encoder
         public EncoderOptions()
         {
             MaxIteration = 100;
-            MinFeatureFreq = 2;
+            MinFeatureFreq = 1;
             MinDifference = 0.0001;
             SlotUsageRateThreshold = 0.95;
             ThreadsNum = 1;

BotSharp.NLP/Models/CRFLite/Encoder/ModelWriter.cs

@@ -43,6 +43,8 @@ namespace BotSharp.Models.CRFLite.Encoder
         //Regenerate feature id and shrink features with lower frequency
         public void Shrink(EncoderTagger[] xList, int freq)
         {
+            Console.WriteLine($"Shrink features lower than {freq} frequency");
+
             var old2new = new CRFLite.Utils.BTreeDictionary<long, long>();
             featureLexicalDict.Shrink(freq);
             maxid_ = featureLexicalDict.RegenerateFeatureId(old2new, y_.Count);
@@ -86,7 +88,7 @@ namespace BotSharp.Models.CRFLite.Encoder
                     var oldValue = Interlocked.Increment(ref arrayEncoderTaggerSize) - 1;
                     arrayEncoderTagger[oldValue] = _x;
 
-                    if (oldValue % 10000 == 0)
+                    if (oldValue % 100 == 0)
                     {
                         //Show current progress on console
                         Console.Write("{0}...", oldValue);
@@ -94,10 +96,11 @@ namespace BotSharp.Models.CRFLite.Encoder
                 }
             });
 
+            Console.WriteLine($"Read {trainCorpusList.Count} records");
+
             trainCorpusList.Clear();
             trainCorpusList = null;
-
-            Console.WriteLine();
+            
             return arrayEncoderTagger;
         }
 
@@ -136,6 +139,8 @@ namespace BotSharp.Models.CRFLite.Encoder
             //Save indexed feature set into file
             da.save(filename_featureset);
 
+            Console.WriteLine($"Saved featureset to {filename_featureset}");
+
             if (string.IsNullOrWhiteSpace(modelFileName))
             {
                 //Clean up all data
@@ -220,6 +225,7 @@ namespace BotSharp.Models.CRFLite.Encoder
 
             tofs.Close();
 
+            Console.WriteLine($"Saved meta data to {filename}");
             return true;
         }
 
@@ -277,6 +283,8 @@ namespace BotSharp.Models.CRFLite.Encoder
 
         bool OpenTemplateFile(string filename)
         {
+            Console.WriteLine($"Open template: {filename}");
+
             var ifs = new StreamReader(filename);
             unigram_templs_ = new List<string>();
             bigram_templs_ = new List<string>();
@@ -305,6 +313,8 @@ namespace BotSharp.Models.CRFLite.Encoder
 
         bool OpenTrainCorpusFile(string strTrainingCorpusFileName)
         {
+            Console.WriteLine($"Open corpus: {strTrainingCorpusFileName}");
+
             var ifs = new StreamReader(strTrainingCorpusFileName);
             y_ = new List<string>();
             trainCorpusList = new List<List<List<string>>>();

BotSharp.NLP/Models/CRFLite/SegDecoderTagger.cs

@@ -13,7 +13,7 @@ namespace BotSharp.Models.CRFLite
             crf_max_word_num = this_crf_max_word_num;
         }
 
-        int seg_termbuf_build(crf_seg_out term_buf)
+        int seg_termbuf_build(CRFSegOut term_buf)
         {
             term_buf.Clear();
 
@@ -42,24 +42,24 @@ namespace BotSharp.Models.CRFLite
                     i == x_.Count - 1)
                 {
                     var tkn = new SegToken();
-                    tkn.length = term_len;
-                    tkn.offset = term_buf.termTotalLength;
+                    tkn.Length = term_len;
+                    tkn.Offset = term_buf.termTotalLength;
 
                     var spos = strTag.IndexOf('_');
                     if (spos < 0)
                     {
                         if (strTag == "NOR")
                         {
-                            tkn.strTag = "";
+                            tkn.Tag = "";
                         }
                         else
                         {
-                            tkn.strTag = strTag;
+                            tkn.Tag = strTag;
                         }
                     }
                     else
                     {
-                        tkn.strTag = strTag.Substring(spos + 1);
+                        tkn.Tag = strTag.Substring(spos + 1);
                     }
 
                     term_buf.termTotalLength += term_len;
@@ -67,10 +67,10 @@ namespace BotSharp.Models.CRFLite
                     switch (vlevel_)
                     {
                         case 0:
-                            tkn.fWeight = 0.0;
+                            tkn.Weight = 0.0;
                             break;
                         case 2:
-                            tkn.fWeight = weight / num;
+                            tkn.Weight = weight / num;
                             weight = 0.0;
                             num = 0;
                             break;
@@ -86,7 +86,7 @@ namespace BotSharp.Models.CRFLite
         }
 
 
-        public int output(crf_seg_out[] pout)
+        public int output(CRFSegOut[] pout)
         {
             var n = 0;
             var ret = 0;

BotSharp.NLP/Txt2Vec/OneHotEncoder.cs

@@ -25,7 +25,7 @@ namespace BotSharp.NLP.Txt2Vec
 
             sentence.Words.ForEach(w =>
             {
-                int index = Words.IndexOf(w.Lemma.ToLower());
+                int index = Words.IndexOf(w.Lemma);
                 if(index > 0)
                 {
                     vector[index] = 1;
@@ -49,7 +49,12 @@ namespace BotSharp.NLP.Txt2Vec
         {
             if (Words == null)
             {
-                // Words = "shuffle,pause,resume,next,stop,previous,continue,mode,repeat,back,music,play,enough,off,them,playlist,skip,restart,favourites,on,add,go,again,turn,save,my,station,favourite,start,by,playing,please,now,running,move".Split(',').ToList();
+                Words = new List<string>();
+                Sentences.ForEach(x =>
+                {
+                    Words.AddRange(x.Words.Where(w => w.IsAlpha).Select(w => w.Lemma));
+                });
+                Words = Words.Distinct().OrderBy(x => x).ToList();
             }
 
             return Words;

BotSharp.RestApi/AgentController.cs

@@ -38,9 +38,22 @@ namespace BotSharp.RestApi
         [HttpGet]
         public ActionResult<List<Agent>> AllAgents()
         {
-            var dc = new DefaultDataContextLoader().GetDefaultDc();
+            List<Agent> agents = new List<Agent>();
 
-            return dc.Table<Agent>().ToList();
+            string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects");
+
+            var names = Directory.EnumerateDirectories(agentDir).Select(x => x.Split(Path.DirectorySeparatorChar).Last()).ToList();
+
+            names.ForEach(name =>
+            {
+                agents.Add(new Agent
+                {
+                    Name = name
+                });
+
+            });
+
+            return agents;
         }
 
         /// <summary>

BotSharp.RestApi/BotSharp.RestApi.xml

@@ -120,7 +120,7 @@
             Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project.
             </summary>
             <param name="model">Model name</param>
-            <param name="project"></param>
+            <param name="project">Agent name or agent id</param>
             <returns></returns>
         </member>
         <member name="P:BotSharp.RestApi.Integrations.FacebookMessenger.WebhookMessageRecipient.Id">

BotSharp.RestApi/Rasa/TrainController.cs

@@ -38,10 +38,10 @@ namespace BotSharp.RestApi.Rasa
         /// Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project.
         /// </summary>
         /// <param name="model">Model name</param>
-        /// <param name="project"></param>
+        /// <param name="project">Agent name or agent id</param>
         /// <returns></returns>
         [HttpPost]
-        public async Task<ActionResult<String>> Train([FromQuery] string model, [FromQuery] string project)
+        public async Task<ActionResult<String>> Train([FromQuery] string project, [FromQuery] string model)
         {
             string body = "";
             using (var reader = new StreamReader(Request.Body))

BotSharp.UI/package-lock.json

@@ -3487,7 +3487,7 @@
           "dev": true,
           "requires": {
             "boom": "2.x.x",
-            "cryptiles": "2.x.x",
+            "cryptiles": "~>4.1.2",
             "hoek": "2.x.x",
             "sntp": "1.x.x"
           }
@@ -8726,7 +8726,7 @@
       "integrity": "sha1-iSIN32t1GuUrX3JISGNShZa7hME=",
       "dev": true,
       "requires": {
-        "macaddress": "^0.2.8"
+        "macaddress": "~>0.2.9"
       }
     },
     "uniqs": {

BotSharp.WebHost/App_Data/CRFLite/template.en

@@ -0,0 +1,18 @@
+# Unigram
+U01:%x[-1,0]
+U02:%x[0,0]
+U03:%x[1,0]
+U05:%x[-1,0]/%x[0,0]
+U06:%x[0,0]/%x[1,0]
+
+U11:%x[-1,1]
+U12:%x[0,1]
+U13:%x[1,1]
+U16:%x[-1,1]/%x[0,1]
+U17:%x[0,1]/%x[1,1]
+
+U20:%x[-1,0]/%x[0,0]/%x[1,0]
+U21:%x[-1,1]/%x[0,1]/%x[1,1]
+
+# Bigram
+B

BotSharp.WebHost/BotSharp.WebHost.csproj

@@ -37,7 +37,6 @@
     <Compile Remove="App_Data\Corpus\**" />
     <Compile Remove="App_Data\DbInitializer\**" />
     <Compile Remove="App_Data\ModelFiles\**" />
-    <Compile Remove="App_Data\NewFolder\**" />
     <Compile Remove="App_Data\Projects\**" />
     <Compile Remove="App_Data\TrainingFiles\**" />
     <Compile Remove="PublishOutput\**" />
@@ -45,7 +44,6 @@
     <Content Remove="App_Data\Corpus\**" />
     <Content Remove="App_Data\DbInitializer\**" />
     <Content Remove="App_Data\ModelFiles\**" />
-    <Content Remove="App_Data\NewFolder\**" />
     <Content Remove="App_Data\Projects\**" />
     <Content Remove="App_Data\TrainingFiles\**" />
     <Content Remove="PublishOutput\**" />
@@ -53,7 +51,6 @@
     <EmbeddedResource Remove="App_Data\Corpus\**" />
     <EmbeddedResource Remove="App_Data\DbInitializer\**" />
     <EmbeddedResource Remove="App_Data\ModelFiles\**" />
-    <EmbeddedResource Remove="App_Data\NewFolder\**" />
     <EmbeddedResource Remove="App_Data\Projects\**" />
     <EmbeddedResource Remove="App_Data\TrainingFiles\**" />
     <EmbeddedResource Remove="PublishOutput\**" />
@@ -61,7 +58,6 @@
     <None Remove="App_Data\Corpus\**" />
     <None Remove="App_Data\DbInitializer\**" />
     <None Remove="App_Data\ModelFiles\**" />
-    <None Remove="App_Data\NewFolder\**" />
     <None Remove="App_Data\Projects\**" />
     <None Remove="App_Data\TrainingFiles\**" />
     <None Remove="PublishOutput\**" />

BotSharp.WebHost/Program.cs

@@ -18,10 +18,20 @@ namespace BotSharp.WebHost
                 .ConfigureAppConfiguration((hostingContext, config) =>
                 {
                     var env = hostingContext.HostingEnvironment;
-                    string dir = Path.GetFullPath(env.ContentRootPath + "/..");
-                    var settings = Directory.GetFiles(Path.Combine(dir, "Settings"), "*.json");
+                    string dir = Path.GetFullPath(env.ContentRootPath);
+                    string settingsFolder = Path.Combine(dir, "Settings");
+
+                    if (!Directory.Exists(settingsFolder))
+                    {
+                        dir = Path.GetFullPath(env.ContentRootPath + "/..");
+                    }
+
+                    settingsFolder = Path.Combine(dir, "Settings");
+                    Console.WriteLine($"Settings folder: {settingsFolder}");
+                    var settings = Directory.GetFiles(settingsFolder, "*.json");
                     settings.ToList().ForEach(setting =>
                     {
+                        Console.WriteLine($"Read {setting}");
                         config.AddJsonFile(setting, optional: false, reloadOnChange: true);
                     });
                 })

Dockerfile

@@ -5,15 +5,13 @@ WORKDIR /source
 # copies the rest of your code
 COPY . .
 RUN dotnet build
-RUN dotnet publish --output /app/ --configuration Debug
+RUN dotnet publish --output /app --configuration RASA
 
-# install facebookresearch fasttext
-RUN wget https://github.com/facebookresearch/fastText/archive/v0.1.0.zip
-RUN apt-get update 
-RUN apt-get install -y unzip make g++
-RUN unzip v0.1.0.zip
-WORKDIR /source/fastText-0.1.0/ 
-RUN make
+# copy Settings folder
+COPY Settings /app/Settings
+
+# App_Data
+COPY BotSharp.WebHost/App_Data /app/App_Data
 
 # stage 2: run
 WORKDIR /app

README.rst

@@ -1,11 +1,14 @@
-The Open Source AI Bot Platform Builder for Enterprise
+The Open Source AI Bot Platform Builder
 ======================================================
 
-*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to our enterprise developers using the BotSharp Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your enterprise robots and precisely control every step of the AI processing pipeline."*
+.. image:: https://img.shields.io/badge/gitter-join%20chat-brightgreen.svg
+    :target: `gitter`_
 
-**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in enterprise information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. 
+*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to our .NET developers using the BotSharp AI BOT Platform Builder to build a CaaP. It opens up as much learning power as possible for your own robots and precisely control every step of the AI processing pipeline."*
 
-It's witten  in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. More friendly to corporate developers. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope. 
+**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. 
+
+It's witten  in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. More friendly to corporate developers. BotSharp adopts machine learning algrithm in C# directly. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope. 
 
 Why we do this? because we all know python is not friendly programming language for enterprise developers, it's not only because it's low performance but also it's a type weak language, it will be a disater if you use python to build your bussiness system.
 
@@ -45,3 +48,4 @@ If you feel that this project is helpful to you, please Star on the project, we
 .. _Docker: https://www.docker.com
 .. _Rasa UI: https://github.com/paschmann/rasa-ui
 .. _Articulate UI: https://spg.ai/projects/articulate
+.. _gitter: https://gitter.im/botsharpcore/Lobby

Settings/bot.json

@@ -7,19 +7,23 @@
     },
 
     "Pipe": {
-      "train": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier",
-      "predict": "BotSharpTokenizer, BotSharpTagger, CRFsuiteEntityRecognizer, BotSharpNBayesClassifier"
+      "train": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier",
+      "predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier"
     },
 	
 	"BotSharpNBayesClassifier": {
 	},
 	
     "BotSharpSVMClassifier": {
-      "wordvec": "C:\\Users\\bpeng\\Desktop\\BoloReborn\\BotSharp\\Data"
+      "wordvec": ""
     },
 
     "BotSharpTagger": {
     },
+	
+	"BotSharpCRFNer": {
+		"template": "|App_Data|CRFLite/template.en"
+	},
 
     "CRFsuiteEntityRecognizer": {
       "fields": "y w pos chk",

Settings/swagger.json

@@ -2,7 +2,7 @@
   "Swagger": {
     "Contact": {
       "Email": "haiping008@gmail.com",
-      "Name": "Haiping Chen",
+      "Name": "BotSharp AI",
       "Url": "https://github.com/Oceania2018"
     },
     "Description": "BotSharp is a chatbot platform written in C# (.net core), and it's developed for enterprise usage.",

docker-compose-articulateui.yml

@@ -2,7 +2,7 @@ version: '3.0'
 
 services:
   api:
-    image: samtecspg/articulate-api:repo-head
+    image: samtecspg/articulate-api:0.12.1
     ports: ['0.0.0.0:7500:7500']
     networks: ['botsharp-network']
     entrypoint: ['node', 'start.js']
@@ -10,7 +10,7 @@ services:
      - SWAGGER_BASE_PATH
 
   ui:
-    image: samtecspg/articulate-ui:repo-head
+    image: samtecspg/articulate-ui:0.12.1
     ports: ['0.0.0.0:3000:3000']
     networks: ['botsharp-network']
     environment:
@@ -21,10 +21,12 @@ services:
     image: botsharpdocker/botsharp:latest
     ports: ['0.0.0.0:5000:5000']
     networks: ['botsharp-network']
+
   duckling:
     image: samtecspg/duckling:0.1.6.0
     ports: ['0.0.0.0:8000:8000']
     networks: ['botsharp-network']
+
   redis:
     image: redis:4.0.6-alpine
     ports: ['0.0.0.0:6379:6379']

docs/agent/import-agent.rst

@@ -0,0 +1,45 @@
+Import Agent
+============
+Designed as a multi-platform framework, BotSharp allows developers to create their own Bot platforms and support multiple Bot platform services. It supports multiple Bot import, export and message reply formats such as Dialogflow and Rasa.
+Support for importing and exporting between platforms.
+
+**First, export agent from other chatbot platform.**
+
+In general, the platform provides the ability to export to a compressed file. Different platform has different export method.
+
+**Second, add meta.json to zip file.**
+
+meta.json is used to tell BotSharp where the agent is exported from. It should looks like below:
+
+.. code-block:: json
+
+    {
+        "Id": "YOURS",
+        "Name": "YOURS",
+        "Platform": "Dialogflow",
+        "ClientAccessToken": "YOURS",
+        "DeveloperAccessToken": "YOURS",
+        "Integrations": []
+    }
+
+Extract zip file and add the meta.json to the zip file.    
+
+1. Google Dialogflow
+::
+
+"Platform": "Dialogflow"
+
+2. RASA
+::
+
+"Platform": "Rasa"
+
+3. Microsoft LUIS
+
+**Last, upload updated zip file.**
+
+Upload zip file in REST API.
+
+|RestoreAgentFromZipScreenshot|
+
+.. |RestoreAgentFromZipScreenshot| image:: /static/screenshots/RestoreAgentFromZip.png

docs/agent/optimize-agent.rst

@@ -0,0 +1,4 @@
+Optimized Robot
+===============
+
+The semantic understanding ability of the robot can be improved by modifying the parameters of the training and the selection of the hyperparameters.

docs/agent/test-agent.rst

@@ -0,0 +1,13 @@
+Test Agent
+===========
+
+After the training is complete, you can start testing the Agent. Enter the robot name and the statement you want to test.
+
+|APITestInputScreenshot|
+
+After clicking the execution, you will get the result returned by the server, which contains the user intent and the entity value.
+
+|APITestResultScreenshot|
+
+.. |APITestInputScreenshot| image:: /static/screenshots/APITestInput.png
+.. |APITestResultScreenshot| image:: /static/screenshots/APITestResult.png

docs/agent/train-agent.rst

@@ -0,0 +1,20 @@
+Train Agent
+===========
+
+When you successfully import the Agent, the next step is to train your Agent and let it run according to your pre-designed process. 
+
+Fill in your Agent name and click the "Train" button for a while (depending on the size of the data).
+
+|APITrainStartScreenshot|
+
+During the training, the console will enter the training status immediately.
+
+|APITrainInProgressScreenshot|
+
+After the training is completed, you will get the details of a model.
+
+|APITrainCompletedScreenshot|
+
+.. |APITrainStartScreenshot| image:: /static/screenshots/APITrainStart.png
+.. |APITrainInProgressScreenshot| image:: /static/screenshots/APITrainInProgress.png
+.. |APITrainCompletedScreenshot| image:: /static/screenshots/APITrainCompleted.png

docs/conf.py

@@ -101,7 +101,7 @@ html_theme = 'sphinx_rtd_theme'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ['static']
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.

docs/index.rst

@@ -6,12 +6,15 @@
 .. image:: https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png
    :height: 30px
    
-The Open Source AI Bot Platform Builder for Enterprise
+.. image:: https://img.shields.io/badge/gitter-join%20chat-brightgreen.svg
+    :target: `gitter`_
+
+The Open Source AI Bot Platform Builder
 ======================================================
 
-*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to our enterprise developers using the BotSharp Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your enterprise robots and precisely control every step of the AI processing pipeline."*
+*"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to .NET developers using the BotSharp Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your robots and precisely control every step of the AI processing pipeline."*
 
-**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in enterprise information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. 
+**BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. 
 
 It's witten  in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. More friendly to corporate developers. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope. 
 
@@ -22,13 +25,16 @@ BotSharp is in accordance with components princple strictly, decouples every par
 Some Features
 -------------
 
+* Integrated debugging is easier without relying on any other machine learning algorithm libraries.
 * Built-in multi-Agents management, easy to build Bot as a Service platform.
 * Context In/ Out with lifespan to make conversion flow be controllable.
-* Use the natural language processing pipeline mechanism and the popular NLP algorithm library to build your own unique robot processing flow.
+* Use the natural language processing pipeline mechanism to work with extensions easily, and build your own unique robot processing flows. 
+* Rewrote NLP algorithm from ground without historical issues.
 * Support export/ import agent from other bot platforms directly. 
 * Support different UI providers like `Rasa UI`_ and `Articulate UI`_.
 * Support for multiple data request and response formats such as Rasa NLU and Dialogflow.
 * Integrate with popular social platforms like Facebook Messenger, Slack and Telegram.
+* Multi-core parallel computing optimization, High-Performance C# on GPUs in Hybridizer.
 
 Indices and tables
 ==================
@@ -36,6 +42,7 @@ The main documentation for the site is organized into a couple sections:
 
 * :ref:`User Documentation <user-docs>`
 * :ref:`Integration Documentation <integration-docs>`
+* :ref:`NLP Documentation <nlp-docs>`
 * :ref:`search`
 
 .. _user-docs:
@@ -45,16 +52,37 @@ The main documentation for the site is organized into a couple sections:
    :caption: User Documentation:
 
    installation
+   agent/import-agent
+   agent/train-agent
+   agent/test-agent
+   agent/optimize-agent
 
 .. _integration-docs:
 
 .. toctree::
-   :maxdepth: 2
-   :caption: Integration Documentation:
+   :maxdepth: 3
+   :caption: Channels Integration Documentation:
 
    integrations/facebook-messenger
+   integrations/slack
+   integrations/telegram
+   integrations/skype
 
-If you feel that this project is helpful to you, please Star on the project, we will be very grateful.
+.. _nlp-docs:
+
+.. toctree::
+   :maxdepth: 2
+   :caption: NLP Documentation:
+
+   models/crf
+   models/nb
+   models/ngram
+   models/svm
+   models/tfidf
+   models/penntreebank
+
+If you feel that this project is helpful to you, please Star us on the project, we will be very grateful.
 
 .. _Rasa UI: https://github.com/paschmann/rasa-ui
 .. _Articulate UI: https://spg.ai/projects/articulate
+.. _gitter: https://gitter.im/botsharpcore/Lobby

docs/installation.rst

@@ -2,7 +2,7 @@ Installation
 ============
 BotSharp strictly follows the modular design principle and adopts a structure in which views and logic are separated. 
 So you can choose the front-end Bot design and management interface. 
-If you want to use the `RASA UI`_ as a front end, you can use the rasaui-specific compose file to quickly experience BotSharp.
+If you want to use the `Articulate UI`_ as a front end, you can use the articulateui-specific compose file to quickly experience BotSharp.
 
 Docker Composer
 ^^^^^^^^^^^^^^^
@@ -20,6 +20,7 @@ You can use docker compose to run, make sure you've got `Docker`_ installed.
  PS D:\BotSharp\> docker-compose -f docker-compose-articulateui.yml up
 
 Point your web browser at http://localhost:3000 and enjoy Articulate-UI with BotSharp.
+|ArticulateHomeScreenshot|
 
 2. Integrate with `Rasa UI`_, you can use docker compose to run.
 
@@ -29,8 +30,14 @@ Point your web browser at http://localhost:3000 and enjoy Articulate-UI with Bot
 
 Point your web browser at http://localhost:5001 and enjoy Rasa-UI with BotSharp.
 
+|RasaUIHomeScreenshot|
+
+3. Integrate with `Rasa Talk`_
+
+
 Building BotSharp
 ^^^^^^^^^^^^^^^^^
+If you are a .NET developer and want to develop extensions or fix bug for BotSharp, you would CTRL + F5 to run it locally in debug mode. 
 Make sure the `Microsoft .NET Core`_ build environment is installed. 
 Building solution using dotnet CLI (preferred).
 
@@ -43,6 +50,8 @@ Building solution using dotnet CLI (preferred).
 Install in docker container
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+If you just want to run BotSharp as a backend NLU engine, you can run it standalone in docker.
+
 ::
  
     PS D:\> git clone https://github.com/Oceania2018/BotSharp
@@ -61,7 +70,9 @@ Start a container:
  PS D:\BotSharp\> docker run -it -p 5000:5000 botsharp
 
  
+Access restful APIs: http://localhost:5000 if you are using RASA response format.
 
+|APIHomeScreenshot|
 
 
 Install in NuGet
@@ -81,5 +92,10 @@ Use BotSharp.NLP as a natural language processing toolkit alone.
 
 .. _Rasa UI: https://github.com/paschmann/rasa-ui
 .. _Articulate UI: https://spg.ai/projects/articulate
+.. _Rasa Talk: https://github.com/jackdh/RasaTalk
 .. _Microsoft .NET Core: https://www.microsoft.com/net/download
-.. _Docker: https://www.docker.com
+.. _Docker: https://www.docker.com
+
+.. |APIHomeScreenshot| image:: /static/screenshots/APIHome.png
+.. |ArticulateHomeScreenshot| image:: /static/screenshots/ArticulateHome.png
+.. |RasaUIHomeScreenshot| image:: /static/screenshots/RasaUIHome.png

docs/integrations/skype.rst

@@ -0,0 +1,2 @@
+Skype Chabot
+============

docs/integrations/slack.rst

@@ -0,0 +1,6 @@
+Slack App
+=========
+
+A bot is a type of Slack App designed to interact with users via conversation.
+
+A bot is the same as a regular app: it can access the same range of APIs and do all of the magical things that a Slack App can do.

docs/integrations/telegram.rst

@@ -0,0 +1,4 @@
+Telegram Bot Platform
+=====================
+
+Bots are simply Telegram accounts operated by software – not people – and they'll often have AI features. They can do anything – teach, play, search, broadcast, remind, connect, integrate with other services, or even pass commands to the Internet of Things.

docs/models/penntreebank.rst

@@ -0,0 +1,2 @@
+Penn Treebank
+=============

docs/models/tfidf.rst

@@ -0,0 +1,2 @@
+TF-IDF
+======