Make pipline part options more configurable.

BotSharp.Core/Engines/BotPredictor.cs

@@ -52,8 +52,7 @@ namespace BotSharp.Core.Engines
             var settings = new PipeSettings
             {
                 ModelDir = dir,
-                ProjectDir = request.AgentDir,
-                AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
+                ProjectDir = request.AgentDir
             };
 
 
@@ -66,7 +65,7 @@ namespace BotSharp.Core.Engines
             for(int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++)
             {
                 var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpPredict;
-                pipe.Configuration = provider.Configuration;
+                pipe.Configuration = provider.Configuration.GetSection(pipelines[pipeIdx]);
                 pipe.Settings = settings;
                 await pipe.Predict(agent, data, meta.Pipeline.FirstOrDefault(x => x.Name == pipelines[pipeIdx]));
             }

BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs

@@ -54,7 +54,7 @@ namespace BotSharp.Core.Engines.BotSharp
             }
 
             string contentDir = AppDomain.CurrentDomain.GetData("DataPath").ToString();
-            string template = Configuration.GetValue<String>($"BotSharpCRFNer:template");
+            string template = Configuration.GetValue<String>($"template");
             template = template.Replace("|App_Data|", contentDir + System.IO.Path.DirectorySeparatorChar);
 
             var encoder = new CRFEncoder();

BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs

@@ -15,17 +15,17 @@ namespace BotSharp.Core.Engines.BotSharp
     {
         public IConfiguration Configuration { get; set; }
         public PipeSettings Settings { get; set; }
-        private TokenizerFactory<TreebankTokenizer> _tokenizer;
+        private TokenizerFactory _tokenizer;
 
         public BotSharpTokenizer()
         {
-            _tokenizer = new TokenizerFactory<TreebankTokenizer>(new TokenizationOptions
-            {
-            }, SupportedLanguage.English);
+
         }
 
         public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
         {
+            Init();
+
             doc.Tokenizer = this;
 
             // same as train
@@ -39,6 +39,8 @@ namespace BotSharp.Core.Engines.BotSharp
 
         public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
         {
+            Init();
+
             doc.Tokenizer = this;
             doc.Sentences = new List<NlpDocSentence>();
 
@@ -54,5 +56,20 @@ namespace BotSharp.Core.Engines.BotSharp
 
             return true;
         }
+
+        private void Init()
+        {
+            if(_tokenizer == null)
+            {
+                _tokenizer = new TokenizerFactory(new TokenizationOptions
+                {
+                    Pattern = Configuration.GetValue<String>("options:pattern")
+                }, SupportedLanguage.English);
+
+                string tokenizerName = Configuration.GetValue<String>($"tokenizer");
+
+                _tokenizer.GetTokenizer(tokenizerName);
+            }
+        }
     }
 }

BotSharp.Core/Engines/BotTrainer.cs

@@ -97,7 +97,8 @@ namespace BotSharp.Core.Engines
             for (int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++)
             {
                 var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpTrain;
-                pipe.Configuration = provider.Configuration;
+                // set configuration to current section
+                pipe.Configuration = provider.Configuration.GetSection(pipelines[pipeIdx]);
                 pipe.Settings = settings;
                 pipeModel = new PipeModel
                 {

BotSharp.NLP/Tokenize/TokenizerFactory.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Reflection;
 using System.Text;
 using System.Threading.Tasks;
 
@@ -12,19 +13,35 @@ namespace BotSharp.NLP.Tokenize
     /// The particular tokenizer requires implement interface 
     /// models to be installed.BotSharp.NLP also provides a simpler, regular-expression based tokenizer, which splits text on whitespace and punctuation.
     /// </summary>
-    public class TokenizerFactory<ITokenize> where ITokenize : ITokenizer, new()
+    public class TokenizerFactory
     {
         private SupportedLanguage _lang;
 
-        private ITokenize _tokenizer;
+        private ITokenizer _tokenizer;
 
         private TokenizationOptions _options;
 
+        public ITokenizer GetTokenizer<ITokenize>() where ITokenize : ITokenizer, new()
+        {
+            return _tokenizer = new ITokenize();
+        }
+
+        public ITokenizer GetTokenizer(string name)
+        {
+            List<Type> types = Assembly.Load(new AssemblyName("BotSharp.NLP"))
+                .GetTypes().Where(x => !x.IsAbstract && !x.FullName.StartsWith("<>f__AnonymousType")).ToList();
+
+            Type type = types.FirstOrDefault(x => x.Name == name);
+            var instance = (ITokenizer)Activator.CreateInstance(type);
+
+            return _tokenizer = instance;
+        }
+
         public TokenizerFactory(TokenizationOptions options, SupportedLanguage lang)
         {
             _lang = lang;
             _options = options;
-            _tokenizer = new ITokenize();
+            
         }
 
         public List<Token> Tokenize(string sentence)

Settings/bot.json

@@ -11,6 +11,10 @@
       "predict": "BotSharpTokenizer, BotSharpTagger, BotSharpCRFNer, BotSharpNBayesClassifier"
     },
 	
+	"BotSharpTokenizer": {
+		"tokenizer": "TreebankTokenizer"
+	},
+	
 	"BotSharpNBayesClassifier": {
 	},
 	
@@ -19,6 +23,7 @@
     },
 
     "BotSharpTagger": {
+		
     },
 	
 	"BotSharpCRFNer": {