diff --git a/BotSharp.Core/Accounts/AccountDbInitializer.cs b/BotSharp.Core/Accounts/AccountDbInitializer.cs index 0e55e6e0..5f90383b 100644 --- a/BotSharp.Core/Accounts/AccountDbInitializer.cs +++ b/BotSharp.Core/Accounts/AccountDbInitializer.cs @@ -1,4 +1,5 @@ using BotSharp.Core.Abstractions; +using BotSharp.NLP.Tokenize; using EntityFrameworkCore.BootKit; using Newtonsoft.Json; using System; @@ -20,8 +21,8 @@ namespace BotSharp.Core.Accounts private void ImportAccount(Database dc) { - var dataPath = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "DbInitializer", "Accounts"); - string json = File.ReadAllText(Path.Join(dataPath, "users.json")); + var dataPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "DbInitializer", "Accounts"); + string json = File.ReadAllText(Path.Combine(dataPath, "users.json")); var users = JsonConvert.DeserializeObject>(json); users.ForEach(user => diff --git a/BotSharp.Core/BotSharp.Core.csproj b/BotSharp.Core/BotSharp.Core.csproj index e0fa9531..26cb7a13 100644 --- a/BotSharp.Core/BotSharp.Core.csproj +++ b/BotSharp.Core/BotSharp.Core.csproj @@ -1,19 +1,20 @@ - + SAK SAK SAK SAK + AnyCPU;x64 - netcoreapp2.1 + netstandard2.0 true Haiping Chen - BotSharp Chatbot Platform - Open source chatbot platform which is written in C# runs on .Net Core and is enterprise oriented. Integrated with multiple bot engines besides BotSharp bot engine. Modulized pipeline design make NLP tasks plugin easily. Abstract platform and NLP task, migrate existed chatbot from a platform into another platform perfectly through dump and restore. + BotSharp AI Bot Platform Builder + Open source AI Bot platform builder which is written in C# runs on .Net Core and is enterprise oriented. Integrated with multiple bot engines besides BotSharp bot engine. Modulized pipeline design make NLP tasks plugin easily. Abstract platform and NLP task, migrate existed chatbot from a platform into another platform perfectly through dump and restore. MIT https://github.com/Oceania2018/BotSharp NLU, Chatbot, Bot, AI Bot, Artificial Intelligence, RPA @@ -30,13 +31,21 @@ TRACE;DEBUG + + TRACE;DEBUG + + TRACE;MODEL_PER_CONTEXTS + + TRACE;MODEL_PER_CONTEXTS + + - - + + @@ -44,6 +53,7 @@ + diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs index 7975ac41..6822a834 100644 --- a/BotSharp.Core/Engines/BotEngineBase.cs +++ b/BotSharp.Core/Engines/BotEngineBase.cs @@ -32,7 +32,7 @@ namespace BotSharp.Core.Engines { dc = new DefaultDataContextLoader().GetDefaultDc(); string dataPath = AppDomain.CurrentDomain.GetData("DataPath").ToString(); - DbInitializerPath = Path.Join(dataPath, $"DbInitializer"); + DbInitializerPath = Path.Combine(dataPath, $"DbInitializer"); } public AIResponse TextRequest(AIRequest request) @@ -91,7 +91,7 @@ namespace BotSharp.Core.Engines /// public bool RestoreAgent(AgentImportHeader agentHeader) where TAgentImporter : IAgentImporter, new() { - string dataDir = Path.Join(DbInitializerPath, "Agents"); + string dataDir = Path.Combine(DbInitializerPath, "Agents"); int row = dc.DbTran(() => { LoadAgentFromFile(dataDir, agentHeader); diff --git a/BotSharp.Core/Engines/BotPredictor.cs b/BotSharp.Core/Engines/BotPredictor.cs index 3d5b7b0f..6a941e2a 100644 --- a/BotSharp.Core/Engines/BotPredictor.cs +++ b/BotSharp.Core/Engines/BotPredictor.cs @@ -20,9 +20,9 @@ namespace BotSharp.Core.Engines public async Task Predict(Agent agent, AIRequest request) { // load model - var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); + var dir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id); Console.WriteLine($"Load model from {dir}"); - var metaJson = File.ReadAllText(Path.Join(dir, "metadata.json")); + var metaJson = File.ReadAllText(Path.Combine(dir, "metadata.json")); var meta = JsonConvert.DeserializeObject(metaJson); // Get NLP Provider @@ -49,8 +49,8 @@ namespace BotSharp.Core.Engines var settings = new PipeSettings { - ProjectDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id), - AlgorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms") + ProjectDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id), + AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms") }; diff --git a/BotSharp.Core/Engines/BotTrainer.cs b/BotSharp.Core/Engines/BotTrainer.cs index 5a083544..b092ed41 100644 --- a/BotSharp.Core/Engines/BotTrainer.cs +++ b/BotSharp.Core/Engines/BotTrainer.cs @@ -66,11 +66,11 @@ namespace BotSharp.Core.Engines var settings = new PipeSettings { - ProjectDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id), - AlgorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms") + ProjectDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agent.Id), + AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms") }; - settings.ModelDir = Path.Join(settings.ProjectDir, "model" + DateTime.UtcNow.ToString("MMddyyyyHHmm")); + settings.ModelDir = Path.Combine(settings.ProjectDir, "model" + DateTime.UtcNow.ToString("MMddyyyyHHmm")); if (!Directory.Exists(settings.ProjectDir)) { @@ -126,7 +126,7 @@ namespace BotSharp.Core.Engines NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver() }); - File.WriteAllText(Path.Join(settings.ModelDir, "metadata.json"), metaJson); + File.WriteAllText(Path.Combine(settings.ModelDir, "metadata.json"), metaJson); Console.WriteLine(metaJson); diff --git a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs index 5549d2a2..f54dcf4e 100644 --- a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs +++ b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs @@ -21,17 +21,17 @@ namespace BotSharp.Core.Engines.Classifiers public async Task Predict(Agent agent, NlpDoc doc, PipeModel meta) { - string modelFileName = Path.Join(Settings.ModelDir, meta.Model); - string predictFileName = Path.Join(Settings.TempDir, "fasttext.txt"); + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); + string predictFileName = Path.Combine(Settings.TempDir, "fasttext.txt"); File.WriteAllText(predictFileName, doc.Sentences[0].Text); - var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"predict-prob {modelFileName}.bin {predictFileName}"); + var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"predict-prob {modelFileName}.bin {predictFileName}"); File.Delete(predictFileName); doc.Sentences[0].Intent = new TextClassificationResult { - Label = output.Split(' ')[0].Split("__label__")[1], + Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1], Confidence = decimal.Parse(output.Split(' ')[1]) }; @@ -42,8 +42,8 @@ namespace BotSharp.Core.Engines.Classifiers { meta.Model = "classification-fasttext.model"; - string parsedTrainingDataFileName = Path.Join(Settings.TempDir, $"classification-fasttext.parsed.txt"); - string modelFileName = Path.Join(Settings.ModelDir, meta.Model); + string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-fasttext.parsed.txt"); + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); // assemble corpus StringBuilder corpus = new StringBuilder(); @@ -51,7 +51,7 @@ namespace BotSharp.Core.Engines.Classifiers File.WriteAllText(parsedTrainingDataFileName, corpus.ToString()); - var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}", false); + var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}", false); Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); diff --git a/BotSharp.Core/Engines/Dialogflow/AIConfiguration.cs b/BotSharp.Core/Engines/Dialogflow/AIConfiguration.cs index 77df4aa2..f09b468a 100644 --- a/BotSharp.Core/Engines/Dialogflow/AIConfiguration.cs +++ b/BotSharp.Core/Engines/Dialogflow/AIConfiguration.cs @@ -1,4 +1,5 @@ -using System; +using BotSharp.NLP; +using System; using System.Collections.Generic; using System.Text; diff --git a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs index 9d687523..029324c4 100644 --- a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs +++ b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs @@ -30,7 +30,7 @@ namespace BotSharp.Core.Engines public Agent LoadAgent(AgentImportHeader agentHeader) { // load agent profile - string data = File.ReadAllText(Path.Join(AgentDir, "Dialogflow", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json")); + string data = File.ReadAllText(Path.Combine(AgentDir, "Dialogflow", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json")); var agent = JsonConvert.DeserializeObject(data); agent.Name = agentHeader.Name; agent.Id = agentHeader.Id; @@ -58,14 +58,14 @@ namespace BotSharp.Core.Engines public void LoadCustomEntities(Agent agent) { agent.Entities = new List(); - string entityDir = Path.Join(AgentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}entities"); + string entityDir = Path.Combine(AgentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}entities"); if (!Directory.Exists(entityDir)) return; Directory.EnumerateFiles(entityDir) .ToList() .ForEach(fileName => { - string entityName = fileName.Split($"{Path.DirectorySeparatorChar}").Last(); + string entityName = fileName.Split(Path.DirectorySeparatorChar).Last(); if (!entityName.Contains("_")) { string entityJson = File.ReadAllText($"{fileName}"); @@ -93,7 +93,7 @@ namespace BotSharp.Core.Engines public void LoadIntents(Agent agent) { agent.Intents = new List(); - string intentDir = Path.Join(AgentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}intents"); + string intentDir = Path.Combine(AgentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}intents"); if (!Directory.Exists(intentDir)) return; Directory.EnumerateFiles(intentDir) diff --git a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs index 3575c9ab..005ca9cb 100644 --- a/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/NERs/CRFsuiteEntityRecognizer.cs @@ -1,6 +1,7 @@ using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; using BotSharp.MachineLearning.NLP; +using BotSharp.NLP.Tokenize; using DotNetToolkit; using EntityFrameworkCore.BootKit; using Microsoft.Extensions.Configuration; @@ -45,9 +46,9 @@ namespace BotSharp.Core.Engines.NERs List> userSays = corpus.UserSays; List> list = new List>(); - string rawTrainingDataFileName = Path.Join(Settings.TempDir, "ner-crf.corpus.txt"); - string parsedTrainingDataFileName = Path.Join(Settings.TempDir, "ner-crf.parsed.txt"); - string modelFileName = Path.Join(Settings.ModelDir, meta.Model); + string rawTrainingDataFileName = Path.Combine(Settings.TempDir, "ner-crf.corpus.txt"); + string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, "ner-crf.parsed.txt"); + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create)) { @@ -74,11 +75,11 @@ namespace BotSharp.Core.Engines.NERs var biFeatures = Configuration.GetValue($"CRFsuiteEntityRecognizer:biFeatures"); new MachineLearning.CRFsuite.Ner() - .NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, fields, uniFeatures.Split(" "), biFeatures.Split(" ")); + .NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, fields, uniFeatures.Split(' '), biFeatures.Split(' ')); - var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms"); + var algorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms"); - CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}", false); // --split=3 -x + CmdHelper.Run(Path.Combine(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}", false); // --split=3 -x Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); meta.Meta["fields"] = fields; @@ -88,7 +89,7 @@ namespace BotSharp.Core.Engines.NERs return true; } - public List Merge(List tokens, List entities) + public List Merge(List tokens, List entities) { List trainingTuple = new List(); HashSet entityWordBag = new HashSet(); @@ -103,7 +104,7 @@ namespace BotSharp.Core.Engines.NERs entities.ForEach(entity => { if (!entityFinded) { - string[] words = entity.Value.Split(" "); + string[] words = entity.Value.Split(' '); for (int j = 0; j < words.Length; j++) { if (tokens[i + j].Text == words[j]) @@ -150,11 +151,11 @@ namespace BotSharp.Core.Engines.NERs var uniFeatures = meta.Meta["uniFeatures"].ToString(); var biFeatures = meta.Meta["biFeatures"].ToString(); string field = meta.Meta["fields"].ToString(); - string[] fields = field.Split(" "); + string[] fields = field.Split(' '); - string rawPredictingDataFileName = Path.Join(Settings.TempDir, "ner-crf.corpus.predict.txt"); - string parsedPredictingDataFileName = Path.Join(Settings.TempDir, "ner-crf.parsed.predict.txt"); - string modelFileName = Path.Join(Settings.ModelDir, meta.Model); + string rawPredictingDataFileName = Path.Combine(Settings.TempDir, "ner-crf.corpus.predict.txt"); + string parsedPredictingDataFileName = Path.Combine(Settings.TempDir, "ner-crf.parsed.predict.txt"); + string modelFileName = Path.Combine(Settings.ModelDir, meta.Model); using (FileStream fs = new FileStream(rawPredictingDataFileName, FileMode.Create)) { @@ -163,7 +164,7 @@ namespace BotSharp.Core.Engines.NERs List curLine = new List(); foreach (NlpDocSentence sentence in doc.Sentences) { - foreach (NlpToken token in sentence.Tokens) + foreach (Token token in sentence.Tokens) { for (int i = 0 ; i < fields.Length; i++) { @@ -191,18 +192,18 @@ namespace BotSharp.Core.Engines.NERs } new MachineLearning.CRFsuite.Ner() - .NerStart(rawPredictingDataFileName, parsedPredictingDataFileName, field, uniFeatures.Split(" "), biFeatures.Split(" ")); + .NerStart(rawPredictingDataFileName, parsedPredictingDataFileName, field, uniFeatures.Split(' '), biFeatures.Split(' ')); - var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "crfsuite"), $"tag -i -m {modelFileName} {parsedPredictingDataFileName}", false); + var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "crfsuite"), $"tag -i -m {modelFileName} {parsedPredictingDataFileName}", false); var entities = new List(); - string[] entityProbabilityPairs = output.Split(Environment.NewLine).Where(x => !String.IsNullOrEmpty(x)).ToArray(); + string[] entityProbabilityPairs = output.Split(new string[] { Environment.NewLine }, StringSplitOptions.None).Where(x => !String.IsNullOrEmpty(x)).ToArray(); for (int i = 0; i < entityProbabilityPairs.Length; i++) { string entityProbabilityPair = entityProbabilityPairs[i]; - string entity = entityProbabilityPair.Split(":")[0]; - decimal probability = decimal.Parse(entityProbabilityPair.Split(":")[1]); + string entity = entityProbabilityPair.Split(':')[0]; + decimal probability = decimal.Parse(entityProbabilityPair.Split(':')[1]); entities.Add(new NlpEntity { Entity = entity, diff --git a/BotSharp.Core/Engines/NlpDoc.cs b/BotSharp.Core/Engines/NlpDoc.cs index 2e69a536..9db6287c 100644 --- a/BotSharp.Core/Engines/NlpDoc.cs +++ b/BotSharp.Core/Engines/NlpDoc.cs @@ -1,4 +1,5 @@ using BotSharp.MachineLearning.NLP; +using BotSharp.NLP.Tokenize; using System; using System.Collections.Generic; using System.Text; @@ -13,7 +14,7 @@ namespace BotSharp.Core.Engines public class NlpDocSentence { public string Text { get; set; } - public List Tokens { get; set; } + public List Tokens { get; set; } public List Entities { get; set; } public TextClassificationResult Intent { get; set; } } diff --git a/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs b/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs index a1cfe329..a200fe7b 100644 --- a/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs +++ b/BotSharp.Core/Engines/Nltk/NltkTokenizer.cs @@ -2,6 +2,7 @@ using BotSharp.Core.Abstractions; using BotSharp.Core.Agents; using BotSharp.Core.Models; using BotSharp.MachineLearning.NLP; +using BotSharp.NLP.Tokenize; using EntityFrameworkCore.BootKit; using Microsoft.Extensions.Configuration; using Newtonsoft.Json; @@ -24,7 +25,7 @@ namespace BotSharp.Core.Engines.SpaCy { var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value); var request = new RestRequest("nltktokenizesentences", Method.POST); - List> tokens = new List>(); + List> tokens = new List>(); Boolean res = true; var dc = new DefaultDataContextLoader().GetDefaultDc(); var corpus = agent.Corpus; @@ -74,7 +75,7 @@ namespace BotSharp.Core.Engines.SpaCy { var client = new RestClient(Configuration.GetSection("NltkProvider:Url").Value); var request = new RestRequest("nltktokenizesentences", Method.POST); - List> tokens = new List>(); + List> tokens = new List>(); Boolean res = true; var corpus = agent.Corpus; @@ -92,7 +93,7 @@ namespace BotSharp.Core.Engines.SpaCy private class Result { - public List> TokensList { get; set; } + public List> TokensList { get; set; } } private class Documents diff --git a/BotSharp.Core/Engines/PipeSettings.cs b/BotSharp.Core/Engines/PipeSettings.cs index 174b6ff0..afe29f18 100644 --- a/BotSharp.Core/Engines/PipeSettings.cs +++ b/BotSharp.Core/Engines/PipeSettings.cs @@ -14,7 +14,7 @@ namespace BotSharp.Core.Engines { get { - return Path.Join(ProjectDir, "Temp"); + return Path.Combine(ProjectDir, "Temp"); } } } diff --git a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs index c64dd703..f3811c40 100644 --- a/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs +++ b/BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs @@ -87,7 +87,7 @@ namespace BotSharp.Core.Engines.Rasa public void LoadIntents(Agent agent) { - string data = File.ReadAllText(Path.Join(AgentDir, "corpus.json")); + string data = File.ReadAllText(Path.Combine(AgentDir, "corpus.json")); var rasa = JsonConvert.DeserializeObject(data); agent.Intents = rasa.UserSays.Select(x => x.Intent).Distinct().Select(x => new Intent { Name = x }).ToList(); diff --git a/BotSharp.Core/Engines/Rasa/RasaAi.cs b/BotSharp.Core/Engines/Rasa/RasaAi.cs index 44eb2b20..fb650bcd 100644 --- a/BotSharp.Core/Engines/Rasa/RasaAi.cs +++ b/BotSharp.Core/Engines/Rasa/RasaAi.cs @@ -198,7 +198,7 @@ namespace BotSharp.Core.Engines rest.AddQueryParameter("model", ctx); string trainingConfig = agent.Language == "zh" ? "config_jieba_mitie_sklearn.yml" : "config_mitie_sklearn.yml"; var contentRootPatch = AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(); - string body = File.ReadAllText(Path.Join(contentRootPatch, "Settings", trainingConfig)); + string body = File.ReadAllText(Path.Combine(contentRootPatch, "Settings", trainingConfig)); body = $"{body}\r\ndata: {json}"; rest.AddParameter("application/x-yml", body, ParameterType.RequestBody); @@ -208,7 +208,7 @@ namespace BotSharp.Core.Engines { var result = JObject.Parse(response.Content); - string modelName = result["info"].Value().Split(": ")[1]; + string modelName = result["info"].Value().Split(new string[] { ": " }, StringSplitOptions.None)[1]; } else { diff --git a/BotSharp.Core/Engines/Rasa/RasaRequestExtension.cs b/BotSharp.Core/Engines/Rasa/RasaRequestExtension.cs index 0a1a063f..bb3afa42 100644 --- a/BotSharp.Core/Engines/Rasa/RasaRequestExtension.cs +++ b/BotSharp.Core/Engines/Rasa/RasaRequestExtension.cs @@ -101,7 +101,7 @@ namespace BotSharp.Core.Engines intentResponse.Parameters.ForEach(p => { string query = request.Query.First(); - var entity = response.Entities.FirstOrDefault(x => x.Entity == p.Name || x.Entity.Split(":").Contains(p.Name)); + var entity = response.Entities.FirstOrDefault(x => x.Entity == p.Name || x.Entity.Split(':').Contains(p.Name)); if (entity != null) { p.Value = query.Substring(entity.Start, entity.End - entity.Start); @@ -168,7 +168,7 @@ namespace BotSharp.Core.Engines if (msg.Speech != "[]") { msg.Speech = msg.Speech.StartsWith("[") ? - ArrayHelper.GetRandom(msg.Speech.Substring(2, msg.Speech.Length - 4).Split("\",\"").ToList()) : + ArrayHelper.GetRandom(msg.Speech.Substring(2, msg.Speech.Length - 4).Split(new string[] { "\",\"" }, StringSplitOptions.None).ToList()) : msg.Speech; msg.Speech = ReplaceParameters4Response(intentResponse.Parameters, msg.Speech); @@ -181,7 +181,7 @@ namespace BotSharp.Core.Engines { var reg = new Regex(@"\$\w+"); - reg.Matches(text).ToList().ForEach(token => { + reg.Matches(text).Cast().ToList().ForEach(token => { var parameter = parameters.FirstOrDefault(x => x.Name == token.Value.Substring(1)); if(parameter != null) { diff --git a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs index 3b6a9fac..ef53368d 100644 --- a/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs +++ b/BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs @@ -31,7 +31,7 @@ namespace BotSharp.Core.Engines public Agent LoadAgent(AgentImportHeader agentHeader) { // load agent profile - string data = File.ReadAllText(Path.Join(AgentDir, "Sebis", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json")); + string data = File.ReadAllText(Path.Combine(AgentDir, "Sebis", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json")); var agent = JsonConvert.DeserializeObject(data); agent.Name = agentHeader.Name; agent.Id = agentHeader.Id; @@ -54,7 +54,7 @@ namespace BotSharp.Core.Engines public void LoadIntents(Agent agent) { - string data = File.ReadAllText(Path.Join(AgentDir, "Sebis", $"{agent.Name}{Path.DirectorySeparatorChar}corpus.json")); + string data = File.ReadAllText(Path.Combine(AgentDir, "Sebis", $"{agent.Name}{Path.DirectorySeparatorChar}corpus.json")); var sentences = JsonConvert.DeserializeObject(data).Sentences; agent.Intents = sentences.Select(x => x.Name).Distinct().Select(x => new Intent{Name = x}).ToList(); diff --git a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs index c8284a8d..ffd70f64 100644 --- a/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs +++ b/BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs @@ -2,6 +2,7 @@ using BotSharp.Core.Agents; using BotSharp.Core.Models; using BotSharp.MachineLearning.NLP; +using BotSharp.NLP.Tokenize; using EntityFrameworkCore.BootKit; using Microsoft.Extensions.Configuration; using Newtonsoft.Json; @@ -24,7 +25,7 @@ namespace BotSharp.Core.Engines.SpaCy { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("tokenizer", Method.POST); - List> tokens = new List>(); + List> tokens = new List>(); Boolean res = true; var corpus = agent.Corpus; @@ -57,7 +58,7 @@ namespace BotSharp.Core.Engines.SpaCy { var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value); var request = new RestRequest("tokenizer", Method.GET); - List> tokens = new List>(); + List> tokens = new List>(); Boolean res = true; var corpus = agent.Corpus; @@ -75,7 +76,7 @@ namespace BotSharp.Core.Engines.SpaCy private class Result { - public List> TokensList { get; set; } + public List> TokensList { get; set; } } private class Documents diff --git a/BotSharp.Core/Intents/Intent.cs b/BotSharp.Core/Intents/Intent.cs index 682b9b81..fda3aa65 100644 --- a/BotSharp.Core/Intents/Intent.cs +++ b/BotSharp.Core/Intents/Intent.cs @@ -41,7 +41,7 @@ namespace BotSharp.Core.Intents { return Contexts == null || Contexts.Count == 0 ? Guid.Empty.ToString("N") - : $"{String.Join(',', Contexts.OrderBy(x => x.Name).Select(x => x.Name))}".GetMd5Hash(); + : $"{String.Join(",", Contexts.OrderBy(x => x.Name).Select(x => x.Name))}".GetMd5Hash(); } } diff --git a/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj b/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj index 207e8624..d838d0ee 100644 --- a/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj +++ b/BotSharp.MachineLearning/BotSharp.MachineLearning.csproj @@ -1,21 +1,12 @@ - + - netcoreapp2.1 + netstandard2.0 + AnyCPU;x64 - - - - - - - ..\..\..\..\..\..\Program Files\dotnet\sdk\NuGetFallbackFolder\microsoft.data.sqlite.core\2.1.0\lib\netstandard2.0\Microsoft.Data.Sqlite.dll - - - diff --git a/BotSharp.MachineLearning/CRFsuite/Crfutils.cs b/BotSharp.MachineLearning/CRFsuite/Crfutils.cs index eb1adcfd..c8f97940 100644 --- a/BotSharp.MachineLearning/CRFsuite/Crfutils.cs +++ b/BotSharp.MachineLearning/CRFsuite/Crfutils.cs @@ -63,7 +63,7 @@ namespace BotSharp.MachineLearning.CRFsuite /// each attribute name in fields /// seperate by - public List>> Readiter (string fiPath, List names, string sep = " ") + public List>> Readiter (string fiPath, List names, char sep = ' ') { List>> Xs = new List>>(); List> X = new List>(); @@ -143,8 +143,8 @@ namespace BotSharp.MachineLearning.CRFsuite { using (StreamWriter sw = new StreamWriter(fs)) { - List F = fields.Split(" ").ToList(); - List>> Xs = Readiter(rawFile, F, " "); + List F = fields.Split(' ').ToList(); + List>> Xs = Readiter(rawFile, F); foreach (List> X in Xs) { diff --git a/BotSharp.NLP.UnitTest/BotSharp.NLP.UnitTest.csproj b/BotSharp.NLP.UnitTest/BotSharp.NLP.UnitTest.csproj new file mode 100644 index 00000000..0fe0560f --- /dev/null +++ b/BotSharp.NLP.UnitTest/BotSharp.NLP.UnitTest.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp2.1 + + false + + AnyCPU;x64 + + + + + + + + + + + + + diff --git a/BotSharp.NLP.UnitTest/RegexpTokenizerTest.cs b/BotSharp.NLP.UnitTest/RegexpTokenizerTest.cs new file mode 100644 index 00000000..42afdc5b --- /dev/null +++ b/BotSharp.NLP.UnitTest/RegexpTokenizerTest.cs @@ -0,0 +1,36 @@ +using BotSharp.NLP.Tokenize; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace BotSharp.NLP.UnitTest +{ + [TestClass] + public class RegexpTokenizerTest + { + [TestMethod] + public void Tokenize() + { + var tokenizer = new TokenizerFactory(); + + var tokens = tokenizer.Tokenize("Chop into pieces, isn't it?", + new TokenizationOptions + { + Pattern = RegexTokenizer.WHITE_SPACE + }); + + Assert.IsTrue(tokens[0].Offset == 0); + Assert.IsTrue(tokens[0].Text == "Chop"); + + Assert.IsTrue(tokens[1].Offset == 5); + Assert.IsTrue(tokens[1].Text == "into"); + + Assert.IsTrue(tokens[2].Offset == 10); + Assert.IsTrue(tokens[2].Text == "pieces,"); + + Assert.IsTrue(tokens[3].Offset == 18); + Assert.IsTrue(tokens[3].Text == "isn't"); + + Assert.IsTrue(tokens[3].Offset == 24); + Assert.IsTrue(tokens[3].Text == "it?"); + } + } +} diff --git a/BotSharp.NLP/BotSharp.NLP.csproj b/BotSharp.NLP/BotSharp.NLP.csproj new file mode 100644 index 00000000..669148b2 --- /dev/null +++ b/BotSharp.NLP/BotSharp.NLP.csproj @@ -0,0 +1,8 @@ + + + + netstandard2.0 + AnyCPU;x64 + + + diff --git a/BotSharp.Core/Engines/Dialogflow/SupportedLanguage.cs b/BotSharp.NLP/SupportedLanguage.cs similarity index 98% rename from BotSharp.Core/Engines/Dialogflow/SupportedLanguage.cs rename to BotSharp.NLP/SupportedLanguage.cs index db09af18..51472d1d 100644 --- a/BotSharp.Core/Engines/Dialogflow/SupportedLanguage.cs +++ b/BotSharp.NLP/SupportedLanguage.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace BotSharp.Core.Models +namespace BotSharp.NLP { public class SupportedLanguage { diff --git a/BotSharp.NLP/Tokenize/ITokenizer.cs b/BotSharp.NLP/Tokenize/ITokenizer.cs new file mode 100644 index 00000000..01248d5b --- /dev/null +++ b/BotSharp.NLP/Tokenize/ITokenizer.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Tokenize +{ + /// + /// A tokenizer is a component used for dividing text intotokens. + /// A tokenizer is language specific and takes into account the peculiarities of the language, e.g. don’t in English is tokenized as two tokens. + /// + public interface ITokenizer + { + /// + /// Language + /// + SupportedLanguage Lang { get; set; } + + /// + /// Tokenize + /// + /// input + /// Options such as: regex expression + /// + Token[] Tokenize(string text, TokenizationOptions options); + } +} diff --git a/BotSharp.NLP/Tokenize/RegexTokenizer.cs b/BotSharp.NLP/Tokenize/RegexTokenizer.cs new file mode 100644 index 00000000..2ca72148 --- /dev/null +++ b/BotSharp.NLP/Tokenize/RegexTokenizer.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace BotSharp.NLP.Tokenize +{ + public class RegexTokenizer : ITokenizer + { + public SupportedLanguage Lang { get; set; } + + /// + /// Tokenize a text into a sequence of alphabetic and non-alphabetic characters + /// + public const string WORD_PUNC = @"\w+|[^\w\s]+"; + + /// + /// Tokenize a string, treating any sequence of blank lines as a delimiter. + /// Blank lines are defined as lines containing no characters, except for space or tab characters. + /// options.IsGap = true + /// + public const string BLANK_LINE = @"\s*\n\s*\n\s*"; + + /// + /// Tokenize a string on whitespace (space, tab, newline). + /// In general, users should use the string ``split()`` method instead. + /// options.IsGap = true + /// + public const string WHITE_SPACE = @"\s+"; + + private Regex _regex; + + public Token[] Tokenize(string text, TokenizationOptions options) + { + _regex = new Regex(options.Pattern); + + var matches = _regex.Matches(text).Cast().ToArray(); + + options.IsGap = new string[] { WHITE_SPACE, BLANK_LINE }.Contains(options.Pattern); + + if (options.IsGap) + { + int pos = 0; + int span = 0; + + var tokens = matches.Select(x => + { + var token = new Token + { + Text = (span == matches.Length - 1) ? text.Substring(pos) : text.Substring(pos, x.Index - pos), + Offset = pos + }; + + pos = x.Index + 1; + + if (span == matches.Length - 1) + { + + } + + span++; + + return token; + }).ToArray(); + + return tokens; + } + else + { + return matches.Select(x => new Token + { + Text = x.Value, + Offset = x.Index + }).ToArray(); + } + + } + } +} diff --git a/BotSharp.MachineLearning/NLP/NlpToken.cs b/BotSharp.NLP/Tokenize/Token.cs similarity index 77% rename from BotSharp.MachineLearning/NLP/NlpToken.cs rename to BotSharp.NLP/Tokenize/Token.cs index e6cb34b8..4de60268 100644 --- a/BotSharp.MachineLearning/NLP/NlpToken.cs +++ b/BotSharp.NLP/Tokenize/Token.cs @@ -2,9 +2,9 @@ using System.Collections.Generic; using System.Text; -namespace BotSharp.MachineLearning.NLP +namespace BotSharp.NLP.Tokenize { - public class NlpToken + public class Token { public string Text { get; set; } public int Offset { get; set; } @@ -15,7 +15,7 @@ namespace BotSharp.MachineLearning.NLP { get { - return Offset + Text.Length; + return Offset + Text.Length - 1; } } } diff --git a/BotSharp.NLP/Tokenize/TokenizationOptions.cs b/BotSharp.NLP/Tokenize/TokenizationOptions.cs new file mode 100644 index 00000000..bb9017d9 --- /dev/null +++ b/BotSharp.NLP/Tokenize/TokenizationOptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Tokenize +{ + public class TokenizationOptions + { + /// + /// Regex pattern + /// + public string Pattern { get; set; } + + /// + /// True if this tokenizer's pattern should be used to find separators between tokens; + /// False if this tokenizer's pattern should be used to find the tokens themselves. + /// + public bool IsGap { get; set; } + } +} diff --git a/BotSharp.NLP/Tokenize/TokenizerFactory.cs b/BotSharp.NLP/Tokenize/TokenizerFactory.cs new file mode 100644 index 00000000..4b5ae3b7 --- /dev/null +++ b/BotSharp.NLP/Tokenize/TokenizerFactory.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotSharp.NLP.Tokenize +{ + /// + /// BotSharp Tokenizer Factory + /// Tokenizers divide strings into lists of substrings. + /// The particular tokenizer requires implement interface + /// models to be installed.BotSharp.NLP also provides a simpler, regular-expression based tokenizer, which splits text on whitespace and punctuation. + /// + public class TokenizerFactory where ITokenize : ITokenizer, new() + { + private ITokenize _tokenizer; + + public TokenizerFactory() + { + _tokenizer = new ITokenize(); + } + + public Token[] Tokenize(string text, TokenizationOptions options) + { + return _tokenizer.Tokenize(text, options); + } + } +} diff --git a/BotSharp.RestApi/AgentController.cs b/BotSharp.RestApi/AgentController.cs index fdf0537d..c32122c1 100644 --- a/BotSharp.RestApi/AgentController.cs +++ b/BotSharp.RestApi/AgentController.cs @@ -48,7 +48,7 @@ namespace BotSharp.RestApi [HttpGet("{agentId}")] public ActionResult Restore([FromRoute] String agentId) { - var botsHeaderFilePath = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), $"DbInitializer{Path.DirectorySeparatorChar}Agents{Path.DirectorySeparatorChar}agents.json"); + var botsHeaderFilePath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), $"DbInitializer{Path.DirectorySeparatorChar}Agents{Path.DirectorySeparatorChar}agents.json"); var agents = JsonConvert.DeserializeObject>(System.IO.File.ReadAllText(botsHeaderFilePath)); var rasa = new BotSharpAi(); diff --git a/BotSharp.RestApi/BotSharp.RestApi.csproj b/BotSharp.RestApi/BotSharp.RestApi.csproj index 4f92570f..e5c46e32 100644 --- a/BotSharp.RestApi/BotSharp.RestApi.csproj +++ b/BotSharp.RestApi/BotSharp.RestApi.csproj @@ -1,7 +1,7 @@ - + - netcoreapp2.1 + netstandard2.0 true Haiping Chen Personal @@ -15,6 +15,7 @@ https://github.com/Oceania2018/BotSharp/blob/master/LICENSE NLU, Chatbot, Bot, AI Bot Restful API for BotSharp.Core + AnyCPU;x64 @@ -22,10 +23,19 @@ TRACE;DEBUG;MODE_RASA + + bin\Debug\netcoreapp2.1\BotSharp.RestApi.xml + TRACE;DEBUG;MODE_RASA + + TRACE;MODE_DIALOGFLOW;RELEASE;NETCOREAPP;NETCOREAPP2_1;RELEASE;NETCOREAPP;NETCOREAPP2_1 + + TRACE;MODE_DIALOGFLOW;RELEASE;NETCOREAPP;NETCOREAPP2_1;RELEASE;NETCOREAPP;NETCOREAPP2_1 + + diff --git a/BotSharp.RestApi/Integrations/FacebookMessenger/FacebookMessengerController.cs b/BotSharp.RestApi/Integrations/FacebookMessenger/FacebookMessengerController.cs index 93f303d6..f8cce5db 100644 --- a/BotSharp.RestApi/Integrations/FacebookMessenger/FacebookMessengerController.cs +++ b/BotSharp.RestApi/Integrations/FacebookMessenger/FacebookMessengerController.cs @@ -2,6 +2,7 @@ using BotSharp.Core.Engines; using BotSharp.Core.Engines.Dialogflow; using BotSharp.Core.Models; +using BotSharp.NLP; using BotSharp.RestApi.Integrations.FacebookMessenger; using DotNetToolkit; using EntityFrameworkCore.BootKit; diff --git a/BotSharp.RestApi/Rasa/ParseController.cs b/BotSharp.RestApi/Rasa/ParseController.cs index 5deb8e76..a18cbffd 100644 --- a/BotSharp.RestApi/Rasa/ParseController.cs +++ b/BotSharp.RestApi/Rasa/ParseController.cs @@ -1,5 +1,6 @@ using BotSharp.Core.Engines; using BotSharp.Core.Models; +using BotSharp.NLP; using Microsoft.AspNetCore.Mvc; using System; using System.Collections.Generic; diff --git a/BotSharp.RestApi/Rasa/TrainController.cs b/BotSharp.RestApi/Rasa/TrainController.cs index 50e42a63..14dcc349 100644 --- a/BotSharp.RestApi/Rasa/TrainController.cs +++ b/BotSharp.RestApi/Rasa/TrainController.cs @@ -38,16 +38,16 @@ namespace BotSharp.RestApi.Rasa } // save corpus to agent dir - var projectPath = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects"); - var dataPath = Path.Join(projectPath, project); - var agentPath = Path.Join(dataPath, "Temp"); + var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects"); + var dataPath = Path.Combine(projectPath, project); + var agentPath = Path.Combine(dataPath, "Temp"); if (!Directory.Exists(agentPath)) { Directory.CreateDirectory(agentPath); } - var fileName = Path.Join(agentPath, "corpus.json"); + var fileName = Path.Combine(agentPath, "corpus.json"); System.IO.File.WriteAllText(fileName, JsonConvert.SerializeObject(request.Corpus, new JsonSerializerSettings { diff --git a/BotSharp.UnitTest/AgentTest.cs b/BotSharp.UnitTest/AgentTest.cs index 0c14e0b0..1b9a690f 100644 --- a/BotSharp.UnitTest/AgentTest.cs +++ b/BotSharp.UnitTest/AgentTest.cs @@ -47,7 +47,7 @@ namespace BotSharp.UnitTest public void RestoreAgentFromDialogflowToRasaTest() { string dataPath = AppDomain.CurrentDomain.GetData("DataPath").ToString(); - var botsHeaderFilePath = Path.Join(dataPath, "DbInitializer", $"Agents{Path.DirectorySeparatorChar}agents.json"); + var botsHeaderFilePath = Path.Combine(dataPath, "DbInitializer", $"Agents{Path.DirectorySeparatorChar}agents.json"); var agents = JsonConvert.DeserializeObject>(File.ReadAllText(botsHeaderFilePath)); agents.ForEach(agentHeader => { diff --git a/BotSharp.UnitTest/BotSharp.UnitTest.csproj b/BotSharp.UnitTest/BotSharp.UnitTest.csproj index ca7277c4..3baff6bd 100644 --- a/BotSharp.UnitTest/BotSharp.UnitTest.csproj +++ b/BotSharp.UnitTest/BotSharp.UnitTest.csproj @@ -1,9 +1,11 @@ - + netcoreapp2.1 false + + AnyCPU;x64 diff --git a/BotSharp.UnitTest/ConversationTest.cs b/BotSharp.UnitTest/ConversationTest.cs index 23a7577d..9326e6ff 100644 --- a/BotSharp.UnitTest/ConversationTest.cs +++ b/BotSharp.UnitTest/ConversationTest.cs @@ -1,6 +1,7 @@ using BotSharp.Core.Engines; using BotSharp.Core.Engines.Dialogflow; using BotSharp.Core.Models; +using BotSharp.NLP; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; diff --git a/BotSharp.UnitTest/TestEssential.cs b/BotSharp.UnitTest/TestEssential.cs index 6be81cfc..b398b2bf 100644 --- a/BotSharp.UnitTest/TestEssential.cs +++ b/BotSharp.UnitTest/TestEssential.cs @@ -26,7 +26,7 @@ namespace BotSharp.UnitTest configurationBuilder.AddJsonFile(setting, optional: false, reloadOnChange: true); }); - AppDomain.CurrentDomain.SetData("DataPath", Path.Join(contentRoot, "App_Data")); + AppDomain.CurrentDomain.SetData("DataPath", Path.Combine(contentRoot, "App_Data")); AppDomain.CurrentDomain.SetData("Configuration", configurationBuilder.Build()); AppDomain.CurrentDomain.SetData("ContentRootPath", contentRoot); AppDomain.CurrentDomain.SetData("Assemblies", new String[] { "BotSharp.Core" }); diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj b/BotSharp.WebHost/BotSharp.WebHost.csproj index 6d94338f..87167357 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj @@ -3,12 +3,17 @@ netcoreapp2.1 Portable;win10-x64;centos.7-x64 + AnyCPU;x64 TRACE;DEBUG;MODE_RASA + + TRACE;DEBUG;MODE_RASA + + diff --git a/BotSharp.WebHost/InitializationLoader.cs b/BotSharp.WebHost/InitializationLoader.cs index f6e475d1..41400f3f 100644 --- a/BotSharp.WebHost/InitializationLoader.cs +++ b/BotSharp.WebHost/InitializationLoader.cs @@ -1,12 +1,9 @@ using BotSharp.Core.Abstractions; using DotNetToolkit; -using EntityFrameworkCore.BootKit; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using System; -using System.Collections.Generic; using System.Linq; -using System.Threading.Tasks; namespace BotSharp.WebHost { @@ -18,7 +15,8 @@ namespace BotSharp.WebHost { var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies"); var appsLoaders1 = TypeHelper.GetInstanceWithInterface(assemblies); - appsLoaders1.ForEach(loader => { + appsLoaders1.ForEach(loader => + { loader.Initialize(Config, Env); }); } diff --git a/BotSharp.WebHost/Program.cs b/BotSharp.WebHost/Program.cs index 4bedf365..82d23ba5 100644 --- a/BotSharp.WebHost/Program.cs +++ b/BotSharp.WebHost/Program.cs @@ -1,12 +1,8 @@ -using System; -using System.Collections.Generic; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using System; using System.IO; using System.Linq; -using System.Threading.Tasks; -using Microsoft.AspNetCore; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; namespace BotSharp.WebHost { @@ -19,11 +15,13 @@ namespace BotSharp.WebHost public static IWebHost BuildWebHost(string[] args) => Microsoft.AspNetCore.WebHost.CreateDefaultBuilder(args) - .ConfigureAppConfiguration((hostingContext, config) => { - + .ConfigureAppConfiguration((hostingContext, config) => + { + var env = hostingContext.HostingEnvironment; var settings = Directory.GetFiles($"{env.ContentRootPath}{Path.DirectorySeparatorChar}Settings", "*.json"); - settings.ToList().ForEach(setting => { + settings.ToList().ForEach(setting => + { config.AddJsonFile(setting, optional: false, reloadOnChange: true); }); }) diff --git a/BotSharp.WebHost/Startup.cs b/BotSharp.WebHost/Startup.cs index 0518ebe5..74d20aa3 100644 --- a/BotSharp.WebHost/Startup.cs +++ b/BotSharp.WebHost/Startup.cs @@ -1,23 +1,19 @@ -using System; -using System.IO; -using System.Linq; +using BotSharp.Core.Agents; using BotSharp.Core.Engines; using DotNetToolkit; +using DotNetToolkit.JwtHelper; using EntityFrameworkCore.BootKit; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.PlatformAbstractions; using Newtonsoft.Json.Serialization; using Swashbuckle.AspNetCore.Swagger; -using BotSharp.Core.Engines.BotSharp; +using System; using System.Collections.Generic; -using Newtonsoft.Json; -using DotNetToolkit.JwtHelper; -using BotSharp.Core.Agents; +using System.IO; +using System.Linq; namespace BotSharp.WebHost { @@ -124,7 +120,7 @@ namespace BotSharp.WebHost app.UseMvc(); - AppDomain.CurrentDomain.SetData("DataPath", Path.Join(env.ContentRootPath, "App_Data")); + AppDomain.CurrentDomain.SetData("DataPath", Path.Combine(env.ContentRootPath, "App_Data")); AppDomain.CurrentDomain.SetData("Configuration", Configuration); AppDomain.CurrentDomain.SetData("ContentRootPath", env.ContentRootPath); AppDomain.CurrentDomain.SetData("Assemblies", Configuration.GetValue("Assemblies").Split(',')); diff --git a/BotSharp.sln b/BotSharp.sln index c3bb81a6..1465645b 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -13,6 +13,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.WebHost", "BotShar EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.MachineLearning", "BotSharp.MachineLearning\BotSharp.MachineLearning.csproj", "{E664115A-AE86-49E9-8AE4-D4589A568CD7}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP", "BotSharp.NLP\BotSharp.NLP.csproj", "{D60A6A0A-4428-4460-868E-18CB5C7DA20F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.NLP.UnitTest", "BotSharp.NLP.UnitTest\BotSharp.NLP.UnitTest.csproj", "{2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -39,6 +43,14 @@ Global {E664115A-AE86-49E9-8AE4-D4589A568CD7}.Debug|Any CPU.Build.0 = Debug|Any CPU {E664115A-AE86-49E9-8AE4-D4589A568CD7}.Release|Any CPU.ActiveCfg = Release|Any CPU {E664115A-AE86-49E9-8AE4-D4589A568CD7}.Release|Any CPU.Build.0 = Release|Any CPU + {D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D60A6A0A-4428-4460-868E-18CB5C7DA20F}.Release|Any CPU.Build.0 = Release|Any CPU + {2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2A8C199C-FD8E-4CB7-A83B-08F50F809AE8}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/README.md b/README.md index 0c764355..20a16a4b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # -### The Open Source AI Chatbot Platform Builder for Enterprise - +### The Open Source AI Bot Platform Builder for Enterprise +#### Open up as much learning power as possible for your enterprise robots and precisely control every step of the AI processing pipeline. **BotSharp** is an open source machine learning framework for AI Bot platform builder. This project involves natural language understanding, computer vision and audio processing technologies, and aims to promote the development and application of intelligent robot assistants in enterprise information systems. Out-of-the-box machine learning algorithms allow ordinary programmers to develop artificial intelligence applications faster and easier. It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. More friendly to corporate developers. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope.