From 6898d851f7e535b1f5ac8d51b8c838a6d5fe27ef Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Fri, 21 Sep 2018 15:11:57 -0500 Subject: [PATCH] Dialogflow compatibility improvement 1 --- .gitignore | 1 + BotSharp.Algorithm/BotSharp.Algorithm.csproj | 5 ++ BotSharp.Core/Abstractions/IBotPlatform.cs | 6 ++ BotSharp.Core/BotSharp.Core.csproj | 5 ++ BotSharp.Core/Engines/BotEngineBase.cs | 41 ++++------- BotSharp.NLP/BotSharp.NLP.csproj | 5 ++ BotSharp.RestApi/AgentController.cs | 12 +++- BotSharp.RestApi/BotSharp.RestApi.csproj | 5 ++ BotSharp.RestApi/BotSharp.RestApi.xml | 70 ++++++------------- .../Dialogflow/QueryController.cs | 7 ++ .../Dialogflow/TrainController.cs | 2 + BotSharp.WebHost/BotSharp.WebHost.csproj | 5 ++ BotSharp.WebHost/BotSharp.WebHost.csproj.user | 3 + BotSharp.WebHost/Program.cs | 4 -- .../Properties/launchSettings.json | 1 - BotSharp.WebHost/Startup.cs | 6 ++ Dockerfile | 2 +- docs/index.rst | 11 +-- 18 files changed, 101 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index b79b8e34..585f8689 100644 --- a/.gitignore +++ b/.gitignore @@ -289,3 +289,4 @@ __pycache__/ /BotSharp.WebHost/PublishOutput /Data /docs/_build +*.RestApi.xml diff --git a/BotSharp.Algorithm/BotSharp.Algorithm.csproj b/BotSharp.Algorithm/BotSharp.Algorithm.csproj index 34524381..98d1bb67 100644 --- a/BotSharp.Algorithm/BotSharp.Algorithm.csproj +++ b/BotSharp.Algorithm/BotSharp.Algorithm.csproj @@ -19,4 +19,9 @@ TRACE;DEBUG;RASA + + DEBUG;TRACE;DIALOGFLOW;NETSTANDARD;NETSTANDARD2_0 + false + + diff --git a/BotSharp.Core/Abstractions/IBotPlatform.cs b/BotSharp.Core/Abstractions/IBotPlatform.cs index f14f3732..aee495ba 100644 --- a/BotSharp.Core/Abstractions/IBotPlatform.cs +++ b/BotSharp.Core/Abstractions/IBotPlatform.cs @@ -27,6 +27,12 @@ namespace BotSharp.Core.Engines /// Agent LoadAgentFromFile(string dataDir); + /// + /// Save agent data to database + /// + /// + String SaveAgentToDb(); + AIResponse TextRequest(AIRequest request); Task Train(BotTrainOptions options); diff --git a/BotSharp.Core/BotSharp.Core.csproj b/BotSharp.Core/BotSharp.Core.csproj index 5440713b..c7b653aa 100644 --- a/BotSharp.Core/BotSharp.Core.csproj +++ b/BotSharp.Core/BotSharp.Core.csproj @@ -45,6 +45,11 @@ If you feel that this project is helpful to you, please Star on the project, we TRACE; + + DEBUG;TRACE;DIALOGFLOW;NETSTANDARD;NETSTANDARD2_0 + false + + diff --git a/BotSharp.Core/Engines/BotEngineBase.cs b/BotSharp.Core/Engines/BotEngineBase.cs index bba696db..8c989c74 100644 --- a/BotSharp.Core/Engines/BotEngineBase.cs +++ b/BotSharp.Core/Engines/BotEngineBase.cs @@ -92,22 +92,6 @@ namespace BotSharp.Core.Engines return agent; } - /// - /// Restore a agent instance from backup json files - /// - /// - /// - /// - public bool RestoreAgent(string dataDir) where TAgentImporter : IAgentImporter, new() - { - int row = dc.DbTran(() => { - LoadAgentFromFile(dataDir); - SaveAgent(); - }); - - return row > 0; - } - public Agent LoadAgentFromFile(string dataDir) { var meta = LoadMeta(dataDir); @@ -159,19 +143,22 @@ namespace BotSharp.Core.Engines return JsonConvert.DeserializeObject(metaJson); } - public String SaveAgent() + public String SaveAgentToDb() { - var existedAgent = dc.Table().FirstOrDefault(x => x.Id == agent.Id || x.Name == agent.Name); - if (existedAgent == null) + dc.DbTran(() => { - dc.Table().Add(agent); - return agent.Id; - } - else - { - agent.Id = existedAgent.Id; - return existedAgent.Id; - } + var existedAgent = dc.Table().FirstOrDefault(x => x.Id == agent.Id || x.Name == agent.Name); + if (existedAgent == null) + { + dc.Table().Add(agent); + } + else + { + agent.Id = existedAgent.Id; + } + }); + + return agent.Id; } public TrainingCorpus GetIntentExpressions(Agent agent) diff --git a/BotSharp.NLP/BotSharp.NLP.csproj b/BotSharp.NLP/BotSharp.NLP.csproj index 41185d49..22ecaa6f 100644 --- a/BotSharp.NLP/BotSharp.NLP.csproj +++ b/BotSharp.NLP/BotSharp.NLP.csproj @@ -35,6 +35,11 @@ Naive Bayes Classifier bin\RASA + + DEBUG;TRACE;DIALOGFLOW;NETSTANDARD;NETSTANDARD2_0 + false + + diff --git a/BotSharp.RestApi/AgentController.cs b/BotSharp.RestApi/AgentController.cs index aee08112..50d1fb7a 100644 --- a/BotSharp.RestApi/AgentController.cs +++ b/BotSharp.RestApi/AgentController.cs @@ -70,6 +70,7 @@ namespace BotSharp.RestApi } var filePath = Path.GetTempFileName(); + Console.WriteLine($"Temp file saved to {filePath}"); using (var stream = new FileStream(filePath, FileMode.Create)) { @@ -77,12 +78,19 @@ namespace BotSharp.RestApi } string dest = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", uploadedFile.FileName.Split('.').First(), "model_" + DateTime.UtcNow.ToString("MMddyyyyHHmm")); + + Console.WriteLine($"Extract zip file to {dest}"); ZipFile.ExtractToDirectory(filePath, dest); System.IO.File.Delete(filePath); + Console.WriteLine($"LoadAgentFromFile {dest}"); var agent = _platform.LoadAgentFromFile(dest); +#if DIALOGFLOW + _platform.SaveAgentToDb(); +#endif + return Ok(agent.Id); } @@ -91,7 +99,7 @@ namespace BotSharp.RestApi /// /// /// - [HttpGet("{agentId}")] + /*[HttpGet("{agentId}")] public string Train([FromRoute] String agentId) { string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agentId); @@ -100,7 +108,7 @@ namespace BotSharp.RestApi _platform.Train(new BotTrainOptions { AgentDir = agentDir, Model = dest.Split(Path.DirectorySeparatorChar).Last() }); return ""; - } + }*/ /// /// Dump agent diff --git a/BotSharp.RestApi/BotSharp.RestApi.csproj b/BotSharp.RestApi/BotSharp.RestApi.csproj index 5094ccc5..9c9589cc 100644 --- a/BotSharp.RestApi/BotSharp.RestApi.csproj +++ b/BotSharp.RestApi/BotSharp.RestApi.csproj @@ -62,6 +62,7 @@ 1 TRACE;DEBUG BotSharp.RestApi.xml + false @@ -70,6 +71,10 @@ + + + + diff --git a/BotSharp.RestApi/BotSharp.RestApi.xml b/BotSharp.RestApi/BotSharp.RestApi.xml index 2231b601..dba99f03 100644 --- a/BotSharp.RestApi/BotSharp.RestApi.xml +++ b/BotSharp.RestApi/BotSharp.RestApi.xml @@ -22,13 +22,6 @@ - - - Train agent - - - - Dump agent @@ -36,6 +29,26 @@ + + + Dialogflow mode query controller + + + + + Initialize dialog controller and get a platform instance + + + + + + The query endpoint is used to process natural language in the form of text. + The query requests return structured data in JSON format with an action and parameters for that action. + Both GET and POST methods return the same JSON response. + + + + Array of additional context objects. Should be sent via a POST /query request. @@ -68,41 +81,6 @@ Time zone from IANA Time Zone Database Examples: America/New_York, Europe/Paris - - - send a text request - - - - - Initialize dialog controller and get a platform instance - - - - - - parse request - - - - - - - This returns all the currently available projects. - - - - - Initialize status controller and get a platform instance - - - - - - Returns a list of available projects the server can use to fulfill /parse requests. - - - You can post your training data to this endpoint to train a new model for a project. @@ -115,14 +93,6 @@ - - - Using the HTTP server, you must specify the project you want to train a new model for to be able to use it during parse requests later on : /train?project=my_project. - - Model name - Agent name or agent id - - PAGE_ID diff --git a/BotSharp.RestApi/Dialogflow/QueryController.cs b/BotSharp.RestApi/Dialogflow/QueryController.cs index 770e09ef..16cfecc1 100644 --- a/BotSharp.RestApi/Dialogflow/QueryController.cs +++ b/BotSharp.RestApi/Dialogflow/QueryController.cs @@ -76,6 +76,13 @@ namespace BotSharp.RestApi.Dialogflow // Load agent string model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last().Split(Path.DirectorySeparatorChar).Last(); string dataDir = Path.Combine(projectPath, model); + + if(!System.IO.File.Exists(Path.Combine(dataDir, "model-meta.json"))) + { + return BadRequest("The agent hasn't been trained. Please train the agent before querying."); + } + + Console.WriteLine($"LoadAgentFromFile {dataDir}"); _platform.LoadAgentFromFile(dataDir); diff --git a/BotSharp.RestApi/Dialogflow/TrainController.cs b/BotSharp.RestApi/Dialogflow/TrainController.cs index b6460cbf..ff0f68ae 100644 --- a/BotSharp.RestApi/Dialogflow/TrainController.cs +++ b/BotSharp.RestApi/Dialogflow/TrainController.cs @@ -44,6 +44,8 @@ namespace BotSharp.RestApi.Rasa var model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last().Split(Path.DirectorySeparatorChar).Last(); string dataDir = Path.Combine(projectPath, model); + Console.WriteLine($"LoadAgentFromFile {dataDir}"); + var agent = _platform.LoadAgentFromFile(dataDir); var info = await trainer.Train(agent, new BotTrainOptions diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj b/BotSharp.WebHost/BotSharp.WebHost.csproj index b72f6cc6..f9268d89 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj @@ -16,6 +16,11 @@ bin\RASA + + false + DEBUG;TRACE;DIALOGFLOW;NETCOREAPP;NETCOREAPP2_1 + + diff --git a/BotSharp.WebHost/BotSharp.WebHost.csproj.user b/BotSharp.WebHost/BotSharp.WebHost.csproj.user index 8e15de60..e1c26c60 100644 --- a/BotSharp.WebHost/BotSharp.WebHost.csproj.user +++ b/BotSharp.WebHost/BotSharp.WebHost.csproj.user @@ -7,4 +7,7 @@ ProjectDebugger + + ProjectDebugger + \ No newline at end of file diff --git a/BotSharp.WebHost/Program.cs b/BotSharp.WebHost/Program.cs index 23601a8c..8bfc511a 100644 --- a/BotSharp.WebHost/Program.cs +++ b/BotSharp.WebHost/Program.cs @@ -35,11 +35,7 @@ namespace BotSharp.WebHost config.AddJsonFile(setting, optional: false, reloadOnChange: true); }); }) -#if RASA .UseUrls("http://0.0.0.0:5000") -#else - .UseUrls("http://0.0.0.0:3112") -#endif .UseStartup() .Build(); } diff --git a/BotSharp.WebHost/Properties/launchSettings.json b/BotSharp.WebHost/Properties/launchSettings.json index ecb75fb4..e869a57a 100644 --- a/BotSharp.WebHost/Properties/launchSettings.json +++ b/BotSharp.WebHost/Properties/launchSettings.json @@ -17,7 +17,6 @@ }, "BotSharp.WebHost": { "commandName": "Project", - "launchBrowser": true, "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" }, diff --git a/BotSharp.WebHost/Startup.cs b/BotSharp.WebHost/Startup.cs index 211e26e5..0e7fb90d 100644 --- a/BotSharp.WebHost/Startup.cs +++ b/BotSharp.WebHost/Startup.cs @@ -91,6 +91,12 @@ namespace BotSharp.WebHost { var info = Configuration.GetSection("Swagger").Get(); +#if DIALOGFLOW + info.Title += " (DIALOGFLOW)"; +#elif RASA + info.Title += " (RASA)"; +#endif + c.SupportedSubmitMethods(SubmitMethod.Get, SubmitMethod.Post, SubmitMethod.Put, SubmitMethod.Patch, SubmitMethod.Delete); c.ShowExtensions(); c.SwaggerEndpoint(Configuration.GetValue("Swagger:Endpoint"), info.Title); diff --git a/Dockerfile b/Dockerfile index 6c205b90..0fe1e9d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /source # copies the rest of your code COPY . . # RUN dotnet build -RUN dotnet publish BotSharp.WebHost/BotSharp.WebHost.csproj --configuration RASA --output /app +RUN dotnet publish BotSharp.WebHost/BotSharp.WebHost.csproj --configuration DIALOGFLOW --output /app # copy Settings folder WORKDIR /app diff --git a/docs/index.rst b/docs/index.rst index 5eef073b..d90292d7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,14 +3,12 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -.. image:: https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png - :height: 30px - +The Open Source AI Bot Platform Builder +====================================================== + .. image:: https://img.shields.io/badge/gitter-join%20chat-brightgreen.svg :target: `gitter`_ -The Open Source AI Bot Platform Builder -====================================================== *"Conversation as a platform (CaaP) is the future, so it's perfect that we're already offering the whole toolkits to .NET developers using the BotSharp Bot Platform Builder to build a CaaP. It opens up as much learning power as possible for your robots and precisely control every step of the AI processing pipeline."* @@ -22,6 +20,9 @@ Why we do this? because we all know python is not friendly programming language BotSharp is in accordance with components princple strictly, decouples every part that needed in the platform builder. So you can choose different UI/UX, or pick up a different NLP Tagger, or select a more advanced algrithm to do NER task. They are all modulized based an unfied interfaces. +.. image:: https://raw.githubusercontent.com/Oceania2018/BotSharp/master/BotSharp.WebHost/wwwroot/images/BotSharp.png + :height: 30px + Some Features -------------