Cancel the dependency on the database, restore the Bot to the file system, and separate the Bot NLP Engine and the pre-Bot Platform system. Complete the AgentImporterInDialogflow function.

BotSharp.Core.UnitTest/AgentTest.cs

@@ -51,7 +51,7 @@ namespace BotSharp.Core.UnitTest
 
             agents.ForEach(agentHeader => {
                 var bot = new BotSharpAi();
-                bot.RestoreAgent<AgentImporterInDialogflow>(agentHeader);
+                bot.RestoreAgent<AgentImporterInDialogflow>();
             });
         }
 

BotSharp.Core/Abstractions/IAgentImporter.cs

@@ -15,9 +15,8 @@ namespace BotSharp.Core.Engines
         /// <summary>
         /// Load agent summary
         /// </summary>
-        /// <param name="agentHeader"></param>
         /// <returns></returns>
-        Agent LoadAgent(AgentImportHeader agentHeader);
+        Agent LoadAgent();
 
         /// <summary>
         /// Load user customized entity type which defined in dictionary

BotSharp.Core/Abstractions/IBotPlatform.cs

@@ -17,7 +17,7 @@ namespace BotSharp.Core.Engines
         /// <returns></returns>
         Agent LoadAgent(string id);
 
-        Agent LoadAgentFromFile<TAgentImporter>(string dataDir, AgentImportHeader agentHeader) where TAgentImporter : IAgentImporter, new();
+        Agent LoadAgentFromFile<TAgentImporter>(string dataDir) where TAgentImporter : IAgentImporter, new();
 
         AIResponse TextRequest(AIRequest request);
 

BotSharp.Core/BotSharp.Core.csproj

@@ -43,6 +43,12 @@
     <DefineConstants>TRACE;MODEL_PER_CONTEXTS</DefineConstants>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Remove="Accounts\**" />
+    <EmbeddedResource Remove="Accounts\**" />
+    <None Remove="Accounts\**" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="BotSharp.NLP" Version="0.2.3" />
     <PackageReference Include="DotNetToolkit" Version="1.6.0" />

BotSharp.Core/Engines/BotEngineBase.cs

@@ -86,28 +86,27 @@ namespace BotSharp.Core.Engines
         /// Restore a agent instance from backup json files
         /// </summary>
         /// <param name="importer"></param>
-        /// <param name="agentHeader"></param>
         /// <param name="dataDir"></param>
         /// <returns></returns>
-        public bool RestoreAgent<TAgentImporter>(AgentImportHeader agentHeader) where TAgentImporter : IAgentImporter, new()
+        public bool RestoreAgent<TAgentImporter>() where TAgentImporter : IAgentImporter, new()
         {
             string dataDir = Path.Combine(DbInitializerPath, "Agents");
 
             int row = dc.DbTran(() => {
-                LoadAgentFromFile<TAgentImporter>(dataDir, agentHeader);
+                LoadAgentFromFile<TAgentImporter>(dataDir);
                 SaveAgent();
             });
 
             return row > 0;
         }
 
-        public Agent LoadAgentFromFile<TAgentImporter>(string dataDir, AgentImportHeader agentHeader) where TAgentImporter : IAgentImporter, new()
+        public Agent LoadAgentFromFile<TAgentImporter>(string dataDir) where TAgentImporter : IAgentImporter, new()
         {
             var importer = new TAgentImporter();
             importer.AgentDir = dataDir;
 
             // Load agent summary
-            agent = importer.LoadAgent(agentHeader);
+            agent = importer.LoadAgent();
 
             // Load user custom entities
             importer.LoadCustomEntities(agent);

BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs

@@ -27,10 +27,15 @@ namespace BotSharp.Core.Engines
         /// <param name="agentName"></param>
         /// <param name="agentDir"></param>
         /// <returns></returns>
-        public Agent LoadAgent(AgentImportHeader agentHeader)
+        public Agent LoadAgent()
         {
+            // load meta
+            string metaJson = File.ReadAllText(Path.Combine(AgentDir, "meta.json"));
+
+            AgentImportHeader agentHeader = JsonConvert.DeserializeObject<AgentImportHeader>(metaJson);
+
             // load agent profile
-            string data = File.ReadAllText(Path.Combine(AgentDir, "Dialogflow", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json"));
+            string data = File.ReadAllText(Path.Combine(AgentDir, "agent.json"));
             var agent = JsonConvert.DeserializeObject<DialogflowAgent>(data);
             agent.Name = agentHeader.Name;
             agent.Id = agentHeader.Id;
@@ -54,7 +59,7 @@ namespace BotSharp.Core.Engines
         public void LoadCustomEntities(Agent agent)
         {
             agent.Entities = new List<EntityType>();
-            string entityDir = Path.Combine(AgentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}entities");
+            string entityDir = Path.Combine(AgentDir, "entities");
             if (!Directory.Exists(entityDir)) return;
 
             Directory.EnumerateFiles(entityDir)
@@ -89,7 +94,7 @@ namespace BotSharp.Core.Engines
         public void LoadIntents(Agent agent)
         {
             agent.Intents = new List<Intent>();
-            string intentDir = Path.Combine(AgentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}intents");
+            string intentDir = Path.Combine(AgentDir, "intents");
             if (!Directory.Exists(intentDir)) return;
 
             Directory.EnumerateFiles(intentDir)

BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs

@@ -16,8 +16,10 @@ namespace BotSharp.Core.Engines.Rasa
     {
         public string AgentDir { get; set; }
 
-        public Agent LoadAgent(AgentImportHeader agentHeader)
+        public Agent LoadAgent()
         {
+            AgentImportHeader agentHeader = null;
+
             var agent = new Agent();
             agent.ClientAccessToken = Guid.NewGuid().ToString("N");
             agent.DeveloperAccessToken = Guid.NewGuid().ToString("N");

BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs

@@ -25,11 +25,12 @@ namespace BotSharp.Core.Engines
         /// <summary>
         /// Load agent meta
         /// </summary>
-        /// <param name="agentName"></param>
         /// <param name="agentDir"></param>
         /// <returns></returns>
-        public Agent LoadAgent(AgentImportHeader agentHeader)
+        public Agent LoadAgent()
         {
+            AgentImportHeader agentHeader = null;
+
             // load agent profile
             string data = File.ReadAllText(Path.Combine(AgentDir, "Sebis", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json"));
             var agent = JsonConvert.DeserializeObject<SebisAgent>(data);

BotSharp.RestApi/AgentController.cs

@@ -46,13 +46,12 @@ namespace BotSharp.RestApi
         /// <summary>
         /// Restore a agent from a uploaded zip file 
         /// </summary>
-        /// <param name="name"></param>
         /// <param name="file"></param>
         /// <returns></returns>
         [HttpPost]
-        public async Task<ActionResult> Restore(IFormFile file)
+        public async Task<ActionResult> Restore(IFormFile uploadedFile)
         {
-            if (file == null || file.Length == 0)
+            if (uploadedFile == null || uploadedFile.Length == 0)
             {
                 return BadRequest();
             }
@@ -61,15 +60,17 @@ namespace BotSharp.RestApi
 
             using (var stream = new FileStream(filePath, FileMode.Create))
             {
-                await file.CopyToAsync(stream);
+                await uploadedFile.CopyToAsync(stream);
             }
 
-            string dest = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", "", DateTime.UtcNow.ToString("MMddyyyyHHmm"));
+            string dest = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", uploadedFile.FileName.Split('.').First(), "model_" + DateTime.UtcNow.ToString("MMddyyyyHHmm"));
             ZipFile.ExtractToDirectory(filePath, dest);
 
-            _platform.LoadAgentFromFile<AgentImporterInDialogflow>("", new AgentImportHeader { });
+            System.IO.File.Delete(filePath);
 
-            return Ok();
+            var agent = _platform.LoadAgentFromFile<AgentImporterInDialogflow>(dest);
+
+            return Ok(agent.Id);
         }
 
         /// <summary>

BotSharp.RestApi/Authentication/AuthenticationController.cs

@@ -1,5 +1,4 @@
-using BotSharp.Core.Accounts;
+using DotNetToolkit;
-using DotNetToolkit;
 using DotNetToolkit.JwtHelper;
 using EntityFrameworkCore.BootKit;
 using Microsoft.AspNetCore.Authorization;

BotSharp.RestApi/BotSharp.RestApi.csproj

@@ -36,6 +36,12 @@
     <DefineConstants>TRACE;MODE_DIALOGFLOW;RELEASE;NETCOREAPP;NETCOREAPP2_1;RELEASE;NETCOREAPP;NETCOREAPP2_1</DefineConstants>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Remove="Authentication\**" />
+    <EmbeddedResource Remove="Authentication\**" />
+    <None Remove="Authentication\**" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.1.2" />
   </ItemGroup>

BotSharp.RestApi/Rasa/ParseController.cs

@@ -52,12 +52,7 @@ namespace BotSharp.RestApi.Rasa
             var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", request.Project);
             var modelPath = Path.Combine(projectPath, request.Model);
 
-            _platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath,
+            _platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath);
-                new AgentImportHeader
-                {
-                    Id = request.Project,
-                    Name = request.Project
-                });
 
             var aIResponse = _platform.TextRequest(new AIRequest
             {

BotSharp.RestApi/Rasa/TrainController.cs

@@ -83,12 +83,7 @@ namespace BotSharp.RestApi.Rasa
                 ContractResolver = new CamelCasePropertyNamesContractResolver()
             }));
 
-            var agent = _platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath,
+            var agent = _platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath);
-                new AgentImportHeader
-                {
-                    Id = request.Project,
-                    Name = project
-                });
 
             var info = await trainer.Train(agent, new BotTrainOptions { Model = request.Model });
 

BotSharp.WebHost/BotSharp.WebHost.csproj

@@ -15,21 +15,33 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <Compile Remove="App_Data\AgentArchive\**" />
+    <Compile Remove="App_Data\Corpus\**" />
+    <Compile Remove="App_Data\DbInitializer\**" />
     <Compile Remove="App_Data\ModelFiles\**" />
     <Compile Remove="App_Data\NewFolder\**" />
     <Compile Remove="App_Data\Projects\**" />
     <Compile Remove="App_Data\TrainingFiles\**" />
     <Compile Remove="PublishOutput\**" />
+    <Content Remove="App_Data\AgentArchive\**" />
+    <Content Remove="App_Data\Corpus\**" />
+    <Content Remove="App_Data\DbInitializer\**" />
     <Content Remove="App_Data\ModelFiles\**" />
     <Content Remove="App_Data\NewFolder\**" />
     <Content Remove="App_Data\Projects\**" />
     <Content Remove="App_Data\TrainingFiles\**" />
     <Content Remove="PublishOutput\**" />
+    <EmbeddedResource Remove="App_Data\AgentArchive\**" />
+    <EmbeddedResource Remove="App_Data\Corpus\**" />
+    <EmbeddedResource Remove="App_Data\DbInitializer\**" />
     <EmbeddedResource Remove="App_Data\ModelFiles\**" />
     <EmbeddedResource Remove="App_Data\NewFolder\**" />
     <EmbeddedResource Remove="App_Data\Projects\**" />
     <EmbeddedResource Remove="App_Data\TrainingFiles\**" />
     <EmbeddedResource Remove="PublishOutput\**" />
+    <None Remove="App_Data\AgentArchive\**" />
+    <None Remove="App_Data\Corpus\**" />
+    <None Remove="App_Data\DbInitializer\**" />
     <None Remove="App_Data\ModelFiles\**" />
     <None Remove="App_Data\NewFolder\**" />
     <None Remove="App_Data\Projects\**" />
@@ -37,44 +49,6 @@
     <None Remove="PublishOutput\**" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Content Remove="App_Data\DbInitializer\Agents\agents.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\agent.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-album.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-album_entries_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-service.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-service_entries_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-sort.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-sort_entries_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\playlist.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\playlist_entries_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\song.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\song_entries_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music.play.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music.play_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_favorite.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_favorite_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_playlist.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_playlist_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.pause.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.pause_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.play.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.play_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.repeat.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.repeat_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.resume.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.resume_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.shuffle.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.shuffle_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_backward.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_backward_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_forward.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_forward_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.stop.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.stop_usersays_en.json" />
-    <Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\package.json" />
-  </ItemGroup>
-
   <ItemGroup>
     <None Remove="App_Data\BotSharp.db" />
   </ItemGroup>
@@ -88,44 +62,6 @@
     <ProjectReference Include="..\BotSharp.RestApi\BotSharp.RestApi.csproj" />
   </ItemGroup>
 
-  <ItemGroup>
-    <None Include="App_Data\DbInitializer\Agents\agents.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\agent.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-album.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-album_entries_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-service.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-service_entries_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-sort.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-sort_entries_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\playlist.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\playlist_entries_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\song.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\song_entries_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music.play.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music.play_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_favorite.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_favorite_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_playlist.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_playlist_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.pause.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.pause_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.play.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.play_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.repeat.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.repeat_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.resume.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.resume_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.shuffle.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.shuffle_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_backward.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_backward_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_forward.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_forward_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.stop.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.stop_usersays_en.json" />
-    <None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\package.json" />
-  </ItemGroup>
-
   <ItemGroup>
     <PackageReference Include="Microsoft.AspNetCore" Version="2.1.3" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc" Version="2.1.2" />
@@ -136,4 +72,8 @@
     <PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="3.0.0" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="App_Data\" />
+  </ItemGroup>
+
 </Project>

BotSharp.WebHost/Settings/config_jieba_mitie_sklearn.yml

@@ -1,11 +0,0 @@
-language: "zh"
-
-pipeline:
-- name: "nlp_mitie"
-  model: "data/total_word_feature_extractor_zh.dat"
-- name: "tokenizer_jieba"
-- name: "ner_mitie"
-- name: "ner_synonyms"
-- name: "intent_entity_featurizer_regex"
-- name: "intent_featurizer_mitie"
-- name: "intent_classifier_sklearn"

BotSharp.WebHost/Settings/config_mitie_sklearn.yml

@@ -1,11 +0,0 @@
-language: "en"
-
-pipeline:
-- name: "nlp_mitie"
-  model: "data/total_word_feature_extractor.dat"
-- name: "tokenizer_mitie"
-- name: "ner_mitie"
-- name: "ner_synonyms"
-- name: "intent_entity_featurizer_regex"
-- name: "intent_featurizer_mitie"
-- name: "intent_classifier_sklearn"

BotSharp.WebHost/Settings/config_spacy.yml

@@ -1,3 +0,0 @@
-language: "en"
-
-pipeline: "spacy_sklearn"

BotSharp.WebHost/SwaggerFileUploadOperation.cs

@@ -11,7 +11,7 @@ namespace BotSharp.WebHost
     {
         public void Apply(Operation operation, OperationFilterContext context)
         {
-            if (operation.OperationId == "V1AgentRestoreByNamePost")
+            if (operation.OperationId == "V1AgentRestorePost")
             {
                 operation.Parameters.Clear();
 
@@ -19,7 +19,7 @@ namespace BotSharp.WebHost
                 {
                     Name = "uploadedFile",
                     In = "formData",
-                    Description = "Upload Zip File",
+                    Description = "Upload Zip File， meta.json is the extra data for customized function.",
                     Required = true,
                     Type = "file"
                 });