SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Cancel the dependency on the database, restore the Bot to the file system, and separate the Bot NLP Engine and the pre-Bot Platform system. Complete the AgentImporterInDialogflow function.

Browse source
This commit is contained in:
Oceania2018 2018-08-27 16:04:00 -05:00
parent 409a150e11
commit 8ee00fab36
18 changed files with 63 additions and 140 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BotSharp.Core.UnitTest/AgentTest.cs
View file

@ -51,7 +51,7 @@ namespace BotSharp.Core.UnitTest
agents.ForEach(agentHeader => {
var bot = new BotSharpAi();
bot.RestoreAgent<AgentImporterInDialogflow>(agentHeader);
bot.RestoreAgent<AgentImporterInDialogflow>();
});
}

3
BotSharp.Core/Abstractions/IAgentImporter.cs
View file

@ -15,9 +15,8 @@ namespace BotSharp.Core.Engines
/// <summary>
/// Load agent summary
/// </summary>
/// <param name="agentHeader"></param>
/// <returns></returns>
Agent LoadAgent(AgentImportHeader agentHeader);
Agent LoadAgent();
/// <summary>
/// Load user customized entity type which defined in dictionary

2
BotSharp.Core/Abstractions/IBotPlatform.cs
View file

@ -17,7 +17,7 @@ namespace BotSharp.Core.Engines
/// <returns></returns>
Agent LoadAgent(string id);
Agent LoadAgentFromFile<TAgentImporter>(string dataDir, AgentImportHeader agentHeader) where TAgentImporter : IAgentImporter, new();
Agent LoadAgentFromFile<TAgentImporter>(string dataDir) where TAgentImporter : IAgentImporter, new();
AIResponse TextRequest(AIRequest request);

6
BotSharp.Core/BotSharp.Core.csproj
View file

@ -43,6 +43,12 @@
<DefineConstants>TRACE;MODEL_PER_CONTEXTS</DefineConstants>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Accounts\**" />
<EmbeddedResource Remove="Accounts\**" />
<None Remove="Accounts\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="BotSharp.NLP" Version="0.2.3" />
<PackageReference Include="DotNetToolkit" Version="1.6.0" />

9
BotSharp.Core/Engines/BotEngineBase.cs
View file

@ -86,28 +86,27 @@ namespace BotSharp.Core.Engines
/// Restore a agent instance from backup json files
/// </summary>
/// <param name="importer"></param>
/// <param name="agentHeader"></param>
/// <param name="dataDir"></param>
/// <returns></returns>
public bool RestoreAgent<TAgentImporter>(AgentImportHeader agentHeader) where TAgentImporter : IAgentImporter, new()
public bool RestoreAgent<TAgentImporter>() where TAgentImporter : IAgentImporter, new()
{
string dataDir = Path.Combine(DbInitializerPath, "Agents");
int row = dc.DbTran(() => {
LoadAgentFromFile<TAgentImporter>(dataDir, agentHeader);
LoadAgentFromFile<TAgentImporter>(dataDir);
SaveAgent();
});
return row > 0;
}
public Agent LoadAgentFromFile<TAgentImporter>(string dataDir, AgentImportHeader agentHeader) where TAgentImporter : IAgentImporter, new()
public Agent LoadAgentFromFile<TAgentImporter>(string dataDir) where TAgentImporter : IAgentImporter, new()
{
var importer = new TAgentImporter();
importer.AgentDir = dataDir;
// Load agent summary
agent = importer.LoadAgent(agentHeader);
agent = importer.LoadAgent();
// Load user custom entities
importer.LoadCustomEntities(agent);

13
BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs
View file

@ -27,10 +27,15 @@ namespace BotSharp.Core.Engines
/// <param name="agentName"></param>
/// <param name="agentDir"></param>
/// <returns></returns>
public Agent LoadAgent(AgentImportHeader agentHeader)
public Agent LoadAgent()
{
// load meta
string metaJson = File.ReadAllText(Path.Combine(AgentDir, "meta.json"));
AgentImportHeader agentHeader = JsonConvert.DeserializeObject<AgentImportHeader>(metaJson);
// load agent profile
string data = File.ReadAllText(Path.Combine(AgentDir, "Dialogflow", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json"));
string data = File.ReadAllText(Path.Combine(AgentDir, "agent.json"));
var agent = JsonConvert.DeserializeObject<DialogflowAgent>(data);
agent.Name = agentHeader.Name;
agent.Id = agentHeader.Id;
@ -54,7 +59,7 @@ namespace BotSharp.Core.Engines
public void LoadCustomEntities(Agent agent)
{
agent.Entities = new List<EntityType>();
string entityDir = Path.Combine(AgentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}entities");
string entityDir = Path.Combine(AgentDir, "entities");
if (!Directory.Exists(entityDir)) return;
Directory.EnumerateFiles(entityDir)
@ -89,7 +94,7 @@ namespace BotSharp.Core.Engines
public void LoadIntents(Agent agent)
{
agent.Intents = new List<Intent>();
string intentDir = Path.Combine(AgentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}intents");
string intentDir = Path.Combine(AgentDir, "intents");
if (!Directory.Exists(intentDir)) return;
Directory.EnumerateFiles(intentDir)

4
BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs
View file

@ -16,8 +16,10 @@ namespace BotSharp.Core.Engines.Rasa
{
public string AgentDir { get; set; }
public Agent LoadAgent(AgentImportHeader agentHeader)
public Agent LoadAgent()
{
AgentImportHeader agentHeader = null;
var agent = new Agent();
agent.ClientAccessToken = Guid.NewGuid().ToString("N");
agent.DeveloperAccessToken = Guid.NewGuid().ToString("N");

5
BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs
View file

@ -25,11 +25,12 @@ namespace BotSharp.Core.Engines
/// <summary>
/// Load agent meta
/// </summary>
/// <param name="agentName"></param>
/// <param name="agentDir"></param>
/// <returns></returns>
public Agent LoadAgent(AgentImportHeader agentHeader)
public Agent LoadAgent()
{
AgentImportHeader agentHeader = null;
// load agent profile
string data = File.ReadAllText(Path.Combine(AgentDir, "Sebis", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json"));
var agent = JsonConvert.DeserializeObject<SebisAgent>(data);

15
BotSharp.RestApi/AgentController.cs
View file

@ -46,13 +46,12 @@ namespace BotSharp.RestApi
/// <summary>
/// Restore a agent from a uploaded zip file
/// </summary>
/// <param name="name"></param>
/// <param name="file"></param>
/// <returns></returns>
[HttpPost]
public async Task<ActionResult> Restore(IFormFile file)
public async Task<ActionResult> Restore(IFormFile uploadedFile)
{
if (file == null || file.Length == 0)
if (uploadedFile == null || uploadedFile.Length == 0)
{
return BadRequest();
}
@ -61,15 +60,17 @@ namespace BotSharp.RestApi
using (var stream = new FileStream(filePath, FileMode.Create))
{
await file.CopyToAsync(stream);
await uploadedFile.CopyToAsync(stream);
}
string dest = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", "", DateTime.UtcNow.ToString("MMddyyyyHHmm"));
string dest = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", uploadedFile.FileName.Split('.').First(), "model_" + DateTime.UtcNow.ToString("MMddyyyyHHmm"));
ZipFile.ExtractToDirectory(filePath, dest);
_platform.LoadAgentFromFile<AgentImporterInDialogflow>("", new AgentImportHeader { });
System.IO.File.Delete(filePath);
return Ok();
var agent = _platform.LoadAgentFromFile<AgentImporterInDialogflow>(dest);
return Ok(agent.Id);
}
/// <summary>

3
BotSharp.RestApi/Authentication/AuthenticationController.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Accounts;
using DotNetToolkit;
using DotNetToolkit;
using DotNetToolkit.JwtHelper;
using EntityFrameworkCore.BootKit;
using Microsoft.AspNetCore.Authorization;

6
BotSharp.RestApi/BotSharp.RestApi.csproj
View file

@ -36,6 +36,12 @@
<DefineConstants>TRACE;MODE_DIALOGFLOW;RELEASE;NETCOREAPP;NETCOREAPP2_1;RELEASE;NETCOREAPP;NETCOREAPP2_1</DefineConstants>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Authentication\**" />
<EmbeddedResource Remove="Authentication\**" />
<None Remove="Authentication\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.1.2" />
</ItemGroup>

7
BotSharp.RestApi/Rasa/ParseController.cs
View file

@ -52,12 +52,7 @@ namespace BotSharp.RestApi.Rasa
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", request.Project);
var modelPath = Path.Combine(projectPath, request.Model);
_platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath,
new AgentImportHeader
{
Id = request.Project,
Name = request.Project
});
_platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath);
var aIResponse = _platform.TextRequest(new AIRequest
{

7
BotSharp.RestApi/Rasa/TrainController.cs
View file

@ -83,12 +83,7 @@ namespace BotSharp.RestApi.Rasa
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
var agent = _platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath,
new AgentImportHeader
{
Id = request.Project,
Name = project
});
var agent = _platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath);
var info = await trainer.Train(agent, new BotTrainOptions { Model = request.Model });

92
BotSharp.WebHost/BotSharp.WebHost.csproj
View file

@ -15,21 +15,33 @@
</PropertyGroup>
<ItemGroup>
<Compile Remove="App_Data\AgentArchive\**" />
<Compile Remove="App_Data\Corpus\**" />
<Compile Remove="App_Data\DbInitializer\**" />
<Compile Remove="App_Data\ModelFiles\**" />
<Compile Remove="App_Data\NewFolder\**" />
<Compile Remove="App_Data\Projects\**" />
<Compile Remove="App_Data\TrainingFiles\**" />
<Compile Remove="PublishOutput\**" />
<Content Remove="App_Data\AgentArchive\**" />
<Content Remove="App_Data\Corpus\**" />
<Content Remove="App_Data\DbInitializer\**" />
<Content Remove="App_Data\ModelFiles\**" />
<Content Remove="App_Data\NewFolder\**" />
<Content Remove="App_Data\Projects\**" />
<Content Remove="App_Data\TrainingFiles\**" />
<Content Remove="PublishOutput\**" />
<EmbeddedResource Remove="App_Data\AgentArchive\**" />
<EmbeddedResource Remove="App_Data\Corpus\**" />
<EmbeddedResource Remove="App_Data\DbInitializer\**" />
<EmbeddedResource Remove="App_Data\ModelFiles\**" />
<EmbeddedResource Remove="App_Data\NewFolder\**" />
<EmbeddedResource Remove="App_Data\Projects\**" />
<EmbeddedResource Remove="App_Data\TrainingFiles\**" />
<EmbeddedResource Remove="PublishOutput\**" />
<None Remove="App_Data\AgentArchive\**" />
<None Remove="App_Data\Corpus\**" />
<None Remove="App_Data\DbInitializer\**" />
<None Remove="App_Data\ModelFiles\**" />
<None Remove="App_Data\NewFolder\**" />
<None Remove="App_Data\Projects\**" />
@ -37,44 +49,6 @@
<None Remove="PublishOutput\**" />
</ItemGroup>
<ItemGroup>
<Content Remove="App_Data\DbInitializer\Agents\agents.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\agent.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-album.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-album_entries_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-service.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-service_entries_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-sort.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-sort_entries_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\playlist.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\playlist_entries_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\song.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\song_entries_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music.play.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music.play_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_favorite.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_favorite_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_playlist.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_playlist_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.pause.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.pause_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.play.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.play_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.repeat.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.repeat_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.resume.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.resume_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.shuffle.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.shuffle_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_backward.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_backward_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_forward.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_forward_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.stop.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.stop_usersays_en.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\package.json" />
</ItemGroup>
<ItemGroup>
<None Remove="App_Data\BotSharp.db" />
</ItemGroup>
@ -88,44 +62,6 @@
<ProjectReference Include="..\BotSharp.RestApi\BotSharp.RestApi.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="App_Data\DbInitializer\Agents\agents.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\agent.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-album.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-album_entries_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-service.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-service_entries_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-sort.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\music-sort_entries_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\playlist.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\playlist_entries_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\song.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\entities\song_entries_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music.play.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music.play_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_favorite.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_favorite_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_playlist.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.add_playlist_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.pause.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.pause_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.play.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.play_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.repeat.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.repeat_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.resume.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.resume_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.shuffle.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.shuffle_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_backward.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_backward_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_forward.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.skip_forward_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.stop.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\intents\music_player_control.stop_usersays_en.json" />
<None Include="App_Data\DbInitializer\Agents\Dialogflow\Spotify\package.json" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore" Version="2.1.3" />
<PackageReference Include="Microsoft.AspNetCore.Mvc" Version="2.1.2" />
@ -136,4 +72,8 @@
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="3.0.0" />
</ItemGroup>
<ItemGroup>
<Folder Include="App_Data\" />
</ItemGroup>
</Project>

11
BotSharp.WebHost/Settings/config_jieba_mitie_sklearn.yml
View file

@ -1,11 +0,0 @@
language: "zh"
pipeline:
- name: "nlp_mitie"
model: "data/total_word_feature_extractor_zh.dat"
- name: "tokenizer_jieba"
- name: "ner_mitie"
- name: "ner_synonyms"
- name: "intent_entity_featurizer_regex"
- name: "intent_featurizer_mitie"
- name: "intent_classifier_sklearn"

11
BotSharp.WebHost/Settings/config_mitie_sklearn.yml
View file

@ -1,11 +0,0 @@
language: "en"
pipeline:
- name: "nlp_mitie"
model: "data/total_word_feature_extractor.dat"
- name: "tokenizer_mitie"
- name: "ner_mitie"
- name: "ner_synonyms"
- name: "intent_entity_featurizer_regex"
- name: "intent_featurizer_mitie"
- name: "intent_classifier_sklearn"

3
BotSharp.WebHost/Settings/config_spacy.yml
View file

@ -1,3 +0,0 @@
language: "en"
pipeline: "spacy_sklearn"

4
BotSharp.WebHost/SwaggerFileUploadOperation.cs
View file

@ -11,7 +11,7 @@ namespace BotSharp.WebHost
{
public void Apply(Operation operation, OperationFilterContext context)
{
if (operation.OperationId == "V1AgentRestoreByNamePost")
if (operation.OperationId == "V1AgentRestorePost")
{
operation.Parameters.Clear();
@ -19,7 +19,7 @@ namespace BotSharp.WebHost
{
Name = "uploadedFile",
In = "formData",
Description = "Upload Zip File",
Description = "Upload Zip File， meta.json is the extra data for customized function.",
Required = true,
Type = "file"
});