SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add data initializer.

Browse source
This commit is contained in:
Oceania2018 2018-08-02 16:14:46 -05:00
parent 69c437224c
commit e22086cc88
25 changed files with 351 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
BotSharp.Core/Abstractions/IHookDbInitializer.cs Normal file
View file

@ -0,0 +1,19 @@
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Abstractions
{
/// <summary>
/// Initialize data for modules
/// </summary>
public interface IHookDbInitializer
{
/// <summary>
/// value smaller is higher priority
/// </summary>
int Priority { get; }
void Load(Database dc);
}
}

14
BotSharp.Core/Abstractions/IInitializationLoader.cs Normal file
View file

@ -0,0 +1,14 @@
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Abstractions
{
public interface IInitializationLoader
{
int Priority { get; }
void Initialize(IConfiguration config, IHostingEnvironment env);
}
}

24
BotSharp.Core/Accounts/AccountDbInitializer.cs Normal file
View file

@ -0,0 +1,24 @@
using BotSharp.Core.Abstractions;
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace BotSharp.Core.Accounts
{
public class AccountDbInitializer : IHookDbInitializer
{
public int Priority => 100;
public void Load(Database dc)
{
ImportAccount(dc);
}
private void ImportAccount(Database dc)
{
var dataPath = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Accounts");
}
}
}

60
BotSharp.Core/Accounts/User.cs Normal file
View file

@ -0,0 +1,60 @@
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace BotSharp.Core.Accounts
{
/// <summary>
/// User profile
/// </summary>
[Table("User")]
public class User : DbRecord, IDbRecord
{
[Required]
[StringLength(64)]
public String UserName { get; set; }
[Required]
[StringLength(64)]
[DataType(DataType.EmailAddress)]
public String Email { get; set; }
[StringLength(32)]
public String FirstName { get; set; }
[StringLength(32)]
public String LastName { get; set; }
[MaxLength(256)]
public String Description { get; set; }
[Required]
[DataType(DataType.DateTime)]
public DateTime SignupDate { get; set; }
[DataType(DataType.Date)]
public DateTime? Birthday { get; set; }
[MaxLength(36)]
public String Nationality { get; set; }
[NotMapped]
public String FullName
{
get
{
return FirstName + (String.IsNullOrEmpty(LastName) ? "" : " " + LastName);
}
}
public UserAuth Authenticaiton { get; set; }
public User()
{
SignupDate = DateTime.UtcNow;
}
}
}

36
BotSharp.Core/Accounts/UserAuth.cs Normal file
View file

@ -0,0 +1,36 @@
using EntityFrameworkCore.BootKit;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace BotSharp.Core.Accounts
{
/// <summary>
/// User authentication
/// </summary>
[Table("UserAuth")]
public class UserAuth : DbRecord, IDbRecord
{
[StringLength(36)]
public String UserId { get; set; }
[Required]
[StringLength(256)]
[DataType(DataType.Password)]
public String Password { get; set; }
[Required]
[StringLength(64)]
public String Salt { get; set; }
[StringLength(32)]
public String ActivationCode { get; set; }
public Boolean IsActivated { get; set; }
[ForeignKey("UserId")]
public User User { get; set; }
}
}

2
BotSharp.Core/BotSharp.Core.csproj
View file

@ -36,12 +36,12 @@
<ItemGroup>
<PackageReference Include="DotNetToolkit" Version="1.4.0" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="1.7.1" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="RestSharp" Version="106.3.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\EntityFrameworkCore.BootKit\EntityFrameworkCore.BootKit\EntityFrameworkCore.BootKit.csproj" />
<ProjectReference Include="..\BotSharp.MachineLearning\BotSharp.MachineLearning.csproj" />
</ItemGroup>

5
BotSharp.Core/Engines/BotEngineBase.cs
View file

@ -28,7 +28,8 @@ namespace BotSharp.Core.Engines
public BotEngineBase()
{
dc = new DefaultDataContextLoader().GetDefaultDc();
DbInitializerPath = $"{Database.ContentRootPath}App_Data{Path.DirectorySeparatorChar}DbInitializer{Path.DirectorySeparatorChar}";
string dataPath = AppDomain.CurrentDomain.GetData("DataPath").ToString();
DbInitializerPath = Path.Join(dataPath, $"DbInitializer");
}
public Agent LoadAgent(string id)
@ -62,7 +63,7 @@ namespace BotSharp.Core.Engines
{
var importer = new TAgentImporter();
string dataDir = $"{DbInitializerPath}Agents{Path.DirectorySeparatorChar}";
string dataDir = Path.Join(DbInitializerPath, "Agents");
int row = dc.DbTran(() => {

13
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -8,6 +8,7 @@ using BotSharp.Core.Intents;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
namespace BotSharp.Core.Engines
@ -43,22 +44,24 @@ namespace BotSharp.Core.Engines
});
// Get NLP Provider
string providerName = Database.Configuration.GetSection($"{config}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, Database.Assemblies) as INlpPipeline;
provider.Configuration = Database.Configuration.GetSection("BotSharpAi");
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
string providerName = config.GetSection($"{config}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpPipeline;
provider.Configuration = config.GetSection("BotSharpAi");
provider.Process(agent, data);
//var corpus = agent.GrabCorpus(dc);
// pipe process
var pipelines = Database.Configuration.GetSection($"{config}:Pipe").Value
var pipelines = config.GetSection($"{config}:Pipe").Value
.Split(',')
.Select(x => x.Trim())
.ToList();
pipelines.ForEach(pipeName =>
{
var pipe = TypeHelper.GetInstance(pipeName, Database.Assemblies) as INlpPipeline;
var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
pipe.Process(agent, data);
});

33
BotSharp.Core/Engines/DbInitializer.cs Normal file
View file

@ -0,0 +1,33 @@
using BotSharp.Core.Abstractions;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.Core.Engines
{
public class DbInitializer : IInitializationLoader
{
public int Priority => 1;
public void Initialize(IConfiguration config, IHostingEnvironment env)
{
var dc = new DefaultDataContextLoader().GetDefaultDc();
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var instances = TypeHelper.GetInstanceWithInterface<IHookDbInitializer>(assemblies);
// initial app db order by priority
instances.OrderBy(x => x.Priority).ToList()
.ForEach(instance =>
{
Console.WriteLine($"DbInitializer: {instance.ToString()}");
dc.Transaction<IDbRecord>(() => instance.Load(dc));
});
}
}
}

6
BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs
View file

@ -28,7 +28,7 @@ namespace BotSharp.Core.Engines
public Agent LoadAgent(AgentImportHeader agentHeader, string agentDir)
{
// load agent profile
string data = File.ReadAllText($"{agentDir}{Path.DirectorySeparatorChar}Dialogflow{Path.DirectorySeparatorChar}{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json");
string data = File.ReadAllText(Path.Join(agentDir, "Dialogflow", $"agentHeader.Name{Path.DirectorySeparatorChar}agent.json"));
var agent = JsonConvert.DeserializeObject<DialogflowAgent>(data);
agent.Name = agentHeader.Name;
agent.Id = agentHeader.Id;
@ -56,7 +56,7 @@ namespace BotSharp.Core.Engines
public void LoadCustomEntities(Agent agent, string agentDir)
{
agent.Entities = new List<EntityType>();
string entityDir = $"{agentDir}{Path.DirectorySeparatorChar}Dialogflow{Path.DirectorySeparatorChar}{agent.Name}{Path.DirectorySeparatorChar}entities";
string entityDir = Path.Join(agentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}entities");
if (!Directory.Exists(entityDir)) return;
Directory.EnumerateFiles(entityDir)
@ -91,7 +91,7 @@ namespace BotSharp.Core.Engines
public void LoadIntents(Agent agent, string agentDir)
{
agent.Intents = new List<Intent>();
string intentDir = $"{agentDir}{Path.DirectorySeparatorChar}Dialogflow{Path.DirectorySeparatorChar}{agent.Name}{Path.DirectorySeparatorChar}intents";
string intentDir = Path.Join(agentDir, "Dialogflow", $"{agent.Name}{Path.DirectorySeparatorChar}intents");
if (!Directory.Exists(intentDir)) return;
Directory.EnumerateFiles(intentDir)

9
BotSharp.Core/Engines/Rasa/RasaAi.cs
View file

@ -83,7 +83,8 @@ namespace BotSharp.Core.Engines
private IRestResponse<RasaResponse> CallRasa(string projectId, string text, string model)
{
var client = new RestClient($"{Database.Configuration.GetSection("Rasa:Nlu").Value}");
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var client = new RestClient($"{config.GetSection("Rasa:Nlu").Value}");
var rest = new RestRequest("parse", Method.POST);
string json = JsonConvert.SerializeObject(new { Project = projectId, Q = text, Model = model },
@ -105,7 +106,8 @@ namespace BotSharp.Core.Engines
};
var corpus = GetIntentExpressions();
var client = new RestClient($"{Database.Configuration.GetSection("Rasa:Nlu").Value}");
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var client = new RestClient($"{config.GetSection("Rasa:Nlu").Value}");
var contextHashs = corpus.UserSays
.Select(x => x.ContextHash)
@ -195,7 +197,8 @@ namespace BotSharp.Core.Engines
rest.AddQueryParameter("project", agent.Id);
rest.AddQueryParameter("model", ctx);
string trainingConfig = agent.Language == "zh" ? "config_jieba_mitie_sklearn.yml" : "config_mitie_sklearn.yml";
string body = File.ReadAllText($"{Database.ContentRootPath}{Path.DirectorySeparatorChar}Settings{Path.DirectorySeparatorChar}{trainingConfig}");
var contentRootPatch = AppDomain.CurrentDomain.GetData("ContentRootPath").ToString();
string body = File.ReadAllText(Path.Join(contentRootPatch, "Settings", trainingConfig));
body = $"{body}\r\ndata: {json}";
rest.AddParameter("application/x-yml", body, ParameterType.RequestBody);

2
BotSharp.RestApi/AgentController.cs
View file

@ -26,7 +26,7 @@ namespace BotSharp.RestApi
[HttpGet("{agentId}")]
public ActionResult Restore([FromRoute] String agentId)
{
var botsHeaderFilePath = $"{Database.ContentRootPath}App_Data{Path.DirectorySeparatorChar}DbInitializer{Path.DirectorySeparatorChar}Agents{Path.DirectorySeparatorChar}agents.json";
var botsHeaderFilePath = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), $"DbInitializer{Path.DirectorySeparatorChar}Agents{Path.DirectorySeparatorChar}agents.json");
var agents = JsonConvert.DeserializeObject<List<AgentImportHeader>>(System.IO.File.ReadAllText(botsHeaderFilePath));
var rasa = new RasaAi();

62
BotSharp.RestApi/Authentication/AuthenticationController.cs Normal file
View file

@ -0,0 +1,62 @@
using BotSharp.Core.Accounts;
using DotNetToolkit;
using DotNetToolkit.JwtHelper;
using EntityFrameworkCore.BootKit;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BotSharp.RestApi.Authentication
{
/// <summary>
/// User authentication
/// </summary>
public class AuthenticationController : ControllerBase
{
/// <summary>
/// Get user token
/// </summary>
/// <param name="userModel"></param>
/// <returns></returns>
[AllowAnonymous]
[HttpPost("/token")]
public IActionResult Token([FromBody] VmUserLogin userModel)
{
if (String.IsNullOrEmpty(userModel.UserName) || String.IsNullOrEmpty(userModel.Password))
{
return new BadRequestObjectResult("Username and password should not be empty.");
}
var dc = new DefaultDataContextLoader().GetDefaultDc();
// validate from local
var user = (from usr in dc.Table<User>()
join auth in dc.Table<UserAuth>() on usr.Id equals auth.UserId
where usr.Email == userModel.UserName
select auth).FirstOrDefault();
if (user != null)
{
// validate password
string hash = PasswordHelper.Hash(userModel.Password, user.Salt);
if (user.Password == hash)
{
return Ok(JwtToken.GenerateToken((IConfiguration)AppDomain.CurrentDomain.GetData("Configuration"), user.UserId));
}
else
{
return BadRequest("Authorization Failed.");
}
}
else
{
return BadRequest("Account doesn't exist");
}
}
}
}

22
BotSharp.RestApi/Authentication/VmUserLogin.cs Normal file
View file

@ -0,0 +1,22 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.RestApi.Authentication
{
/// <summary>
/// User login view model
/// </summary>
public class VmUserLogin
{
/// <summary>
/// User identity, email or phone
/// </summary>
public String UserName { get; set; }
/// <summary>
/// User password
/// </summary>
public String Password { get; set; }
}
}

2
BotSharp.UI/README.md
View file

@ -4,7 +4,7 @@ Voiceweb AI Chatbot Web Console
## How to install
#### Download source code
````sh
git clone https://github.com/voicecoin/voiceweb-chatbot-vue
git clone https://github.com/Oceania2018/BotSharp
````
#### Run node.js server
````sh

3
BotSharp.UI/src/config/config.js
View file

@ -2,8 +2,7 @@ import Env from './env';
let config = {
env: Env,
authURL: 'http://0.0.0.0:3112',
baseURL: (Env == 'development' ? `http://0.0.0.0:3112` : `http://0.0.0.0:3112`),
baseURL: (Env == 'development' ? `http://localhost:3112` : `http://localhost:3112`),
testAccount: {username: `botsharp@gmail.com`, password: (Env == 'development' ? `botsharp` : `botsharp`)}
};
export default config;

2
BotSharp.UI/src/main.js
View file

@ -88,7 +88,7 @@ const store = new Vuex.Store({
authenticated(state, token) {
localStorage.setItem('token', token);
router.push('/agent/agents');
HTTP.get('/account', { baseURL: config.authURL })
HTTP.get('/account', { baseURL: config.baseURL })
.then(response => {
state.user = response.data;
localStorage.setItem('user', JSON.stringify(response.data));

2
BotSharp.UI/src/views/account/login.vue
View file

@ -56,7 +56,7 @@
})
},
getToken() {
this.$ajax.post(`/token`, this.formInline, { baseURL: this.$config.authURL })
this.$ajax.post(`/token`, this.formInline, { baseURL: this.$config.baseURL })
.then(response => {
this.$store.commit('authenticated', response.data)
});

4
BotSharp.UI/src/views/account/settings.vue
View file

@ -52,7 +52,7 @@
},
created() {
let agentId = this.$route.query.agentId;
this.$ajax.get(`/Account`, { baseURL: this.$config.authURL })
this.$ajax.get(`/Account`, { baseURL: this.$config.baseURL })
.then(response => {
this.user = response.data;
});
@ -61,7 +61,7 @@
updateAgent(agentId){
this.$ajax.put('/v1/Agents/' + agentId, this.agent)
.then(response => {
this.$Message.info("保存成功");
this.$Message.info("Saved");
});
}
}

3
BotSharp.UnitTest/AgentTest.cs
View file

@ -45,7 +45,8 @@ namespace BotSharp.UnitTest
[TestMethod]
public void RestoreAgentFromDialogflowToRasaTest()
{
var botsHeaderFilePath = $"{Database.ContentRootPath}App_Data{Path.DirectorySeparatorChar}DbInitializer{Path.DirectorySeparatorChar}Agents{Path.DirectorySeparatorChar}agents.json";
string dataPath = AppDomain.CurrentDomain.GetData("DataPath").ToString();
var botsHeaderFilePath = Path.Join(dataPath, "DbInitializer", $"Agents{Path.DirectorySeparatorChar}agents.json");
var agents = JsonConvert.DeserializeObject<List<AgentImportHeader>>(File.ReadAllText(botsHeaderFilePath));
agents.ForEach(agentHeader => {

7
BotSharp.UnitTest/TestEssential.cs
View file

@ -25,10 +25,11 @@ namespace BotSharp.UnitTest
{
configurationBuilder.AddJsonFile(setting, optional: false, reloadOnChange: true);
});
Database.Configuration = configurationBuilder.Build();
Database.Assemblies = new String[] { "BotSharp.Core" };
Database.ContentRootPath = contentRoot;
AppDomain.CurrentDomain.SetData("DataPath", Path.Join(contentRoot, "App_Data"));
AppDomain.CurrentDomain.SetData("Configuration", configurationBuilder.Build());
AppDomain.CurrentDomain.SetData("ContentRootPath", contentRoot);
AppDomain.CurrentDomain.SetData("Assemblies", new String[] { "BotSharp.Core" });
dc = new DefaultDataContextLoader().GetDefaultDc();
}

1
BotSharp.WebHost/BotSharp.WebHost.csproj
View file

@ -57,6 +57,7 @@
<ItemGroup>
<Folder Include="App_Data\DbInitializer\Agents\Rasa\" />
<Folder Include="App_Data\DbInitializer\Accounts\" />
</ItemGroup>
<ItemGroup>

26
BotSharp.WebHost/InitializationLoader.cs Normal file
View file

@ -0,0 +1,26 @@
using BotSharp.Core.Abstractions;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace BotSharp.WebHost
{
public class InitializationLoader
{
public IHostingEnvironment Env { get; set; }
public IConfiguration Config { get; set; }
public void Load()
{
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var appsLoaders1 = TypeHelper.GetInstanceWithInterface<IInitializationLoader>(assemblies);
appsLoaders1.ForEach(loader => {
loader.Initialize(Config, Env);
});
}
}
}

18
BotSharp.WebHost/Startup.cs
View file

@ -50,8 +50,10 @@ namespace BotSharp.WebHost
// register platform dependency
services.AddTransient<IBotPlatform>((provider) =>
{
var implements = TypeHelper.GetClassesWithInterface<IBotPlatform>(Database.Assemblies);
string platform = Database.Configuration.GetValue<String>("BotPlatform");
var assemblies = (String[])AppDomain.CurrentDomain.GetData("Assemblies");
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var implements = TypeHelper.GetClassesWithInterface<IBotPlatform>(assemblies);
string platform = config.GetValue<String>("BotPlatform");
var implement = implements.FirstOrDefault(x => x.Name.Split('.').Last() == platform);
var instance = (IBotPlatform)Activator.CreateInstance(implement);
@ -100,9 +102,15 @@ namespace BotSharp.WebHost
app.UseMvc();
Database.Configuration = Configuration;
Database.ContentRootPath = env.ContentRootPath;
Database.Assemblies = Configuration.GetValue<String>("Assemblies").Split(',');
AppDomain.CurrentDomain.SetData("DataPath", Path.Join(env.ContentRootPath, "App_Data"));
AppDomain.CurrentDomain.SetData("Configuration", Configuration);
AppDomain.CurrentDomain.SetData("ContentRootPath", env.ContentRootPath);
AppDomain.CurrentDomain.SetData("Assemblies", Configuration.GetValue<String>("Assemblies").Split(','));
InitializationLoader loader = new InitializationLoader();
loader.Env = env;
loader.Config = Configuration;
loader.Load();
}
}
}

8
BotSharp.sln
View file

@ -11,7 +11,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.RestApi", "BotShar
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.WebHost", "BotSharp.WebHost\BotSharp.WebHost.csproj", "{03DCA427-327A-4FC9-9A2F-57D17F16708C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.MachineLearning", "BotSharp.MachineLearning\BotSharp.MachineLearning.csproj", "{E664115A-AE86-49E9-8AE4-D4589A568CD7}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.MachineLearning", "BotSharp.MachineLearning\BotSharp.MachineLearning.csproj", "{E664115A-AE86-49E9-8AE4-D4589A568CD7}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EntityFrameworkCore.BootKit", "..\EntityFrameworkCore.BootKit\EntityFrameworkCore.BootKit\EntityFrameworkCore.BootKit.csproj", "{A5F6D16F-E6F4-449E-AA52-7830B4897F88}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -39,6 +41,10 @@ Global
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E664115A-AE86-49E9-8AE4-D4589A568CD7}.Release|Any CPU.Build.0 = Release|Any CPU
{A5F6D16F-E6F4-449E-AA52-7830B4897F88}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A5F6D16F-E6F4-449E-AA52-7830B4897F88}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A5F6D16F-E6F4-449E-AA52-7830B4897F88}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A5F6D16F-E6F4-449E-AA52-7830B4897F88}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE