SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refactor training.

Browse source
This commit is contained in:
botsharp2018 2018-08-28 09:58:35 -05:00
parent 8ee00fab36
commit 867ebc71f9
13 changed files with 130 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
BotSharp.Core.UnitTest/BotTrainerTest.cs
View file

@ -12,11 +12,11 @@ namespace BotSharp.Core.UnitTest
public class BotTrainerTest : TestEssential
{
[TestMethod]
public void TrainingTest()
public async void TrainingTest()
{
var ai = new BotSharpAi();
ai.LoadAgent(BOT_ID);
ai.Train();
await ai.Train(new BotTrainOptions { });
}
}
}

2
BotSharp.Core/Abstractions/IAgentImporter.cs
View file

@ -16,7 +16,7 @@ namespace BotSharp.Core.Engines
/// Load agent summary
/// </summary>
/// <returns></returns>
Agent LoadAgent();
Agent LoadAgent(AgentImportHeader agentHeader);
/// <summary>
/// Load user customized entity type which defined in dictionary

10
BotSharp.Core/Abstractions/IBotPlatform.cs
View file

@ -17,10 +17,16 @@ namespace BotSharp.Core.Engines
/// <returns></returns>
Agent LoadAgent(string id);
Agent LoadAgentFromFile<TAgentImporter>(string dataDir) where TAgentImporter : IAgentImporter, new();
/// <summary>
/// Load agent from files.
/// There must contain a meta.json
/// </summary>
/// <param name="dataDir"></param>
/// <returns></returns>
Agent LoadAgentFromFile(string dataDir);
AIResponse TextRequest(AIRequest request);
Task Train();
Task Train(BotTrainOptions options);
}
}

106
BotSharp.Core/Engines/BotEngineBase.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines.Rasa;
using BotSharp.Core.Entities;
using BotSharp.Core.Intents;
using BotSharp.Core.Models;
@ -93,20 +94,37 @@ namespace BotSharp.Core.Engines
string dataDir = Path.Combine(DbInitializerPath, "Agents");
int row = dc.DbTran(() => {
LoadAgentFromFile<TAgentImporter>(dataDir);
LoadAgentFromFile(dataDir);
SaveAgent();
});
return row > 0;
}
public Agent LoadAgentFromFile<TAgentImporter>(string dataDir) where TAgentImporter : IAgentImporter, new()
public Agent LoadAgentFromFile(string dataDir)
{
var importer = new TAgentImporter();
var meta = LoadMeta(dataDir);
IAgentImporter importer = null;
switch (meta.Platform)
{
case "Dialogflow":
importer = new AgentImporterInDialogflow();
break;
case "Rasa":
importer = new AgentImporterInRasa();
break;
case "Sebis":
importer = new AgentImporterInSebis();
break;
default:
break;
}
importer.AgentDir = dataDir;
// Load agent summary
agent = importer.LoadAgent();
agent = importer.LoadAgent(meta);
// Load user custom entities
importer.LoadCustomEntities(agent);
@ -123,6 +141,14 @@ namespace BotSharp.Core.Engines
return agent;
}
private AgentImportHeader LoadMeta(string dataDir)
{
// load meta
string metaJson = File.ReadAllText(Path.Combine(dataDir, "meta.json"));
return JsonConvert.DeserializeObject<AgentImportHeader>(metaJson);
}
public String SaveAgent()
{
var existedAgent = dc.Table<Agent>().FirstOrDefault(x => x.Id == agent.Id || x.Name == agent.Name);
@ -138,6 +164,76 @@ namespace BotSharp.Core.Engines
}
}
public TrainingCorpus GetIntentExpressions(Agent agent)
{
TrainingCorpus corpus = new TrainingCorpus()
{
UserSays = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>(),
Entities = new List<TrainingEntity>()
};
var expressParts = new List<IntentExpressionPart>();
var intents = agent.Intents;
intents.ForEach(intent =>
{
intent.UserSays.ForEach(exp =>
{
exp.Data = exp.Data.OrderBy(x => x.UpdatedTime).ToList();
var say = new TrainingIntentExpression<TrainingIntentExpressionPart>
{
Intent = intent.Name,
Text = String.Join("", exp.Data.Select(x => x.Text)),
ContextHash = intent.ContextHash
};
// convert entity format
exp.Data.Where(x => !String.IsNullOrEmpty(x.Meta))
.ToList()
.ForEach(x =>
{
var part = new TrainingIntentExpressionPart
{
Value = x.Text,
Entity = $"{x.Meta}:{x.Alias}",
Start = x.Start
};
if (say.Entities == null) say.Entities = new List<TrainingIntentExpressionPart>();
say.Entities.Add(part);
// assemble entity synonmus
/*if (!trainingData.Entities.Any(y => y.EntityType == x.Alias && y.EntityValue == x.Text))
{
var allSynonyms = (from e in dc.Table<EntityType>()
join ee in dc.Table<EntityEntry>() on e.Id equals ee.EntityId
join ees in dc.Table<EntrySynonym>() on ee.Id equals ees.EntityEntryId
where e.Name == x.Alias && ee.Value == x.Text & ees.Synonym != x.Text
select ees.Synonym).ToList();
var te = new TrainingEntity
{
EntityType = $"{x.Meta}:{x.Alias}",
EntityValue = x.Text,
Synonyms = allSynonyms
};
trainingData.Entities.Add(te);
}*/
});
corpus.UserSays.Add(say);
});
});
// remove Default Fallback Intent
corpus.UserSays = corpus.UserSays.Where(x => x.Intent != "Default Fallback Intent").ToList();
return corpus;
}
public TrainingCorpus GetIntentExpressions()
{
TrainingCorpus corpus = new TrainingCorpus()
@ -212,7 +308,7 @@ namespace BotSharp.Core.Engines
return corpus;
}
public virtual Task Train()
public virtual Task Train(BotTrainOptions options)
{
return Task.CompletedTask;
}

6
BotSharp.Core/Engines/BotSharp/BotSharpAi.cs
View file

@ -8,11 +8,11 @@ namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpAi : BotEngineBase, IBotPlatform
{
public override async Task Train()
public override async Task Train(BotTrainOptions options)
{
agent.Corpus = GetIntentExpressions();
agent.Corpus = GetIntentExpressions(agent);
var trainer = new BotTrainer(agent.Id, dc);
await trainer.Train(agent, new BotTrainOptions { });
await trainer.Train(agent, options);
}
}
}

5
BotSharp.Core/Engines/BotTrainOptions.cs
View file

@ -6,6 +6,11 @@ namespace BotSharp.Core.Engines
{
public class BotTrainOptions
{
/// <summary>
/// Agent data direcotry
/// </summary>
public string AgentDir { get; set; }
/// <summary>
/// Model Name
/// </summary>

9
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -35,15 +35,6 @@ namespace BotSharp.Core.Engines
public async Task<ModelMetaData> Train(Agent agent, BotTrainOptions options)
{
/*agent.Intents = dc.Table<Intent>()
.Include(x => x.Contexts)
.Include(x => x.Responses).ThenInclude(x => x.Contexts)
.Include(x => x.Responses).ThenInclude(x => x.Parameters).ThenInclude(x => x.Prompts)
.Include(x => x.Responses).ThenInclude(x => x.Messages)
.Include(x => x.UserSays).ThenInclude(x => x.Data)
.Where(x => x.AgentId == agentId)
.ToList();*/
var data = new NlpDoc();
// Get NLP Provider

7
BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs
View file

@ -27,13 +27,8 @@ namespace BotSharp.Core.Engines
/// <param name="agentName"></param>
/// <param name="agentDir"></param>
/// <returns></returns>
public Agent LoadAgent()
public Agent LoadAgent(AgentImportHeader agentHeader)
{
// load meta
string metaJson = File.ReadAllText(Path.Combine(AgentDir, "meta.json"));
AgentImportHeader agentHeader = JsonConvert.DeserializeObject<AgentImportHeader>(metaJson);
// load agent profile
string data = File.ReadAllText(Path.Combine(AgentDir, "agent.json"));
var agent = JsonConvert.DeserializeObject<DialogflowAgent>(data);

4
BotSharp.Core/Engines/Rasa/AgentImporterInRasa.cs
View file

@ -16,10 +16,8 @@ namespace BotSharp.Core.Engines.Rasa
{
public string AgentDir { get; set; }
public Agent LoadAgent()
public Agent LoadAgent(AgentImportHeader agentHeader)
{
AgentImportHeader agentHeader = null;
var agent = new Agent();
agent.ClientAccessToken = Guid.NewGuid().ToString("N");
agent.DeveloperAccessToken = Guid.NewGuid().ToString("N");

4
BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs
View file

@ -27,10 +27,8 @@ namespace BotSharp.Core.Engines
/// </summary>
/// <param name="agentDir"></param>
/// <returns></returns>
public Agent LoadAgent()
public Agent LoadAgent(AgentImportHeader agentHeader)
{
AgentImportHeader agentHeader = null;
// load agent profile
string data = File.ReadAllText(Path.Combine(AgentDir, "Sebis", $"{agentHeader.Name}{Path.DirectorySeparatorChar}agent.json"));
var agent = JsonConvert.DeserializeObject<SebisAgent>(data);

8
BotSharp.RestApi/AgentController.cs
View file

@ -68,7 +68,7 @@ namespace BotSharp.RestApi
System.IO.File.Delete(filePath);
var agent = _platform.LoadAgentFromFile<AgentImporterInDialogflow>(dest);
var agent = _platform.LoadAgentFromFile(dest);
return Ok(agent.Id);
}
@ -81,8 +81,10 @@ namespace BotSharp.RestApi
[HttpGet("{agentId}")]
public string Train([FromRoute] String agentId)
{
_platform.LoadAgent(agentId);
_platform.Train();
string agentDir = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agentId);
string dest = Directory.GetDirectories(agentDir).Last();
var agent = _platform.LoadAgentFromFile(dest);
_platform.Train(new BotTrainOptions { AgentDir = agentDir, Model = dest.Split(Path.DirectorySeparatorChar).Last() });
return "";
}

2
BotSharp.RestApi/Rasa/ParseController.cs
View file

@ -52,7 +52,7 @@ namespace BotSharp.RestApi.Rasa
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", request.Project);
var modelPath = Path.Combine(projectPath, request.Model);
_platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath);
_platform.LoadAgentFromFile(modelPath);
var aIResponse = _platform.TextRequest(new AIRequest
{

2
BotSharp.RestApi/Rasa/TrainController.cs
View file

@ -83,7 +83,7 @@ namespace BotSharp.RestApi.Rasa
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
var agent = _platform.LoadAgentFromFile<AgentImporterInRasa>(modelPath);
var agent = _platform.LoadAgentFromFile(modelPath);
var info = await trainer.Train(agent, new BotTrainOptions { Model = request.Model });