SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

make IAgentStorage configurable in json.

Browse source
This commit is contained in:
Oceania2018 2018-10-01 16:28:40 -05:00
parent f796df748e
commit b4a8d4919c
10 changed files with 60 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
BotSharp.Core.UnitTest/BotTrainerTest.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Core.Engines;
using BotSharp.Core.Engines.BotSharp;
using BotSharp.Core.Models;
using BotSharp.Platform.Models;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;

65
BotSharp.Core/Engines/BotSharp/BotSharpCBOWClassifier.cs
View file

@ -1,65 +0,0 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpCBOWClassifier : INlpTrain, INlpPredict
{
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
{
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
string predictFileName = Path.Combine(Settings.TempDir, "fasttext.txt");
File.WriteAllText(predictFileName, doc.Sentences[0].Text);
var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"predict-prob \"{modelFileName}.bin\" \"{predictFileName}\"");
File.Delete(predictFileName);
if (!String.IsNullOrEmpty(output))
{
doc.Sentences[0].Intent = new TextClassificationResult
{
Classifier = "FasttextClassifier",
Label = output.Split(' ')[0].Split(new string[] { "__label__" }, StringSplitOptions.None)[1],
Confidence = decimal.Parse(output.Split(' ')[1])
};
}
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
{
meta.Model = "classification-fasttext.model";
string parsedTrainingDataFileName = Path.Combine(Settings.TempDir, $"classification-fasttext.parsed.txt");
string modelFileName = Path.Combine(Settings.ModelDir, meta.Model);
// assemble corpus
StringBuilder corpus = new StringBuilder();
agent.Corpus.UserSays.ForEach(x => corpus.AppendLine($"__label__{x.Intent} {x.Text}"));
File.WriteAllText(parsedTrainingDataFileName, corpus.ToString());
var output = CmdHelper.Run(Path.Combine(Settings.AlgorithmDir, "fasttext"), $"supervised -input \"{parsedTrainingDataFileName}\" -output \"{modelFileName}\"", false);
Console.WriteLine($"Saved model to {modelFileName}");
return true;
}
}
}

2
BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs
View file

@ -31,7 +31,7 @@ namespace BotSharp.Core.Engines.BotSharp
List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
List<List<TrainingData>> list = new List<List<TrainingData>>();
string rawTrainingDataFileName = System.IO.Path.Combine(Settings.TempDir, "ner-crf.corpus.txt");
string rawTrainingDataFileName = System.IO.Path.Combine(Settings.ModelDir, "ner-crf.corpus.txt");
string modelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model);
using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create))

8
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -58,8 +58,7 @@ namespace BotSharp.Core.Engines
var settings = new PipeSettings
{
ProjectDir = options.AgentDir,
AlgorithmDir = Path.Combine(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms")
ProjectDir = options.AgentDir
};
settings.ModelDir = Path.Combine(options.AgentDir, options.Model);
@ -69,11 +68,6 @@ namespace BotSharp.Core.Engines
Directory.CreateDirectory(settings.ProjectDir);
}
if (!Directory.Exists(settings.TempDir))
{
Directory.CreateDirectory(settings.TempDir);
}
if (!Directory.Exists(settings.ModelDir))
{
Directory.CreateDirectory(settings.ModelDir);

8
BotSharp.Core/Engines/PipeSettings.cs
View file

@ -9,13 +9,5 @@ namespace BotSharp.Core.Engines
{
public string ProjectDir { get; set; }
public string ModelDir { get; set; }
public string AlgorithmDir { get; set; }
public string TempDir
{
get
{
return Path.Combine(ProjectDir, "Temp");
}
}
}
}

3
BotSharp.NLP/Classify/SVMClassifier.cs
View file

@ -66,8 +66,9 @@ namespace BotSharp.NLP.Classify
int numberOfClasses = train.Y.OrderBy(x => x).Distinct().Count();
if (numberOfClasses == 1)
{
throw new ArgumentException("Number of classes can't be one!");
Console.Write("Number of classes must greater than one!");
}
if (svm == SvmType.C_SVC)
{
for (int i = 0; i < numberOfClasses; i++)

1
BotSharp.RestApi/Rasa/TrainController.cs
View file

@ -1,6 +1,7 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines.Rasa;
using BotSharp.Platform.Models;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

49
Platform.Articulate/ArticulateAi.cs
View file

@ -1,6 +1,9 @@
using BotSharp.Core;
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Entities;
using BotSharp.Core.Intents;
using BotSharp.Core.Models;
using BotSharp.Platform.Abstraction;
using BotSharp.Platform.Models;
using DotNetToolkit;
@ -139,11 +142,53 @@ namespace Platform.Articulate
var trainer = new BotTrainer();
var parsedAgent = agent.ToObject<Agent>();
var info = await trainer.Train(parsedAgent, new BotTrainOptions
var intents = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>();
foreach (DomainModel domain in (agent as AgentModel).Domains)
{
foreach (IntentModel intent in domain.Intents)
{
foreach (IntentExampleModel example in intent.Examples)
{
var parsedIntent = new TrainingIntentExpression<TrainingIntentExpressionPart>
{
Intent = intent.IntentName,
Text = example.UserSays,
Entities = example.Entities.Select(x => new TrainingIntentExpressionPart
{
Entity = x.Entity,
Start = x.Start,
Value = x.Value
}).ToList()
};
intents.Add(parsedIntent);
}
}
}
parsedAgent.Corpus = new TrainingCorpus
{
Entities = (agent as AgentModel).Entities.Select(x => new TrainingEntity
{
Entity = x.EntityName,
Values = x.Examples.Select(y => new TrainingEntitySynonym
{
Value = y.Value,
Synonyms = y.Synonyms
}).ToList()
}).ToList(),
UserSays = intents
};
var trainOptions = new BotTrainOptions
{
AgentDir = projectPath,
Model = model
});
};
var info = await trainer.Train(parsedAgent, trainOptions);
return true;
}

4
Platform.Articulate/Controllers/ParseControllercs.cs
View file

@ -13,9 +13,9 @@ namespace Platform.Articulate.Controllers
private readonly IConfiguration configuration;
[HttpGet("/agent/{agentId}/converse")]
public void ParseText([FromRoute] string agentId, [FromQuery] string text, [FromQuery] string sessionId)
public ActionResult ParseText([FromRoute] string agentId, [FromQuery] string text, [FromQuery] string sessionId)
{
return Ok();
}
}
#endif

5
Platform.Articulate/Controllers/TrainController.cs
View file

@ -26,10 +26,15 @@ namespace Platform.Articulate.Controllers
public async Task<AgentModel> TrainAgent([FromRoute] string agentId)
{
var agent = builder.GetAgentById(agentId);
var corpus = builder.ExtractorCorpus(agent);
await builder.Train(agent, corpus);
agent.Status = "Ready";
builder.SaveAgent(agent);
return agent;
}
}