SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add persist and load model function.

Browse source
This commit is contained in:
haiping008@gmail.com 2018-08-08 16:39:00 -05:00
parent 18253d6fea
commit ec67e768a0
24 changed files with 310 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -297,3 +297,7 @@ __pycache__/
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.parsed.txt
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.model
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.corpus.txt
/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.model
/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/metadata.json
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.parsed.txt
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.corpus.txt

3
BotSharp.Core/Abstractions/IBotPlatform.cs
View file

@ -4,6 +4,7 @@ using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines
{
@ -18,6 +19,6 @@ namespace BotSharp.Core.Engines
AIResponse TextRequest(AIRequest request);
void Train();
Task Train();
}
}

12
BotSharp.Core/Abstractions/INlpPipeline.cs
View file

@ -1,9 +1,11 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Abstractions
{
@ -14,6 +16,14 @@ namespace BotSharp.Core.Abstractions
{
IConfiguration Configuration { get; set; }
bool ProcessAsync(Agent agent, JObject data);
/// <summary>
/// Process
/// </summary>
/// <param name="agent"></param>
/// <param name="data">Intermediate result</param>
/// <param name="meta">Meta data which is packed to model</param>
/// <returns></returns>
Task<bool> Train(Agent agent, JObject data, PipeModel meta);
Task<bool> Predict(Agent agent, JObject data, PipeModel meta);
}
}

13
BotSharp.Core/Engines/BotEngineBase.cs
View file

@ -4,11 +4,13 @@ using BotSharp.Core.Intents;
using BotSharp.Core.Models;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines
{
@ -32,6 +34,13 @@ namespace BotSharp.Core.Engines
DbInitializerPath = Path.Join(dataPath, $"DbInitializer");
}
public AIResponse TextRequest(AIRequest request)
{
var preditor = new BotPreditor();
var text = preditor.Predict(agent, request);
return null;
}
public Agent LoadAgent(string id)
{
if (agent == null)
@ -177,9 +186,9 @@ namespace BotSharp.Core.Engines
return corpus;
}
public virtual void Train()
public virtual Task Train()
{
return Task.CompletedTask;
}
}

53
BotSharp.Core/Engines/BotPreditor.cs Normal file
View file

@ -0,0 +1,53 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Models;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines
{
public class BotPreditor
{
public async Task<string> Predict(Agent agent, AIRequest request)
{
// load model
var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
Console.WriteLine($"Load model from {dir}");
var metaJson = File.ReadAllText(Path.Join(dir, "metadata.json"));
var meta = JsonConvert.DeserializeObject<ModelMetaData>(metaJson);
// Get NLP Provider
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var providerPipe = meta.Pipeline.First();
var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpPipeline;
provider.Configuration = config.GetSection(meta.Platform);
var data = JObject.FromObject(new
{
});
await provider.Train(agent, data, providerPipe);
meta.Pipeline.RemoveAt(0);
// pipe process
meta.Pipeline.ForEach(async pipeMeta =>
{
var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
await pipe.Predict(agent, data, pipeMeta);
});
return "";
}
}
}

9
BotSharp.Core/Engines/BotSharp/BotSharpAi.cs
View file

@ -1,21 +1,18 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Models;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpAi : BotEngineBase, IBotPlatform
{
public AIResponse TextRequest(AIRequest request)
{
throw new NotImplementedException();
}
public override void Train()
public override async Task Train()
{
agent.Corpus = GetIntentExpressions();
var trainer = new BotTrainer(agent.Id, dc);
trainer.Train(agent);
await trainer.Train(agent);
}
}
}

51
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Intents;
@ -9,7 +11,9 @@ using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
namespace BotSharp.Core.Engines
{
@ -25,7 +29,7 @@ namespace BotSharp.Core.Engines
this.agentId = agentId;
}
public string Train(Agent agent)
public async Task<string> Train(Agent agent)
{
agent.Intents = dc.Table<Intent>()
.Include(x => x.Contexts)
@ -47,9 +51,25 @@ namespace BotSharp.Core.Engines
string providerName = config.GetSection($"{platform}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpPipeline;
provider.Configuration = config.GetSection(platform);
provider.ProcessAsync(agent, data);
//var corpus = agent.GrabCorpus(dc);
var pipeModel = new PipeModel
{
Name = providerName,
Class = provider.ToString(),
Meta = new JObject(),
Time = DateTime.UtcNow
};
await provider.Train(agent, data, pipeModel);
var meta = new ModelMetaData
{
Platform = platform,
Language = agent.Language,
TrainingDate = DateTime.UtcNow,
Version = config.GetValue<String>($"Version"),
Pipeline = new List<PipeModel>() { pipeModel }
};
// pipe process
var pipelines = provider.Configuration.GetSection($"Pipe").Value
@ -57,15 +77,34 @@ namespace BotSharp.Core.Engines
.Select(x => x.Trim())
.ToList();
pipelines.ForEach(pipeName =>
pipelines.ForEach(async pipeName =>
{
var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
pipe.ProcessAsync(agent, data);
pipeModel = new PipeModel
{
Name = pipeName,
Class = pipe.ToString(),
Time = DateTime.UtcNow
};
meta.Pipeline.Add(pipeModel);
await pipe.Train(agent, data, pipeModel);
});
// save model meta data
var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
var metaJson = JsonConvert.SerializeObject(meta, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
File.WriteAllText(Path.Join(dir, "metadata.json"), metaJson);
return "";
Console.WriteLine(metaJson);
return metaJson;
}
}
}

43
BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs
View file

@ -21,7 +21,7 @@ namespace BotSharp.Core.Engines.CRFsuite
{
public IConfiguration Configuration { get; set; }
public bool ProcessAsync(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var dc = new DefaultDataContextLoader().GetDefaultDc();
var corpus = agent.Corpus;
@ -30,11 +30,22 @@ namespace BotSharp.Core.Engines.CRFsuite
List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
List<List<TrainingData>> list = new List<List<TrainingData>>();
var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles");
string rawTrainingDataFileName = Path.Join(dir, $"{agent.Id}.corpus.txt");
string parsedTrainingDataFileName = Path.Join(dir, $"{agent.Id}.parsed.txt");
string modelFileName = Path.Join(dir, $"{agent.Id}.model");
string logFileName = Path.Join(dir, $"{agent.Id}.log.txt");
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
if (!Directory.Exists(dirTrain))
{
Directory.CreateDirectory(dirTrain);
}
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
if (!Directory.Exists(dirModel))
{
Directory.CreateDirectory(dirModel);
}
string rawTrainingDataFileName = Path.Join(dirTrain, "ner-crf.corpus.txt");
string parsedTrainingDataFileName = Path.Join(dirTrain, "ner-crf.parsed.txt");
string modelFileName = Path.Join(dirModel, $"ner-crf.model");
string logFileName = Path.Join(dirTrain, $"ner-crf.log.txt");
using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create))
{
@ -56,18 +67,23 @@ namespace BotSharp.Core.Engines.CRFsuite
}
}
var uniFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:uniFeatures").Split(" ");
var biFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:biFeatures").Split(" ");
var fields = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:fields");
var uniFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:uniFeatures");
var biFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:biFeatures");
new MachineLearning.CRFsuite.Ner()
.NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, uniFeatures, biFeatures);
.NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, fields, uniFeatures.Split(" "), biFeatures.Split(" "));
var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();
meta.Meta["model"] = $"ner-crf.model";
meta.Meta["fields"] = fields;
meta.Meta["uniFeatures"] = uniFeatures;
meta.Meta["biFeatures"] = biFeatures;
return true;
}
@ -125,8 +141,13 @@ namespace BotSharp.Core.Engines.CRFsuite
i = i + wordCandidateCount - 1;
}
}
return trainingTuple;
return trainingTuple;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
}

17
BotSharp.Core/Engines/ModelMetaData.cs Normal file
View file

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
public class ModelMetaData
{
public string Platform { get; set; }
public string Language { get; set; }
public string Version { get; set; }
public DateTime TrainingDate { get; set; }
public List<PipeModel> Pipeline { get; set; }
}
}

27
BotSharp.Core/Engines/PipeModel.cs Normal file
View file

@ -0,0 +1,27 @@
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
public class PipeModel
{
/// <summary>
/// Pipe name
/// </summary>
public string Name { get; set; }
/// <summary>
/// Pipe type name
/// </summary>
public string Class { get; set; }
public DateTime Time { get; set; }
/// <summary>
/// Extra meta data according to pipe
/// </summary>
public JObject Meta { get; set; }
}
}

8
BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Models;
@ -15,7 +16,12 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool ProcessAsync(Agent agent, JObject data)
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("entitize", Method.GET);

8
BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs
View file

@ -9,6 +9,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
@ -17,7 +18,7 @@ namespace BotSharp.Core.Engines.SpaCy
List<String> entitiesInTrainingSet = new List<string>();
public IConfiguration Configuration { get; set; }
public bool ProcessAsync(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
String modelPath = "./entity_rec_output";
String newModelName = "test";
@ -54,6 +55,11 @@ namespace BotSharp.Core.Engines.SpaCy
return true;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
}
public class Result

24
BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs
View file

@ -1,26 +1,44 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
public class SpaCyProvider : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public bool ProcessAsync(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("load", Method.GET);
var response = client.Execute(request);
var response = client.Execute<Result>(request);
meta.Meta = JObject.FromObject(response.Data);
return response.IsSuccessful;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
private class Result
{
[JsonProperty("spaCy ver")]
public string Version { get; set; }
[JsonProperty("models")]
public string Models { get; set; }
[JsonProperty("python ver")]
public string Python { get; set; }
}
}
}

11
BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs
View file

@ -8,6 +8,7 @@ using System;
using System.Collections.Generic;
using System.Text;
using BotSharp.MachineLearning.NLP;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
@ -15,8 +16,7 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool ProcessAsync(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("tagger", Method.GET);
@ -36,7 +36,12 @@ namespace BotSharp.Core.Engines.SpaCy
return res;
}
public class Result
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
private class Result
{
public List<String> Tags { get; set; }
}

8
BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs
View file

@ -9,6 +9,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
@ -16,7 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool ProcessAsync(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
//var input = new List<Tuple<String, JObject>>();
@ -62,6 +63,11 @@ namespace BotSharp.Core.Engines.SpaCy
return true;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
public class Result
{
public String ModelName { get; set; }

11
BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs
View file

@ -10,6 +10,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
@ -17,7 +18,7 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool ProcessAsync(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("tokenizer", Method.GET);
@ -41,9 +42,13 @@ namespace BotSharp.Core.Engines.SpaCy
return res;
}
public class Result
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
private class Result
{
public List<NlpToken> Tokens { get; set; }
}

8
BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using EntityFrameworkCore.BootKit;
@ -14,7 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool ProcessAsync(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("featurize", Method.GET);
@ -35,6 +36,11 @@ namespace BotSharp.Core.Engines.SpaCy
return res;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
public class Result
{
public List<decimal> Vectors { get; set; }

5
BotSharp.MachineLearning/CRFsuite/Ner.cs
View file

@ -11,8 +11,7 @@ namespace BotSharp.MachineLearning.CRFsuite
{
// Separator of field values.
string separator = " ";
// Field names of the input data.
string fields = "y w pos chk";
Template templates = new Template();
public string GetShape (string token)
@ -515,7 +514,7 @@ namespace BotSharp.MachineLearning.CRFsuite
}
}
public void NerStart (string rawFile, string parsedName, string[] uniFeatures, string[] biFeatures)
public void NerStart(string rawFile, string parsedName, string fields, string[] uniFeatures, string[] biFeatures)
{
InitialTemplate(uniFeatures, biFeatures);
new Crfutils().CRFFileGenerator(FeatureExtractor, fields, rawFile, parsedName, separator);

9
BotSharp.RestApi/Dialogs/DialogController.cs
View file

@ -13,6 +13,7 @@ namespace BotSharp.RestApi.Dialogs
/// <summary>
/// Conversation controller
/// </summary>
[Authorize]
[Route("v1/[controller]")]
public class DialogController : ControllerBase
{
@ -33,16 +34,10 @@ namespace BotSharp.RestApi.Dialogs
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
[AllowAnonymous]
[HttpPost("/v1/query")]
public ActionResult<AIResponse> Query([FromBody] QueryModel request)
{
String clientAccessToken = Request.Headers["Authorization"];
if (String.IsNullOrEmpty(clientAccessToken))
{
return Unauthorized();
}
String clientAccessToken = Request.Headers["ClientAccessToken"];
var config = new AIConfiguration(clientAccessToken, SupportedLanguage.English);
config.SessionId = request.SessionId;

1
BotSharp.WebHost/App_Data/DbInitializer/Accounts/users.json
View file

@ -1,5 +1,6 @@
[
{
"id": "54cc19ee-e3d5-4d59-a011-fa0121450e36",
"userName": "botsharp",
"email": "support@botsharp.io",
"firstName": "Support",

8
BotSharp.WebHost/BotSharp.WebHost.csproj
View file

@ -6,10 +6,18 @@
</PropertyGroup>
<ItemGroup>
<Compile Remove="App_Data\ModelFiles\**" />
<Compile Remove="App_Data\NewFolder\**" />
<Compile Remove="App_Data\TrainingFiles\**" />
<Content Remove="App_Data\ModelFiles\**" />
<Content Remove="App_Data\NewFolder\**" />
<Content Remove="App_Data\TrainingFiles\**" />
<EmbeddedResource Remove="App_Data\ModelFiles\**" />
<EmbeddedResource Remove="App_Data\NewFolder\**" />
<EmbeddedResource Remove="App_Data\TrainingFiles\**" />
<None Remove="App_Data\ModelFiles\**" />
<None Remove="App_Data\NewFolder\**" />
<None Remove="App_Data\TrainingFiles\**" />
</ItemGroup>
<ItemGroup>

3
BotSharp.WebHost/Settings/app.json
View file

@ -1,4 +1,5 @@
{
"Assemblies": "BotSharp.Core",
"BotPlatform": "BotSharpAi"
"BotPlatform": "BotSharpAi",
"Version": "0.1.0"
}

1
BotSharp.WebHost/Settings/bot.json
View file

@ -11,6 +11,7 @@
},
"Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer",
"CRFsuiteEntityRecognizer": {
"fields": "y w pos chk",
"uniFeatures": "w wl pos chk shape shaped type p1 p2 p3 p4 s1 s2 s3 s4 2d 4d d&a d&- d&/ d&, d&. up iu au al ad ao cu cl ca cd cs",
"biFeatures": "w pos chk shaped type"
}

74
BotSharp.WebHost/Startup.cs
View file

@ -16,6 +16,8 @@ using Swashbuckle.AspNetCore.Swagger;
using BotSharp.Core.Engines.BotSharp;
using System.Collections.Generic;
using Newtonsoft.Json;
using DotNetToolkit.JwtHelper;
using BotSharp.Core.Agents;
namespace BotSharp.WebHost
{
@ -31,6 +33,7 @@ namespace BotSharp.WebHost
public void ConfigureServices(IServiceCollection services)
{
services.AddCors();
services.AddJwtAuth(Configuration);
services.AddMvc(options =>
{
@ -43,6 +46,18 @@ namespace BotSharp.WebHost
services.AddSwaggerGen(c =>
{
c.AddSecurityDefinition("Bearer", new ApiKeyScheme()
{
In = "header",
Description = "Please insert JWT with Bearer schema. Example: \"Authorization: Bearer {token}\"",
Name = "Authorization",
Type = "apiKey"
});
c.AddSecurityRequirement(new Dictionary<string, IEnumerable<string>> {
{ "Bearer", Enumerable.Empty<string>() },
});
var info = Configuration.GetSection("Swagger").Get<Info>();
c.SwaggerDoc(info.Version, info);
@ -74,10 +89,7 @@ namespace BotSharp.WebHost
app.UseDefaultFiles();
app.UseStaticFiles();
app.UseSwagger(c =>
{
});
app.UseSwagger();
app.UseSwaggerUI(c =>
{
var info = Configuration.GetSection("Swagger").Get<Info>();
@ -95,8 +107,15 @@ namespace BotSharp.WebHost
app.Use(async (context, next) =>
{
string token = context.Request.Headers["Authorization"];
if (string.IsNullOrWhiteSpace(token))
if (!string.IsNullOrWhiteSpace(token) && (token = token.Split(' ').Last()).Length == 32)
{
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
context.Request.Headers["ClientAccessToken"] = token;
var dc = new DefaultDataContextLoader().GetDefaultDc();
var userId = dc.Table<Agent>().FirstOrDefault(x => x.ClientAccessToken == token)?.UserId;
context.Request.Headers["Authorization"] = "Bearer " + JwtToken.GenerateToken(config, userId);
}
await next.Invoke();
@ -114,51 +133,6 @@ namespace BotSharp.WebHost
loader.Env = env;
loader.Config = Configuration;
loader.Load();
/*Runcmd();
var ai = new BotSharpAi();
ai.LoadAgent("6a9fd374-c43d-447a-97f2-f37540d0c725");
ai.Train();*/
}
public void Runcmd ()
{
string cmd = "/home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite learn -m /home/bolo/Desktop/BotSharp/TrainingFiles/bolo.model /home/bolo/Desktop/BotSharp/TrainingFiles/1.txt";
System.Diagnostics.Process p = new System.Diagnostics.Process();
p.StartInfo.FileName = "sh";
p.StartInfo.UseShellExecute = false; //是否使用操作系统shell启动
p.StartInfo.RedirectStandardInput = true;//接受来自调用程序的输入信息
p.StartInfo.RedirectStandardOutput = true;//由调用程序获取输出信息
p.StartInfo.RedirectStandardError = true;//重定向标准错误输出
p.StartInfo.CreateNoWindow = false;//不显示程序窗口
p.Start();//启动程序
//向cmd窗口发送输入信息
p.StandardInput.WriteLine(cmd + "&exit");
p.StandardInput.AutoFlush = false;
//p.StandardInput.WriteLine("exit");
//向标准输入写入要执行的命令。这里使用&是批处理命令的符号，表示前面一个命令不管是否执行成功都执行后面(exit)命令，如果不执行exit命令，后面调用ReadToEnd()方法会假死
//同类的符号还有&&和||前者表示必须前一个命令执行成功才会执行后面的命令，后者表示必须前一个命令执行失败才会执行后面的命令
//获取cmd窗口的输出信息
string output = p.StandardOutput.ReadToEnd();
//StreamReader reader = p.StandardOutput;
//string line=reader.ReadLine();
//while (!reader.EndOfStream)
//{
// str += line + " ";
// line = reader.ReadLine();
//}
p.WaitForExit();//等待程序执行完退出进程
p.Close();
Console.WriteLine(output);
}
}
}