SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

update code with Oceania

Browse source
This commit is contained in:
Bolo 2018-08-09 16:06:40 -05:00
parent 5486eb43a2
commit b72717cedf
36 changed files with 4782 additions and 4455 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.gitignore vendored
View file

@ -294,3 +294,12 @@ __pycache__/
/BotSharp.UnitTest/App_Data/BotSharp.db
/BotSharp.WebHost/App_Data/BotSharp.db
/BotSharp.UI
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.parsed.txt
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.model
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.corpus.txt
/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.model
/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/metadata.json
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.parsed.txt
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318/ner-crf.corpus.txt
/BotSharp.WebHost/App_Data/ModelFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318

3
BotSharp.Core/Abstractions/IBotPlatform.cs
View file

@ -4,6 +4,7 @@ using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines
{
@ -18,6 +19,6 @@ namespace BotSharp.Core.Engines
AIResponse TextRequest(AIRequest request);
void Train();
Task Train();
}
}

12
BotSharp.Core/Abstractions/INlpPipeline.cs
View file

@ -1,9 +1,11 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Abstractions
{
@ -14,6 +16,14 @@ namespace BotSharp.Core.Abstractions
{
IConfiguration Configuration { get; set; }
bool Process(Agent agent, JObject data);
/// <summary>
/// Process
/// </summary>
/// <param name="agent"></param>
/// <param name="data">Intermediate result</param>
/// <param name="meta">Meta data which is packed to model</param>
/// <returns></returns>
Task<bool> Train(Agent agent, JObject data, PipeModel meta);
Task<bool> Predict(Agent agent, JObject data, PipeModel meta);
}
}

6
BotSharp.Core/BotSharp.Core.csproj
View file

@ -35,7 +35,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="DotNetToolkit" Version="1.4.0" />
<PackageReference Include="DotNetToolkit" Version="1.5.0" />
<PackageReference Include="EntityFrameworkCore.BootKit" Version="1.8.0" />
<PackageReference Include="Microsoft.AspNetCore.Cryptography.KeyDerivation" Version="2.1.1" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
@ -46,4 +46,8 @@
<ProjectReference Include="..\BotSharp.MachineLearning\BotSharp.MachineLearning.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Engines\CoreNlp\" />
</ItemGroup>
</Project>

13
BotSharp.Core/Engines/BotEngineBase.cs
View file

@ -4,11 +4,13 @@ using BotSharp.Core.Intents;
using BotSharp.Core.Models;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines
{
@ -32,6 +34,13 @@ namespace BotSharp.Core.Engines
DbInitializerPath = Path.Join(dataPath, $"DbInitializer");
}
public AIResponse TextRequest(AIRequest request)
{
var preditor = new BotPreditor();
var text = preditor.Predict(agent, request);
return null;
}
public Agent LoadAgent(string id)
{
if (agent == null)
@ -177,9 +186,9 @@ namespace BotSharp.Core.Engines
return corpus;
}
public virtual void Train()
public virtual Task Train()
{
return Task.CompletedTask;
}
}

53
BotSharp.Core/Engines/BotPreditor.cs Normal file
View file

@ -0,0 +1,53 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Models;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines
{
public class BotPreditor
{
public async Task<string> Predict(Agent agent, AIRequest request)
{
// load model
var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
Console.WriteLine($"Load model from {dir}");
var metaJson = File.ReadAllText(Path.Join(dir, "metadata.json"));
var meta = JsonConvert.DeserializeObject<ModelMetaData>(metaJson);
// Get NLP Provider
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var providerPipe = meta.Pipeline.First();
var provider = TypeHelper.GetInstance(providerPipe.Name, assemblies) as INlpPipeline;
provider.Configuration = config.GetSection(meta.Platform);
var data = JObject.FromObject(new
{
});
await provider.Train(agent, data, providerPipe);
meta.Pipeline.RemoveAt(0);
// pipe process
meta.Pipeline.ForEach(async pipeMeta =>
{
var pipe = TypeHelper.GetInstance(pipeMeta.Name, assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
await pipe.Predict(agent, data, pipeMeta);
});
return "";
}
}
}

9
BotSharp.Core/Engines/BotSharp/BotSharpAi.cs
View file

@ -1,21 +1,18 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Models;
namespace BotSharp.Core.Engines.BotSharp
{
public class BotSharpAi : BotEngineBase, IBotPlatform
{
public AIResponse TextRequest(AIRequest request)
{
throw new NotImplementedException();
}
public override void Train()
public override async Task Train()
{
agent.Corpus = GetIntentExpressions();
var trainer = new BotTrainer(agent.Id, dc);
trainer.Train(agent);
await trainer.Train(agent);
}
}
}

63
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Intents;
@ -9,7 +11,9 @@ using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
namespace BotSharp.Core.Engines
{
@ -25,7 +29,7 @@ namespace BotSharp.Core.Engines
this.agentId = agentId;
}
public string Train(Agent agent)
public async Task<string> Train(Agent agent)
{
agent.Intents = dc.Table<Intent>()
.Include(x => x.Contexts)
@ -47,9 +51,37 @@ namespace BotSharp.Core.Engines
string providerName = config.GetSection($"{platform}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpPipeline;
provider.Configuration = config.GetSection(platform);
provider.Process(agent, data);
//var corpus = agent.GrabCorpus(dc);
var pipeModel = new PipeModel
{
Name = providerName,
Class = provider.ToString(),
Meta = new JObject(),
Time = DateTime.UtcNow
};
await provider.Train(agent, data, pipeModel);
var meta = new ModelMetaData
{
Platform = platform,
Language = agent.Language,
TrainingDate = DateTime.UtcNow,
Version = config.GetValue<String>($"Version"),
Pipeline = new List<PipeModel>() { pipeModel }
};
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
if (!Directory.Exists(dirTrain))
{
Directory.CreateDirectory(dirTrain);
}
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
if (!Directory.Exists(dirModel))
{
Directory.CreateDirectory(dirModel);
}
// pipe process
var pipelines = provider.Configuration.GetSection($"Pipe").Value
@ -57,15 +89,34 @@ namespace BotSharp.Core.Engines
.Select(x => x.Trim())
.ToList();
pipelines.ForEach(pipeName =>
pipelines.ForEach(async pipeName =>
{
var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline;
pipe.Configuration = provider.Configuration;
pipe.Process(agent, data);
pipeModel = new PipeModel
{
Name = pipeName,
Class = pipe.ToString(),
Time = DateTime.UtcNow
};
meta.Pipeline.Add(pipeModel);
await pipe.Train(agent, data, pipeModel);
});
// save model meta data
var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
var metaJson = JsonConvert.SerializeObject(meta, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
File.WriteAllText(Path.Join(dir, "metadata.json"), metaJson);
return "";
Console.WriteLine(metaJson);
return metaJson;
}
}
}

141
BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs
View file

@ -1,17 +1,19 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.MachineLearning.NLP;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.CRFsuite
{
@ -19,73 +21,70 @@ namespace BotSharp.Core.Engines.CRFsuite
{
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var dc = new DefaultDataContextLoader().GetDefaultDc();
var corpus = agent.Corpus;
List<List<String>> tags = data["Tags"].ToObject<List<List<String>>>();
meta.Model = "ner-crf.model";
List<List<NlpToken>> tokens = data["Tokens"].ToObject<List<List<NlpToken>>>();
List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
List<List<TrainingData>> list = new List<List<TrainingData>>();
var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles");
FileStream fs = new FileStream(Path.Join(dir, "rawTrain.txt"), FileMode.Create);
StreamWriter sw = new StreamWriter(fs);
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
string rawTrainingDataFileName = Path.Join(dirTrain, "ner-crf.corpus.txt");
string parsedTrainingDataFileName = Path.Join(dirTrain, "ner-crf.parsed.txt");
string modelFileName = Path.Join(dirModel, meta.Model);
for (int i = 0 ; i < tags.Count; i++)
using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create))
{
List<TrainingData> curLine = Merge(tokens[i], tags[i], userSays[i].Entities);
list.Add(curLine);
curLine.ForEach(trainingData =>{
string[] wordParams = {trainingData.Entity, trainingData.Token, trainingData.Tag, trainingData.Chunk};
string wordStr = string.Join(" ", wordParams);
sw.Write(wordStr + "\n");
});
sw.Write("\n");
}
sw.Flush();
sw.Close();
fs.Close();
new MachineLearning.CRFsuite.Ner().NerStart();
Runcmd();
using (StreamWriter sw = new StreamWriter(fs))
{
for (int i = 0; i < tokens.Count; i++)
{
List<TrainingData> curLine = Merge(tokens[i], userSays[i].Entities);
curLine.ForEach(trainingData =>
{
string[] wordParams = { trainingData.Entity, trainingData.Token, trainingData.Pos, trainingData.Chunk };
string wordStr = string.Join(" ", wordParams);
sw.Write(wordStr + "\n");
});
list.Add(curLine);
sw.Write("\n");
}
sw.Flush();
}
}
var fields = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:fields");
var uniFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:uniFeatures");
var biFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:biFeatures");
new MachineLearning.CRFsuite.Ner()
.NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, fields, uniFeatures.Split(" "), biFeatures.Split(" "));
var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();
meta.Meta["fields"] = fields;
meta.Meta["uniFeatures"] = uniFeatures;
meta.Meta["biFeatures"] = biFeatures;
return true;
}
public void Runcmd ()
{
var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
var dataDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles");
string cmd = $"{algorithmDir}/crfsuite learn -m {dataDir}/crfsuite/bolo.model {dataDir}/crfsuite/1.txt";
System.Diagnostics.Process p = new System.Diagnostics.Process();
p.StartInfo.FileName = "sh";
p.StartInfo.UseShellExecute = false;
p.StartInfo.RedirectStandardInput = true;
p.StartInfo.RedirectStandardOutput = true;
p.StartInfo.RedirectStandardError = true;
p.StartInfo.CreateNoWindow = false;
p.Start();
p.StandardInput.WriteLine(cmd + "&exit");
p.StandardInput.AutoFlush = false;
string output = p.StandardOutput.ReadToEnd();
p.WaitForExit();//等待程序执行完退出进程
p.Close();
Console.WriteLine(output);
}
public List<TrainingData> Merge(List<NlpToken> sentence, List<string> tags, List<TrainingIntentExpressionPart> entities)
public List<TrainingData> Merge(List<NlpToken> tokens, List<TrainingIntentExpressionPart> entities)
{
List<TrainingData> trainingTuple = new List<TrainingData>();
HashSet<String> entityWordBag = new HashSet<String>();
int wordCandidateCount = 0;
for (int i = 0; i < sentence.Count; i++)
for (int i = 0; i < tokens.Count; i++)
{
TrainingIntentExpressionPart curEntity = null;
if (entities != null)
@ -97,7 +96,7 @@ namespace BotSharp.Core.Engines.CRFsuite
string[] words = entity.Value.Split(" ");
for (int j = 0; j < words.Length; j++)
{
if (sentence[i + j].Text == words[j])
if (tokens[i + j].Text == words[j])
{
wordCandidateCount++;
if (j == words.Length - 1)
@ -116,7 +115,7 @@ namespace BotSharp.Core.Engines.CRFsuite
String entityName = curEntity.Entity.Contains(":")? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1): curEntity.Entity;
foreach(string s in words)
{
trainingTuple.Add(new TrainingData(entityName, s, tags[i], "I"));
trainingTuple.Add(new TrainingData(entityName, s, tokens[i].Pos, "I"));
}
entityFinded = true;
}
@ -125,50 +124,36 @@ namespace BotSharp.Core.Engines.CRFsuite
}
if (wordCandidateCount == 0)
{
trainingTuple.Add(new TrainingData("O", sentence[i].Text, tags[i], "O"));
trainingTuple.Add(new TrainingData("O", tokens[i].Text, tokens[i].Pos, "O"));
}
else
{
i = i + wordCandidateCount - 1;
}
}
return trainingTuple;
return trainingTuple;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
}
public class TrainingData
{
public String Token { get; set; }
public String Entity { get; set; }
public String Tag { get; set; }
public String Pos { get; set; }
public String Chunk { get; set; }
public TrainingData(string entity, string token, string tag, string chunk)
public TrainingData(string entity, string token, string pos, string chunk)
{
this.Token = token;
this.Entity = entity;
this.Tag = tag;
this.Chunk = chunk;
Token = token;
Entity = entity;
Pos = pos;
Chunk = chunk;
}
}
public class Token
{
public String Text { get; set; }
public int Offset { get; set; }
public int End { get; set; }
}
public class Entity
{
public String EntityName { get; set; }
public String Value { get; set; }
public int Start { get; set; }
public int End { get; set; }
}
}

49
BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs Normal file
View file

@ -0,0 +1,49 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.Classifiers
{
public class FasttextClassifier : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
throw new NotImplementedException();
}
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
meta.Model = "classification-fasttext.model";
var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
var dirTrain = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles", agent.Id);
var dirModel = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "ModelFiles", agent.Id);
string parsedTrainingDataFileName = Path.Join(dirTrain, $"classification-fasttext.parsed.txt");
string modelFileName = Path.Join(dirModel, meta.Model);
// assemble corpus
StringBuilder corpus = new StringBuilder();
agent.Corpus.UserSays.ForEach(x => corpus.AppendLine($"__label__{x.Intent} {x.Text}"));
File.WriteAllText(parsedTrainingDataFileName, corpus.ToString());
var output = CmdHelper.Run(Path.Join(algorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}");
Console.WriteLine($"Saved model to {modelFileName}");
meta.Meta = new JObject();
meta.Meta["compiled at"] = "Aug 3, 2018";
return true;
}
}
}

17
BotSharp.Core/Engines/ModelMetaData.cs Normal file
View file

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
public class ModelMetaData
{
public string Platform { get; set; }
public string Language { get; set; }
public string Version { get; set; }
public DateTime TrainingDate { get; set; }
public List<PipeModel> Pipeline { get; set; }
}
}

29
BotSharp.Core/Engines/PipeModel.cs Normal file
View file

@ -0,0 +1,29 @@
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines
{
public class PipeModel
{
/// <summary>
/// Pipe name
/// </summary>
public string Name { get; set; }
/// <summary>
/// Pipe type name
/// </summary>
public string Class { get; set; }
public DateTime Time { get; set; }
public string Model { get; set; }
/// <summary>
/// Extra meta data according to pipe
/// </summary>
public JObject Meta { get; set; }
}
}

2
BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs
View file

@ -105,7 +105,7 @@ namespace BotSharp.Core.Engines
Text = say.Text.Substring(entity.Start, entity.Value.Length)
});
pos = entity.End;
pos = entity.End + 1;
if (pos < say.Text.Length && entityIdx == say.Entities.Count - 1)
{

8
BotSharp.Core/Engines/SpaCy/SpaCyEntitizer.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Core.Models;
@ -15,7 +16,12 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("entitize", Method.GET);

8
BotSharp.Core/Engines/SpaCy/SpaCyEntityRecognizer.cs
View file

@ -9,6 +9,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
@ -17,7 +18,7 @@ namespace BotSharp.Core.Engines.SpaCy
List<String> entitiesInTrainingSet = new List<string>();
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
String modelPath = "./entity_rec_output";
String newModelName = "test";
@ -54,6 +55,11 @@ namespace BotSharp.Core.Engines.SpaCy
return true;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
}
public class Result

26
BotSharp.Core/Engines/SpaCy/SpaCyProvider.cs
View file

@ -1,26 +1,46 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
public class SpaCyProvider : INlpPipeline
{
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("load", Method.GET);
var response = client.Execute(request);
var response = client.Execute<Result>(request);
meta.Meta = JObject.FromObject(response.Data);
meta.Meta["models"] = null;
meta.Model = response.Data.Models;
return response.IsSuccessful;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
private class Result
{
[JsonProperty("spaCy ver")]
public string Version { get; set; }
[JsonProperty("models")]
public string Models { get; set; }
[JsonProperty("python ver")]
public string Python { get; set; }
}
}
}

11
BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs
View file

@ -8,6 +8,7 @@ using System;
using System.Collections.Generic;
using System.Text;
using BotSharp.MachineLearning.NLP;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
@ -15,8 +16,7 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("tagger", Method.GET);
@ -36,7 +36,12 @@ namespace BotSharp.Core.Engines.SpaCy
return res;
}
public class Result
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
private class Result
{
public List<String> Tags { get; set; }
}

8
BotSharp.Core/Engines/SpaCy/SpaCyTextCategorizer.cs
View file

@ -9,6 +9,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
@ -16,7 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
//var input = new List<Tuple<String, JObject>>();
@ -62,6 +63,11 @@ namespace BotSharp.Core.Engines.SpaCy
return true;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
public class Result
{
public String ModelName { get; set; }

20
BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs
View file

@ -10,6 +10,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BotSharp.Core.Engines.SpaCy
{
@ -17,32 +18,37 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("tokenize", Method.GET);
var request = new RestRequest("tokenizer", Method.GET);
List<List<NlpToken>> tokens = new List<List<NlpToken>>();
Boolean res = true;
var dc = new DefaultDataContextLoader().GetDefaultDc();
var corpus = agent.Corpus;
corpus.UserSays.ForEach(usersay => {
Console.WriteLine(usersay.Text);
request.AddParameter("text", usersay.Text);
var response = client.Execute<Result>(request);
tokens.Add(response.Data.Tokens);
res = res && response.IsSuccessful;
});
data.Add("Tokens", JToken.FromObject(tokens));
return res;
}
public class Result
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
private class Result
{
public List<NlpToken> Tokens { get; set; }
}

8
BotSharp.Core/Engines/SpaCy/SpacyFeaturizer.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using EntityFrameworkCore.BootKit;
@ -14,7 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
public async Task<bool> Train(Agent agent, JObject data, PipeModel meta)
{
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("featurize", Method.GET);
@ -35,6 +36,11 @@ namespace BotSharp.Core.Engines.SpaCy
return res;
}
public async Task<bool> Predict(Agent agent, JObject data, PipeModel meta)
{
return true;
}
public class Result
{
public List<decimal> Vectors { get; set; }

11
BotSharp.MachineLearning/CRFsuite/Crfutils.cs
View file

@ -135,16 +135,19 @@ namespace BotSharp.MachineLearning.CRFsuite
/// <param name="FeatureExtractor">an extractor which to do the feature extracting work</param>
/// <param name="fields">attributes name seperated by space</param>
/// <param name="sep">string whihch seperated by</param>
public void CRFFileGenerator (System.Action<List<Dictionary<string, Object>>> FeatureExtractor, string fields, string sep= " ")
public void CRFFileGenerator (System.Action<List<Dictionary<string, Object>>> FeatureExtractor, string fields, string rawFile, string parsedName, string sep= " ")
{
String fiPath = "/home/bolo/Desktop/BotSharp/TrainingFiles/rawTrain.txt";
FileStream fs = new FileStream("/home/bolo/Desktop/BotSharp/TrainingFiles/1.txt", FileMode.Create);
FileStream fs = new FileStream(parsedName, FileMode.Create);
StreamWriter sw = new StreamWriter(fs);
List<string> F = fields.Split(" ").ToList();
List<List<Dictionary<string, Object>>> Xs = Readiter(fiPath, F, " ");
List<List<Dictionary<string, Object>>> Xs = Readiter(rawFile, F, " ");
foreach (List<Dictionary<string, Object>> X in Xs)
{
if (X.Any(x => x["w"].ToString() == ""))
{
}
FeatureExtractor(X);
OutputFeatures(sw, X, "y");
}

22
BotSharp.MachineLearning/CRFsuite/Ner.cs
View file

@ -11,8 +11,7 @@ namespace BotSharp.MachineLearning.CRFsuite
{
// Separator of field values.
string separator = " ";
// Field names of the input data.
string fields = "y w pos chk";
Template templates = new Template();
public string GetShape (string token)
@ -469,15 +468,9 @@ namespace BotSharp.MachineLearning.CRFsuite
}
}
string[] Uique = new string[]{"w", "wl", "pos", "chk", "shape", "shaped", "type",
"p1", "p2", "p3", "p4","s1", "s2", "s3", "s4",
"2d", "4d", "d&a", "d&-", "d&/", "d&,", "d&.", "up",
"iu", "au", "al", "ad", "ao", "cu", "cl", "ca", "cd", "cs"};
string[] Bi = new string[]{"w", "pos", "chk", "shaped", "type"};
public void InitialTemplate ()
public void InitialTemplate (string[] uniFeatures, string[] biFeatures)
{
foreach (string name in Uique)
foreach (string name in uniFeatures)
{
for (int i = -2 ; i < 3; i++)
{
@ -487,7 +480,7 @@ namespace BotSharp.MachineLearning.CRFsuite
}
}
foreach (string name in Bi)
foreach (string name in biFeatures)
{
for (int i = -2 ; i < 2; i++)
{
@ -509,7 +502,6 @@ namespace BotSharp.MachineLearning.CRFsuite
// Apply the feature templates.
new Crfutils().ApplyTemplates(X, templates);
// Append disjunctive features.
for (int t = 0; t < X.Count ; t++)
{
DisJunctive(X, t, "w", -4, -1);
@ -522,10 +514,10 @@ namespace BotSharp.MachineLearning.CRFsuite
}
}
public void NerStart ()
public void NerStart(string rawFile, string parsedName, string fields, string[] uniFeatures, string[] biFeatures)
{
InitialTemplate();
new Crfutils().CRFFileGenerator(FeatureExtractor, fields, separator);
InitialTemplate(uniFeatures, biFeatures);
new Crfutils().CRFFileGenerator(FeatureExtractor, fields, rawFile, parsedName, separator);
}

3
BotSharp.MachineLearning/NLP/NlpToken.cs
View file

@ -8,6 +8,9 @@ namespace BotSharp.MachineLearning.NLP
{
public string Text { get; set; }
public int Offset { get; set; }
public string Pos { get; set; }
public string Tag { get; set; }
public string Lemma { get; set; }
public int End
{
get

148
BotSharp.MachineLearning/SpaCy/server.py
View file

@ -1,25 +1,35 @@
from bottle import route, run, request
from spacy.tokenizer import Tokenizer
from spacy.pipeline import EntityRecognizer
from spacy.pipeline import TextCategorizer
from spacy.gold import GoldParse
#import plac
import random
import spacy
nlp = spacy.load('en')
tokenizer = Tokenizer(nlp.vocab)
ner = EntityRecognizer(nlp.vocab)
@route('/load')
def load():
pass
@route('/tokenize')
@route('/tokenizer')
def tokenize():
tokens = tokenizer(request.query.text)
doc = nlp(request.query.text)
tokens = []
for token in doc:
tokens.append({'text': token.text, 'offset': token.idx, 'pos': token.pos_, 'tag': token.tag_, 'lemma': token.lemma_})
return {'tokens': tokens}
@route('/tagger')
def tagger():
doc = nlp(request.query.text)
list = []
for token in tokens:
print(token)
list.append({'text': token.text, 'offset': token.idx})
return {'tokens': list}
for token in doc:
list.append(token.tag_)
return {'tags': list}
@route('/featurize')
def tokenize():
@ -46,13 +56,16 @@ def textcategorizer():
texts = request.json["Texts"]
golds = request.json["Golds"]
labels = request.json["Labels"]
print(labels)
i = 1
train_data = []
for index in range(len(texts)):
tuple =(texts[index], golds[index])
train_data.append(tuple)
print("training data body is: {0}".format(train_data))
'''
for tup in train_data:
print("cur tuple {0}, training data body: {1}".format(i, train_data))
i = i + 1
'''
textcat = nlp.create_pipe('textcat')
nlp.add_pipe(textcat, last=True)
for label in labels:
@ -68,8 +81,8 @@ def textcategorizer():
return {'ModelName':'textcat_try'}
@route('/predict')
def predict():
@route('/textcategorizerpredict')
def textcategorizerpredict():
textcat = TextCategorizer(nlp.vocab)
textcat.from_disk('./textcat_try')
@ -78,15 +91,114 @@ def predict():
doc = nlp(request.query.text)
#scores = textcat.predict([request.query.text])
#print(scores)
print(doc.cats)
list = []
for label, confidence in doc.cats:
print(label)
list.append({'Label': label, 'Confidence': confidence})
for key in doc.cats:
print(key)
list.append({'Label': key, 'Confidence': doc.cats[key]})
print (list)
return {'Labels': list}
run(host='0.0.0.0', port=5005, debug=True)
@route('/entityrecognizer', method='POST')
def entityrecognizer():
model = request.json["ModelPath"]
new_model_name = request.json["NewModelName"]
output_dir = request.json["OutputDir"]
n_iter = request.json["IterTimes"]
raw_data = request.json["TrainingData"]
# generate training_data from raw_data
training_data = []
for node in raw_data:
labels = []
for entity in node['Labels']:
label = (entity['Start'], entity['End'], entity['Name'])
labels.append(label)
tup = (node['Text'], labels)
training_data.append(tup)
print(training_data)
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
nlp = spacy.blank('en') # create blank Language class
print("Created blank 'en' model")
# Add entity recognizer to model if it's not in the pipeline
# nlp.create_pipe works for built-ins that are registered with spaCy
if 'ner' not in nlp.pipe_names:
ner = nlp.create_pipe('ner')
nlp.add_pipe(ner)
print("ner created succeed!")
# otherwise, get it, so we can add labels to it
else:
ner = nlp.get_pipe('ner')
print("ner loaded succeed!")
# check whether there are new labels
entities_in_training_set = request.json["EntitiesInTrainingSet"]
en_labels = ["PERSON","NORP","FAC","ORG","GPE","LOC","PRODUCT",\
"EVENT","WORK_OF_ART","LAW","LANGUAGE","DATE","TIME","PERCENT",\
"MONEY","QUANTITY","ORDINAL","CARDINAL"]
extra_labels = nlp.entity.cfg[u'extra_labels'] \
if ('extra_labels' in nlp.entity.cfg) else []
labels = []
for entity in entities_in_training_set:
if (entity in en_labels or entity in extra_labels):
continue
labels.append(entity)
for label in labels:
ner.add_label(label) # add new entity label to entity recognizer
print("label added succeed!")
if model is None:
optimizer = nlp.begin_training()
else:
# Note that 'begin_training' initializes the models, so it'll zero out
# existing entity types.
optimizer = nlp.entity.create_optimizer()
# get names of other pipes to disable them during training
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
with nlp.disable_pipes(*other_pipes): # only train NER
for itn in range(n_iter):
random.shuffle(training_data)
losses = {}
for text, annotations in training_data:
print(text)
print(annotations)
#
doc = nlp.make_doc(text)
gold = GoldParse(doc, entities=annotations)
#
nlp.update([doc], [gold], sgd=optimizer, drop=0.35)#,losses=losses)
#print(losses)
# save model to output directory
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
nlp.meta['name'] = new_model_name # rename model
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
return True
@route('/entityrecognizerpredict')
def entityrecognizerpredict():
print("Loading from", './entity_rec_output')
nlp2 = spacy.load('./entity_rec_output')
doc2 = nlp2(request.query.text)
for ent in doc2.ents:
print(ent.label_, ent.text)
run(host='0.0.0.0', port=5005, debug=True)

9
BotSharp.RestApi/Dialogs/DialogController.cs
View file

@ -13,6 +13,7 @@ namespace BotSharp.RestApi.Dialogs
/// <summary>
/// Conversation controller
/// </summary>
[Authorize]
[Route("v1/[controller]")]
public class DialogController : ControllerBase
{
@ -33,16 +34,10 @@ namespace BotSharp.RestApi.Dialogs
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
[AllowAnonymous]
[HttpPost("/v1/query")]
public ActionResult<AIResponse> Query([FromBody] QueryModel request)
{
String clientAccessToken = Request.Headers["Authorization"];
if (String.IsNullOrEmpty(clientAccessToken))
{
return Unauthorized();
}
String clientAccessToken = Request.Headers["ClientAccessToken"];
var config = new AIConfiguration(clientAccessToken, SupportedLanguage.English);
config.SessionId = request.SessionId;

BIN
BotSharp.WebHost/Algorithms/crfsuite.exe Normal file
View file

Binary file not shown.

BIN
BotSharp.WebHost/Algorithms/fasttext.exe Normal file
View file

Binary file not shown.

1
BotSharp.WebHost/App_Data/DbInitializer/Accounts/users.json
View file

@ -1,5 +1,6 @@
[
{
"id": "54cc19ee-e3d5-4d59-a011-fa0121450e36",
"userName": "botsharp",
"email": "support@botsharp.io",
"firstName": "Support",

0
BotSharp.WebHost/App_Data/DbInitializer/Agents/Sebis/Airport/agent.json → BotSharp.WebHost/App_Data/DbInitializer/Agents/Sebis/Chatbot/agent.json
View file

8444
BotSharp.WebHost/App_Data/DbInitializer/Agents/Sebis/Airport/corpus.json → BotSharp.WebHost/App_Data/DbInitializer/Agents/Sebis/Chatbot/corpus.json
View file

File diff suppressed because it is too large Load diff

2
BotSharp.WebHost/App_Data/DbInitializer/Agents/agents.json
View file

@ -16,7 +16,7 @@
},
{
"Id": "bff7605c-3db5-44dc-9ba7-1c9be2832318",
"Name": "Airport",
"Name": "Chatbot",
"UserId": "8da9e1e0-42dc-420a-8016-79b04c1297d0",
"ClientAccessToken": "6ba8a06865944f14981ce18d229283f5",
"DeveloperAccessToken": "f12fbdb0da5a4616b18fa7582d32f6e3",

16
BotSharp.WebHost/BotSharp.WebHost.csproj
View file

@ -5,6 +5,21 @@
<RuntimeIdentifiers>Portable;win10-x64;centos.7-x64</RuntimeIdentifiers>
</PropertyGroup>
<ItemGroup>
<Compile Remove="App_Data\ModelFiles\**" />
<Compile Remove="App_Data\NewFolder\**" />
<Compile Remove="App_Data\TrainingFiles\**" />
<Content Remove="App_Data\ModelFiles\**" />
<Content Remove="App_Data\NewFolder\**" />
<Content Remove="App_Data\TrainingFiles\**" />
<EmbeddedResource Remove="App_Data\ModelFiles\**" />
<EmbeddedResource Remove="App_Data\NewFolder\**" />
<EmbeddedResource Remove="App_Data\TrainingFiles\**" />
<None Remove="App_Data\ModelFiles\**" />
<None Remove="App_Data\NewFolder\**" />
<None Remove="App_Data\TrainingFiles\**" />
</ItemGroup>
<ItemGroup>
<Content Remove="App_Data\DbInitializer\Agents\agents.json" />
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\agent.json" />
@ -57,7 +72,6 @@
<ItemGroup>
<Folder Include="App_Data\DbInitializer\Agents\Rasa\" />
<Folder Include="Algorithms\" />
</ItemGroup>
<ItemGroup>

3
BotSharp.WebHost/Settings/app.json
View file

@ -1,4 +1,5 @@
{
"Assemblies": "BotSharp.Core",
"BotPlatform": "BotSharpAi"
"BotPlatform": "BotSharpAi",
"Version": "0.1.0"
}

7
BotSharp.WebHost/Settings/bot.json
View file

@ -9,6 +9,11 @@
"SpaCyProvider": {
"Url": "http://10.2.21.200:5005"
},
"Pipe": "SpaCyTokenizer, SpaCyTagger, CRFsuiteEntityRecognizer"
"Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer, FasttextClassifier",
"CRFsuiteEntityRecognizer": {
"fields": "y w pos chk",
"uniFeatures": "w wl pos chk shape shaped type p1 p2 p3 p4 s1 s2 s3 s4 2d 4d d&a d&- d&/ d&, d&. up iu au al ad ao cu cl ca cd cs",
"biFeatures": "w pos chk shaped type"
}
}
}

74
BotSharp.WebHost/Startup.cs
View file

@ -16,6 +16,8 @@ using Swashbuckle.AspNetCore.Swagger;
using BotSharp.Core.Engines.BotSharp;
using System.Collections.Generic;
using Newtonsoft.Json;
using DotNetToolkit.JwtHelper;
using BotSharp.Core.Agents;
namespace BotSharp.WebHost
{
@ -31,6 +33,7 @@ namespace BotSharp.WebHost
public void ConfigureServices(IServiceCollection services)
{
services.AddCors();
services.AddJwtAuth(Configuration);
services.AddMvc(options =>
{
@ -43,6 +46,18 @@ namespace BotSharp.WebHost
services.AddSwaggerGen(c =>
{
c.AddSecurityDefinition("Bearer", new ApiKeyScheme()
{
In = "header",
Description = "Please insert JWT with Bearer schema. Example: \"Authorization: Bearer {token}\"",
Name = "Authorization",
Type = "apiKey"
});
c.AddSecurityRequirement(new Dictionary<string, IEnumerable<string>> {
{ "Bearer", Enumerable.Empty<string>() },
});
var info = Configuration.GetSection("Swagger").Get<Info>();
c.SwaggerDoc(info.Version, info);
@ -74,10 +89,7 @@ namespace BotSharp.WebHost
app.UseDefaultFiles();
app.UseStaticFiles();
app.UseSwagger(c =>
{
});
app.UseSwagger();
app.UseSwaggerUI(c =>
{
var info = Configuration.GetSection("Swagger").Get<Info>();
@ -95,8 +107,15 @@ namespace BotSharp.WebHost
app.Use(async (context, next) =>
{
string token = context.Request.Headers["Authorization"];
if (string.IsNullOrWhiteSpace(token))
if (!string.IsNullOrWhiteSpace(token) && (token = token.Split(' ').Last()).Length == 32)
{
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
context.Request.Headers["ClientAccessToken"] = token;
var dc = new DefaultDataContextLoader().GetDefaultDc();
var userId = dc.Table<Agent>().FirstOrDefault(x => x.ClientAccessToken == token)?.UserId;
context.Request.Headers["Authorization"] = "Bearer " + JwtToken.GenerateToken(config, userId);
}
await next.Invoke();
@ -114,51 +133,6 @@ namespace BotSharp.WebHost
loader.Env = env;
loader.Config = Configuration;
loader.Load();
/*Runcmd();
var ai = new BotSharpAi();
ai.LoadAgent("6a9fd374-c43d-447a-97f2-f37540d0c725");
ai.Train();*/
}
public void Runcmd ()
{
string cmd = "/home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite learn -m /home/bolo/Desktop/BotSharp/TrainingFiles/bolo.model /home/bolo/Desktop/BotSharp/TrainingFiles/1.txt";
System.Diagnostics.Process p = new System.Diagnostics.Process();
p.StartInfo.FileName = "sh";
p.StartInfo.UseShellExecute = false; //是否使用操作系统shell启动
p.StartInfo.RedirectStandardInput = true;//接受来自调用程序的输入信息
p.StartInfo.RedirectStandardOutput = true;//由调用程序获取输出信息
p.StartInfo.RedirectStandardError = true;//重定向标准错误输出
p.StartInfo.CreateNoWindow = false;//不显示程序窗口
p.Start();//启动程序
//向cmd窗口发送输入信息
p.StandardInput.WriteLine(cmd + "&exit");
p.StandardInput.AutoFlush = false;
//p.StandardInput.WriteLine("exit");
//向标准输入写入要执行的命令。这里使用&是批处理命令的符号，表示前面一个命令不管是否执行成功都执行后面(exit)命令，如果不执行exit命令，后面调用ReadToEnd()方法会假死
//同类的符号还有&&和||前者表示必须前一个命令执行成功才会执行后面的命令，后者表示必须前一个命令执行失败才会执行后面的命令
//获取cmd窗口的输出信息
string output = p.StandardOutput.ReadToEnd();
//StreamReader reader = p.StandardOutput;
//string line=reader.ReadLine();
//while (!reader.EndOfStream)
//{
// str += line + " ";
// line = reader.ReadLine();
//}
p.WaitForExit();//等待程序执行完退出进程
p.Close();
Console.WriteLine(output);
}
}
}

2
README.md
View file

@ -2,7 +2,7 @@
### The Open Source AI Chatbot Platform Builder for Enterprise
###
**BotSharp** is an open source AI chatbot platform builder not only a bot builder, it's a complete out of box toolkit for building up a functional chabot platform which utilize aritifical intelligence. It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope.
**BotSharp** is an open source AI chatbot platform builder's framework. It's not only a bot builder but also a complete out of box toolkit for building up a functional chabot platform which utilize aritifical intelligence. It's witten in C# running on .Net Core that is full cross-platform framework. C# is a enterprise grade programming language which is widely used to code business logic in information management related system. BotSharp adopts machine learning algrithm in C/C++ interfaces directly which skips the python interfaces. That will facilitate the feature of the typed language C#, and be more easier when refactoring code in system scope.
Why we do this? because we all know python is not friendly programming language for enterprise developers, it's not only because it's low performance but also it's a type weak language, it will be a disater if you use python to build your bussiness system.