Fix cqdb_writer_close strncpy_s memory issue in Windows.
This commit is contained in:
haiping008@gmail.com
parent
b389d837f2
commit
2927aef12e
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -294,3 +294,6 @@ __pycache__/
|
|||
/BotSharp.UnitTest/App_Data/BotSharp.db
|
||||
/BotSharp.WebHost/App_Data/BotSharp.db
|
||||
/BotSharp.UI
|
||||
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.parsed.txt
|
||||
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.model
|
||||
/BotSharp.WebHost/App_Data/TrainingFiles/bff7605c-3db5-44dc-9ba7-1c9be2832318.corpus.txt
|
||||
|
|
|
|||
|
|
@ -14,6 +14,6 @@ namespace BotSharp.Core.Abstractions
|
|||
{
|
||||
IConfiguration Configuration { get; set; }
|
||||
|
||||
bool Process(Agent agent, JObject data);
|
||||
bool ProcessAsync(Agent agent, JObject data);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ namespace BotSharp.Core.Engines
|
|||
string providerName = config.GetSection($"{platform}:Provider").Value;
|
||||
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpPipeline;
|
||||
provider.Configuration = config.GetSection(platform);
|
||||
provider.Process(agent, data);
|
||||
provider.ProcessAsync(agent, data);
|
||||
|
||||
//var corpus = agent.GrabCorpus(dc);
|
||||
|
||||
|
|
@ -61,7 +61,7 @@ namespace BotSharp.Core.Engines
|
|||
{
|
||||
var pipe = TypeHelper.GetInstance(pipeName, assemblies) as INlpPipeline;
|
||||
pipe.Configuration = provider.Configuration;
|
||||
pipe.Process(agent, data);
|
||||
pipe.ProcessAsync(agent, data);
|
||||
});
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,10 +8,12 @@ using Newtonsoft.Json.Linq;
|
|||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace BotSharp.Core.Engines.CRFsuite
|
||||
{
|
||||
|
|
@ -19,73 +21,98 @@ namespace BotSharp.Core.Engines.CRFsuite
|
|||
{
|
||||
public IConfiguration Configuration { get; set; }
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
public bool ProcessAsync(Agent agent, JObject data)
|
||||
{
|
||||
var dc = new DefaultDataContextLoader().GetDefaultDc();
|
||||
var corpus = agent.Corpus;
|
||||
|
||||
List<List<String>> tags = data["Tags"].ToObject<List<List<String>>>();
|
||||
|
||||
List<List<NlpToken>> tokens = data["Tokens"].ToObject<List<List<NlpToken>>>();
|
||||
List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
|
||||
List<List<TrainingData>> list = new List<List<TrainingData>>();
|
||||
|
||||
var dir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles");
|
||||
FileStream fs = new FileStream(Path.Join(dir, "rawTrain.txt"), FileMode.Create);
|
||||
StreamWriter sw = new StreamWriter(fs);
|
||||
string rawTrainingDataFileName = Path.Join(dir, $"{agent.Id}.corpus.txt");
|
||||
string parsedTrainingDataFileName = Path.Join(dir, $"{agent.Id}.parsed.txt");
|
||||
string modelFileName = Path.Join(dir, $"{agent.Id}.model");
|
||||
string logFileName = Path.Join(dir, $"{agent.Id}.log.txt");
|
||||
|
||||
for (int i = 0 ; i < tags.Count; i++)
|
||||
using (FileStream fs = new FileStream(rawTrainingDataFileName, FileMode.Create))
|
||||
{
|
||||
List<TrainingData> curLine = Merge(tokens[i], tags[i], userSays[i].Entities);
|
||||
list.Add(curLine);
|
||||
curLine.ForEach(trainingData =>{
|
||||
string[] wordParams = {trainingData.Entity, trainingData.Token, trainingData.Tag, trainingData.Chunk};
|
||||
string wordStr = string.Join(" ", wordParams);
|
||||
sw.Write(wordStr + "\n");
|
||||
});
|
||||
sw.Write("\n");
|
||||
}
|
||||
sw.Flush();
|
||||
sw.Close();
|
||||
fs.Close();
|
||||
|
||||
new MachineLearning.CRFsuite.Ner().NerStart();
|
||||
|
||||
Runcmd();
|
||||
using (StreamWriter sw = new StreamWriter(fs))
|
||||
{
|
||||
for (int i = 0; i < tokens.Count; i++)
|
||||
{
|
||||
List<TrainingData> curLine = Merge(tokens[i], userSays[i].Entities);
|
||||
curLine.ForEach(trainingData =>
|
||||
{
|
||||
string[] wordParams = { trainingData.Entity, trainingData.Token, trainingData.Pos, trainingData.Chunk };
|
||||
string wordStr = string.Join(" ", wordParams);
|
||||
sw.Write(wordStr + "\n");
|
||||
});
|
||||
list.Add(curLine);
|
||||
sw.Write("\n");
|
||||
}
|
||||
sw.Flush();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var uniFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:uniFeatures").Split(" ");
|
||||
var biFeatures = Configuration.GetValue<String>($"CRFsuiteEntityRecognizer:biFeatures").Split(" ");
|
||||
|
||||
new MachineLearning.CRFsuite.Ner()
|
||||
.NerStart(rawTrainingDataFileName, parsedTrainingDataFileName, uniFeatures, biFeatures);
|
||||
|
||||
var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
|
||||
|
||||
CallCommandLine(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x
|
||||
|
||||
Console.WriteLine($"Saved model to {modelFileName}");
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public void Runcmd ()
|
||||
public void CallCommandLine(string fileName, string arguments)
|
||||
{
|
||||
var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
|
||||
var dataDir = Path.Join(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "TrainingFiles");
|
||||
Console.WriteLine($"{fileName} {arguments}");
|
||||
|
||||
string cmd = $"{algorithmDir}/crfsuite learn -m {dataDir}/crfsuite/bolo.model {dataDir}/crfsuite/1.txt";
|
||||
System.Diagnostics.Process p = new System.Diagnostics.Process();
|
||||
p.StartInfo.FileName = "sh";
|
||||
p.StartInfo.UseShellExecute = false;
|
||||
p.StartInfo.RedirectStandardInput = true;
|
||||
p.StartInfo.RedirectStandardOutput = true;
|
||||
p.StartInfo.RedirectStandardError = true;
|
||||
p.StartInfo.CreateNoWindow = false;
|
||||
p.Start();
|
||||
ProcessStartInfo procStartInfo = new ProcessStartInfo(fileName);
|
||||
procStartInfo.Arguments = arguments;
|
||||
// The following commands are needed to redirect the standard output.
|
||||
// This means that it will be redirected to the Process.StandardOutput StreamReader.
|
||||
procStartInfo.RedirectStandardOutput = true;
|
||||
procStartInfo.UseShellExecute = false;
|
||||
// Do not create the black window.
|
||||
procStartInfo.CreateNoWindow = true;
|
||||
if (procStartInfo.EnvironmentVariables["OS"] == "Windows_NT")
|
||||
{
|
||||
procStartInfo.FileName = fileName;
|
||||
}
|
||||
else
|
||||
{
|
||||
procStartInfo.FileName = "sh";
|
||||
}
|
||||
|
||||
p.StandardInput.WriteLine(cmd + "&exit");
|
||||
p.StandardInput.AutoFlush = false;
|
||||
Process proc = new Process();
|
||||
proc.StartInfo = procStartInfo;
|
||||
proc.Start();
|
||||
|
||||
string output = p.StandardOutput.ReadToEnd();
|
||||
|
||||
p.WaitForExit();//等待程序执行完退出进程
|
||||
p.Close();
|
||||
Console.WriteLine(output);
|
||||
string output = String.Empty;
|
||||
while (!proc.HasExited)
|
||||
{
|
||||
Thread.Sleep(1);
|
||||
output = proc.StandardOutput.ReadLine();
|
||||
Console.WriteLine(output);
|
||||
}
|
||||
}
|
||||
public List<TrainingData> Merge(List<NlpToken> sentence, List<string> tags, List<TrainingIntentExpressionPart> entities)
|
||||
|
||||
public List<TrainingData> Merge(List<NlpToken> tokens, List<TrainingIntentExpressionPart> entities)
|
||||
{
|
||||
List<TrainingData> trainingTuple = new List<TrainingData>();
|
||||
HashSet<String> entityWordBag = new HashSet<String>();
|
||||
int wordCandidateCount = 0;
|
||||
|
||||
for (int i = 0; i < sentence.Count; i++)
|
||||
for (int i = 0; i < tokens.Count; i++)
|
||||
{
|
||||
TrainingIntentExpressionPart curEntity = null;
|
||||
if (entities != null)
|
||||
|
|
@ -97,7 +124,7 @@ namespace BotSharp.Core.Engines.CRFsuite
|
|||
string[] words = entity.Value.Split(" ");
|
||||
for (int j = 0; j < words.Length; j++)
|
||||
{
|
||||
if (sentence[i + j].Text == words[j])
|
||||
if (tokens[i + j].Text == words[j])
|
||||
{
|
||||
wordCandidateCount++;
|
||||
if (j == words.Length - 1)
|
||||
|
|
@ -116,7 +143,7 @@ namespace BotSharp.Core.Engines.CRFsuite
|
|||
String entityName = curEntity.Entity.Contains(":")? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1): curEntity.Entity;
|
||||
foreach(string s in words)
|
||||
{
|
||||
trainingTuple.Add(new TrainingData(entityName, s, tags[i], "I"));
|
||||
trainingTuple.Add(new TrainingData(entityName, s, tokens[i].Pos, "I"));
|
||||
}
|
||||
entityFinded = true;
|
||||
}
|
||||
|
|
@ -125,7 +152,7 @@ namespace BotSharp.Core.Engines.CRFsuite
|
|||
}
|
||||
if (wordCandidateCount == 0)
|
||||
{
|
||||
trainingTuple.Add(new TrainingData("O", sentence[i].Text, tags[i], "O"));
|
||||
trainingTuple.Add(new TrainingData("O", tokens[i].Text, tokens[i].Pos, "O"));
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -141,33 +168,15 @@ namespace BotSharp.Core.Engines.CRFsuite
|
|||
{
|
||||
public String Token { get; set; }
|
||||
public String Entity { get; set; }
|
||||
public String Tag { get; set; }
|
||||
public String Pos { get; set; }
|
||||
public String Chunk { get; set; }
|
||||
|
||||
public TrainingData(string entity, string token, string tag, string chunk)
|
||||
public TrainingData(string entity, string token, string pos, string chunk)
|
||||
{
|
||||
this.Token = token;
|
||||
this.Entity = entity;
|
||||
this.Tag = tag;
|
||||
this.Pos = pos;
|
||||
this.Chunk = chunk;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public class Token
|
||||
{
|
||||
public String Text { get; set; }
|
||||
public int Offset { get; set; }
|
||||
public int End { get; set; }
|
||||
}
|
||||
|
||||
public class Entity
|
||||
{
|
||||
public String EntityName { get; set; }
|
||||
public String Value { get; set; }
|
||||
public int Start { get; set; }
|
||||
public int End { get; set; }
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ namespace BotSharp.Core.Engines.SpaCy
|
|||
{
|
||||
public IConfiguration Configuration { get; set; }
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
public bool ProcessAsync(Agent agent, JObject data)
|
||||
{
|
||||
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
|
||||
var request = new RestRequest("entitize", Method.GET);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ namespace BotSharp.Core.Engines.SpaCy
|
|||
List<String> entitiesInTrainingSet = new List<string>();
|
||||
public IConfiguration Configuration { get; set; }
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
public bool ProcessAsync(Agent agent, JObject data)
|
||||
{
|
||||
String modelPath = "./entity_rec_output";
|
||||
String newModelName = "test";
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ namespace BotSharp.Core.Engines.SpaCy
|
|||
{
|
||||
public IConfiguration Configuration { get; set; }
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
public bool ProcessAsync(Agent agent, JObject data)
|
||||
{
|
||||
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
|
||||
var request = new RestRequest("load", Method.GET);
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ namespace BotSharp.Core.Engines.SpaCy
|
|||
public IConfiguration Configuration { get; set; }
|
||||
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
public bool ProcessAsync(Agent agent, JObject data)
|
||||
{
|
||||
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
|
||||
var request = new RestRequest("tagger", Method.GET);
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ namespace BotSharp.Core.Engines.SpaCy
|
|||
{
|
||||
public IConfiguration Configuration { get; set; }
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
public bool ProcessAsync(Agent agent, JObject data)
|
||||
{
|
||||
//var input = new List<Tuple<String, JObject>>();
|
||||
|
||||
|
|
|
|||
|
|
@ -17,25 +17,26 @@ namespace BotSharp.Core.Engines.SpaCy
|
|||
{
|
||||
public IConfiguration Configuration { get; set; }
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
public bool ProcessAsync(Agent agent, JObject data)
|
||||
{
|
||||
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
|
||||
var request = new RestRequest("tokenize", Method.GET);
|
||||
var request = new RestRequest("tokenizer", Method.GET);
|
||||
List<List<NlpToken>> tokens = new List<List<NlpToken>>();
|
||||
Boolean res = true;
|
||||
var dc = new DefaultDataContextLoader().GetDefaultDc();
|
||||
var corpus = agent.Corpus;
|
||||
|
||||
corpus.UserSays.ForEach(usersay => {
|
||||
Console.WriteLine(usersay.Text);
|
||||
request.AddParameter("text", usersay.Text);
|
||||
var response = client.Execute<Result>(request);
|
||||
|
||||
tokens.Add(response.Data.Tokens);
|
||||
|
||||
res = res && response.IsSuccessful;
|
||||
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
data.Add("Tokens", JToken.FromObject(tokens));
|
||||
|
||||
return res;
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ namespace BotSharp.Core.Engines.SpaCy
|
|||
{
|
||||
public IConfiguration Configuration { get; set; }
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
public bool ProcessAsync(Agent agent, JObject data)
|
||||
{
|
||||
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
|
||||
var request = new RestRequest("featurize", Method.GET);
|
||||
|
|
|
|||
|
|
@ -135,16 +135,19 @@ namespace BotSharp.MachineLearning.CRFsuite
|
|||
/// <param name="FeatureExtractor">an extractor which to do the feature extracting work</param>
|
||||
/// <param name="fields">attributes name seperated by space</param>
|
||||
/// <param name="sep">string whihch seperated by</param>
|
||||
public void CRFFileGenerator (System.Action<List<Dictionary<string, Object>>> FeatureExtractor, string fields, string sep= " ")
|
||||
public void CRFFileGenerator (System.Action<List<Dictionary<string, Object>>> FeatureExtractor, string fields, string rawFile, string parsedName, string sep= " ")
|
||||
{
|
||||
String fiPath = "/home/bolo/Desktop/BotSharp/TrainingFiles/rawTrain.txt";
|
||||
FileStream fs = new FileStream("/home/bolo/Desktop/BotSharp/TrainingFiles/1.txt", FileMode.Create);
|
||||
FileStream fs = new FileStream(parsedName, FileMode.Create);
|
||||
StreamWriter sw = new StreamWriter(fs);
|
||||
List<string> F = fields.Split(" ").ToList();
|
||||
List<List<Dictionary<string, Object>>> Xs = Readiter(fiPath, F, " ");
|
||||
List<List<Dictionary<string, Object>>> Xs = Readiter(rawFile, F, " ");
|
||||
|
||||
foreach (List<Dictionary<string, Object>> X in Xs)
|
||||
{
|
||||
if (X.Any(x => x["w"].ToString() == ""))
|
||||
{
|
||||
|
||||
}
|
||||
FeatureExtractor(X);
|
||||
OutputFeatures(sw, X, "y");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -469,15 +469,9 @@ namespace BotSharp.MachineLearning.CRFsuite
|
|||
}
|
||||
}
|
||||
|
||||
string[] Uique = new string[]{"w", "wl", "pos", "chk", "shape", "shaped", "type",
|
||||
"p1", "p2", "p3", "p4","s1", "s2", "s3", "s4",
|
||||
"2d", "4d", "d&a", "d&-", "d&/", "d&,", "d&.", "up",
|
||||
"iu", "au", "al", "ad", "ao", "cu", "cl", "ca", "cd", "cs"};
|
||||
string[] Bi = new string[]{"w", "pos", "chk", "shaped", "type"};
|
||||
|
||||
public void InitialTemplate ()
|
||||
public void InitialTemplate (string[] uniFeatures, string[] biFeatures)
|
||||
{
|
||||
foreach (string name in Uique)
|
||||
foreach (string name in uniFeatures)
|
||||
{
|
||||
for (int i = -2 ; i < 3; i++)
|
||||
{
|
||||
|
|
@ -487,7 +481,7 @@ namespace BotSharp.MachineLearning.CRFsuite
|
|||
}
|
||||
}
|
||||
|
||||
foreach (string name in Bi)
|
||||
foreach (string name in biFeatures)
|
||||
{
|
||||
for (int i = -2 ; i < 2; i++)
|
||||
{
|
||||
|
|
@ -509,7 +503,6 @@ namespace BotSharp.MachineLearning.CRFsuite
|
|||
// Apply the feature templates.
|
||||
new Crfutils().ApplyTemplates(X, templates);
|
||||
|
||||
// Append disjunctive features.
|
||||
for (int t = 0; t < X.Count ; t++)
|
||||
{
|
||||
DisJunctive(X, t, "w", -4, -1);
|
||||
|
|
@ -522,10 +515,10 @@ namespace BotSharp.MachineLearning.CRFsuite
|
|||
}
|
||||
}
|
||||
|
||||
public void NerStart ()
|
||||
public void NerStart (string rawFile, string parsedName, string[] uniFeatures, string[] biFeatures)
|
||||
{
|
||||
InitialTemplate();
|
||||
new Crfutils().CRFFileGenerator(FeatureExtractor, fields, separator);
|
||||
InitialTemplate(uniFeatures, biFeatures);
|
||||
new Crfutils().CRFFileGenerator(FeatureExtractor, fields, rawFile, parsedName, separator);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,9 @@ namespace BotSharp.MachineLearning.NLP
|
|||
{
|
||||
public string Text { get; set; }
|
||||
public int Offset { get; set; }
|
||||
public string Pos { get; set; }
|
||||
public string Tag { get; set; }
|
||||
public string Lemma { get; set; }
|
||||
public int End
|
||||
{
|
||||
get
|
||||
|
|
|
|||
|
|
@ -1,25 +1,35 @@
|
|||
from bottle import route, run, request
|
||||
from spacy.tokenizer import Tokenizer
|
||||
from spacy.pipeline import EntityRecognizer
|
||||
from spacy.pipeline import TextCategorizer
|
||||
from spacy.gold import GoldParse
|
||||
#import plac
|
||||
import random
|
||||
import spacy
|
||||
|
||||
nlp = spacy.load('en')
|
||||
tokenizer = Tokenizer(nlp.vocab)
|
||||
ner = EntityRecognizer(nlp.vocab)
|
||||
|
||||
@route('/load')
|
||||
def load():
|
||||
pass
|
||||
|
||||
@route('/tokenize')
|
||||
@route('/tokenizer')
|
||||
def tokenize():
|
||||
tokens = tokenizer(request.query.text)
|
||||
doc = nlp(request.query.text)
|
||||
tokens = []
|
||||
for token in doc:
|
||||
tokens.append({'text': token.text, 'offset': token.idx, 'pos': token.pos_, 'tag': token.tag_, 'lemma': token.lemma_})
|
||||
return {'tokens': tokens}
|
||||
|
||||
|
||||
@route('/tagger')
|
||||
def tagger():
|
||||
doc = nlp(request.query.text)
|
||||
list = []
|
||||
for token in tokens:
|
||||
print(token)
|
||||
list.append({'text': token.text, 'offset': token.idx})
|
||||
return {'tokens': list}
|
||||
for token in doc:
|
||||
list.append(token.tag_)
|
||||
return {'tags': list}
|
||||
|
||||
|
||||
@route('/featurize')
|
||||
def tokenize():
|
||||
|
|
@ -46,13 +56,16 @@ def textcategorizer():
|
|||
texts = request.json["Texts"]
|
||||
golds = request.json["Golds"]
|
||||
labels = request.json["Labels"]
|
||||
print(labels)
|
||||
i = 1
|
||||
train_data = []
|
||||
for index in range(len(texts)):
|
||||
tuple =(texts[index], golds[index])
|
||||
train_data.append(tuple)
|
||||
|
||||
print("training data body is: {0}".format(train_data))
|
||||
'''
|
||||
for tup in train_data:
|
||||
print("cur tuple {0}, training data body: {1}".format(i, train_data))
|
||||
i = i + 1
|
||||
'''
|
||||
textcat = nlp.create_pipe('textcat')
|
||||
nlp.add_pipe(textcat, last=True)
|
||||
for label in labels:
|
||||
|
|
@ -68,8 +81,8 @@ def textcategorizer():
|
|||
|
||||
return {'ModelName':'textcat_try'}
|
||||
|
||||
@route('/predict')
|
||||
def predict():
|
||||
@route('/textcategorizerpredict')
|
||||
def textcategorizerpredict():
|
||||
textcat = TextCategorizer(nlp.vocab)
|
||||
textcat.from_disk('./textcat_try')
|
||||
|
||||
|
|
@ -78,15 +91,114 @@ def predict():
|
|||
doc = nlp(request.query.text)
|
||||
|
||||
#scores = textcat.predict([request.query.text])
|
||||
#print(scores)
|
||||
print(doc.cats)
|
||||
|
||||
list = []
|
||||
for label, confidence in doc.cats:
|
||||
print(label)
|
||||
list.append({'Label': label, 'Confidence': confidence})
|
||||
for key in doc.cats:
|
||||
print(key)
|
||||
list.append({'Label': key, 'Confidence': doc.cats[key]})
|
||||
|
||||
print (list)
|
||||
|
||||
return {'Labels': list}
|
||||
|
||||
run(host='0.0.0.0', port=5005, debug=True)
|
||||
@route('/entityrecognizer', method='POST')
|
||||
def entityrecognizer():
|
||||
model = request.json["ModelPath"]
|
||||
new_model_name = request.json["NewModelName"]
|
||||
output_dir = request.json["OutputDir"]
|
||||
n_iter = request.json["IterTimes"]
|
||||
raw_data = request.json["TrainingData"]
|
||||
# generate training_data from raw_data
|
||||
training_data = []
|
||||
for node in raw_data:
|
||||
labels = []
|
||||
for entity in node['Labels']:
|
||||
label = (entity['Start'], entity['End'], entity['Name'])
|
||||
labels.append(label)
|
||||
tup = (node['Text'], labels)
|
||||
training_data.append(tup)
|
||||
print(training_data)
|
||||
|
||||
if model is not None:
|
||||
nlp = spacy.load(model) # load existing spaCy model
|
||||
print("Loaded model '%s'" % model)
|
||||
else:
|
||||
nlp = spacy.blank('en') # create blank Language class
|
||||
print("Created blank 'en' model")
|
||||
|
||||
# Add entity recognizer to model if it's not in the pipeline
|
||||
# nlp.create_pipe works for built-ins that are registered with spaCy
|
||||
if 'ner' not in nlp.pipe_names:
|
||||
ner = nlp.create_pipe('ner')
|
||||
nlp.add_pipe(ner)
|
||||
print("ner created succeed!")
|
||||
# otherwise, get it, so we can add labels to it
|
||||
else:
|
||||
ner = nlp.get_pipe('ner')
|
||||
print("ner loaded succeed!")
|
||||
|
||||
# check whether there are new labels
|
||||
entities_in_training_set = request.json["EntitiesInTrainingSet"]
|
||||
en_labels = ["PERSON","NORP","FAC","ORG","GPE","LOC","PRODUCT",\
|
||||
"EVENT","WORK_OF_ART","LAW","LANGUAGE","DATE","TIME","PERCENT",\
|
||||
"MONEY","QUANTITY","ORDINAL","CARDINAL"]
|
||||
|
||||
extra_labels = nlp.entity.cfg[u'extra_labels'] \
|
||||
if ('extra_labels' in nlp.entity.cfg) else []
|
||||
|
||||
labels = []
|
||||
for entity in entities_in_training_set:
|
||||
if (entity in en_labels or entity in extra_labels):
|
||||
continue
|
||||
labels.append(entity)
|
||||
|
||||
for label in labels:
|
||||
ner.add_label(label) # add new entity label to entity recognizer
|
||||
print("label added succeed!")
|
||||
|
||||
if model is None:
|
||||
optimizer = nlp.begin_training()
|
||||
else:
|
||||
# Note that 'begin_training' initializes the models, so it'll zero out
|
||||
# existing entity types.
|
||||
optimizer = nlp.entity.create_optimizer()
|
||||
|
||||
# get names of other pipes to disable them during training
|
||||
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
|
||||
with nlp.disable_pipes(*other_pipes): # only train NER
|
||||
for itn in range(n_iter):
|
||||
random.shuffle(training_data)
|
||||
losses = {}
|
||||
for text, annotations in training_data:
|
||||
print(text)
|
||||
print(annotations)
|
||||
#
|
||||
doc = nlp.make_doc(text)
|
||||
gold = GoldParse(doc, entities=annotations)
|
||||
#
|
||||
nlp.update([doc], [gold], sgd=optimizer, drop=0.35)#,losses=losses)
|
||||
#print(losses)
|
||||
|
||||
# save model to output directory
|
||||
if output_dir is not None:
|
||||
output_dir = Path(output_dir)
|
||||
if not output_dir.exists():
|
||||
output_dir.mkdir()
|
||||
nlp.meta['name'] = new_model_name # rename model
|
||||
nlp.to_disk(output_dir)
|
||||
print("Saved model to", output_dir)
|
||||
return True
|
||||
|
||||
@route('/entityrecognizerpredict')
|
||||
def entityrecognizerpredict():
|
||||
|
||||
print("Loading from", './entity_rec_output')
|
||||
nlp2 = spacy.load('./entity_rec_output')
|
||||
doc2 = nlp2(request.query.text)
|
||||
for ent in doc2.ents:
|
||||
print(ent.label_, ent.text)
|
||||
|
||||
|
||||
|
||||
run(host='0.0.0.0', port=5005, debug=True)
|
||||
BIN
BotSharp.WebHost/Algorithms/crfsuite.exe
Normal file
BIN
BotSharp.WebHost/Algorithms/crfsuite.exe
Normal file
Binary file not shown.
File diff suppressed because it is too large
Load diff
|
|
@ -16,7 +16,7 @@
|
|||
},
|
||||
{
|
||||
"Id": "bff7605c-3db5-44dc-9ba7-1c9be2832318",
|
||||
"Name": "Airport",
|
||||
"Name": "Chatbot",
|
||||
"UserId": "8da9e1e0-42dc-420a-8016-79b04c1297d0",
|
||||
"ClientAccessToken": "6ba8a06865944f14981ce18d229283f5",
|
||||
"DeveloperAccessToken": "f12fbdb0da5a4616b18fa7582d32f6e3",
|
||||
|
|
|
|||
|
|
@ -5,6 +5,13 @@
|
|||
<RuntimeIdentifiers>Portable;win10-x64;centos.7-x64</RuntimeIdentifiers>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Remove="App_Data\NewFolder\**" />
|
||||
<Content Remove="App_Data\NewFolder\**" />
|
||||
<EmbeddedResource Remove="App_Data\NewFolder\**" />
|
||||
<None Remove="App_Data\NewFolder\**" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Remove="App_Data\DbInitializer\Agents\agents.json" />
|
||||
<Content Remove="App_Data\DbInitializer\Agents\Dialogflow\Spotify\agent.json" />
|
||||
|
|
|
|||
|
|
@ -9,6 +9,10 @@
|
|||
"SpaCyProvider": {
|
||||
"Url": "http://10.2.21.200:5005"
|
||||
},
|
||||
"Pipe": "SpaCyTokenizer, SpaCyTagger, CRFsuiteEntityRecognizer"
|
||||
"Pipe": "SpaCyTokenizer, CRFsuiteEntityRecognizer",
|
||||
"CRFsuiteEntityRecognizer": {
|
||||
"uniFeatures": "w wl pos chk shape shaped type p1 p2 p3 p4 s1 s2 s3 s4 2d 4d d&a d&- d&/ d&, d&. up iu au al ad ao cu cl ca cd cs",
|
||||
"biFeatures": "w pos chk shaped type"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue