SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

build spacy tagger in pipeline/create ner in crf/ build wrapper:Crfutils

Browse source
This commit is contained in:
Bolo 2018-08-01 15:25:50 -05:00
parent dbd0d9ac04
commit 1f28cdc15f
17 changed files with 9850 additions and 249 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.vscode/launch.json vendored
View file

@ -10,9 +10,9 @@
"request": "launch",
"preLaunchTask": "build",
// If you have changed target frameworks, make sure to update the program path.
"program": "${workspaceFolder}/BotSharp.UnitTest/bin/Debug/netcoreapp2.0/BotSharp.UnitTest.dll",
"program": "${workspaceFolder}/BotSharp.WebHost/bin/Debug/netcoreapp2.1/BotSharp.WebHost.dll",
"args": [],
"cwd": "${workspaceFolder}/BotSharp.UnitTest",
"cwd": "${workspaceFolder}/BotSharp.WebHost",
// For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window
"console": "internalConsole",
"stopAtEntry": false,

6
BotSharp.Core/Agents/Agent.cs
View file

@ -1,4 +1,5 @@
using BotSharp.Core.Entities;
using BotSharp.Core.Engines;
using BotSharp.Core.Entities;
using BotSharp.Core.Intents;
using EntityFrameworkCore.BootKit;
using Newtonsoft.Json;
@ -72,5 +73,8 @@ namespace BotSharp.Core.Agents
[ForeignKey("AgentId")]
public AgentMlConfig MlConfig { get; set; }
[NotMapped]
public TrainingCorpus Corpus { get; set; }
}
}

3
BotSharp.Core/Engines/BotEngineBase.cs
View file

@ -181,8 +181,7 @@ namespace BotSharp.Core.Engines
public virtual void Train()
{
var trainer = new BotTrainer(agent.Id, dc);
trainer.Train(agent);
}
}

5
BotSharp.Core/Engines/BotSharp/BotSharpAi.cs
View file

@ -11,10 +11,11 @@ namespace BotSharp.Core.Engines.BotSharp
{
throw new NotImplementedException();
}
public override void Train()
{
agent.Corpus = GetIntentExpressions();
var trainer = new BotTrainer(agent.Id, dc);
trainer.Train(agent);
}
}
}

337
BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs
View file

@ -1,13 +1,17 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.MachineLearning.NLP;
using EntityFrameworkCore.BootKit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
namespace BotSharp.Core.Engines.CRFsuite
{
@ -18,271 +22,130 @@ namespace BotSharp.Core.Engines.CRFsuite
public bool Process(Agent agent, JObject data)
{
var dc = new DefaultDataContextLoader().GetDefaultDc();
//var corpus = agent.GrabCorpus(dc);
// Mock Data
List<TrainingData> train_sent = new List<TrainingData>();
train_sent.Add(new TrainingData("Melbourne", "NP", "B-LOC"));
train_sent.Add(new TrainingData("(", "Fpa", "O"));
train_sent.Add(new TrainingData("Australia", "NP", "B-LOC"));
train_sent.Add(new TrainingData(")", "Fpt", "O"));
train_sent.Add(new TrainingData(",", "Fc", "O"));
train_sent.Add(new TrainingData("25", "Z", "O"));
train_sent.Add(new TrainingData("may", "NC", "O"));
train_sent.Add(new TrainingData("(", "Fpa", "O"));
train_sent.Add(new TrainingData("EFE", "NC", "B-ORG"));
train_sent.Add(new TrainingData(")", "Fpt", "O"));
train_sent.Add(new TrainingData(".", "Fp", "O"));
List<List<TrainingData>> train_sents = new List<List<TrainingData>>();
train_sents.Add(train_sent);
List<ItemSequence> X_train = new List<ItemSequence>();
train_sents.ForEach(cur_sent => X_train.Add(new ItemSequence(sent2features(cur_sent))));
StringList sl = new StringList();
List<StringList> y_train = new List<StringList>();
train_sents.ForEach(cur_sent => y_train.Add(new StringList(sent2labels(cur_sent))));
Fit(X_train,y_train);
var corpus = agent.Corpus;
List<List<String>> tags = data["Tags"].ToObject<List<List<String>>>();
List<List<NlpToken>> tokens = data["Tokens"].ToObject<List<List<NlpToken>>>();
List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
List<List<TrainingData>> list =new List<List<TrainingData>>();
FileStream fs = new FileStream("/home/bolo/Desktop/BotSharp/TrainingFiles/rawTrain.txt", FileMode.Create);
StreamWriter sw = new StreamWriter(fs);
for (int i = 0 ; i < tags.Count; i++)
{
List<TrainingData> curLine = Merge(tokens[i], tags[i], userSays[i].Entities);
list.Add(curLine);
curLine.ForEach(trainingData =>{
string[] wordParams = {trainingData.Entity, trainingData.Token, trainingData.Tag, trainingData.Chunk};
string wordStr = string.Join(" ", wordParams);
sw.Write(wordStr + "\n");
});
sw.Write("\n");
}
sw.Flush();
sw.Close();
fs.Close();
new Ner().NerStart();
Runcmd();
return true;
}
/*
public List<TrainingData> Merge(List<Token> sentence, List<Entity> entities )
public void Runcmd ()
{
string cmd = "/home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite learn -m /home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite/bolo.model /home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite/1.txt";
System.Diagnostics.Process p = new System.Diagnostics.Process();
p.StartInfo.FileName = "sh";
p.StartInfo.UseShellExecute = false;
p.StartInfo.RedirectStandardInput = true;
p.StartInfo.RedirectStandardOutput = true;
p.StartInfo.RedirectStandardError = true;
p.StartInfo.CreateNoWindow = false;
p.Start();
p.StandardInput.WriteLine(cmd + "&exit");
p.StandardInput.AutoFlush = false;
string output = p.StandardOutput.ReadToEnd();
p.WaitForExit();//等待程序执行完退出进程
p.Close();
Console.WriteLine(output);
}
public List<TrainingData> Merge(List<NlpToken> sentence, List<string> tags, List<TrainingIntentExpressionPart> entities)
{
List<TrainingData> trainingTuple = new List<TrainingData>();
HashSet<String> entityWordBag = new HashSet<String>();
entities.ForEach(entity =>
int wordCandidateCount = 0;
for (int i = 0; i < sentence.Count; i++)
{
String[] words = entity.Value.Split();
foreach (string word in words)
TrainingIntentExpressionPart curEntity = null;
if (entities != null)
{
entityWordBag.Add(word);
bool entityFinded = false;
entities.ForEach(entity => {
if (!entityFinded)
{
string[] words = entity.Value.Split(" ");
for (int j = 0; j < words.Length; j++)
{
if (sentence[i + j].Text == words[j])
{
wordCandidateCount++;
if (j == words.Length - 1)
{
curEntity = entity;
}
}
else
{
wordCandidateCount = 0;
break;
}
}
if (wordCandidateCount != 0)
{
String entityName = curEntity.Entity.Contains(":")? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1): curEntity.Entity;
foreach(string s in words)
{
trainingTuple.Add(new TrainingData(entityName, s, tags[i], "I"));
}
entityFinded = true;
}
}
});
}
});
sentence.ForEach(token => {
if (!entityWordBag.Contains(token.Text))
if (wordCandidateCount == 0)
{
trainingTuple.Add(new TrainingData(token.Text, "O", token.Offset));
trainingTuple.Add(new TrainingData("O", sentence[i].Text, tags[i], "O"));
}
});
entities.ForEach(entity => trainingTuple.Add(new TrainingData(entity.Value, entity.EntityName, entity.Start)));
trainingTuple.Sort((left, right) => {
if (left.Start > right.Start)
return 1;
else if (left.Start < right.Start)
return -1;
else
return 0;
});
{
i = i + wordCandidateCount - 1;
}
}
return trainingTuple;
}
*/
/* Train a model.
* Parameters
* ----------
* X : list of lists of dicts
Feature dicts for several documents (in a python-crfsuite format).
* y : list of lists of strings
Labels for several documents.
* X_dev : (optional) list of lists of dicts
Feature dicts used for testing.
* y_dev : (optional) list of lists of strings
Labels corresponding to X_dev.
*/
public void Fit(List<ItemSequence> X, List<StringList> y, List<ItemSequence> X_dev = null, List<StringList> y_dev = null) {
Trainer trainer = new Trainer();
for (int i = 0; i < Math.Min(X.Count, y.Count); i++)
{
// group ?
trainer.append(X[i], y[i], 0);
}
trainer.train("model_test", X_dev == null ? -1 : 1);
}
public Item Word2Features(List<TrainingData> sent, int i) {
string word = sent[i].Token;
string postag = sent[i].Tag;
float bias = 1.0F;
String wordLower = word.ToLower();
String wordLast3Char = wordLower.Length >= 3 ? wordLower.Substring(wordLower.Length - 3) : wordLower;
string patternAllCaptain = @"^[A-Z]+$";
Boolean isSupper = new Regex(patternAllCaptain).IsMatch(word);
string patternFirstCaptain = @"^[A-Z]{1}[a-z]+$";
Boolean isTitle = new Regex(patternFirstCaptain).IsMatch(word);
string patternAllDigit = @"^[0-9]+$";
Boolean isDigit = new Regex(patternAllDigit).IsMatch(word);
String posTag = postag;
String postagFirst2Char = postag.Length >= 2 ? postag.Substring(0,2) : postag.Substring(0);
Feature feature = new Feature(bias, wordLower, wordLast3Char, isSupper, isTitle, isDigit, posTag, postagFirst2Char);
Item curItem = new Item();
if (i > 0)
{
string minusWord = sent[i - 1].Token;
string minusPostag = sent[i - 1].Tag;
feature.MinusWordLower = minusWord;
feature.MinusIsTitle = new Regex(patternFirstCaptain).IsMatch(minusWord);
feature.MinusIsSupper = new Regex(patternAllCaptain).IsMatch(minusWord);
feature.MinusPostag = minusPostag;
feature.MinusPostagFirst2Char = minusPostag.Length >= 2 ? minusPostag.Substring(0, 2) : minusPostag.Substring(0);
feature.Items.Add(new Attribute($"minusWordLower:{minusWord}", 1.0));
feature.Items.Add(new Attribute($"minusIsTitle", feature.MinusIsTitle ? 1.0 : 0.0));
feature.Items.Add(new Attribute($"minusIsSupper", feature.MinusIsSupper ? 1.0 : 0.0));
feature.Items.Add(new Attribute($"minusPostag:{minusPostag}", 1.0));
feature.Items.Add(new Attribute($"minusPostagFirst2Char:{feature.MinusPostagFirst2Char}", 1.0));
}
else {
feature.BOS = true;
feature.Items.Add(new Attribute($"BOS", feature.BOS? 1.0 : 0.0));
}
if ( i < sent.Count - 1)
{
string plusWord = sent[i + 1].Token;
string plusPostag = sent[i + 1].Tag;
feature.PlusWordLower = plusWord;
feature.PlusIsTitle = new Regex(patternFirstCaptain).IsMatch(plusWord);
feature.PlusIsSupper = new Regex(patternAllCaptain).IsMatch(plusWord);
feature.PlusPostag = plusPostag;
feature.PlusPostagFirst2Char = plusPostag.Length >= 2 ? plusPostag.Substring(0, 2) : plusPostag.Substring(0);
feature.Items.Add(new Attribute($"minusWordLower:{plusWord}", 1.0));
feature.Items.Add(new Attribute($"minusIsTitle", feature.PlusIsTitle ? 1.0 : 0.0));
feature.Items.Add(new Attribute($"minusIsSupper", feature.PlusIsSupper ? 1.0 : 0.0));
feature.Items.Add(new Attribute($"minusPostag:{plusPostag}", 1.0));
feature.Items.Add(new Attribute($"minusPostagFirst2Char:{feature.PlusPostagFirst2Char}", 1.0));
}
return feature.ToItems();
}
public List<Item> sent2features(List<TrainingData> sent)
{
List<Item> list = new List<Item>();
for (int i = 0 ; i < sent.Count; i++ )
{
list.Add(Word2Features(sent, i));
}
return list;
}
public List<String> sent2labels(List<TrainingData> sent)
{
List<String> list = new List<String>();
sent.ForEach(tuple => list.Add(tuple.Entity));
return list;
}
public List<String> sent2tokens(List<TrainingData> sent)
{
List<String> list = new List<String>();
sent.ForEach(tuple => list.Add(tuple.Token));
return list;
}
}
public class Feature
{
public float Bias { get; set; }
public String WordLower { get; set; }
public String WordLast3Char { get; set; }
public Boolean IsSupper { get; set; }
public Boolean IsTitle { get; set; }
public Boolean IsDigit { get; set; }
public String Postag { get; set; }
public String PostagFirst2Char { get; set; }
public Boolean BOS { get; set; }
public Boolean EOS { get; set; }
public String PlusWordLower { get; set; }
public String PlusLast3Char { get; set; }
public Boolean PlusIsSupper { get; set; }
public Boolean PlusIsTitle { get; set; }
public Boolean PlusIsDigit { get; set; }
public String PlusPostag { get; set; }
public String PlusPostagFirst2Char { get; set; }
public String MinusWordLower { get; set; }
public String MinusLast3Char { get; set; }
public Boolean MinusIsSupper { get; set; }
public Boolean MinusIsTitle { get; set; }
public Boolean MinusIsDigit { get; set; }
public String MinusPostag { get; set; }
public String MinusPostagFirst2Char { get; set; }
public Item Items { get; set; }
public Feature(float bias, String wordLower, String wordLast3Char, Boolean isSupper, Boolean isTitle, Boolean isDigit, String posTag, String postagFirst2Char)
{
this.Bias = bias;
this.WordLower = wordLower;
this.WordLast3Char = wordLast3Char;
this.IsSupper = isSupper;
this.IsTitle = isTitle;
this.IsDigit = isDigit;
this.Postag = posTag;
this.PostagFirst2Char = postagFirst2Char;
this.Items = new Item();
Items.Add(new Attribute($"bias", bias));
Items.Add(new Attribute($"wordLower:{wordLower}", 1.0));
Items.Add(new Attribute($"wordLast3Char:{wordLast3Char}", 1.0));
Items.Add(new Attribute($"isSupper", isSupper? 1.0 : 0.0));
Items.Add(new Attribute($"isTitle", isTitle ? 1.0 : 0.0));
Items.Add(new Attribute($"isDigit", isDigit ? 1.0 : 0.0));
Items.Add(new Attribute($"posTag={posTag}", 1.0));
Items.Add(new Attribute($"postagFirst2Char:{postagFirst2Char}", 1.0));
}
public Item ToItems()
{
return this.Items;
}
}
public class TrainingData
{
public String Token { get; set; }
public String Entity { get; set; }
public String Tag { get; set; }
public String Chunk { get; set; }
public TrainingData(string token, string entity, string tag)
public TrainingData(string entity, string token, string tag, string chunk)
{
this.Token = token;
this.Entity = entity;
this.Tag = tag;
this.Chunk = chunk;
}
}

163
BotSharp.Core/Engines/CRFsuite/Crfutils.cs Normal file
View file

@ -0,0 +1,163 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.MachineLearning.NLP;
using EntityFrameworkCore.BootKit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.Core.Engines.CRFsuite
{
public class Crfutils
{
/// <summary>
/// Generate features for an item sequence by applying feature templates.
/// A feature template consists of a tuple of (name, offset) pairs,
/// where name and offset specify a field name and offset from which
/// the template extracts a feature valreaditerue. Generated features are stored
/// in the 'F' field of each item in the sequence.
/// </summary>
/// <param name="X">Token features for a sentence</param>
/// <param name="template">the template which contains what feature to extract</param>
public void ApplyTemplates (List<Dictionary<string, Object>> X, Template templates)
{
foreach (List<CRFFeature> template in templates.Features)
{
List<string> list = new List<string>();
template.ForEach(t => list.Add($"{t.Field}[{t.Offset}]"));
string name = string.Join("|", list);
for (int t = 0 ; t < X.Count() ; t++) {
List<string> values = new List<string>();
foreach (CRFFeature crffeature in template)
{
string field = crffeature.Field;
int offset = crffeature.Offset;
int p = t + offset;
if (p < 0 || p >= X.Count)
{
values.Clear();
break;
}
values.Add(X[p][field].ToString());
}
if (values != null && values.Count > 0)
{
string value = string.Join("|", values);
((List<string>)X[t]["F"]).Add($"{name}={value}");
}
}
}
}
/// <summary>
/// Return an iterator for item sequences read from a file object.
/// This function reads a sequence from a file object L{fi}, and
/// yields the sequence as a list of mapping objects. Each line
/// (item) from the file object is split by the separator character
/// L{sep}. Separated values of the item are named by L{names},
/// and stored in a mapping object. Every item has a field 'F' that
/// is reserved for storing features.
/// </summary>
/// <param name="fiPath">source file which contains crf style training data</param>
/// <param name="names">each attribute name in fields</param>
/// <param name="sep">seperate by</param>
public List<List<Dictionary<string, Object>>> Readiter (string fiPath, List<string> names, string sep = " ")
{
List<List<Dictionary<string, Object>>> Xs = new List<List<Dictionary<string, Object>>>();
List<Dictionary<string, Object>> X = new List<Dictionary<string, Object>>();
StreamReader sr = new StreamReader(fiPath, Encoding.Default);
string line;
while ((line = sr.ReadLine()) != null)
{
line = line.Replace("\n","");
if (line == null || line.Length == 0)
{
Xs.Add(new List<Dictionary<string, Object>>(X));
X.Clear();
}
else
{
String[] fields = line.Split(sep);
if (fields.Count() < names.Count)
{
// Error Exception
}
Dictionary<string, Object> item = new Dictionary<string, Object>();
item.Add("F", new List<string>());
for (int i = 0 ; i < names.Count ; i++)
{
item.Add(names[i], fields[i]);
}
X.Add(item);
}
}
return Xs;
}
/// <summary>
/// Escape colon characters from feature names.
/// </summary>
/// <param name="src">a feature name</param>
public string Escape(string src)
{
return src.Replace(":", "__COLON__");
}
/// <summary>
/// Output features (and reference labels) of a sequence in CRFSuite
/// format. For each item in the sequence, this function writes a
/// reference label (if L{field} is a non-empty string) and features.
/// </summary>
/// <param name="sw">destination file stream writer</param>
/// <param name="X">Token features for a sentence</param>
/// <param name="field">one attribute name in fields</param>
public void OutputFeatures (StreamWriter sw, List<Dictionary<string, Object>> X, string field = "")
{
for (int t = 0; t < X.Count; t++)
{
if (field.Length != 0)
{
sw.Write(X[t][field]);
}
foreach (string a in (List<string>)X[t]["F"])
{
sw.Write($"\t{Escape(a)}");
}
sw.Write("\n");
}
sw.Write("\n");
}
/// <summary>
/// CRFFileGenerator
/// </summary>
/// <param name="FeatureExtractor">an extractor which to do the feature extracting work</param>
/// <param name="fields">attributes name seperated by space</param>
/// <param name="sep">string whihch seperated by</param>
public void CRFFileGenerator (System.Action<List<Dictionary<string, Object>>> FeatureExtractor, string fields, string sep= " ")
{
String fiPath = "/home/bolo/Desktop/BotSharp/TrainingFiles/rawTrain.txt";
FileStream fs = new FileStream("/home/bolo/Desktop/BotSharp/TrainingFiles/1.txt", FileMode.Create);
StreamWriter sw = new StreamWriter(fs);
List<string> F = fields.Split(" ").ToList();
List<List<Dictionary<string, Object>>> Xs = Readiter(fiPath, F, " ");
foreach (List<Dictionary<string, Object>> X in Xs)
{
FeatureExtractor(X);
OutputFeatures(sw, X, "y");
}
sw.Flush();
sw.Close();
fs.Close();
}
}
}

596
BotSharp.Core/Engines/CRFsuite/Ner.cs Normal file
View file

@ -0,0 +1,596 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.MachineLearning.NLP;
using EntityFrameworkCore.BootKit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.Core.Engines.CRFsuite
{
public class Ner
{
// Separator of field values.
string separator = " ";
// Field names of the input data.
string fields = "y w pos chk";
Template templates = new Template();
public string GetShape (string token)
{
string r = "";
foreach (char c in token.ToCharArray())
{
if (IsSupperChar(c))
{
r += "U";
}
else if (IsLowerChar(c))
{
r += "L";
}
else if (IsDigitChar(c))
{
r += "D";
}
else if (((IList)new char[]{'.', ','}).Contains(c))
{
r += ".";
}
else if (((IList)new char[]{';', ':', '?', '!'}).Contains(c))
{
r += ";";
}
else if (((IList)new char[]{'+', '-', '*', '/', '=', '|', '_'}).Contains(c))
{
r += "-";
}
else if (((IList)new char[]{'(', '{', '[', '<'}).Contains(c))
{
r += "(";
}
else if (((IList)new char[]{')', '}', ']', '>'}).Contains(c))
{
r += ")";
}
else
{
r += c;
}
}
return r;
}
public string Degenerate (string src)
{
string dst = "";
foreach (char c in src)
{
if (dst.Trim() == "" || char.Parse(dst.Substring(dst.Length - 1, 1)) != c)
{
dst += c;
}
}
return dst;
}
public string GetType (string token)
{
List<string> T = new List<String>{"AllUpper", "AllDigit", "AllSymbol","AllUpperDigit", "AllUpperSymbol", "AllDigitSymbol",
"AllUpperDigitSymbol","InitUpper","AllLetter","AllAlnum"};
HashSet<string> R = new HashSet<string>(T);
if (token == null || token.Trim() == "")
{
return "EMPTY";
}
for (int i = 0 ; i < token.Length ; i++)
{
char c = token[i];
if (IsSupperChar(c))
{
if (R.Contains("AllDigit"))
{
R.Remove("AllDigit");
}
if (R.Contains("AllSymbol"))
{
R.Remove("AllSymbol");
}
if (R.Contains("AllDigitSymbol"))
{
R.Remove("AllDigitSymbol");
}
}
else if (IsDigitChar(c) || ((IList)new char[]{'.', ','}).Contains(c))
{
if (R.Contains("AllUpper"))
{
R.Remove("AllUpper");
}
if (R.Contains("AllSymbol"))
{
R.Remove("AllSymbol");
}
if (R.Contains("AllUpperSymbol"))
{
R.Remove("AllUpperSymbol");
}
if (R.Contains("AllLetter"))
{
R.Remove("AllLetter");
}
}
else if (IsLowerChar(c))
{
if (R.Contains("AllUpper"))
{
R.Remove("AllUpper");
}
if (R.Contains("AllDigit"))
{
R.Remove("AllDigit");
}
if (R.Contains("AllSymbol"))
{
R.Remove("AllSymbol");
}
if (R.Contains("AllUpperDigit"))
{
R.Remove("AllUpperDigit");
}
if (R.Contains("AllUpperSymbol"))
{
R.Remove("AllUpperSymbol");
}
if (R.Contains("AllDigitSymbol"))
{
R.Remove("AllDigitSymbol");
}
if (R.Contains("AllUpperDigitSymbol"))
{
R.Remove("AllUpperDigitSymbol");
}
}
else
{
if (R.Contains("AllUpper"))
{
R.Remove("AllUpper");
}
if (R.Contains("AllDigit"))
{
R.Remove("AllDigit");
}
if (R.Contains("AllUpperDigit"))
{
R.Remove("AllUpperDigit");
}
if (R.Contains("AllLetter"))
{
R.Remove("AllLetter");
}
if (R.Contains("AllAlnum"))
{
R.Remove("AllAlnum");
}
}
if (i == 0 && !IsSupperChar(c))
{
if (R.Contains("InitUpper"))
{
R.Remove("InitUpper");
}
}
}
foreach (string tag in T)
{
if (R.Contains(tag))
{
return tag;
}
}
return "NO";
}
public Boolean Get2d (string token)
{
return token.Length == 2 && IsDigit(token);
}
public Boolean Get4d (string token)
{
return token.Length == 4 && IsDigit(token);
}
// is token digit, alpha or not
public Boolean GetDa (string token)
{
Boolean bd = false;
Boolean ba = false;
foreach (char c in token)
{
if (IsDigitChar(c))
{
bd = true;
}
else if (IsAlphaChar(c))
{
ba = true;
}
else
{
return false;
}
}
return bd && ba;
}
public Boolean GetDand (string token, char p)
{
Boolean bd = false;
Boolean bdd = false;
foreach (char c in token)
{
if (IsDigitChar(c))
{
bd = true;
}
else if (c == p)
{
bdd = true;
}
else
{
return false;
}
}
return bd && bdd;
}
public Boolean GetAllOther (string token)
{
foreach (char c in token)
{
if (IsDigitChar(c) || IsAlphaChar(c))
{
return false;
}
}
return true;
}
public Boolean GetCapPeriod (string token)
{
return token.Length == 2 && IsSupperChar(token[0]) && token[1] == '.';
}
public Boolean ContainsUpper (string token)
{
foreach (char c in token)
{
if (IsSupperChar(c))
{
return true;
}
}
return false;
}
public Boolean ContainsLower (string token)
{
foreach (char c in token)
{
if (IsLowerChar(c))
{
return true;
}
}
return false;
}
public Boolean ContainsAlpha (string token)
{
foreach (char c in token)
{
if (IsAlphaChar(c))
{
return true;
}
}
return false;
}
public Boolean ContainsDigit (string token)
{
foreach (char c in token)
{
if (IsDigitChar(c))
{
return true;
}
}
return false;
}
public Boolean ContainsSymbol (string token)
{
foreach (char c in token)
{
if (!IsAlnumChar(c))
{
return true;
}
}
return false;
}
public string B (Boolean v)
{
return v ? "yes" : "no";
}
public void Observation (Dictionary<string, Object> v, string defval = "")
{
// Lowercased token.
v.Add("wl", v["w"].ToString().ToLower());
// Token shape.
v.Add("shape", GetShape(v["w"].ToString()));
// Token shape degenerated.
v.Add("shaped", Degenerate(v["shape"].ToString()));
// Token type.
v.Add("type", GetType(v["w"].ToString()));
// Prefixes (length between one to four).
if (v["w"].ToString().Length >= 1)
{
v.Add("p1", v["w"].ToString().Substring(0, 1));
}
else
{
v.Add("p1", defval);
}
if (v["w"].ToString().Length >= 2)
{
v.Add("p2", v["w"].ToString().Substring(0, 2));
}
else
{
v.Add("p2", defval);
}
if (v["w"].ToString().Length >= 3)
{
v.Add("p3", v["w"].ToString().Substring(0, 3));
}
else
{
v.Add("p3", defval);
}
if (v["w"].ToString().Length >= 4)
{
v.Add("p4", v["w"].ToString().Substring(0, 4));
}
else
{
v.Add("p4", defval);
}
// Suffixes (length between one to four).
string word = v["w"].ToString();
if (v["w"].ToString().Length >= 1)
{
v.Add("s1", word.Substring(word.Length - 1, 1));
}
else
{
v.Add("s1", defval);
}
if (v["w"].ToString().Length >= 2)
{
v.Add("s2", word.Substring(word.Length - 2, 2));
}
else
{
v.Add("s2", defval);
}
if (v["w"].ToString().Length >= 3)
{
v.Add("s3", word.Substring(word.Length - 3, 3));
}
else
{
v.Add("s3", defval);
}
if (v["w"].ToString().Length >= 4)
{
v.Add("s4", word.Substring(word.Length - 4, 4));
}
else
{
v.Add("s4", defval);
}
// Two digits
v.Add("2d", B(Get2d(v["w"].ToString())));
// Four digits
v.Add("4d", B(Get4d(v["w"].ToString())));
// Alphanumeric token.
v.Add("d&a", B(GetDa(v["w"].ToString())));
// Digits and '-'.
v.Add("d&-", B(GetDand(v["w"].ToString(), '-')));
// Digits and '/'.
v.Add("d&/", B(GetDand(v["w"].ToString(), '/')));
// Digits and ','.
v.Add("d&,", B(GetDand(v["w"].ToString(), ',')));
// Digits and '.'.
v.Add("d&.", B(GetDand(v["w"].ToString(), '.')));
// A uppercase letter followed by '.'
v.Add("up", B(GetCapPeriod(v["w"].ToString())));
// An initial uppercase letter.
v.Add("iu", B(IsSupperChar(v["w"].ToString()[0])));
// All uppercase letters.
v.Add("au", B(IsSupper(v["w"].ToString())));
// All lowercase letters.
v.Add("al", B(IsLower(v["w"].ToString())));
// All digit letters.
v.Add("ad", B(IsDigit(v["w"].ToString())));
// All other (non-alphanumeric) letters.
v.Add("ao",B(GetAllOther(v["w"].ToString())));
// Contains a uppercase letter.
v.Add("cu", B(ContainsUpper(v["w"].ToString())));
// Contains a lowercase letter.
v.Add("cl", B(ContainsLower(v["w"].ToString())));
// Contains a alphabet letter.
v.Add("ca", B(ContainsAlpha(v["w"].ToString())));
// Contains a digit.
v.Add("cd", B(ContainsUpper(v["w"].ToString())));
// Contains a symbol.
v.Add("cs", B(ContainsSymbol(v["w"].ToString())));
}
public void DisJunctive(List<Dictionary<string, Object>> X, int t, string field, int begin, int end)
{
string name = $"{field}[{begin}..{end}]";
for (int offset = begin; offset < end + 1; offset++)
{
int p = t + offset;
if (p < 0 || p >= X.Count)
{
continue;
}
List<string> F = (List<string>)X[t]["F"];
F.Add($"{name}={X[p][field]}");
}
}
string[] Uique = new string[]{"w", "wl", "pos", "chk", "shape", "shaped", "type",
"p1", "p2", "p3", "p4","s1", "s2", "s3", "s4",
"2d", "4d", "d&a", "d&-", "d&/", "d&,", "d&.", "up",
"iu", "au", "al", "ad", "ao", "cu", "cl", "ca", "cd", "cs"};
string[] Bi = new string[]{"w", "pos", "chk", "shaped", "type"};
public void InitialTemplate ()
{
foreach (string name in Uique)
{
for (int i = -2 ; i < 3; i++)
{
List<CRFFeature> templateRowFeature = new List<CRFFeature>();
templateRowFeature.Add(new CRFFeature(name, i));
templates.Features.Add(templateRowFeature);
}
}
foreach (string name in Bi)
{
for (int i = -2 ; i < 2; i++)
{
List<CRFFeature> templateRowFeature = new List<CRFFeature>();
templateRowFeature.Add(new CRFFeature(name, i));
templateRowFeature.Add(new CRFFeature(name, i + 1));
templates.Features.Add(templateRowFeature);
}
}
}
public void FeatureExtractor (List<Dictionary<string, Object>> X)
{
// Append observations.
foreach (Dictionary<string, Object> d in X)
{
Observation(d);
}
// Apply the feature templates.
new Crfutils().ApplyTemplates(X, templates);
// Append disjunctive features.
for (int t = 0; t < X.Count ; t++)
{
DisJunctive(X, t, "w", -4, -1);
DisJunctive(X, t, "w", 1, 4);
}
if (X != null && X.Count > 0) {
((List<string>)X[0]["F"]).Add("__BOS__");
((List<string>)X[X.Count - 1]["F"]).Add("__EOS__");
}
}
public void NerStart ()
{
InitialTemplate();
new Crfutils().CRFFileGenerator(FeatureExtractor, fields, separator);
}
private Boolean IsSupperChar(char c)
{
return ((int) c - 'A' >= 0) && ('Z' - (int) c >= 0) ;
}
private Boolean IsLowerChar(char c)
{
return ((int) c - 'a' >= 0)&&('z' - (int) c >= 0) ;
}
private Boolean IsDigitChar(char c)
{
return ((int) c - '0' >= 0)&&('9' - (int) c >= 0) ;
}
private Boolean IsAlphaChar (char c)
{
return IsLowerChar(c) || IsSupperChar(c);
}
private Boolean IsAlnumChar (char c)
{
return IsAlphaChar(c) || IsDigitChar(c);
}
private Boolean IsDigit (String s)
{
string patternAllDigit = @"^[0-9]+$";
return new Regex(patternAllDigit).IsMatch(s);
}
private Boolean IsSupper (string s)
{
string patternAllCaptain = @"^[A-Z]+$";
return new Regex(patternAllCaptain).IsMatch(s);
}
private Boolean IsLower (string s)
{
string patternAllCaptain = @"^[a-z]+$";
return new Regex(patternAllCaptain).IsMatch(s);
}
}
public class Template
{
public List<List<CRFFeature>> Features{ get; set; }
public Template ()
{
this.Features = new List<List<CRFFeature>>();
}
}
public class CRFFeature
{
public string Field { get; set; }
public int Offset { get; set; }
public CRFFeature (string field, int offset)
{
this.Field = field;
this.Offset = offset;
}
}
}

28
BotSharp.Core/Engines/SpaCy/SpaCyTagger.cs
View file

@ -1,10 +1,13 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using EntityFrameworkCore.BootKit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using RestSharp;
using System;
using System.Collections.Generic;
using System.Text;
using BotSharp.MachineLearning.NLP;
namespace BotSharp.Core.Engines.SpaCy
{
@ -12,9 +15,30 @@ namespace BotSharp.Core.Engines.SpaCy
{
public IConfiguration Configuration { get; set; }
public bool Process(Agent agent, JObject data)
public bool Process(Agent agent, JObject data)
{
throw new NotImplementedException();
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
var request = new RestRequest("tagger", Method.GET);
List<List<String>> tags = new List<List<String>>();
Boolean res = true;
var dc = new DefaultDataContextLoader().GetDefaultDc();
var corpus = agent.Corpus;
corpus.UserSays.ForEach(usersay => {
request.AddParameter("text", usersay.Text);
var response = client.Execute<Result>(request);
tags.Add(response.Data.Tags);
res = res && response.IsSuccessful;
});
data.Add("Tags", JToken.FromObject(tags));
return res;
}
public class Result
{
public List<String> Tags { get; set; }
}
}
}

4
BotSharp.Core/Engines/SpaCy/SpaCyTokenizer.cs
View file

@ -24,14 +24,14 @@ namespace BotSharp.Core.Engines.SpaCy
List<List<NlpToken>> tokens = new List<List<NlpToken>>();
Boolean res = true;
var dc = new DefaultDataContextLoader().GetDefaultDc();
/*var corpus = ;
var corpus = agent.Corpus;
corpus.UserSays.ForEach(usersay => {
request.AddParameter("text", usersay.Text);
var response = client.Execute<Result>(request);
tokens.Add(response.Data.Tokens);
res = res && response.IsSuccessful;
});*/
});

2
BotSharp.WebHost/Settings/bot.json
View file

@ -9,6 +9,6 @@
"SpaCyProvider": {
"Url": "http://10.2.21.200:5005"
},
"Pipe": "SpaCyTokenizer, SpacyFeaturizer, CRFsuiteEntityRecognizer" //SpaCyEntitizer, SpaCyTextCategorizer, SpaCyEntityRecognizer
"Pipe": "SpaCyTokenizer, SpaCyTagger, CRFsuiteEntityRecognizer"
}
}

47
BotSharp.WebHost/Startup.cs
View file

@ -10,6 +10,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.PlatformAbstractions;
using Newtonsoft.Json.Serialization;
using Swashbuckle.AspNetCore.Swagger;
using BotSharp.Core.Engines.BotSharp;
namespace BotSharp.WebHost
{
@ -78,6 +79,52 @@ namespace BotSharp.WebHost
Database.Configuration = Configuration;
Database.ContentRootPath = env.ContentRootPath;
Database.Assemblies = Configuration.GetValue<String>("Assemblies").Split(',');
Runcmd();
var ai = new BotSharpAi();
ai.LoadAgent("6a9fd374-c43d-447a-97f2-f37540d0c725");
ai.Train();
}
public void Runcmd ()
{
string cmd = "/home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite learn -m /home/bolo/Desktop/BotSharp/TrainingFiles/bolo.model /home/bolo/Desktop/BotSharp/TrainingFiles/1.txt";
System.Diagnostics.Process p = new System.Diagnostics.Process();
p.StartInfo.FileName = "sh";
p.StartInfo.UseShellExecute = false; //是否使用操作系统shell启动
p.StartInfo.RedirectStandardInput = true;//接受来自调用程序的输入信息
p.StartInfo.RedirectStandardOutput = true;//由调用程序获取输出信息
p.StartInfo.RedirectStandardError = true;//重定向标准错误输出
p.StartInfo.CreateNoWindow = false;//不显示程序窗口
p.Start();//启动程序
//向cmd窗口发送输入信息
p.StandardInput.WriteLine(cmd + "&exit");
p.StandardInput.AutoFlush = false;
//p.StandardInput.WriteLine("exit");
//向标准输入写入要执行的命令。这里使用&是批处理命令的符号，表示前面一个命令不管是否执行成功都执行后面(exit)命令，如果不执行exit命令，后面调用ReadToEnd()方法会假死
//同类的符号还有&&和||前者表示必须前一个命令执行成功才会执行后面的命令，后者表示必须前一个命令执行失败才会执行后面的命令
//获取cmd窗口的输出信息
string output = p.StandardOutput.ReadToEnd();
//StreamReader reader = p.StandardOutput;
//string line=reader.ReadLine();
//while (!reader.EndOfStream)
//{
// str += line + " ";
// line = reader.ReadLine();
//}
p.WaitForExit();//等待程序执行完退出进程
p.Close();
Console.WriteLine(output);
}
}
}

2226
TrainingFiles/1.txt Normal file
View file

File diff suppressed because it is too large Load diff

BIN
TrainingFiles/bolo.model Normal file
View file

Binary file not shown.

BIN
TrainingFiles/crfsuite Executable file
View file

Binary file not shown.

2226
TrainingFiles/rawTrain.txt Normal file
View file

File diff suppressed because it is too large Load diff

2226
TrainingFiles/song_train_file.txt Normal file
View file

File diff suppressed because it is too large Load diff

2226
TrainingFiles/train.crfsuite.txt Normal file
View file

File diff suppressed because it is too large Load diff