Oceania
GitHub
commit
dab33025c4
4
.vscode/launch.json
vendored
4
.vscode/launch.json
vendored
|
|
@ -10,9 +10,9 @@
|
|||
"request": "launch",
|
||||
"preLaunchTask": "build",
|
||||
// If you have changed target frameworks, make sure to update the program path.
|
||||
"program": "${workspaceFolder}/BotSharp.UnitTest/bin/Debug/netcoreapp2.0/BotSharp.UnitTest.dll",
|
||||
"program": "${workspaceFolder}/BotSharp.WebHost/bin/Debug/netcoreapp2.1/BotSharp.WebHost.dll",
|
||||
"args": [],
|
||||
"cwd": "${workspaceFolder}/BotSharp.UnitTest",
|
||||
"cwd": "${workspaceFolder}/BotSharp.WebHost",
|
||||
// For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window
|
||||
"console": "internalConsole",
|
||||
"stopAtEntry": false,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using BotSharp.Core.Entities;
|
||||
using BotSharp.Core.Engines;
|
||||
using BotSharp.Core.Entities;
|
||||
using BotSharp.Core.Intents;
|
||||
using EntityFrameworkCore.BootKit;
|
||||
using Newtonsoft.Json;
|
||||
|
|
@ -73,6 +74,9 @@ namespace BotSharp.Core.Agents
|
|||
[ForeignKey("AgentId")]
|
||||
public AgentMlConfig MlConfig { get; set; }
|
||||
|
||||
[NotMapped]
|
||||
public TrainingCorpus Corpus { get; set; }
|
||||
|
||||
[ForeignKey("AgentId")]
|
||||
public List<AgentIntegration> Integrations { get; set; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -176,8 +176,7 @@ namespace BotSharp.Core.Engines
|
|||
|
||||
public virtual void Train()
|
||||
{
|
||||
var trainer = new BotTrainer(agent.Id, dc);
|
||||
trainer.Train(agent);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,10 +11,11 @@ namespace BotSharp.Core.Engines.BotSharp
|
|||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override void Train()
|
||||
{
|
||||
|
||||
agent.Corpus = GetIntentExpressions();
|
||||
var trainer = new BotTrainer(agent.Id, dc);
|
||||
trainer.Train(agent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,17 @@
|
|||
using BotSharp.Core.Abstractions;
|
||||
using BotSharp.Core.Agents;
|
||||
using BotSharp.MachineLearning.NLP;
|
||||
using EntityFrameworkCore.BootKit;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
|
||||
namespace BotSharp.Core.Engines.CRFsuite
|
||||
{
|
||||
|
|
@ -18,257 +22,114 @@ namespace BotSharp.Core.Engines.CRFsuite
|
|||
public bool Process(Agent agent, JObject data)
|
||||
{
|
||||
var dc = new DefaultDataContextLoader().GetDefaultDc();
|
||||
//var corpus = agent.GrabCorpus(dc);
|
||||
|
||||
// Mock Data
|
||||
List<TrainingData> train_sent = new List<TrainingData>();
|
||||
train_sent.Add(new TrainingData("Melbourne", "NP", "B-LOC"));
|
||||
train_sent.Add(new TrainingData("(", "Fpa", "O"));
|
||||
train_sent.Add(new TrainingData("Australia", "NP", "B-LOC"));
|
||||
train_sent.Add(new TrainingData(")", "Fpt", "O"));
|
||||
train_sent.Add(new TrainingData(",", "Fc", "O"));
|
||||
train_sent.Add(new TrainingData("25", "Z", "O"));
|
||||
train_sent.Add(new TrainingData("may", "NC", "O"));
|
||||
train_sent.Add(new TrainingData("(", "Fpa", "O"));
|
||||
train_sent.Add(new TrainingData("EFE", "NC", "B-ORG"));
|
||||
train_sent.Add(new TrainingData(")", "Fpt", "O"));
|
||||
train_sent.Add(new TrainingData(".", "Fp", "O"));
|
||||
|
||||
List<List<TrainingData>> train_sents = new List<List<TrainingData>>();
|
||||
train_sents.Add(train_sent);
|
||||
|
||||
List<ItemSequence> X_train = new List<ItemSequence>();
|
||||
train_sents.ForEach(cur_sent => X_train.Add(new ItemSequence(sent2features(cur_sent))));
|
||||
|
||||
StringList sl = new StringList();
|
||||
|
||||
|
||||
List<StringList> y_train = new List<StringList>();
|
||||
train_sents.ForEach(cur_sent => y_train.Add(new StringList(sent2labels(cur_sent))));
|
||||
|
||||
Fit(X_train,y_train);
|
||||
|
||||
|
||||
var corpus = agent.Corpus;
|
||||
|
||||
List<List<String>> tags = data["Tags"].ToObject<List<List<String>>>();
|
||||
List<List<NlpToken>> tokens = data["Tokens"].ToObject<List<List<NlpToken>>>();
|
||||
List<TrainingIntentExpression<TrainingIntentExpressionPart>> userSays = corpus.UserSays;
|
||||
List<List<TrainingData>> list =new List<List<TrainingData>>();
|
||||
|
||||
FileStream fs = new FileStream("/home/bolo/Desktop/BotSharp/TrainingFiles/rawTrain.txt", FileMode.Create);
|
||||
StreamWriter sw = new StreamWriter(fs);
|
||||
|
||||
for (int i = 0 ; i < tags.Count; i++)
|
||||
{
|
||||
List<TrainingData> curLine = Merge(tokens[i], tags[i], userSays[i].Entities);
|
||||
list.Add(curLine);
|
||||
curLine.ForEach(trainingData =>{
|
||||
string[] wordParams = {trainingData.Entity, trainingData.Token, trainingData.Tag, trainingData.Chunk};
|
||||
string wordStr = string.Join(" ", wordParams);
|
||||
sw.Write(wordStr + "\n");
|
||||
});
|
||||
sw.Write("\n");
|
||||
}
|
||||
sw.Flush();
|
||||
sw.Close();
|
||||
fs.Close();
|
||||
|
||||
new MachineLearning.CRFsuite.Ner().NerStart();
|
||||
|
||||
Runcmd();
|
||||
|
||||
return true;
|
||||
}
|
||||
/*
|
||||
public List<TrainingData> Merge(List<Token> sentence, List<Entity> entities )
|
||||
|
||||
public void Runcmd ()
|
||||
{
|
||||
string cmd = "/home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite learn -m /home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite/bolo.model /home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite/1.txt";
|
||||
System.Diagnostics.Process p = new System.Diagnostics.Process();
|
||||
p.StartInfo.FileName = "sh";
|
||||
p.StartInfo.UseShellExecute = false;
|
||||
p.StartInfo.RedirectStandardInput = true;
|
||||
p.StartInfo.RedirectStandardOutput = true;
|
||||
p.StartInfo.RedirectStandardError = true;
|
||||
p.StartInfo.CreateNoWindow = false;
|
||||
p.Start();
|
||||
|
||||
p.StandardInput.WriteLine(cmd + "&exit");
|
||||
p.StandardInput.AutoFlush = false;
|
||||
|
||||
string output = p.StandardOutput.ReadToEnd();
|
||||
|
||||
p.WaitForExit();//等待程序执行完退出进程
|
||||
p.Close();
|
||||
Console.WriteLine(output);
|
||||
}
|
||||
public List<TrainingData> Merge(List<NlpToken> sentence, List<string> tags, List<TrainingIntentExpressionPart> entities)
|
||||
{
|
||||
List<TrainingData> trainingTuple = new List<TrainingData>();
|
||||
|
||||
HashSet<String> entityWordBag = new HashSet<String>();
|
||||
|
||||
entities.ForEach(entity =>
|
||||
int wordCandidateCount = 0;
|
||||
|
||||
for (int i = 0; i < sentence.Count; i++)
|
||||
{
|
||||
String[] words = entity.Value.Split();
|
||||
foreach (string word in words)
|
||||
TrainingIntentExpressionPart curEntity = null;
|
||||
if (entities != null)
|
||||
{
|
||||
entityWordBag.Add(word);
|
||||
bool entityFinded = false;
|
||||
entities.ForEach(entity => {
|
||||
if (!entityFinded)
|
||||
{
|
||||
string[] words = entity.Value.Split(" ");
|
||||
for (int j = 0; j < words.Length; j++)
|
||||
{
|
||||
if (sentence[i + j].Text == words[j])
|
||||
{
|
||||
wordCandidateCount++;
|
||||
if (j == words.Length - 1)
|
||||
{
|
||||
curEntity = entity;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
wordCandidateCount = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (wordCandidateCount != 0)
|
||||
{
|
||||
String entityName = curEntity.Entity.Contains(":")? curEntity.Entity.Substring(curEntity.Entity.IndexOf(":") + 1): curEntity.Entity;
|
||||
foreach(string s in words)
|
||||
{
|
||||
trainingTuple.Add(new TrainingData(entityName, s, tags[i], "I"));
|
||||
}
|
||||
entityFinded = true;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
sentence.ForEach(token => {
|
||||
if (!entityWordBag.Contains(token.Text))
|
||||
if (wordCandidateCount == 0)
|
||||
{
|
||||
trainingTuple.Add(new TrainingData(token.Text, "O", token.Offset));
|
||||
trainingTuple.Add(new TrainingData("O", sentence[i].Text, tags[i], "O"));
|
||||
}
|
||||
});
|
||||
|
||||
entities.ForEach(entity => trainingTuple.Add(new TrainingData(entity.Value, entity.EntityName, entity.Start)));
|
||||
|
||||
trainingTuple.Sort((left, right) => {
|
||||
if (left.Start > right.Start)
|
||||
return 1;
|
||||
else if (left.Start < right.Start)
|
||||
return -1;
|
||||
else
|
||||
return 0;
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
{
|
||||
i = i + wordCandidateCount - 1;
|
||||
}
|
||||
}
|
||||
return trainingTuple;
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/* Train a model.
|
||||
|
||||
* Parameters
|
||||
* ----------
|
||||
* X : list of lists of dicts
|
||||
Feature dicts for several documents (in a python-crfsuite format).
|
||||
|
||||
* y : list of lists of strings
|
||||
Labels for several documents.
|
||||
|
||||
* X_dev : (optional) list of lists of dicts
|
||||
Feature dicts used for testing.
|
||||
|
||||
* y_dev : (optional) list of lists of strings
|
||||
Labels corresponding to X_dev.
|
||||
*/
|
||||
public void Fit(List<ItemSequence> X, List<StringList> y, List<ItemSequence> X_dev = null, List<StringList> y_dev = null) {
|
||||
Trainer trainer = new Trainer();
|
||||
for (int i = 0; i < Math.Min(X.Count, y.Count); i++)
|
||||
{
|
||||
// group ?
|
||||
trainer.append(X[i], y[i], 0);
|
||||
}
|
||||
|
||||
trainer.train("model_test", X_dev == null ? -1 : 1);
|
||||
}
|
||||
|
||||
public Item Word2Features(List<TrainingData> sent, int i) {
|
||||
string word = sent[i].Token;
|
||||
string postag = sent[i].Tag;
|
||||
|
||||
float bias = 1.0F;
|
||||
String wordLower = word.ToLower();
|
||||
String wordLast3Char = wordLower.Length >= 3 ? wordLower.Substring(wordLower.Length - 3) : wordLower;
|
||||
string patternAllCaptain = @"^[A-Z]+$";
|
||||
Boolean isSupper = new Regex(patternAllCaptain).IsMatch(word);
|
||||
string patternFirstCaptain = @"^[A-Z]{1}[a-z]+$";
|
||||
Boolean isTitle = new Regex(patternFirstCaptain).IsMatch(word);
|
||||
string patternAllDigit = @"^[0-9]+$";
|
||||
Boolean isDigit = new Regex(patternAllDigit).IsMatch(word);
|
||||
String posTag = postag;
|
||||
String postagFirst2Char = postag.Length >= 2 ? postag.Substring(0,2) : postag.Substring(0);
|
||||
|
||||
Feature feature = new Feature(bias, wordLower, wordLast3Char, isSupper, isTitle, isDigit, posTag, postagFirst2Char);
|
||||
|
||||
Item curItem = new Item();
|
||||
|
||||
|
||||
|
||||
if (i > 0)
|
||||
{
|
||||
string minusWord = sent[i - 1].Token;
|
||||
string minusPostag = sent[i - 1].Tag;
|
||||
|
||||
feature.MinusWordLower = minusWord;
|
||||
feature.MinusIsTitle = new Regex(patternFirstCaptain).IsMatch(minusWord);
|
||||
feature.MinusIsSupper = new Regex(patternAllCaptain).IsMatch(minusWord);
|
||||
feature.MinusPostag = minusPostag;
|
||||
feature.MinusPostagFirst2Char = minusPostag.Length >= 2 ? minusPostag.Substring(0, 2) : minusPostag.Substring(0);
|
||||
|
||||
feature.Items.Add(new Attribute($"minusWordLower:{minusWord}", 1.0));
|
||||
feature.Items.Add(new Attribute($"minusIsTitle", feature.MinusIsTitle ? 1.0 : 0.0));
|
||||
feature.Items.Add(new Attribute($"minusIsSupper", feature.MinusIsSupper ? 1.0 : 0.0));
|
||||
feature.Items.Add(new Attribute($"minusPostag:{minusPostag}", 1.0));
|
||||
feature.Items.Add(new Attribute($"minusPostagFirst2Char:{feature.MinusPostagFirst2Char}", 1.0));
|
||||
}
|
||||
else {
|
||||
feature.BOS = true;
|
||||
feature.Items.Add(new Attribute($"BOS", feature.BOS? 1.0 : 0.0));
|
||||
}
|
||||
|
||||
if ( i < sent.Count - 1)
|
||||
{
|
||||
string plusWord = sent[i + 1].Token;
|
||||
string plusPostag = sent[i + 1].Tag;
|
||||
|
||||
feature.PlusWordLower = plusWord;
|
||||
feature.PlusIsTitle = new Regex(patternFirstCaptain).IsMatch(plusWord);
|
||||
feature.PlusIsSupper = new Regex(patternAllCaptain).IsMatch(plusWord);
|
||||
feature.PlusPostag = plusPostag;
|
||||
feature.PlusPostagFirst2Char = plusPostag.Length >= 2 ? plusPostag.Substring(0, 2) : plusPostag.Substring(0);
|
||||
|
||||
feature.Items.Add(new Attribute($"minusWordLower:{plusWord}", 1.0));
|
||||
feature.Items.Add(new Attribute($"minusIsTitle", feature.PlusIsTitle ? 1.0 : 0.0));
|
||||
feature.Items.Add(new Attribute($"minusIsSupper", feature.PlusIsSupper ? 1.0 : 0.0));
|
||||
feature.Items.Add(new Attribute($"minusPostag:{plusPostag}", 1.0));
|
||||
feature.Items.Add(new Attribute($"minusPostagFirst2Char:{feature.PlusPostagFirst2Char}", 1.0));
|
||||
}
|
||||
return feature.ToItems();
|
||||
}
|
||||
|
||||
public List<Item> sent2features(List<TrainingData> sent)
|
||||
{
|
||||
List<Item> list = new List<Item>();
|
||||
for (int i = 0 ; i < sent.Count; i++ )
|
||||
{
|
||||
list.Add(Word2Features(sent, i));
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public List<String> sent2labels(List<TrainingData> sent)
|
||||
{
|
||||
List<String> list = new List<String>();
|
||||
sent.ForEach(tuple => list.Add(tuple.Entity));
|
||||
return list;
|
||||
}
|
||||
|
||||
public List<String> sent2tokens(List<TrainingData> sent)
|
||||
{
|
||||
List<String> list = new List<String>();
|
||||
sent.ForEach(tuple => list.Add(tuple.Token));
|
||||
return list;
|
||||
}
|
||||
}
|
||||
public class Feature
|
||||
{
|
||||
public float Bias { get; set; }
|
||||
public String WordLower { get; set; }
|
||||
public String WordLast3Char { get; set; }
|
||||
public Boolean IsSupper { get; set; }
|
||||
public Boolean IsTitle { get; set; }
|
||||
public Boolean IsDigit { get; set; }
|
||||
public String Postag { get; set; }
|
||||
public String PostagFirst2Char { get; set; }
|
||||
|
||||
public Boolean BOS { get; set; }
|
||||
public Boolean EOS { get; set; }
|
||||
|
||||
public String PlusWordLower { get; set; }
|
||||
public String PlusLast3Char { get; set; }
|
||||
public Boolean PlusIsSupper { get; set; }
|
||||
public Boolean PlusIsTitle { get; set; }
|
||||
public Boolean PlusIsDigit { get; set; }
|
||||
public String PlusPostag { get; set; }
|
||||
public String PlusPostagFirst2Char { get; set; }
|
||||
|
||||
public String MinusWordLower { get; set; }
|
||||
public String MinusLast3Char { get; set; }
|
||||
public Boolean MinusIsSupper { get; set; }
|
||||
public Boolean MinusIsTitle { get; set; }
|
||||
public Boolean MinusIsDigit { get; set; }
|
||||
public String MinusPostag { get; set; }
|
||||
public String MinusPostagFirst2Char { get; set; }
|
||||
|
||||
public Item Items { get; set; }
|
||||
|
||||
public Feature(float bias, String wordLower, String wordLast3Char, Boolean isSupper, Boolean isTitle, Boolean isDigit, String posTag, String postagFirst2Char)
|
||||
{
|
||||
this.Bias = bias;
|
||||
this.WordLower = wordLower;
|
||||
this.WordLast3Char = wordLast3Char;
|
||||
this.IsSupper = isSupper;
|
||||
this.IsTitle = isTitle;
|
||||
this.IsDigit = isDigit;
|
||||
this.Postag = posTag;
|
||||
this.PostagFirst2Char = postagFirst2Char;
|
||||
this.Items = new Item();
|
||||
|
||||
Items.Add(new Attribute($"bias", bias));
|
||||
Items.Add(new Attribute($"wordLower:{wordLower}", 1.0));
|
||||
Items.Add(new Attribute($"wordLast3Char:{wordLast3Char}", 1.0));
|
||||
Items.Add(new Attribute($"isSupper", isSupper? 1.0 : 0.0));
|
||||
Items.Add(new Attribute($"isTitle", isTitle ? 1.0 : 0.0));
|
||||
Items.Add(new Attribute($"isDigit", isDigit ? 1.0 : 0.0));
|
||||
Items.Add(new Attribute($"posTag={posTag}", 1.0));
|
||||
Items.Add(new Attribute($"postagFirst2Char:{postagFirst2Char}", 1.0));
|
||||
}
|
||||
|
||||
public Item ToItems()
|
||||
{
|
||||
return this.Items;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -277,12 +138,14 @@ namespace BotSharp.Core.Engines.CRFsuite
|
|||
public String Token { get; set; }
|
||||
public String Entity { get; set; }
|
||||
public String Tag { get; set; }
|
||||
public String Chunk { get; set; }
|
||||
|
||||
public TrainingData(string token, string entity, string tag)
|
||||
public TrainingData(string entity, string token, string tag, string chunk)
|
||||
{
|
||||
this.Token = token;
|
||||
this.Entity = entity;
|
||||
this.Tag = tag;
|
||||
this.Chunk = chunk;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
using BotSharp.Core.Abstractions;
|
||||
using BotSharp.Core.Agents;
|
||||
using EntityFrameworkCore.BootKit;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using RestSharp;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using BotSharp.MachineLearning.NLP;
|
||||
|
||||
namespace BotSharp.Core.Engines.SpaCy
|
||||
{
|
||||
|
|
@ -12,9 +15,30 @@ namespace BotSharp.Core.Engines.SpaCy
|
|||
{
|
||||
public IConfiguration Configuration { get; set; }
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
|
||||
public bool Process(Agent agent, JObject data)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
var client = new RestClient(Configuration.GetSection("SpaCyProvider:Url").Value);
|
||||
var request = new RestRequest("tagger", Method.GET);
|
||||
List<List<String>> tags = new List<List<String>>();
|
||||
Boolean res = true;
|
||||
var dc = new DefaultDataContextLoader().GetDefaultDc();
|
||||
var corpus = agent.Corpus;
|
||||
|
||||
corpus.UserSays.ForEach(usersay => {
|
||||
request.AddParameter("text", usersay.Text);
|
||||
var response = client.Execute<Result>(request);
|
||||
tags.Add(response.Data.Tags);
|
||||
res = res && response.IsSuccessful;
|
||||
});
|
||||
data.Add("Tags", JToken.FromObject(tags));
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
public class Result
|
||||
{
|
||||
public List<String> Tags { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,14 +24,14 @@ namespace BotSharp.Core.Engines.SpaCy
|
|||
List<List<NlpToken>> tokens = new List<List<NlpToken>>();
|
||||
Boolean res = true;
|
||||
var dc = new DefaultDataContextLoader().GetDefaultDc();
|
||||
/*var corpus = ;
|
||||
var corpus = agent.Corpus;
|
||||
|
||||
corpus.UserSays.ForEach(usersay => {
|
||||
request.AddParameter("text", usersay.Text);
|
||||
var response = client.Execute<Result>(request);
|
||||
tokens.Add(response.Data.Tokens);
|
||||
res = res && response.IsSuccessful;
|
||||
});*/
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
156
BotSharp.MachineLearning/CRFsuite/Crfutils.cs
Normal file
156
BotSharp.MachineLearning/CRFsuite/Crfutils.cs
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BotSharp.MachineLearning.CRFsuite
|
||||
{
|
||||
public class Crfutils
|
||||
{
|
||||
/// <summary>
|
||||
/// Generate features for an item sequence by applying feature templates.
|
||||
/// A feature template consists of a tuple of (name, offset) pairs,
|
||||
/// where name and offset specify a field name and offset from which
|
||||
/// the template extracts a feature valreaditerue. Generated features are stored
|
||||
/// in the 'F' field of each item in the sequence.
|
||||
/// </summary>
|
||||
/// <param name="X">Token features for a sentence</param>
|
||||
/// <param name="template">the template which contains what feature to extract</param>
|
||||
|
||||
public void ApplyTemplates (List<Dictionary<string, Object>> X, Template templates)
|
||||
{
|
||||
foreach (List<CRFFeature> template in templates.Features)
|
||||
{
|
||||
List<string> list = new List<string>();
|
||||
template.ForEach(t => list.Add($"{t.Field}[{t.Offset}]"));
|
||||
string name = string.Join("|", list);
|
||||
|
||||
for (int t = 0 ; t < X.Count() ; t++) {
|
||||
List<string> values = new List<string>();
|
||||
foreach (CRFFeature crffeature in template)
|
||||
{
|
||||
string field = crffeature.Field;
|
||||
int offset = crffeature.Offset;
|
||||
int p = t + offset;
|
||||
if (p < 0 || p >= X.Count)
|
||||
{
|
||||
values.Clear();
|
||||
break;
|
||||
}
|
||||
values.Add(X[p][field].ToString());
|
||||
}
|
||||
if (values != null && values.Count > 0)
|
||||
{
|
||||
string value = string.Join("|", values);
|
||||
((List<string>)X[t]["F"]).Add($"{name}={value}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// Return an iterator for item sequences read from a file object.
|
||||
/// This function reads a sequence from a file object L{fi}, and
|
||||
/// yields the sequence as a list of mapping objects. Each line
|
||||
/// (item) from the file object is split by the separator character
|
||||
/// L{sep}. Separated values of the item are named by L{names},
|
||||
/// and stored in a mapping object. Every item has a field 'F' that
|
||||
/// is reserved for storing features.
|
||||
/// </summary>
|
||||
/// <param name="fiPath">source file which contains crf style training data</param>
|
||||
/// <param name="names">each attribute name in fields</param>
|
||||
/// <param name="sep">seperate by</param>
|
||||
|
||||
public List<List<Dictionary<string, Object>>> Readiter (string fiPath, List<string> names, string sep = " ")
|
||||
{
|
||||
List<List<Dictionary<string, Object>>> Xs = new List<List<Dictionary<string, Object>>>();
|
||||
List<Dictionary<string, Object>> X = new List<Dictionary<string, Object>>();
|
||||
StreamReader sr = new StreamReader(fiPath, Encoding.Default);
|
||||
string line;
|
||||
while ((line = sr.ReadLine()) != null)
|
||||
{
|
||||
line = line.Replace("\n","");
|
||||
if (line == null || line.Length == 0)
|
||||
{
|
||||
Xs.Add(new List<Dictionary<string, Object>>(X));
|
||||
X.Clear();
|
||||
}
|
||||
else
|
||||
{
|
||||
String[] fields = line.Split(sep);
|
||||
if (fields.Count() < names.Count)
|
||||
{
|
||||
// Error Exception
|
||||
}
|
||||
Dictionary<string, Object> item = new Dictionary<string, Object>();
|
||||
item.Add("F", new List<string>());
|
||||
for (int i = 0 ; i < names.Count ; i++)
|
||||
{
|
||||
item.Add(names[i], fields[i]);
|
||||
}
|
||||
X.Add(item);
|
||||
}
|
||||
}
|
||||
return Xs;
|
||||
}
|
||||
/// <summary>
|
||||
/// Escape colon characters from feature names.
|
||||
/// </summary>
|
||||
/// <param name="src">a feature name</param>
|
||||
|
||||
public string Escape(string src)
|
||||
{
|
||||
return src.Replace(":", "__COLON__");
|
||||
}
|
||||
/// <summary>
|
||||
/// Output features (and reference labels) of a sequence in CRFSuite
|
||||
/// format. For each item in the sequence, this function writes a
|
||||
/// reference label (if L{field} is a non-empty string) and features.
|
||||
/// </summary>
|
||||
/// <param name="sw">destination file stream writer</param>
|
||||
/// <param name="X">Token features for a sentence</param>
|
||||
/// <param name="field">one attribute name in fields</param>
|
||||
|
||||
public void OutputFeatures (StreamWriter sw, List<Dictionary<string, Object>> X, string field = "")
|
||||
{
|
||||
for (int t = 0; t < X.Count; t++)
|
||||
{
|
||||
if (field.Length != 0)
|
||||
{
|
||||
sw.Write(X[t][field]);
|
||||
}
|
||||
foreach (string a in (List<string>)X[t]["F"])
|
||||
{
|
||||
sw.Write($"\t{Escape(a)}");
|
||||
}
|
||||
sw.Write("\n");
|
||||
}
|
||||
sw.Write("\n");
|
||||
}
|
||||
/// <summary>
|
||||
/// CRFFileGenerator
|
||||
/// </summary>
|
||||
/// <param name="FeatureExtractor">an extractor which to do the feature extracting work</param>
|
||||
/// <param name="fields">attributes name seperated by space</param>
|
||||
/// <param name="sep">string whihch seperated by</param>
|
||||
public void CRFFileGenerator (System.Action<List<Dictionary<string, Object>>> FeatureExtractor, string fields, string sep= " ")
|
||||
{
|
||||
String fiPath = "/home/bolo/Desktop/BotSharp/TrainingFiles/rawTrain.txt";
|
||||
FileStream fs = new FileStream("/home/bolo/Desktop/BotSharp/TrainingFiles/1.txt", FileMode.Create);
|
||||
StreamWriter sw = new StreamWriter(fs);
|
||||
List<string> F = fields.Split(" ").ToList();
|
||||
List<List<Dictionary<string, Object>>> Xs = Readiter(fiPath, F, " ");
|
||||
|
||||
foreach (List<Dictionary<string, Object>> X in Xs)
|
||||
{
|
||||
FeatureExtractor(X);
|
||||
OutputFeatures(sw, X, "y");
|
||||
}
|
||||
sw.Flush();
|
||||
sw.Close();
|
||||
fs.Close();
|
||||
}
|
||||
}
|
||||
}
|
||||
589
BotSharp.MachineLearning/CRFsuite/Ner.cs
Normal file
589
BotSharp.MachineLearning/CRFsuite/Ner.cs
Normal file
|
|
@ -0,0 +1,589 @@
|
|||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BotSharp.MachineLearning.CRFsuite
|
||||
{
|
||||
public class Ner
|
||||
{
|
||||
// Separator of field values.
|
||||
string separator = " ";
|
||||
// Field names of the input data.
|
||||
string fields = "y w pos chk";
|
||||
Template templates = new Template();
|
||||
|
||||
public string GetShape (string token)
|
||||
{
|
||||
string r = "";
|
||||
foreach (char c in token.ToCharArray())
|
||||
{
|
||||
if (IsSupperChar(c))
|
||||
{
|
||||
r += "U";
|
||||
}
|
||||
else if (IsLowerChar(c))
|
||||
{
|
||||
r += "L";
|
||||
}
|
||||
else if (IsDigitChar(c))
|
||||
{
|
||||
r += "D";
|
||||
}
|
||||
else if (((IList)new char[]{'.', ','}).Contains(c))
|
||||
{
|
||||
r += ".";
|
||||
}
|
||||
else if (((IList)new char[]{';', ':', '?', '!'}).Contains(c))
|
||||
{
|
||||
r += ";";
|
||||
}
|
||||
else if (((IList)new char[]{'+', '-', '*', '/', '=', '|', '_'}).Contains(c))
|
||||
{
|
||||
r += "-";
|
||||
}
|
||||
else if (((IList)new char[]{'(', '{', '[', '<'}).Contains(c))
|
||||
{
|
||||
r += "(";
|
||||
}
|
||||
else if (((IList)new char[]{')', '}', ']', '>'}).Contains(c))
|
||||
{
|
||||
r += ")";
|
||||
}
|
||||
else
|
||||
{
|
||||
r += c;
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
public string Degenerate (string src)
|
||||
{
|
||||
string dst = "";
|
||||
foreach (char c in src)
|
||||
{
|
||||
if (dst.Trim() == "" || char.Parse(dst.Substring(dst.Length - 1, 1)) != c)
|
||||
{
|
||||
dst += c;
|
||||
}
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
|
||||
public string GetType (string token)
|
||||
{
|
||||
List<string> T = new List<String>{"AllUpper", "AllDigit", "AllSymbol","AllUpperDigit", "AllUpperSymbol", "AllDigitSymbol",
|
||||
"AllUpperDigitSymbol","InitUpper","AllLetter","AllAlnum"};
|
||||
HashSet<string> R = new HashSet<string>(T);
|
||||
if (token == null || token.Trim() == "")
|
||||
{
|
||||
return "EMPTY";
|
||||
}
|
||||
for (int i = 0 ; i < token.Length ; i++)
|
||||
{
|
||||
char c = token[i];
|
||||
if (IsSupperChar(c))
|
||||
{
|
||||
if (R.Contains("AllDigit"))
|
||||
{
|
||||
R.Remove("AllDigit");
|
||||
}
|
||||
if (R.Contains("AllSymbol"))
|
||||
{
|
||||
R.Remove("AllSymbol");
|
||||
}
|
||||
if (R.Contains("AllDigitSymbol"))
|
||||
{
|
||||
R.Remove("AllDigitSymbol");
|
||||
}
|
||||
}
|
||||
else if (IsDigitChar(c) || ((IList)new char[]{'.', ','}).Contains(c))
|
||||
{
|
||||
if (R.Contains("AllUpper"))
|
||||
{
|
||||
R.Remove("AllUpper");
|
||||
}
|
||||
if (R.Contains("AllSymbol"))
|
||||
{
|
||||
R.Remove("AllSymbol");
|
||||
}
|
||||
if (R.Contains("AllUpperSymbol"))
|
||||
{
|
||||
R.Remove("AllUpperSymbol");
|
||||
}
|
||||
if (R.Contains("AllLetter"))
|
||||
{
|
||||
R.Remove("AllLetter");
|
||||
}
|
||||
}
|
||||
else if (IsLowerChar(c))
|
||||
{
|
||||
if (R.Contains("AllUpper"))
|
||||
{
|
||||
R.Remove("AllUpper");
|
||||
}
|
||||
if (R.Contains("AllDigit"))
|
||||
{
|
||||
R.Remove("AllDigit");
|
||||
}
|
||||
if (R.Contains("AllSymbol"))
|
||||
{
|
||||
R.Remove("AllSymbol");
|
||||
}
|
||||
if (R.Contains("AllUpperDigit"))
|
||||
{
|
||||
R.Remove("AllUpperDigit");
|
||||
}
|
||||
if (R.Contains("AllUpperSymbol"))
|
||||
{
|
||||
R.Remove("AllUpperSymbol");
|
||||
}
|
||||
if (R.Contains("AllDigitSymbol"))
|
||||
{
|
||||
R.Remove("AllDigitSymbol");
|
||||
}
|
||||
if (R.Contains("AllUpperDigitSymbol"))
|
||||
{
|
||||
R.Remove("AllUpperDigitSymbol");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (R.Contains("AllUpper"))
|
||||
{
|
||||
R.Remove("AllUpper");
|
||||
}
|
||||
if (R.Contains("AllDigit"))
|
||||
{
|
||||
R.Remove("AllDigit");
|
||||
}
|
||||
if (R.Contains("AllUpperDigit"))
|
||||
{
|
||||
R.Remove("AllUpperDigit");
|
||||
}
|
||||
if (R.Contains("AllLetter"))
|
||||
{
|
||||
R.Remove("AllLetter");
|
||||
}
|
||||
if (R.Contains("AllAlnum"))
|
||||
{
|
||||
R.Remove("AllAlnum");
|
||||
}
|
||||
}
|
||||
|
||||
if (i == 0 && !IsSupperChar(c))
|
||||
{
|
||||
if (R.Contains("InitUpper"))
|
||||
{
|
||||
R.Remove("InitUpper");
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach (string tag in T)
|
||||
{
|
||||
if (R.Contains(tag))
|
||||
{
|
||||
return tag;
|
||||
}
|
||||
}
|
||||
return "NO";
|
||||
}
|
||||
|
||||
public Boolean Get2d (string token)
|
||||
{
|
||||
return token.Length == 2 && IsDigit(token);
|
||||
}
|
||||
|
||||
public Boolean Get4d (string token)
|
||||
{
|
||||
return token.Length == 4 && IsDigit(token);
|
||||
}
|
||||
|
||||
// is token digit, alpha or not
|
||||
public Boolean GetDa (string token)
|
||||
{
|
||||
Boolean bd = false;
|
||||
Boolean ba = false;
|
||||
foreach (char c in token)
|
||||
{
|
||||
if (IsDigitChar(c))
|
||||
{
|
||||
bd = true;
|
||||
}
|
||||
else if (IsAlphaChar(c))
|
||||
{
|
||||
ba = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return bd && ba;
|
||||
}
|
||||
|
||||
public Boolean GetDand (string token, char p)
|
||||
{
|
||||
Boolean bd = false;
|
||||
Boolean bdd = false;
|
||||
foreach (char c in token)
|
||||
{
|
||||
if (IsDigitChar(c))
|
||||
{
|
||||
bd = true;
|
||||
}
|
||||
else if (c == p)
|
||||
{
|
||||
bdd = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return bd && bdd;
|
||||
}
|
||||
|
||||
public Boolean GetAllOther (string token)
|
||||
{
|
||||
foreach (char c in token)
|
||||
{
|
||||
if (IsDigitChar(c) || IsAlphaChar(c))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public Boolean GetCapPeriod (string token)
|
||||
{
|
||||
return token.Length == 2 && IsSupperChar(token[0]) && token[1] == '.';
|
||||
}
|
||||
|
||||
public Boolean ContainsUpper (string token)
|
||||
{
|
||||
foreach (char c in token)
|
||||
{
|
||||
if (IsSupperChar(c))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public Boolean ContainsLower (string token)
|
||||
{
|
||||
foreach (char c in token)
|
||||
{
|
||||
if (IsLowerChar(c))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public Boolean ContainsAlpha (string token)
|
||||
{
|
||||
foreach (char c in token)
|
||||
{
|
||||
if (IsAlphaChar(c))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public Boolean ContainsDigit (string token)
|
||||
{
|
||||
foreach (char c in token)
|
||||
{
|
||||
if (IsDigitChar(c))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public Boolean ContainsSymbol (string token)
|
||||
{
|
||||
foreach (char c in token)
|
||||
{
|
||||
if (!IsAlnumChar(c))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public string B (Boolean v)
|
||||
{
|
||||
return v ? "yes" : "no";
|
||||
}
|
||||
|
||||
public void Observation (Dictionary<string, Object> v, string defval = "")
|
||||
{
|
||||
// Lowercased token.
|
||||
v.Add("wl", v["w"].ToString().ToLower());
|
||||
// Token shape.
|
||||
v.Add("shape", GetShape(v["w"].ToString()));
|
||||
// Token shape degenerated.
|
||||
v.Add("shaped", Degenerate(v["shape"].ToString()));
|
||||
// Token type.
|
||||
v.Add("type", GetType(v["w"].ToString()));
|
||||
// Prefixes (length between one to four).
|
||||
if (v["w"].ToString().Length >= 1)
|
||||
{
|
||||
v.Add("p1", v["w"].ToString().Substring(0, 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
v.Add("p1", defval);
|
||||
}
|
||||
|
||||
if (v["w"].ToString().Length >= 2)
|
||||
{
|
||||
v.Add("p2", v["w"].ToString().Substring(0, 2));
|
||||
}
|
||||
else
|
||||
{
|
||||
v.Add("p2", defval);
|
||||
}
|
||||
|
||||
if (v["w"].ToString().Length >= 3)
|
||||
{
|
||||
v.Add("p3", v["w"].ToString().Substring(0, 3));
|
||||
}
|
||||
else
|
||||
{
|
||||
v.Add("p3", defval);
|
||||
}
|
||||
|
||||
if (v["w"].ToString().Length >= 4)
|
||||
{
|
||||
v.Add("p4", v["w"].ToString().Substring(0, 4));
|
||||
}
|
||||
else
|
||||
{
|
||||
v.Add("p4", defval);
|
||||
}
|
||||
|
||||
// Suffixes (length between one to four).
|
||||
string word = v["w"].ToString();
|
||||
if (v["w"].ToString().Length >= 1)
|
||||
{
|
||||
v.Add("s1", word.Substring(word.Length - 1, 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
v.Add("s1", defval);
|
||||
}
|
||||
|
||||
if (v["w"].ToString().Length >= 2)
|
||||
{
|
||||
v.Add("s2", word.Substring(word.Length - 2, 2));
|
||||
}
|
||||
else
|
||||
{
|
||||
v.Add("s2", defval);
|
||||
}
|
||||
|
||||
if (v["w"].ToString().Length >= 3)
|
||||
{
|
||||
v.Add("s3", word.Substring(word.Length - 3, 3));
|
||||
}
|
||||
else
|
||||
{
|
||||
v.Add("s3", defval);
|
||||
}
|
||||
|
||||
if (v["w"].ToString().Length >= 4)
|
||||
{
|
||||
v.Add("s4", word.Substring(word.Length - 4, 4));
|
||||
}
|
||||
else
|
||||
{
|
||||
v.Add("s4", defval);
|
||||
}
|
||||
|
||||
// Two digits
|
||||
v.Add("2d", B(Get2d(v["w"].ToString())));
|
||||
// Four digits
|
||||
v.Add("4d", B(Get4d(v["w"].ToString())));
|
||||
// Alphanumeric token.
|
||||
v.Add("d&a", B(GetDa(v["w"].ToString())));
|
||||
// Digits and '-'.
|
||||
v.Add("d&-", B(GetDand(v["w"].ToString(), '-')));
|
||||
// Digits and '/'.
|
||||
v.Add("d&/", B(GetDand(v["w"].ToString(), '/')));
|
||||
// Digits and ','.
|
||||
v.Add("d&,", B(GetDand(v["w"].ToString(), ',')));
|
||||
// Digits and '.'.
|
||||
v.Add("d&.", B(GetDand(v["w"].ToString(), '.')));
|
||||
// A uppercase letter followed by '.'
|
||||
v.Add("up", B(GetCapPeriod(v["w"].ToString())));
|
||||
// An initial uppercase letter.
|
||||
v.Add("iu", B(IsSupperChar(v["w"].ToString()[0])));
|
||||
// All uppercase letters.
|
||||
v.Add("au", B(IsSupper(v["w"].ToString())));
|
||||
// All lowercase letters.
|
||||
v.Add("al", B(IsLower(v["w"].ToString())));
|
||||
// All digit letters.
|
||||
v.Add("ad", B(IsDigit(v["w"].ToString())));
|
||||
// All other (non-alphanumeric) letters.
|
||||
v.Add("ao",B(GetAllOther(v["w"].ToString())));
|
||||
|
||||
// Contains a uppercase letter.
|
||||
v.Add("cu", B(ContainsUpper(v["w"].ToString())));
|
||||
// Contains a lowercase letter.
|
||||
v.Add("cl", B(ContainsLower(v["w"].ToString())));
|
||||
// Contains a alphabet letter.
|
||||
v.Add("ca", B(ContainsAlpha(v["w"].ToString())));
|
||||
// Contains a digit.
|
||||
v.Add("cd", B(ContainsUpper(v["w"].ToString())));
|
||||
// Contains a symbol.
|
||||
v.Add("cs", B(ContainsSymbol(v["w"].ToString())));
|
||||
}
|
||||
|
||||
public void DisJunctive(List<Dictionary<string, Object>> X, int t, string field, int begin, int end)
|
||||
{
|
||||
string name = $"{field}[{begin}..{end}]";
|
||||
for (int offset = begin; offset < end + 1; offset++)
|
||||
{
|
||||
int p = t + offset;
|
||||
if (p < 0 || p >= X.Count)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
List<string> F = (List<string>)X[t]["F"];
|
||||
F.Add($"{name}={X[p][field]}");
|
||||
}
|
||||
}
|
||||
|
||||
string[] Uique = new string[]{"w", "wl", "pos", "chk", "shape", "shaped", "type",
|
||||
"p1", "p2", "p3", "p4","s1", "s2", "s3", "s4",
|
||||
"2d", "4d", "d&a", "d&-", "d&/", "d&,", "d&.", "up",
|
||||
"iu", "au", "al", "ad", "ao", "cu", "cl", "ca", "cd", "cs"};
|
||||
string[] Bi = new string[]{"w", "pos", "chk", "shaped", "type"};
|
||||
|
||||
public void InitialTemplate ()
|
||||
{
|
||||
foreach (string name in Uique)
|
||||
{
|
||||
for (int i = -2 ; i < 3; i++)
|
||||
{
|
||||
List<CRFFeature> templateRowFeature = new List<CRFFeature>();
|
||||
templateRowFeature.Add(new CRFFeature(name, i));
|
||||
templates.Features.Add(templateRowFeature);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (string name in Bi)
|
||||
{
|
||||
for (int i = -2 ; i < 2; i++)
|
||||
{
|
||||
List<CRFFeature> templateRowFeature = new List<CRFFeature>();
|
||||
templateRowFeature.Add(new CRFFeature(name, i));
|
||||
templateRowFeature.Add(new CRFFeature(name, i + 1));
|
||||
templates.Features.Add(templateRowFeature);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void FeatureExtractor (List<Dictionary<string, Object>> X)
|
||||
{
|
||||
// Append observations.
|
||||
foreach (Dictionary<string, Object> d in X)
|
||||
{
|
||||
Observation(d);
|
||||
}
|
||||
// Apply the feature templates.
|
||||
new Crfutils().ApplyTemplates(X, templates);
|
||||
|
||||
// Append disjunctive features.
|
||||
for (int t = 0; t < X.Count ; t++)
|
||||
{
|
||||
DisJunctive(X, t, "w", -4, -1);
|
||||
DisJunctive(X, t, "w", 1, 4);
|
||||
}
|
||||
|
||||
if (X != null && X.Count > 0) {
|
||||
((List<string>)X[0]["F"]).Add("__BOS__");
|
||||
((List<string>)X[X.Count - 1]["F"]).Add("__EOS__");
|
||||
}
|
||||
}
|
||||
|
||||
public void NerStart ()
|
||||
{
|
||||
InitialTemplate();
|
||||
new Crfutils().CRFFileGenerator(FeatureExtractor, fields, separator);
|
||||
}
|
||||
|
||||
|
||||
private Boolean IsSupperChar(char c)
|
||||
{
|
||||
return ((int) c - 'A' >= 0) && ('Z' - (int) c >= 0) ;
|
||||
}
|
||||
private Boolean IsLowerChar(char c)
|
||||
{
|
||||
return ((int) c - 'a' >= 0)&&('z' - (int) c >= 0) ;
|
||||
}
|
||||
private Boolean IsDigitChar(char c)
|
||||
{
|
||||
return ((int) c - '0' >= 0)&&('9' - (int) c >= 0) ;
|
||||
}
|
||||
private Boolean IsAlphaChar (char c)
|
||||
{
|
||||
return IsLowerChar(c) || IsSupperChar(c);
|
||||
}
|
||||
private Boolean IsAlnumChar (char c)
|
||||
{
|
||||
return IsAlphaChar(c) || IsDigitChar(c);
|
||||
}
|
||||
private Boolean IsDigit (String s)
|
||||
{
|
||||
string patternAllDigit = @"^[0-9]+$";
|
||||
return new Regex(patternAllDigit).IsMatch(s);
|
||||
}
|
||||
private Boolean IsSupper (string s)
|
||||
{
|
||||
string patternAllCaptain = @"^[A-Z]+$";
|
||||
return new Regex(patternAllCaptain).IsMatch(s);
|
||||
}
|
||||
private Boolean IsLower (string s)
|
||||
{
|
||||
string patternAllCaptain = @"^[a-z]+$";
|
||||
return new Regex(patternAllCaptain).IsMatch(s);
|
||||
}
|
||||
}
|
||||
|
||||
public class Template
|
||||
{
|
||||
public List<List<CRFFeature>> Features{ get; set; }
|
||||
public Template ()
|
||||
{
|
||||
this.Features = new List<List<CRFFeature>>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class CRFFeature
|
||||
{
|
||||
public string Field { get; set; }
|
||||
public int Offset { get; set; }
|
||||
public CRFFeature (string field, int offset)
|
||||
{
|
||||
this.Field = field;
|
||||
this.Offset = offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,6 @@
|
|||
"SpaCyProvider": {
|
||||
"Url": "http://10.2.21.200:5005"
|
||||
},
|
||||
"Pipe": "SpaCyTokenizer, SpacyFeaturizer, CRFsuiteEntityRecognizer" //SpaCyEntitizer, SpaCyTextCategorizer, SpaCyEntityRecognizer
|
||||
"Pipe": "SpaCyTokenizer, SpaCyTagger, CRFsuiteEntityRecognizer"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ using Microsoft.Extensions.DependencyInjection;
|
|||
using Microsoft.Extensions.PlatformAbstractions;
|
||||
using Newtonsoft.Json.Serialization;
|
||||
using Swashbuckle.AspNetCore.Swagger;
|
||||
using BotSharp.Core.Engines.BotSharp;
|
||||
|
||||
namespace BotSharp.WebHost
|
||||
{
|
||||
|
|
@ -103,6 +104,52 @@ namespace BotSharp.WebHost
|
|||
Database.Configuration = Configuration;
|
||||
Database.ContentRootPath = env.ContentRootPath;
|
||||
Database.Assemblies = Configuration.GetValue<String>("Assemblies").Split(',');
|
||||
|
||||
Runcmd();
|
||||
|
||||
var ai = new BotSharpAi();
|
||||
ai.LoadAgent("6a9fd374-c43d-447a-97f2-f37540d0c725");
|
||||
ai.Train();
|
||||
}
|
||||
|
||||
public void Runcmd ()
|
||||
{
|
||||
string cmd = "/home/bolo/Desktop/BotSharp/TrainingFiles/crfsuite learn -m /home/bolo/Desktop/BotSharp/TrainingFiles/bolo.model /home/bolo/Desktop/BotSharp/TrainingFiles/1.txt";
|
||||
System.Diagnostics.Process p = new System.Diagnostics.Process();
|
||||
p.StartInfo.FileName = "sh";
|
||||
p.StartInfo.UseShellExecute = false; //是否使用操作系统shell启动
|
||||
p.StartInfo.RedirectStandardInput = true;//接受来自调用程序的输入信息
|
||||
p.StartInfo.RedirectStandardOutput = true;//由调用程序获取输出信息
|
||||
p.StartInfo.RedirectStandardError = true;//重定向标准错误输出
|
||||
p.StartInfo.CreateNoWindow = false;//不显示程序窗口
|
||||
p.Start();//启动程序
|
||||
|
||||
//向cmd窗口发送输入信息
|
||||
p.StandardInput.WriteLine(cmd + "&exit");
|
||||
|
||||
p.StandardInput.AutoFlush = false;
|
||||
//p.StandardInput.WriteLine("exit");
|
||||
//向标准输入写入要执行的命令。这里使用&是批处理命令的符号，表示前面一个命令不管是否执行成功都执行后面(exit)命令，如果不执行exit命令，后面调用ReadToEnd()方法会假死
|
||||
//同类的符号还有&&和||前者表示必须前一个命令执行成功才会执行后面的命令，后者表示必须前一个命令执行失败才会执行后面的命令
|
||||
|
||||
|
||||
|
||||
//获取cmd窗口的输出信息
|
||||
string output = p.StandardOutput.ReadToEnd();
|
||||
|
||||
//StreamReader reader = p.StandardOutput;
|
||||
//string line=reader.ReadLine();
|
||||
//while (!reader.EndOfStream)
|
||||
//{
|
||||
// str += line + " ";
|
||||
// line = reader.ReadLine();
|
||||
//}
|
||||
|
||||
p.WaitForExit();//等待程序执行完退出进程
|
||||
p.Close();
|
||||
|
||||
|
||||
Console.WriteLine(output);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
2226
TrainingFiles/1.txt
Normal file
2226
TrainingFiles/1.txt
Normal file
File diff suppressed because it is too large
Load diff
BIN
TrainingFiles/bolo.model
Normal file
BIN
TrainingFiles/bolo.model
Normal file
Binary file not shown.
BIN
TrainingFiles/crfsuite
Executable file
BIN
TrainingFiles/crfsuite
Executable file
Binary file not shown.
2226
TrainingFiles/rawTrain.txt
Normal file
2226
TrainingFiles/rawTrain.txt
Normal file
File diff suppressed because it is too large
Load diff
2226
TrainingFiles/song_train_file.txt
Normal file
2226
TrainingFiles/song_train_file.txt
Normal file
File diff suppressed because it is too large
Load diff
2226
TrainingFiles/train.crfsuite.txt
Normal file
2226
TrainingFiles/train.crfsuite.txt
Normal file
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue