diff --git a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs index d5bbae28..e288489a 100644 --- a/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs +++ b/BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs @@ -67,7 +67,6 @@ namespace BotSharp.Core.Engines.CRFsuite var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms"); CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x - Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); meta.Meta["fields"] = fields; @@ -195,12 +194,15 @@ namespace BotSharp.Core.Engines.CRFsuite entities.Add(new NlpEntity { Entity = entity, + Start = doc.Sentences[0].Tokens[i].Offset, Value = doc.Sentences[0].Tokens[i].Text, Confidence = probability }); } - doc.Sentences[0].Entities = entities.Where(x => x.Entity != "O").ToList(); + List unionedEntities = MergeEntity(entities); + + doc.Sentences[0].Entities = unionedEntities.Where(x => x.Entity != "O").ToList(); if(File.Exists(rawPredictingDataFileName)) { @@ -213,6 +215,32 @@ namespace BotSharp.Core.Engines.CRFsuite return true; } + + public List MergeEntity (List tokens) + { + List res = new List(); + for (int i = 0; i < tokens.Count ; i++) + { + NlpEntity nlpEntity = new NlpEntity(); + StringBuilder unionValue = new StringBuilder(tokens[i].Value); + StringBuilder unionEntity = new StringBuilder(tokens[i].Entity); + decimal unoinConfidence = tokens[i].Confidence; + + int j = i + 1; + while (j < tokens.Count && tokens[j].Entity == tokens[i].Entity && tokens[i].Entity != "O") + { + unionValue.Append(" " + tokens[j].Value); + j++; + } + nlpEntity.Entity = unionEntity.ToString(); + nlpEntity.Start = tokens[i].Start; + nlpEntity.Value = unionValue.ToString(); + nlpEntity.Confidence = unoinConfidence; + res.Add(nlpEntity); + i = j - 1; + } + return res; + } } public class TrainingData diff --git a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs index 7de0432e..f91fd12b 100644 --- a/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs +++ b/BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs @@ -5,8 +5,10 @@ using Microsoft.Extensions.Configuration; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Text; +using System.Threading; using System.Threading.Tasks; namespace BotSharp.Core.Engines.Classifiers @@ -23,7 +25,7 @@ namespace BotSharp.Core.Engines.Classifiers string predictFileName = Path.Join(Settings.PredictDir, "fasttext.txt"); File.WriteAllText(predictFileName, doc.Sentences[0].Text); - var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"predict-prob {modelFileName}.bin {predictFileName}"); + var output = Engines.Classifiers.CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"predict-prob {modelFileName}.bin {predictFileName}"); File.Delete(predictFileName); @@ -49,7 +51,7 @@ namespace BotSharp.Core.Engines.Classifiers File.WriteAllText(parsedTrainingDataFileName, corpus.ToString()); - var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}"); + var output = Engines.Classifiers.CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}", false); Console.WriteLine($"Saved model to {modelFileName}"); meta.Meta = new JObject(); @@ -59,4 +61,72 @@ namespace BotSharp.Core.Engines.Classifiers return true; } } + + public static class CmdHelper + { + public static string Run(string fileName, string arguments, bool outputAsync = true) + { + Console.WriteLine($"{fileName} {arguments}"); + + System.Diagnostics.Process proc = new System.Diagnostics.Process(); + ProcessStartInfo procStartInfo = new ProcessStartInfo(fileName); + // procStartInfo.Arguments = arguments; + // The following commands are needed to redirect the standard output. + // This means that it will be redirected to the Process.StandardOutput StreamReader. + procStartInfo.RedirectStandardOutput = true; + procStartInfo.RedirectStandardError = true; + procStartInfo.UseShellExecute = false; + // Do not create the black window. + //procStartInfo.CreateNoWindow = true; + if (procStartInfo.EnvironmentVariables.ContainsKey("OS") && procStartInfo.EnvironmentVariables["OS"] == "Windows_NT") + { + procStartInfo.FileName = fileName + ".exe"; + } + else + { + procStartInfo.FileName = "sh"; + procStartInfo.RedirectStandardInput = true; + procStartInfo.CreateNoWindow = false; + } + proc.StartInfo = procStartInfo; + + string output = String.Empty; + + proc.Start(); + if (procStartInfo.EnvironmentVariables.ContainsKey("OS") && procStartInfo.EnvironmentVariables["OS"] == "Windows_NT") + { + + } + else + { + proc.StandardInput.WriteLine($"{fileName} {arguments}" + "&exit"); + proc.StandardInput.AutoFlush = false; + } + + using (StreamReader reader = proc.StandardOutput) + { + if (outputAsync) + { + string buffer = String.Empty; + while (!proc.HasExited) + { + Thread.Sleep(1); + buffer = proc.StandardOutput.ReadLine(); + output += buffer; + Console.WriteLine(buffer); + } + } + else + { + output = reader.ReadToEnd(); + Console.WriteLine(output); + } + } + + proc.WaitForExit(); + proc.Close(); + + return output; + } + } } diff --git a/BotSharp.WebHost/Algorithms/crfsuite b/BotSharp.WebHost/Algorithms/crfsuite new file mode 100755 index 00000000..3fab0c91 Binary files /dev/null and b/BotSharp.WebHost/Algorithms/crfsuite differ diff --git a/BotSharp.WebHost/crfsuite.exe b/BotSharp.WebHost/crfsuite.exe new file mode 100644 index 00000000..74c7ad44 Binary files /dev/null and b/BotSharp.WebHost/crfsuite.exe differ