Added MergeEntity API to realize union entity prediction

BotSharp.Core/Engines/CRFsuite/CRFsuiteEntityRecognizer.cs

@@ -67,7 +67,6 @@ namespace BotSharp.Core.Engines.CRFsuite
             var algorithmDir = Path.Join(AppDomain.CurrentDomain.GetData("ContentRootPath").ToString(), "Algorithms");
 
             CmdHelper.Run(Path.Join(algorithmDir, "crfsuite"), $"learn -m {modelFileName} {parsedTrainingDataFileName}"); // --split=3 -x
-
             Console.WriteLine($"Saved model to {modelFileName}");
             meta.Meta = new JObject();
             meta.Meta["fields"] = fields;
@@ -195,12 +194,15 @@ namespace BotSharp.Core.Engines.CRFsuite
                 entities.Add(new NlpEntity
                 {
                     Entity = entity,
+                    Start = doc.Sentences[0].Tokens[i].Offset,
                     Value = doc.Sentences[0].Tokens[i].Text,
                     Confidence = probability
                 });
             }
 
-            doc.Sentences[0].Entities = entities.Where(x => x.Entity != "O").ToList();
+            List<NlpEntity> unionedEntities = MergeEntity(entities);
+
+            doc.Sentences[0].Entities = unionedEntities.Where(x => x.Entity != "O").ToList();
             
             if(File.Exists(rawPredictingDataFileName))
             {
@@ -213,6 +215,32 @@ namespace BotSharp.Core.Engines.CRFsuite
 
             return true;
         }
+
+        public List<NlpEntity> MergeEntity (List<NlpEntity> tokens)
+        {
+            List<NlpEntity> res = new List<NlpEntity>();
+            for (int i = 0; i < tokens.Count ; i++) 
+            {
+                NlpEntity nlpEntity = new NlpEntity();
+                StringBuilder unionValue = new StringBuilder(tokens[i].Value);
+                StringBuilder unionEntity = new StringBuilder(tokens[i].Entity);
+                decimal unoinConfidence = tokens[i].Confidence;
+
+                int j = i + 1;
+                while (j < tokens.Count && tokens[j].Entity == tokens[i].Entity && tokens[i].Entity != "O") 
+                {
+                    unionValue.Append(" " + tokens[j].Value);
+                    j++;
+                }
+                nlpEntity.Entity = unionEntity.ToString();
+                nlpEntity.Start = tokens[i].Start;
+                nlpEntity.Value = unionValue.ToString();
+                nlpEntity.Confidence = unoinConfidence;
+                res.Add(nlpEntity);
+                i = j - 1;
+            }
+            return res;
+        }
     }
 
     public class TrainingData

BotSharp.Core/Engines/Classifiers/FasttextClassifier.cs

@@ -5,8 +5,10 @@ using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace BotSharp.Core.Engines.Classifiers
@@ -23,7 +25,7 @@ namespace BotSharp.Core.Engines.Classifiers
             string predictFileName = Path.Join(Settings.PredictDir, "fasttext.txt");
             File.WriteAllText(predictFileName, doc.Sentences[0].Text);
 
-            var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"predict-prob {modelFileName}.bin {predictFileName}");
+            var output = Engines.Classifiers.CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"predict-prob {modelFileName}.bin {predictFileName}");
 
             File.Delete(predictFileName);
 
@@ -49,7 +51,7 @@ namespace BotSharp.Core.Engines.Classifiers
 
             File.WriteAllText(parsedTrainingDataFileName, corpus.ToString());
 
-            var output = CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}");
+            var output = Engines.Classifiers.CmdHelper.Run(Path.Join(Settings.AlgorithmDir, "fasttext"), $"supervised -input {parsedTrainingDataFileName} -output {modelFileName}", false);
 
             Console.WriteLine($"Saved model to {modelFileName}");
             meta.Meta = new JObject();
@@ -59,4 +61,72 @@ namespace BotSharp.Core.Engines.Classifiers
             return true;
         }
     }
+
+    public static class CmdHelper
+    {
+        public static string Run(string fileName, string arguments, bool outputAsync = true)
+        {
+            Console.WriteLine($"{fileName} {arguments}");
+
+            System.Diagnostics.Process proc = new System.Diagnostics.Process();
+            ProcessStartInfo procStartInfo = new ProcessStartInfo(fileName);
+            // procStartInfo.Arguments = arguments;
+            // The following commands are needed to redirect the standard output.
+            // This means that it will be redirected to the Process.StandardOutput StreamReader.
+            procStartInfo.RedirectStandardOutput = true;
+            procStartInfo.RedirectStandardError = true;
+            procStartInfo.UseShellExecute = false;
+            // Do not create the black window.
+            //procStartInfo.CreateNoWindow = true;
+            if (procStartInfo.EnvironmentVariables.ContainsKey("OS") && procStartInfo.EnvironmentVariables["OS"] == "Windows_NT")
+            {
+                procStartInfo.FileName = fileName + ".exe";
+            }
+            else
+            {
+                procStartInfo.FileName = "sh";
+                procStartInfo.RedirectStandardInput = true;
+                procStartInfo.CreateNoWindow = false;
+            }
+            proc.StartInfo = procStartInfo;
+
+            string output = String.Empty;
+
+            proc.Start();
+            if (procStartInfo.EnvironmentVariables.ContainsKey("OS") && procStartInfo.EnvironmentVariables["OS"] == "Windows_NT")
+            {
+            
+            }
+            else
+            {
+                proc.StandardInput.WriteLine($"{fileName} {arguments}" + "&exit");
+                proc.StandardInput.AutoFlush = false;
+            }
+
+            using (StreamReader reader = proc.StandardOutput)
+            {
+                if (outputAsync)
+                {
+                    string buffer = String.Empty;
+                    while (!proc.HasExited)
+                    {
+                        Thread.Sleep(1);
+                        buffer = proc.StandardOutput.ReadLine();
+                        output += buffer;
+                        Console.WriteLine(buffer);
+                    }
+                }
+                else
+                {
+                    output = reader.ReadToEnd();
+                    Console.WriteLine(output);
+                }
+            }
+            
+            proc.WaitForExit();
+            proc.Close();
+
+            return output;
+        }
+    }
 }