Parse Sebis benchmark corpus.

.gitignore

@@ -293,3 +293,4 @@ __pycache__/
 /BotSharp.UnitTest/App_Data/DbInitializer/Agents
 /BotSharp.UnitTest/App_Data/BotSharp.db
 /BotSharp.WebHost/App_Data/BotSharp.db
+/BotSharp.UI

BotSharp.Core/Engines/Sebis/AgentImporterInSebis.cs

@@ -16,7 +16,7 @@ using Newtonsoft.Json.Linq;
 namespace BotSharp.Core.Engines
 {
     /// <summary>
-    /// Import agent from Dialogflow
+    /// Import Sebis NLU benchmark corpus
     /// </summary>
     public class AgentImporterInSebis : IAgentImporter
     {
@@ -58,48 +58,74 @@ namespace BotSharp.Core.Engines
             agent.Intents = sentences.Select(x => x.Name).Distinct().Select(x => new Intent{Name = x}).ToList();
             
             agent.Intents.ForEach(intent => {
-                intent.UserSays = new List<IntentExpression>();
+                ImportIntentUserSays(agent, intent, sentences);
-
-                var userSays = sentences.Where(x => x.Name == intent.Name).ToList();
-
-                userSays.ForEach(say => 
-                {
-                    var expression = new IntentExpression();
-
-                    expression.Data = new List<IntentExpressionPart>();
-
-                    for(int index = 0; index < say.Entities.Count; index++)
-                    {
-                        
-                    }
-
-                    intent.UserSays.Add(expression);
-                });
             });
         }
 
-        private Intent ImportIntentUserSays(Agent agent, Intent intent, string fileName)
+        private void ImportIntentUserSays(Agent agent, Intent intent, List<SebisIntent> sentences)
         {
-            // void id confict
+            intent.UserSays = new List<IntentExpression>();
-            intent.Id = Guid.NewGuid().ToString();
-            intent.Name = intent.Name.Replace("/", "_");
-            // load user expressions
 
-            string expressionFileName = fileName.Replace(intent.Name, $"{intent.Name}_usersays_{agent.Language}");
+            var userSays = sentences.Where(x => x.Name == intent.Name).ToList();
-            if (File.Exists(expressionFileName))
+
+            userSays.ForEach(say =>
             {
-                string expressionJson = File.ReadAllText($"{expressionFileName}");
+                var expression = new IntentExpression();
-                intent.UserSays = JsonConvert.DeserializeObject<List<IntentExpression>>(expressionJson);
-            }
 
-            return null;
+                say.Entities = say.Entities.OrderBy(x => x.Start).ToList();
+
+                int pos = 0;
+                say.Entities.ForEach(x =>
+                {
+                    ConvertWordPosToCharPos(say.Text, x, pos);
+                    pos = x.End;
+                });
+
+                expression.Data = new List<IntentExpressionPart>();
+
+                pos = 0;
+                for (int entityIdx = 0; entityIdx < say.Entities.Count; entityIdx++)
+                {
+                    var entity = say.Entities[entityIdx];
+
+                    // previous
+                    expression.Data.Add(new IntentExpressionPart
+                    {
+                        Text = say.Text.Substring(pos, entity.Start - pos)
+                    });
+
+                    // self
+                    expression.Data.Add(new IntentExpressionPart
+                    {
+                        Alias = entity.Entity,
+                        Meta = entity.Entity,
+                        Text = say.Text.Substring(entity.Start, entity.Value.Length)
+                    });
+
+                    pos = entity.End + 1;
+
+                    if (pos < say.Text.Length && entityIdx == say.Entities.Count - 1)
+                    {
+                        // end
+                        expression.Data.Add(new IntentExpressionPart
+                        {
+                            Text = say.Text.Substring(pos)
+                        });
+                    }
+                }
+            });
+        }
+
+        private void ConvertWordPosToCharPos(string text, SebisIntentExpressionPart entity, int start)
+        {
+            entity.Start = text.IndexOf(entity.Value, start);
+            entity.End = entity.Start + entity.Value.Length;
         }
 
         public void LoadBuildinEntities(Agent agent, string agentDir)
         {
             agent.Intents.ForEach(intent =>
             {
-
                 if (intent.UserSays != null)
                 {
                     intent.UserSays.ForEach(us =>

BotSharp.Core/Engines/Sebis/SebisAgent.cs

@@ -10,9 +10,9 @@ namespace BotSharp.Core.Adapters.Sebis
     {
         public String Id { get; set; }
         public String Name { get; set; }
-        [JsonProperty("desc")]
+        
         public String Description { get; set; }
-        [JsonProperty("lang")]
+        
         public String Language { get; set; }
 
         public List<SebisIntent> Sentences { get; set; }

BotSharp.WebHost/App_Data/DbInitializer/Accounts/users.json

@@ -1,6 +1,6 @@
 [
   {
-    "userName": "support@botsharp.io",
+    "userName": "botsharp",
     "email": "support@botsharp.io",
     "firstName": "Support",
     "lastName": "Botsharp",

BotSharp.WebHost/App_Data/DbInitializer/Agents/Sebis/Airport/corpus.json

@@ -394,13 +394,13 @@
         {
           "entity": "StationStart",
           "start": 6,
-          "stop": 10,
+          "stop": 8,
           "text": "kurt-eisner-straße"
         },
         {
           "entity": "StationDest",
-          "start": 12,
+          "start": 10,
-          "stop": 13,
+          "stop": 11,
           "text": "garching forschungszentrum"
         }
       ]
@@ -586,13 +586,13 @@
         {
           "entity": "StationStart",
           "start": 5,
-          "stop": 9,
+          "stop": 5,
           "text": "kurt-eisner-straße"
         },
         {
           "entity": "StationDest",
-          "start": 11,
+          "start": 7,
-          "stop": 12,
+          "stop": 8,
           "text": "garching forschungszentrum"
         }
       ]