diff --git a/BotSharp.Core/Abstractions/IAgentImporter.cs b/BotSharp.Core/Abstractions/IAgentImporter.cs
index 4a1299bf..584b4e22 100644
--- a/BotSharp.Core/Abstractions/IAgentImporter.cs
+++ b/BotSharp.Core/Abstractions/IAgentImporter.cs
@@ -15,8 +15,25 @@ namespace BotSharp.Core.Engines
///
Agent LoadAgent(string agentId, string agentDir);
- void LoadEntities(Agent agent, string agentDir);
+ ///
+ /// Load user customized entity type
+ ///
+ ///
+ ///
+ void LoadCustomEntities(Agent agent, string agentDir);
+ ///
+ /// Load user customized intents
+ ///
+ ///
+ ///
void LoadIntents(Agent agent, string agentDir);
+
+ ///
+ /// add to user customized entities
+ ///
+ ///
+ ///
+ void LoadBuildinEntities(Agent agent, string agentDir);
}
}
diff --git a/BotSharp.Core/Agents/AgentDriver.cs b/BotSharp.Core/Agents/AgentDriver.cs
index c100b2f8..7b4079bd 100644
--- a/BotSharp.Core/Agents/AgentDriver.cs
+++ b/BotSharp.Core/Agents/AgentDriver.cs
@@ -45,12 +45,15 @@ namespace BotSharp.Core.Agents
// Load agent summary
var agent = importer.LoadAgent(agentId, dataDir);
- // Load agent entities
- importer.LoadEntities(agent, dataDir);
+ // Load user custom entities
+ importer.LoadCustomEntities(agent, dataDir);
// Load agent intents
importer.LoadIntents(agent, dataDir);
+ // Load system buildin entities
+ importer.LoadBuildinEntities(agent, dataDir);
+
return agent;
}
@@ -106,7 +109,7 @@ namespace BotSharp.Core.Agents
var part = new RasaIntentExpressionPart
{
Value = x.Text,
- Entity = x.Alias,
+ Entity = $"{x.Meta}:{x.Alias}",
Start = start,
End = start + x.Text.Length
};
@@ -125,7 +128,7 @@ namespace BotSharp.Core.Agents
var te = new RasaTraningEntity
{
- EntityType = x.Alias,
+ EntityType = $"{x.Meta}:{x.Alias}",
EntityValue = x.Text,
Synonyms = allSynonyms
};
diff --git a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs
index dc05f813..73375431 100644
--- a/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs
+++ b/BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs
@@ -14,8 +14,17 @@ using Newtonsoft.Json.Linq;
namespace BotSharp.Core.Engines
{
+ ///
+ /// Import agent from Dialogflow
+ ///
public class AgentImporterInDialogflow : IAgentImporter
{
+ ///
+ /// Load agent meta
+ ///
+ ///
+ ///
+ ///
public Agent LoadAgent(string agentId, string agentDir)
{
// load agent profile
@@ -27,7 +36,7 @@ namespace BotSharp.Core.Engines
return agent.ToObject();
}
- public void LoadEntities(Agent agent, string agentDir)
+ public void LoadCustomEntities(Agent agent, string agentDir)
{
agent.Entities = new List();
string entityDir = $"{agentDir}{Path.DirectorySeparatorChar}Dialogflow{Path.DirectorySeparatorChar}{agent.Name}{Path.DirectorySeparatorChar}entities";
@@ -83,90 +92,173 @@ namespace BotSharp.Core.Engines
intentJson = intentJson.Replace("\"prompts\":", "\"promptList\":");
var intent = JsonConvert.DeserializeObject(intentJson);
- // void id confict
- intent.Id = Guid.NewGuid().ToString();
- intent.Name = intent.Name.Replace("/","_");
- // load user expressions
- if (fileName.Contains("Default Fallback Intent"))
- {
- intent.UserSays = (intent.Responses[0].MessageList[0].Speech as JArray)
- .Select(x => new DialogflowIntentExpression
- {
- Data = new List
- {
- new DialogflowIntentExpressionPart
- {
- Text = x.ToString()
- }
- }
- }).ToList();
- }
- else
- {
- string expressionFileName = fileName.Replace(intent.Name, $"{intent.Name}_usersays_{agent.Language}");
- if (File.Exists(expressionFileName))
- {
- string expressionJson = File.ReadAllText($"{expressionFileName}");
- intent.UserSays = JsonConvert.DeserializeObject>(expressionJson);
-
- // remove @sys.ignore
- intent.UserSays.ForEach(say =>
- {
- say.Data.Where(x => x.Meta == "@sys.ignore").ToList().ForEach(x => x.Meta = null);
- });
- }
- }
-
-
- var newIntent = intent.ToObject();
- intent.Responses.ForEach(res =>
- {
- var newResponse = newIntent.Responses.First(x => x.Id == res.Id);
-
- newResponse.Contexts = res.AffectedContexts.Select(x => new IntentResponseContext {
- Name = x.Name,
- Lifespan = x.Lifespan
- }).ToList();
-
- newResponse.Messages = res.MessageList.Where(x => x.Speech != null || x.Payload != null)
- .Select(x =>
- {
- if(x.Type == AIResponseMessageType.Custom)
- {
- return new IntentResponseMessage
- {
- Payload = JObject.FromObject(x.Payload),
- PayloadJson = JsonConvert.SerializeObject(x.Payload),
- Type = x.Type
- };
- } else
- {
- var speech = JsonConvert.SerializeObject(x.Speech.GetType().Equals(typeof(String)) ?
- new List { x.Speech.ToString() } :
- (x.Speech as JArray).Select(s => s.Value()).ToList());
-
- return new IntentResponseMessage
- {
- Speech = speech,
- Type = x.Type
- };
- }
-
- }).ToList();
-
- newResponse.Parameters = res.Parameters.Select(p =>
- {
- var rp = p.ToObject();
- rp.Prompts = p.PromptList.Select(pl => new ResponseParameterPrompt { Prompt = pl.Value }).ToList();
- return rp;
- }).ToList();
- });
-
- newIntent.Contexts = intent.ContextList.Select(x => new IntentInputContext { Name = x }).ToList();
-
+ var newIntent = ImportIntentUserSays(agent, intent, fileName);
agent.Intents.Add(newIntent);
}
});
}
+
+ private Intent ImportIntentUserSays(Agent agent, DialogflowIntent intent, string fileName)
+ {
+ // void id confict
+ intent.Id = Guid.NewGuid().ToString();
+ intent.Name = intent.Name.Replace("/", "_");
+ // load user expressions
+ if (fileName.Contains("Default Fallback Intent"))
+ {
+ intent.UserSays = (intent.Responses[0].MessageList[0].Speech as JArray)
+ .Select(x => new DialogflowIntentExpression
+ {
+ Data = new List
+ {
+ new DialogflowIntentExpressionPart
+ {
+ Text = x.ToString()
+ }
+ }
+ }).ToList();
+ }
+ else
+ {
+ string expressionFileName = fileName.Replace(intent.Name, $"{intent.Name}_usersays_{agent.Language}");
+ if (File.Exists(expressionFileName))
+ {
+ string expressionJson = File.ReadAllText($"{expressionFileName}");
+ intent.UserSays = JsonConvert.DeserializeObject>(expressionJson);
+
+ intent.UserSays.ForEach(say =>
+ {
+ // remove @sys.ignore
+ say.Data.Where(x => x.Meta == "@sys.ignore").ToList().ForEach(x => x.Meta = null);
+
+ // remove @sys.
+ say.Data.Where(x => x.Meta != null && x.Meta.StartsWith("@sys.")).ToList().ForEach(x => x.Meta = x.Meta.Substring(5));
+
+ // remove @
+ say.Data.Where(x => x.Meta != null && x.Meta.StartsWith("@")).ToList().ForEach(x => x.Meta = x.Meta.Substring(1));
+ });
+ }
+ }
+
+ var newIntent = ImportIntentResponse(agent, intent);
+
+ newIntent.Contexts = intent.ContextList.Select(x => new IntentInputContext { Name = x }).ToList();
+
+ return newIntent;
+ }
+
+ private Intent ImportIntentResponse(Agent agent, DialogflowIntent intent)
+ {
+ var newIntent = intent.ToObject();
+
+ intent.Responses.ForEach(res =>
+ {
+ var newResponse = newIntent.Responses.First(x => x.Id == res.Id);
+
+ newResponse.Contexts = res.AffectedContexts.Select(x => new IntentResponseContext
+ {
+ Name = x.Name,
+ Lifespan = x.Lifespan
+ }).ToList();
+
+ newResponse.Messages = res.MessageList.Where(x => x.Speech != null || x.Payload != null)
+ .Select(x =>
+ {
+ if (x.Type == AIResponseMessageType.Custom)
+ {
+ return new IntentResponseMessage
+ {
+ Payload = JObject.FromObject(x.Payload),
+ PayloadJson = JsonConvert.SerializeObject(x.Payload),
+ Type = x.Type
+ };
+ }
+ else
+ {
+ var speech = JsonConvert.SerializeObject(x.Speech.GetType().Equals(typeof(String)) ?
+ new List { x.Speech.ToString() } :
+ (x.Speech as JArray).Select(s => s.Value()).ToList());
+
+ return new IntentResponseMessage
+ {
+ Speech = speech,
+ Type = x.Type
+ };
+ }
+
+ }).ToList();
+
+ newResponse.Parameters = res.Parameters.Select(p =>
+ {
+ var rp = p.ToObject();
+
+ // remove @sys.
+ if (rp.DataType.StartsWith("@sys."))
+ {
+ rp.DataType = rp.DataType.Substring(5);
+ }
+
+ if (rp.DataType.StartsWith("@"))
+ {
+ rp.DataType = rp.DataType.Substring(1);
+ }
+
+ rp.Prompts = p.PromptList.Select(pl => new ResponseParameterPrompt { Prompt = pl.Value }).ToList();
+ return rp;
+ }).ToList();
+ });
+
+ return newIntent;
+ }
+
+ public void LoadBuildinEntities(Agent agent, string agentDir)
+ {
+ agent.Intents.ForEach(intent => {
+
+ intent.UserSays.ForEach(us => {
+
+ us.Data.Where(data => data.Meta != null)
+ .ToList()
+ .ForEach(data =>
+ {
+ LoadBuildinEntityTypePerUserSay(agent, data);
+ });
+ });
+
+ });
+ }
+
+ private void LoadBuildinEntityTypePerUserSay(Agent agent, IntentExpressionPart data)
+ {
+ var existedEntityType = agent.Entities.FirstOrDefault(x => x.Name == data.Meta);
+
+ if (existedEntityType == null)
+ {
+ existedEntityType = new EntityType
+ {
+ Name = data.Meta,
+ Entries = new List(),
+ IsOverridable = true
+ };
+
+ agent.Entities.Add(existedEntityType);
+ }
+
+ var entries = existedEntityType.Entries.Select(x => x.Value.ToLower()).ToList();
+ if (!entries.Contains(data.Text.ToLower()))
+ {
+ existedEntityType.Entries.Add(new EntityEntry
+ {
+ Value = data.Text,
+ Synonyms = new List
+ {
+ new EntrySynonym
+ {
+ Synonym = data.Text
+ }
+ }
+ });
+ }
+ }
}
}
diff --git a/BotSharp.Core/Engines/Rasa/RasaAi.cs b/BotSharp.Core/Engines/Rasa/RasaAi.cs
index 42d23ab2..74114f42 100644
--- a/BotSharp.Core/Engines/Rasa/RasaAi.cs
+++ b/BotSharp.Core/Engines/Rasa/RasaAi.cs
@@ -152,11 +152,23 @@ namespace BotSharp.Core.Engines
}
});
+ // set empty synonym to null
+ data.Entities
+ .Where(x => x.Synonyms != null)
+ .ToList()
+ .ForEach(entity =>
+ {
+ if (entity.Synonyms.Count == 0)
+ {
+ entity.Synonyms = null;
+ }
+ });
+
string json = JsonConvert.SerializeObject(new { rasa_nlu_data = data },
new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
- NullValueHandling = NullValueHandling.Ignore
+ NullValueHandling = NullValueHandling.Ignore,
});
var rest = new RestRequest("train", Method.POST);
diff --git a/BotSharp.Core/Engines/RequestExtension.cs b/BotSharp.Core/Engines/RequestExtension.cs
index 7ca82f5e..0854d3f4 100644
--- a/BotSharp.Core/Engines/RequestExtension.cs
+++ b/BotSharp.Core/Engines/RequestExtension.cs
@@ -161,17 +161,17 @@ namespace BotSharp.Core.Engines
intentResponse.Parameters.ForEach(p => {
string query = request.Query.First();
- var entity = response.Entities.FirstOrDefault(x => x.Entity == p.Name);
+ var entity = response.Entities.FirstOrDefault(x => x.Entity == p.Name || x.Entity.Split(":").Contains(p.Name));
if (entity != null)
{
p.Value = query.Substring(entity.Start, entity.End - entity.Start);
}
// convert to Standard entity value
- if (!String.IsNullOrEmpty(p.Value) && !p.DataType.StartsWith("@sys."))
+ if (!String.IsNullOrEmpty(p.Value) && !p.DataType.StartsWith("sys."))
{
p.Value = agent.Entities
- .FirstOrDefault(x => x.Name == p.DataType.Substring(1))
+ .FirstOrDefault(x => x.Name == p.DataType)
.Entries
.FirstOrDefault((entry) =>
{
diff --git a/BotSharp.UnitTest/IntentTest.cs b/BotSharp.UnitTest/ConversationTest.cs
similarity index 98%
rename from BotSharp.UnitTest/IntentTest.cs
rename to BotSharp.UnitTest/ConversationTest.cs
index 436ccdd4..0452d544 100644
--- a/BotSharp.UnitTest/IntentTest.cs
+++ b/BotSharp.UnitTest/ConversationTest.cs
@@ -9,7 +9,7 @@ using System.Text;
namespace BotSharp.UnitTest
{
[TestClass]
- public class IntentTest : TestEssential
+ public class ConversationTest : TestEssential
{
[TestMethod]
public void TextRequest()
diff --git a/BotSharp.UnitTest/Settings/config_mitie_sklearn.yml b/BotSharp.UnitTest/Settings/config_mitie_sklearn.yml
new file mode 100644
index 00000000..5cab1429
--- /dev/null
+++ b/BotSharp.UnitTest/Settings/config_mitie_sklearn.yml
@@ -0,0 +1,11 @@
+language: "en"
+
+pipeline:
+- name: "nlp_mitie"
+ model: "data/total_word_feature_extractor.dat"
+- name: "tokenizer_mitie"
+- name: "ner_mitie"
+- name: "ner_synonyms"
+- name: "intent_entity_featurizer_regex"
+- name: "intent_featurizer_mitie"
+- name: "intent_classifier_sklearn"
\ No newline at end of file