SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add missing entity type when importing from Dialogflow

Browse source
This commit is contained in:
haiping008@gmail.com 2018-06-22 14:42:53 -05:00
parent e4fe4b2a1e
commit d29c74b3ad
7 changed files with 227 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
BotSharp.Core/Abstractions/IAgentImporter.cs
View file

@ -15,8 +15,25 @@ namespace BotSharp.Core.Engines
/// <returns></returns>
Agent LoadAgent(string agentId, string agentDir);
void LoadEntities(Agent agent, string agentDir);
/// <summary>
/// Load user customized entity type
/// </summary>
/// <param name="agent"></param>
/// <param name="agentDir"></param>
void LoadCustomEntities(Agent agent, string agentDir);
/// <summary>
/// Load user customized intents
/// </summary>
/// <param name="agent"></param>
/// <param name="agentDir"></param>
void LoadIntents(Agent agent, string agentDir);
/// <summary>
/// add to user customized entities
/// </summary>
/// <param name="agent"></param>
/// <param name="agentDir"></param>
void LoadBuildinEntities(Agent agent, string agentDir);
}
}

11
BotSharp.Core/Agents/AgentDriver.cs
View file

@ -45,12 +45,15 @@ namespace BotSharp.Core.Agents
// Load agent summary
var agent = importer.LoadAgent(agentId, dataDir);
// Load agent entities
importer.LoadEntities(agent, dataDir);
// Load user custom entities
importer.LoadCustomEntities(agent, dataDir);
// Load agent intents
importer.LoadIntents(agent, dataDir);
// Load system buildin entities
importer.LoadBuildinEntities(agent, dataDir);
return agent;
}
@ -106,7 +109,7 @@ namespace BotSharp.Core.Agents
var part = new RasaIntentExpressionPart
{
Value = x.Text,
Entity = x.Alias,
Entity = $"{x.Meta}:{x.Alias}",
Start = start,
End = start + x.Text.Length
};
@ -125,7 +128,7 @@ namespace BotSharp.Core.Agents
var te = new RasaTraningEntity
{
EntityType = x.Alias,
EntityType = $"{x.Meta}:{x.Alias}",
EntityValue = x.Text,
Synonyms = allSynonyms
};

256
BotSharp.Core/Engines/Dialogflow/AgentImporterInDialogflow.cs
View file

@ -14,8 +14,17 @@ using Newtonsoft.Json.Linq;
namespace BotSharp.Core.Engines
{
/// <summary>
/// Import agent from Dialogflow
/// </summary>
public class AgentImporterInDialogflow : IAgentImporter
{
/// <summary>
/// Load agent meta
/// </summary>
/// <param name="agentId"></param>
/// <param name="agentDir"></param>
/// <returns></returns>
public Agent LoadAgent(string agentId, string agentDir)
{
// load agent profile
@ -27,7 +36,7 @@ namespace BotSharp.Core.Engines
return agent.ToObject<Agent>();
}
public void LoadEntities(Agent agent, string agentDir)
public void LoadCustomEntities(Agent agent, string agentDir)
{
agent.Entities = new List<EntityType>();
string entityDir = $"{agentDir}{Path.DirectorySeparatorChar}Dialogflow{Path.DirectorySeparatorChar}{agent.Name}{Path.DirectorySeparatorChar}entities";
@ -83,90 +92,173 @@ namespace BotSharp.Core.Engines
intentJson = intentJson.Replace("\"prompts\":", "\"promptList\":");
var intent = JsonConvert.DeserializeObject<DialogflowIntent>(intentJson);
// void id confict
intent.Id = Guid.NewGuid().ToString();
intent.Name = intent.Name.Replace("/","_");
// load user expressions
if (fileName.Contains("Default Fallback Intent"))
{
intent.UserSays = (intent.Responses[0].MessageList[0].Speech as JArray)
.Select(x => new DialogflowIntentExpression
{
Data = new List<DialogflowIntentExpressionPart>
{
new DialogflowIntentExpressionPart
{
Text = x.ToString()
}
}
}).ToList();
}
else
{
string expressionFileName = fileName.Replace(intent.Name, $"{intent.Name}_usersays_{agent.Language}");
if (File.Exists(expressionFileName))
{
string expressionJson = File.ReadAllText($"{expressionFileName}");
intent.UserSays = JsonConvert.DeserializeObject<List<DialogflowIntentExpression>>(expressionJson);
// remove @sys.ignore
intent.UserSays.ForEach(say =>
{
say.Data.Where(x => x.Meta == "@sys.ignore").ToList().ForEach(x => x.Meta = null);
});
}
}
var newIntent = intent.ToObject<Intent>();
intent.Responses.ForEach(res =>
{
var newResponse = newIntent.Responses.First(x => x.Id == res.Id);
newResponse.Contexts = res.AffectedContexts.Select(x => new IntentResponseContext {
Name = x.Name,
Lifespan = x.Lifespan
}).ToList();
newResponse.Messages = res.MessageList.Where(x => x.Speech != null || x.Payload != null)
.Select(x =>
{
if(x.Type == AIResponseMessageType.Custom)
{
return new IntentResponseMessage
{
Payload = JObject.FromObject(x.Payload),
PayloadJson = JsonConvert.SerializeObject(x.Payload),
Type = x.Type
};
} else
{
var speech = JsonConvert.SerializeObject(x.Speech.GetType().Equals(typeof(String)) ?
new List<String> { x.Speech.ToString() } :
(x.Speech as JArray).Select(s => s.Value<String>()).ToList());
return new IntentResponseMessage
{
Speech = speech,
Type = x.Type
};
}
}).ToList();
newResponse.Parameters = res.Parameters.Select(p =>
{
var rp = p.ToObject<IntentResponseParameter>();
rp.Prompts = p.PromptList.Select(pl => new ResponseParameterPrompt { Prompt = pl.Value }).ToList();
return rp;
}).ToList();
});
newIntent.Contexts = intent.ContextList.Select(x => new IntentInputContext { Name = x }).ToList();
var newIntent = ImportIntentUserSays(agent, intent, fileName);
agent.Intents.Add(newIntent);
}
});
}
private Intent ImportIntentUserSays(Agent agent, DialogflowIntent intent, string fileName)
{
// void id confict
intent.Id = Guid.NewGuid().ToString();
intent.Name = intent.Name.Replace("/", "_");
// load user expressions
if (fileName.Contains("Default Fallback Intent"))
{
intent.UserSays = (intent.Responses[0].MessageList[0].Speech as JArray)
.Select(x => new DialogflowIntentExpression
{
Data = new List<DialogflowIntentExpressionPart>
{
new DialogflowIntentExpressionPart
{
Text = x.ToString()
}
}
}).ToList();
}
else
{
string expressionFileName = fileName.Replace(intent.Name, $"{intent.Name}_usersays_{agent.Language}");
if (File.Exists(expressionFileName))
{
string expressionJson = File.ReadAllText($"{expressionFileName}");
intent.UserSays = JsonConvert.DeserializeObject<List<DialogflowIntentExpression>>(expressionJson);
intent.UserSays.ForEach(say =>
{
// remove @sys.ignore
say.Data.Where(x => x.Meta == "@sys.ignore").ToList().ForEach(x => x.Meta = null);
// remove @sys.
say.Data.Where(x => x.Meta != null && x.Meta.StartsWith("@sys.")).ToList().ForEach(x => x.Meta = x.Meta.Substring(5));
// remove @
say.Data.Where(x => x.Meta != null && x.Meta.StartsWith("@")).ToList().ForEach(x => x.Meta = x.Meta.Substring(1));
});
}
}
var newIntent = ImportIntentResponse(agent, intent);
newIntent.Contexts = intent.ContextList.Select(x => new IntentInputContext { Name = x }).ToList();
return newIntent;
}
private Intent ImportIntentResponse(Agent agent, DialogflowIntent intent)
{
var newIntent = intent.ToObject<Intent>();
intent.Responses.ForEach(res =>
{
var newResponse = newIntent.Responses.First(x => x.Id == res.Id);
newResponse.Contexts = res.AffectedContexts.Select(x => new IntentResponseContext
{
Name = x.Name,
Lifespan = x.Lifespan
}).ToList();
newResponse.Messages = res.MessageList.Where(x => x.Speech != null || x.Payload != null)
.Select(x =>
{
if (x.Type == AIResponseMessageType.Custom)
{
return new IntentResponseMessage
{
Payload = JObject.FromObject(x.Payload),
PayloadJson = JsonConvert.SerializeObject(x.Payload),
Type = x.Type
};
}
else
{
var speech = JsonConvert.SerializeObject(x.Speech.GetType().Equals(typeof(String)) ?
new List<String> { x.Speech.ToString() } :
(x.Speech as JArray).Select(s => s.Value<String>()).ToList());
return new IntentResponseMessage
{
Speech = speech,
Type = x.Type
};
}
}).ToList();
newResponse.Parameters = res.Parameters.Select(p =>
{
var rp = p.ToObject<IntentResponseParameter>();
// remove @sys.
if (rp.DataType.StartsWith("@sys."))
{
rp.DataType = rp.DataType.Substring(5);
}
if (rp.DataType.StartsWith("@"))
{
rp.DataType = rp.DataType.Substring(1);
}
rp.Prompts = p.PromptList.Select(pl => new ResponseParameterPrompt { Prompt = pl.Value }).ToList();
return rp;
}).ToList();
});
return newIntent;
}
public void LoadBuildinEntities(Agent agent, string agentDir)
{
agent.Intents.ForEach(intent => {
intent.UserSays.ForEach(us => {
us.Data.Where(data => data.Meta != null)
.ToList()
.ForEach(data =>
{
LoadBuildinEntityTypePerUserSay(agent, data);
});
});
});
}
private void LoadBuildinEntityTypePerUserSay(Agent agent, IntentExpressionPart data)
{
var existedEntityType = agent.Entities.FirstOrDefault(x => x.Name == data.Meta);
if (existedEntityType == null)
{
existedEntityType = new EntityType
{
Name = data.Meta,
Entries = new List<EntityEntry>(),
IsOverridable = true
};
agent.Entities.Add(existedEntityType);
}
var entries = existedEntityType.Entries.Select(x => x.Value.ToLower()).ToList();
if (!entries.Contains(data.Text.ToLower()))
{
existedEntityType.Entries.Add(new EntityEntry
{
Value = data.Text,
Synonyms = new List<EntrySynonym>
{
new EntrySynonym
{
Synonym = data.Text
}
}
});
}
}
}
}

14
BotSharp.Core/Engines/Rasa/RasaAi.cs
View file

@ -152,11 +152,23 @@ namespace BotSharp.Core.Engines
}
});
// set empty synonym to null
data.Entities
.Where(x => x.Synonyms != null)
.ToList()
.ForEach(entity =>
{
if (entity.Synonyms.Count == 0)
{
entity.Synonyms = null;
}
});
string json = JsonConvert.SerializeObject(new { rasa_nlu_data = data },
new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
NullValueHandling = NullValueHandling.Ignore
NullValueHandling = NullValueHandling.Ignore,
});
var rest = new RestRequest("train", Method.POST);

6
BotSharp.Core/Engines/RequestExtension.cs
View file

@ -161,17 +161,17 @@ namespace BotSharp.Core.Engines
intentResponse.Parameters.ForEach(p => {
string query = request.Query.First();
var entity = response.Entities.FirstOrDefault(x => x.Entity == p.Name);
var entity = response.Entities.FirstOrDefault(x => x.Entity == p.Name || x.Entity.Split(":").Contains(p.Name));
if (entity != null)
{
p.Value = query.Substring(entity.Start, entity.End - entity.Start);
}
// convert to Standard entity value
if (!String.IsNullOrEmpty(p.Value) && !p.DataType.StartsWith("@sys."))
if (!String.IsNullOrEmpty(p.Value) && !p.DataType.StartsWith("sys."))
{
p.Value = agent.Entities
.FirstOrDefault(x => x.Name == p.DataType.Substring(1))
.FirstOrDefault(x => x.Name == p.DataType)
.Entries
.FirstOrDefault((entry) =>
{

2
BotSharp.UnitTest/IntentTest.cs → BotSharp.UnitTest/ConversationTest.cs
View file

@ -9,7 +9,7 @@ using System.Text;
namespace BotSharp.UnitTest
{
[TestClass]
public class IntentTest : TestEssential
public class ConversationTest : TestEssential
{
[TestMethod]
public void TextRequest()

11
BotSharp.UnitTest/Settings/config_mitie_sklearn.yml Normal file
View file

@ -0,0 +1,11 @@
language: "en"
pipeline:
- name: "nlp_mitie"
model: "data/total_word_feature_extractor.dat"
- name: "tokenizer_mitie"
- name: "ner_mitie"
- name: "ner_synonyms"
- name: "intent_entity_featurizer_regex"
- name: "intent_featurizer_mitie"
- name: "intent_classifier_sklearn"