SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

dialogflow upload and training

Browse source
This commit is contained in:
Oceania2018 2018-10-03 01:00:07 -05:00
parent 201a5f2ea0
commit d656f4a75f
38 changed files with 269 additions and 384 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
BotSharp.Core.UnitTest/AgentTest.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Microsoft.VisualStudio.TestTools.UnitTesting;

4
BotSharp.Core.UnitTest/Performance/Spotify.cs
View file

@ -1,6 +1,6 @@
using Bigtree.Algorithm.Extensions;
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
@ -40,7 +40,7 @@ namespace BotSharp.Core.UnitTest.Performance
//double accuracy = correct / (Samples.Count + 0.0);
}
private Agent LoadAgent()
private AgentBase LoadAgent()
{
//_platform = new BotSharpAi();

3
BotSharp.Core/Abstractions/IBotPlatform.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Agents;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.AiRequest;
using BotSharp.Platform.Models.AiResponse;
using Newtonsoft.Json.Linq;

3
BotSharp.Core/Abstractions/INlpPipeline.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;

6
BotSharp.Core/Abstractions/INlpPredict.cs
View file

@ -1,5 +1,5 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using System;
using System.Collections.Generic;
using System.Text;
@ -9,6 +9,6 @@ namespace BotSharp.Core.Abstractions
{
public interface INlpPredict : INlpPipeline
{
Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta);
Task<bool> Predict(AgentBase agent, NlpDoc doc, PipeModel meta);
}
}

6
BotSharp.Core/Abstractions/INlpProvider.cs
View file

@ -1,5 +1,5 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using System;
using System.Collections.Generic;
using System.Text;
@ -9,6 +9,6 @@ namespace BotSharp.Core.Abstractions
{
public interface INlpProvider : INlpPipeline
{
Task<bool> Load(Agent agent, PipeModel meta);
Task<bool> Load(AgentBase agent, PipeModel meta);
}
}

6
BotSharp.Core/Abstractions/INlpTrain.cs
View file

@ -1,5 +1,5 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using System;
using System.Collections.Generic;
using System.Text;
@ -16,6 +16,6 @@ namespace BotSharp.Core.Abstractions
/// <param name="doc">Intermediate result</param>
/// <param name="meta">Meta data which is packed to model</param>
/// <returns></returns>
Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta);
Task<bool> Train(AgentBase agent, NlpDoc doc, PipeModel meta);
}
}

75
BotSharp.Core/Agents/Agent.cs
View file

@ -1,75 +0,0 @@
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using EntityFrameworkCore.BootKit;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace BotSharp.Core.Agents
{
[Table("Bot_Agent")]
public class Agent : DbRecord, IDbRecord
{
public Agent()
{
CreatedDate = DateTime.UtcNow;
}
[Required]
[MaxLength(64)]
public String Name { get; set; }
[MaxLength(256)]
public String Description { get; set; }
public Boolean Published { get; set; }
[Required]
[MaxLength(5)]
public String Language { get; set; }
/// <summary>
/// Only access text/ audio rquest
/// </summary>
[StringLength(32)]
public String ClientAccessToken { get; set; }
/// <summary>
/// Developer can access more APIs
/// </summary>
[StringLength(32)]
public String DeveloperAccessToken { get; set; }
[ForeignKey("AgentId")]
public List<Platform.Models.Intents.Intent> Intents { get; set; }
/*[ForeignKey("AgentId")]
[JsonProperty("entity_types")]
public List<EntityType> Entities { get; set; }*/
public String Birthday
{
get
{
return CreatedDate.ToShortDateString();
}
}
[Required]
public DateTime CreatedDate { get; set; }
public Boolean IsSkillSet { get; set; }
[ForeignKey("AgentId")]
public AgentMlConfig MlConfig { get; set; }
[NotMapped]
public TrainingCorpus Corpus { get; set; }
[ForeignKey("AgentId")]
public List<AgentIntegration> Integrations { get; set; }
}
}

4
BotSharp.Core/BotSharp.Core.csproj
View file

@ -89,4 +89,8 @@ If you feel that this project is helpful to you, please Star on the project, we
<ProjectReference Include="..\BotSharp.Platform.Models\BotSharp.Platform.Models.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="Agents\" />
</ItemGroup>
</Project>

16
BotSharp.Core/Engines/BotEngineBase.cs
View file

@ -1,9 +1,9 @@
using BotSharp.Core.Agents;
using BotSharp.Models.NLP;
using BotSharp.Models.NLP;
using BotSharp.Platform.Abstraction;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.AiRequest;
using BotSharp.Platform.Models.AiResponse;
using BotSharp.Platform.Models.Intents;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
@ -26,7 +26,7 @@ namespace BotSharp.Core.Engines
{
protected Database dc;
protected Agent agent { get; set; }
protected AgentBase agent { get; set; }
public BotEngineBase()
{
@ -75,7 +75,7 @@ namespace BotSharp.Core.Engines
};
}
public TrainingCorpus GetIntentExpressions(Agent agent)
/*public TrainingCorpus GetIntentExpressions(AgentBase agent)
{
TrainingCorpus corpus = new TrainingCorpus()
{
@ -83,7 +83,7 @@ namespace BotSharp.Core.Engines
Entities = new List<TrainingEntity>()
};
//var expressParts = new List<IntentExpressionPart>();
var expressParts = new List<IntentExpressionPart>();
var intents = agent.Intents;
@ -116,7 +116,7 @@ namespace BotSharp.Core.Engines
say.Entities.Add(part);
// assemble entity synonmus
/*if (!trainingData.Entities.Any(y => y.EntityType == x.Alias && y.EntityValue == x.Text))
if (!trainingData.Entities.Any(y => y.EntityType == x.Alias && y.EntityValue == x.Text))
{
var allSynonyms = (from e in dc.Table<EntityType>()
join ee in dc.Table<EntityEntry>() on e.Id equals ee.EntityId
@ -132,7 +132,7 @@ namespace BotSharp.Core.Engines
};
trainingData.Entities.Add(te);
}*/
}
});
corpus.UserSays.Add(say);
@ -143,7 +143,7 @@ namespace BotSharp.Core.Engines
corpus.UserSays = corpus.UserSays.Where(x => x.Intent != "Default Fallback Intent").ToList();
return corpus;
}
}*/
public TrainingCorpus GetIntentExpressions()
{

4
BotSharp.Core/Engines/BotPredictor.cs
View file

@ -1,5 +1,5 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.AiRequest;
using DotNetToolkit;
using Microsoft.Extensions.Configuration;
@ -19,7 +19,7 @@ namespace BotSharp.Core.Engines
{
public class BotPredictor
{
public async Task<NlpDoc> Predict(Agent agent, AiRequest request)
public async Task<NlpDoc> Predict(AgentBase agent, AiRequest request)
{
// load model
var dir = Path.Combine(request.AgentDir, request.Model);

5
BotSharp.Core/Engines/BotSharp/BotSharpAi.cs
View file

@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Agents;
using BotSharp.Platform.Models;
namespace BotSharp.Core.Engines.BotSharp
@ -11,9 +10,9 @@ namespace BotSharp.Core.Engines.BotSharp
{
public override async Task Train(BotTrainOptions options)
{
agent.Corpus = GetIntentExpressions(agent);
/*agent.Corpus = GetIntentExpressions(agent);
var trainer = new BotTrainer(agent.Id, dc);
await trainer.Train(agent, options);
await trainer.Train(agent, options);*/
}
}
}

5
BotSharp.Core/Engines/BotSharp/BotSharpCRFNer.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using Bigtree.Algorithm.CRFLite;
using Bigtree.Algorithm.CRFLite.Decoder;
using Bigtree.Algorithm.CRFLite.Encoder;
@ -22,7 +21,7 @@ namespace BotSharp.Core.Engines.BotSharp
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
public async Task<bool> Train(AgentBase agent, NlpDoc doc, PipeModel meta)
{
var corpus = agent.Corpus;
@ -157,7 +156,7 @@ namespace BotSharp.Core.Engines.BotSharp
return trainingTuple;
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
public async Task<bool> Predict(AgentBase agent, NlpDoc doc, PipeModel meta)
{
var decoder = new CRFDecoder();
var options = new DecoderOptions

6
BotSharp.Core/Engines/BotSharp/BotSharpIntentClassifier.cs
View file

@ -1,8 +1,8 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Classify;
using BotSharp.NLP.Txt2Vec;
using BotSharp.Platform.Models;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
@ -20,7 +20,7 @@ namespace BotSharp.Core.Engines.BotSharp
public PipeSettings Settings { get; set; }
private ClassifierFactory<SentenceFeatureExtractor> _classifier;
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
public async Task<bool> Train(AgentBase agent, NlpDoc doc, PipeModel meta)
{
Init(meta);
@ -38,7 +38,7 @@ namespace BotSharp.Core.Engines.BotSharp
return true;
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
public async Task<bool> Predict(AgentBase agent, NlpDoc doc, PipeModel meta)
{
Init(meta);

4
BotSharp.Core/Engines/BotSharp/BotSharpProvider.cs
View file

@ -1,5 +1,5 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Platform.Models;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json.Linq;
using System;
@ -14,7 +14,7 @@ namespace BotSharp.Core.Engines.BotSharp
public IConfiguration Configuration { get; set; }
public PipeSettings Settings { get; set; }
public async Task<bool> Load(Agent agent, PipeModel meta)
public async Task<bool> Load(AgentBase agent, PipeModel meta)
{
meta.Meta = JObject.FromObject(new { version = "0.1.0" });

6
BotSharp.Core/Engines/BotSharp/BotSharpTagger.cs
View file

@ -1,9 +1,9 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Corpus;
using BotSharp.NLP.Tag;
using BotSharp.NLP.Tokenize;
using BotSharp.Platform.Models;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
@ -25,7 +25,7 @@ namespace BotSharp.Core.Engines.BotSharp
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
public async Task<bool> Predict(AgentBase agent, NlpDoc doc, PipeModel meta)
{
Init();
@ -38,7 +38,7 @@ namespace BotSharp.Core.Engines.BotSharp
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
public async Task<bool> Train(AgentBase agent, NlpDoc doc, PipeModel meta)
{
Init();

6
BotSharp.Core/Engines/BotSharp/BotSharpTokenizer.cs
View file

@ -1,7 +1,7 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Tokenize;
using BotSharp.Platform.Models;
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
@ -21,7 +21,7 @@ namespace BotSharp.Core.Engines.BotSharp
}
public async Task<bool> Predict(Agent agent, NlpDoc doc, PipeModel meta)
public async Task<bool> Predict(AgentBase agent, NlpDoc doc, PipeModel meta)
{
Init();
@ -36,7 +36,7 @@ namespace BotSharp.Core.Engines.BotSharp
return true;
}
public async Task<bool> Train(Agent agent, NlpDoc doc, PipeModel meta)
public async Task<bool> Train(AgentBase agent, NlpDoc doc, PipeModel meta)
{
Init();

3
BotSharp.Core/Engines/BotTrainer.cs
View file

@ -5,7 +5,6 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.Platform.Models;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
@ -33,7 +32,7 @@ namespace BotSharp.Core.Engines
this.agentId = agentId;
}
public async Task<ModelMetaData> Train(Agent agent, BotTrainOptions options)
public async Task<ModelMetaData> Train(AgentBase agent, BotTrainOptions options)
{
var data = new NlpDoc();

1
BotSharp.Core/Engines/Jieba.NET/JiebaTagger.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP;
using BotSharp.NLP.Tag;
using JiebaNet.Segmenter;

1
BotSharp.Core/Engines/Jieba.NET/JiebaTokenizer.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Abstractions;
using BotSharp.Core.Agents;
using BotSharp.NLP.Tokenize;
using JiebaNet.Segmenter;
using Microsoft.Extensions.Configuration;

5
BotSharp.Core/PlatformBuilderBase.cs
View file

@ -45,10 +45,7 @@ namespace BotSharp.Core
// Load system buildin entities
importer.LoadBuildinEntities(agent);
// Generate corpus
importer.AssembleTrainData(agent);
return default(TAgent);
return agent;
}
private AgentImportHeader LoadMeta(string dataDir)

6
BotSharp.Platform.Abstraction/IAgentImporter.cs
View file

@ -35,11 +35,5 @@ namespace BotSharp.Platform.Abstraction
/// </summary>
/// <param name="agent"></param>
void LoadBuildinEntities(TAgent agent);
/// <summary>
/// generate training data
/// </summary>
/// <param name="agent"></param>
void AssembleTrainData(TAgent agent);
}
}

10
BotSharp.Platform.Models/AgentBase.cs
View file

@ -7,6 +7,11 @@ namespace BotSharp.Platform.Models
{
public abstract class AgentBase
{
public AgentBase()
{
CreatedDate = DateTime.UtcNow;
}
/// <summary>
/// Guid
/// </summary>
@ -32,5 +37,10 @@ namespace BotSharp.Platform.Models
[Required]
[MaxLength(5)]
public String Language { get; set; }
[Required]
public DateTime CreatedDate { get; set; }
public TrainingCorpus Corpus { get; set; }
}
}

8
BotSharp.Core/Agents/AgentMlConfig.cs → BotSharp.Platform.Models/MachineLearning/AgentMlConfig.cs
View file

@ -1,14 +1,12 @@
using EntityFrameworkCore.BootKit;
using System;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Text;
namespace BotSharp.Core.Agents
namespace BotSharp.Platform.Models.MachineLearning
{
[Table("Bot_AgentMlConfig")]
public class AgentMlConfig : DbRecord, IDbRecord
public class AgentMlConfig
{
[Required]
[StringLength(36)]

3
BotSharp.RestApi/Integrations/FacebookMessenger/FacebookMessengerController.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines;
using BotSharp.NLP;
using BotSharp.Platform.Models;
using BotSharp.RestApi.Integrations.FacebookMessenger;

1
BotSharp.WebHost/Program.cs
View file

@ -18,7 +18,6 @@ namespace BotSharp.WebHost
.ConfigureAppConfiguration((hostingContext, config) =>
{
var env = hostingContext.HostingEnvironment;
Console.WriteLine($"ContentRootPath: {env.ContentRootPath}");
string dir = Path.GetFullPath(env.ContentRootPath);
string settingsFolder = Path.Combine(dir, "Settings");

3
BotSharp.WebHost/Startup.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines;
using DotNetToolkit;
using DotNetToolkit.JwtHelper;
using EntityFrameworkCore.BootKit;

5
Platform.Articulate/ArticulateAi.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core;
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Models.NLP;
using BotSharp.Platform.Abstraction;
@ -144,7 +143,7 @@ namespace Platform.Articulate
var modelPath = Path.Combine(projectPath, model);
var trainer = new BotTrainer();
var parsedAgent = agent.ToObject<Agent>();
var parsedAgent = agent.ToObject<AgentModel>();
var intents = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>();
@ -211,7 +210,7 @@ namespace Platform.Articulate
var agent = GetAgentById(request.AgentId);
var preditor = new BotPredictor();
var doc = preditor.Predict(agent.ToObject<Agent>(), request).Result;
var doc = preditor.Predict(agent, request).Result;
var parameters = new Dictionary<String, Object>();
if (doc.Sentences[0].Entities == null)

1
Platform.Articulate/Controllers/AgentController.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core;
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Platform.Abstraction;
using BotSharp.Platform.Models;

57
Platform.Dialogflow/AgentImporterInDialogflow.cs
View file

@ -3,11 +3,11 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using BotSharp.Core.Agents;
using BotSharp.Platform.Abstraction;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.AiResponse;
using BotSharp.Platform.Models.Intents;
using BotSharp.Platform.Models.MachineLearning;
using DotNetToolkit;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
@ -32,12 +32,12 @@ namespace Platform.Dialogflow
{
// load agent profile
string data = File.ReadAllText(Path.Combine(AgentDir, "agent.json"));
var agent = JsonConvert.DeserializeObject<DialogflowAgent>(data);
var agent = JsonConvert.DeserializeObject<DialogflowAgentImportModel>(data);
agent.Name = agentHeader.Name;
agent.Id = agentHeader.Id;
var result = agent.ToObject<TAgent>();
/*result.ClientAccessToken = agentHeader.ClientAccessToken;
result.ClientAccessToken = agentHeader.ClientAccessToken;
result.DeveloperAccessToken = agentHeader.DeveloperAccessToken;
result.MlConfig = agent.ToObject<AgentMlConfig>();
@ -47,14 +47,14 @@ namespace Platform.Dialogflow
{
agentHeader.Integrations.ForEach(x => x.AgentId = agent.Id);
result.Integrations = agentHeader.Integrations;
}*/
}
return result;
}
public void LoadCustomEntities(TAgent agent)
{
//agent.Entities = new List<EntityType>();
agent.Entities = new List<EntityType>();
string entityDir = Path.Combine(AgentDir, "entities");
if (!Directory.Exists(entityDir)) return;
@ -82,14 +82,14 @@ namespace Platform.Dialogflow
}
var entityType = entity.ToObject<EntityType>();
//agent.Entities.Add(entityType);
agent.Entities.Add(entityType);
}
});
}
public void LoadIntents(TAgent agent)
{
//agent.Intents = new List<Intent>();
agent.Intents = new List<Intent>();
string intentDir = Path.Combine(AgentDir, "intents");
if (!Directory.Exists(intentDir)) return;
@ -109,7 +109,7 @@ namespace Platform.Dialogflow
var intent = JsonConvert.DeserializeObject<DialogflowIntent>(intentJson);
var newIntent = ImportIntentUserSays(agent, intent, fileName);
//agent.Intents.Add(newIntent);
agent.Intents.Add(newIntent);
}
});
}
@ -239,7 +239,7 @@ namespace Platform.Dialogflow
public void LoadBuildinEntities(TAgent agent)
{
/*agent.Intents.ForEach(intent =>
agent.Intents.ForEach(intent =>
{
if (intent.UserSays != null)
{
@ -254,7 +254,7 @@ namespace Platform.Dialogflow
});
}
});*/
});
}
private void LoadBuildinEntityTypePerUserSay(TAgent agent, IntentExpressionPart data)
@ -263,17 +263,17 @@ namespace Platform.Dialogflow
if (existedEntityType == null)
{
/*existedEntityType = new EntityType
existedEntityType = new EntityType
{
Name = data.Meta,
Entries = new List<EntityEntry>(),
IsOverridable = true
};*/
};
agent.Entities.Add(existedEntityType);
}
/*var entries = existedEntityType.Entries.Select(x => x.Value.ToLower()).ToList();
var entries = existedEntityType.Entries.Select(x => x.Value.ToLower()).ToList();
if (!entries.Contains(data.Text.ToLower()))
{
existedEntityType.Entries.Add(new EntityEntry
@ -287,36 +287,7 @@ namespace Platform.Dialogflow
}
}
});
}*/
}
public void AssembleTrainData(TAgent agent)
{
// convert agent to training corpus
/*agent.Corpus = new TrainingCorpus
{
Entities = new List<TrainingEntity>(),
UserSays = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>()
};
agent.Intents.ForEach(intent =>
{
intent.UserSays.ForEach(say => {
agent.Corpus.UserSays.Add(new TrainingIntentExpression<TrainingIntentExpressionPart>
{
Intent = intent.Name,
Text = String.Join("", say.Data.Select(x => x.Text)),
Entities = say.Data.Where(x => !String.IsNullOrEmpty(x.Meta))
.Select(x => new TrainingIntentExpressionPart
{
Value = x.Text,
Entity = x.Meta,
Start = x.Start
})
.ToList()
});
});
});*/
}
}
}
}

33
Platform.Dialogflow/ApiAi.cs
View file

@ -1,33 +0,0 @@
using Platform.Dialogflow.Models;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace BotSharp.Core.Engines.Dialogflow
{
public class ApiAi : ApiAiBase
{
private AIDataService dataService;
public AIResponse TextRequest(AIRequest request)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
if(dataService == null)
{
// dataService = new AIDataService(AiConfig);
}
return dataService.Request(request);
}
public void Train()
{
}
}
}

75
Platform.Dialogflow/ApiAiBase.cs
View file

@ -1,75 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Core.Engines.Dialogflow
{
public class ApiAiBase
{
protected float[] TrimSilence(float[] samples)
{
if (samples == null)
{
return null;
}
const float min = 0.000001f;
var startIndex = 0;
var endIndex = samples.Length;
for (var i = 0; i < samples.Length; i++)
{
if (Math.Abs(samples[i]) > min)
{
startIndex = i;
break;
}
}
for (var i = samples.Length - 1; i > 0; i--)
{
if (Math.Abs(samples[i]) > min)
{
endIndex = i;
break;
}
}
if (endIndex <= startIndex)
{
return null;
}
var result = new float[endIndex - startIndex];
Array.Copy(samples, startIndex, result, 0, endIndex - startIndex);
return result;
}
protected static byte[] ConvertArrayShortToBytes(short[] array)
{
var numArray = new byte[array.Length * 2];
Buffer.BlockCopy(array, 0, numArray, 0, numArray.Length);
return numArray;
}
protected static short[] ConvertIeeeToPcm16(float[] source)
{
var resultBuffer = new short[source.Length];
for (var i = 0; i < source.Length; i++)
{
var f = source[i] * 32768f;
if (f > (double)short.MaxValue)
f = short.MaxValue;
else if (f < (double)short.MinValue)
f = short.MinValue;
resultBuffer[i] = Convert.ToInt16(f);
}
return resultBuffer;
}
}
}

13
Platform.Dialogflow/Controllers/AgentController.cs
View file

@ -1,5 +1,4 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.AspNetCore.Http;
@ -58,15 +57,19 @@ namespace Platform.Dialogflow.Controllers
}
string dest = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", uploadedFile.FileName.Split('.').First(), "tmp");
System.IO.Directory.Delete(dest, true);
if (Directory.Exists(dest))
{
System.IO.Directory.Delete(dest, true);
}
Console.WriteLine($"Extract zip file to {dest}");
ZipFile.ExtractToDirectory(filePath, dest);
System.IO.File.Delete(filePath);
Console.WriteLine($"LoadAgentFromFile {dest}");
Console.WriteLine($"Loading agent from folder {dest}");
var agent = builder.LoadAgentFromFile<AgentImporterInDialogflow<AgentModel>>(dest);
builder.SaveAgent(agent);
return Ok(agent.Id);
}
@ -77,7 +80,7 @@ namespace Platform.Dialogflow.Controllers
/// <param name="agentId"></param>
/// <returns></returns>
[HttpGet("{agentId}")]
public ActionResult<Agent> Dump([FromRoute] String agentId)
public ActionResult Dump([FromRoute] String agentId)
{
return Ok();
}

42
Platform.Dialogflow/Controllers/TrainController.cs
View file

@ -1,10 +1,11 @@
using BotSharp.Core.Agents;
using BotSharp.Core.Engines;
using BotSharp.Core.Engines;
using BotSharp.Platform.Models;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using Platform.Dialogflow.Models;
using System;
using System.Collections.Generic;
using System.IO;
@ -23,40 +24,29 @@ namespace Platform.Dialogflow.Controllers
[Route("v1/[controller]")]
public class TrainController : ControllerBase
{
private readonly IBotPlatform _platform;
private DialogflowAi<AgentModel> builder;
/// <summary>
/// Initialize dialog controller and get a platform instance
/// </summary>
/// <param name="platform"></param>
public TrainController(IBotPlatform platform)
public TrainController(IConfiguration configuration)
{
_platform = platform;
builder = new DialogflowAi<AgentModel>();
builder.PlatformConfig = configuration.GetSection("DialogflowAi");
}
[HttpPost]
public async Task<ActionResult<String>> Train([FromQuery] string agentId)
public async Task<ActionResult<AgentModel>> Train([FromQuery] string agentId)
{
var trainer = new BotTrainer();
var agent = builder.GetAgentById(agentId);
// save corpus to agent dir
var projectPath = Path.Combine(AppDomain.CurrentDomain.GetData("DataPath").ToString(), "Projects", agentId);
var model = Directory.GetDirectories(projectPath).Where(x => x.Contains("model_")).Last().Split(Path.DirectorySeparatorChar).Last();
string dataDir = Path.Combine(projectPath, model);
Console.WriteLine($"LoadAgentFromFile {dataDir}");
/*var agent = _platform.LoadAgentFromFile(dataDir);
var info = await trainer.Train(agent, new BotTrainOptions
if(agent == null)
{
AgentDir = projectPath,
Model = model
});
agent = builder.GetAgentByName(agentId);
}
return Ok(new { info = info });*/
var corpus = builder.ExtractorCorpus(agent);
return Ok();
await builder.Train(agent, corpus);
return agent;
}
}
#endif

165
Platform.Dialogflow/DialogflowAi.cs Normal file
View file

@ -0,0 +1,165 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core;
using BotSharp.Core.Engines;
using BotSharp.NLP;
using BotSharp.Platform.Abstraction;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.AiRequest;
using BotSharp.Platform.Models.AiResponse;
using DotNetToolkit;
using Platform.Dialogflow.Models;
namespace Platform.Dialogflow
{
public class DialogflowAi<TAgent> :
PlatformBuilderBase<TAgent>,
IPlatformBuilder<TAgent>
where TAgent : AgentModel
{
public TrainingCorpus ExtractorCorpus(TAgent agent)
{
var corpus = new TrainingCorpus
{
Entities = new List<TrainingEntity>(),
UserSays = new List<TrainingIntentExpression<TrainingIntentExpressionPart>>()
};
agent.Entities.ForEach(entity =>
{
corpus.Entities.Add(new TrainingEntity
{
Entity = entity.Name,
Values = entity.Entries.Select(x => new TrainingEntitySynonym
{
Value = x.Value,
Synonyms = x.Synonyms.Select(y => y.Synonym).ToList()
}).ToList()
});
});
agent.Intents.ForEach(intent =>
{
intent.UserSays.ForEach(say => {
corpus.UserSays.Add(new TrainingIntentExpression<TrainingIntentExpressionPart>
{
Intent = intent.Name,
Text = String.Join("", say.Data.Select(x => x.Text)),
Entities = say.Data.Where(x => !String.IsNullOrEmpty(x.Meta))
.Select(x => new TrainingIntentExpressionPart
{
Value = x.Text,
Entity = x.Meta,
Start = x.Start
})
.ToList()
});
});
});
return corpus;
}
public AiResponse TextRequest(AiRequest request)
{
var dataService = new AIDataService(new AIConfiguration("TOKEN", SupportedLanguage.English)
{
AgentId = request.AgentId,
Language = SupportedLanguage.English,
SessionId = request.SessionId
});
var response = dataService.Request(new AIRequest
{
SessionId = request.SessionId,
Query = new string[] { request.Text }
});
return response.ToObject<AiResponse>();
}
public async Task<bool> Train(TAgent agent, TrainingCorpus corpus)
{
var trainer = new BotTrainer();
var trainOptions = new BotTrainOptions
{
//AgentDir = projectPath,
//Model = model
};
var info = await trainer.Train(agent, trainOptions);
return true;
}
protected float[] TrimSilence(float[] samples)
{
if (samples == null)
{
return null;
}
const float min = 0.000001f;
var startIndex = 0;
var endIndex = samples.Length;
for (var i = 0; i < samples.Length; i++)
{
if (Math.Abs(samples[i]) > min)
{
startIndex = i;
break;
}
}
for (var i = samples.Length - 1; i > 0; i--)
{
if (Math.Abs(samples[i]) > min)
{
endIndex = i;
break;
}
}
if (endIndex <= startIndex)
{
return null;
}
var result = new float[endIndex - startIndex];
Array.Copy(samples, startIndex, result, 0, endIndex - startIndex);
return result;
}
protected static byte[] ConvertArrayShortToBytes(short[] array)
{
var numArray = new byte[array.Length * 2];
Buffer.BlockCopy(array, 0, numArray, 0, numArray.Length);
return numArray;
}
protected static short[] ConvertIeeeToPcm16(float[] source)
{
var resultBuffer = new short[source.Length];
for (var i = 0; i < source.Length; i++)
{
var f = source[i] * 32768f;
if (f > (double)short.MaxValue)
f = short.MaxValue;
else if (f < (double)short.MinValue)
f = short.MinValue;
resultBuffer[i] = Convert.ToInt16(f);
}
return resultBuffer;
}
}
}

22
Platform.Dialogflow/Models/AgentModel.cs
View file

@ -1,6 +1,6 @@
using BotSharp.Core.Agents;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.Intents;
using BotSharp.Platform.Models.MachineLearning;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
@ -13,22 +13,11 @@ namespace Platform.Dialogflow.Models
{
public AgentModel()
{
CreatedDate = DateTime.UtcNow;
}
[Required]
[MaxLength(64)]
public String Name { get; set; }
[MaxLength(256)]
public String Description { get; set; }
public Boolean Published { get; set; }
[Required]
[MaxLength(5)]
public String Language { get; set; }
/// <summary>
/// Only access text/ audio rquest
/// </summary>
@ -54,15 +43,10 @@ namespace Platform.Dialogflow.Models
}
}
[Required]
public DateTime CreatedDate { get; set; }
public Boolean IsSkillSet { get; set; }
public AgentMlConfig MlConfig { get; set; }
public TrainingCorpus Corpus { get; set; }
public List<AgentIntegration> Integrations { get; set; }
}
}

2
Platform.Dialogflow/Models/DialogflowAgent.cs → Platform.Dialogflow/Models/DialogflowAgentImportModel.cs
View file

@ -4,7 +4,7 @@ using System.Text;
namespace Platform.Dialogflow.Models
{
public class DialogflowAgent
public class DialogflowAgentImportModel
{
public String Id { get; set; }
public String Name { get; set; }

35
Platform.Dialogflow/Models/DialogflowAi.cs
View file

@ -1,35 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core;
using BotSharp.Core.Engines;
using BotSharp.Platform.Abstraction;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.AiRequest;
using BotSharp.Platform.Models.AiResponse;
using Platform.Dialogflow.Models;
namespace Platform.Dialogflow.Models
{
public class DialogflowAi<TAgent> :
PlatformBuilderBase<TAgent>,
IPlatformBuilder<TAgent>
where TAgent : AgentModel
{
public TrainingCorpus ExtractorCorpus(TAgent agent)
{
throw new NotImplementedException();
}
public AiResponse TextRequest(AiRequest request)
{
throw new NotImplementedException();
}
public Task<bool> Train(TAgent agent, TrainingCorpus corpus)
{
throw new NotImplementedException();
}
}
}