SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add intent classifier in routing speeder.

Browse source
This commit is contained in:
Wenbo Cao 2023-08-31 10:48:59 -05:00
parent 5f0c3106bb
commit 890df8743a
14 changed files with 53 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBaseSettings.cs → src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs
View file

@ -1,4 +1,4 @@
namespace BotSharp.Core.Plugins.Knowledges;
namespace BotSharp.Abstraction.Knowledges.Settings;
public class KnowledgeBaseSettings
{

1
src/Infrastructure/BotSharp.Core/Plugins/Knowledges/KnowledgeBasePlugin.cs
View file

@ -1,3 +1,4 @@
using BotSharp.Abstraction.Knowledges.Settings;
using BotSharp.Core.Plugins.Knowledges.Services;
using Microsoft.Extensions.Configuration;

2
src/Infrastructure/BotSharp.Core/Plugins/Knowledges/Services/KnowledgeService.cs
View file

@ -1,7 +1,7 @@
using BotSharp.Abstraction.Knowledges.Models;
using BotSharp.Abstraction.Knowledges.Settings;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.VectorStorage;
using System.Text.Json;
namespace BotSharp.Core.Plugins.Knowledges.Services;

2
src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeController.cs
View file

@ -4,7 +4,7 @@ using Microsoft.AspNetCore.Http;
using UglyToad.PdfPig.Content;
using UglyToad.PdfPig;
using BotSharp.Core.Plugins.Knowledges;
using BotSharp.Abstraction.Knowledges.Settings;
namespace BotSharp.OpenAPI.Controllers;

6
src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj
View file

@ -8,13 +8,11 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
<PackageReference Include="TensorFlow.Keras" Version="0.11.2" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="FastText.NetWrapper" Version="1.3.0" />
<PackageReference Include="TensorFlow.Keras" Version="0.11.2" />
<PackageReference Include="TensorFlow.NET" Version="0.110.2" />
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
</Project>

31
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/DialogueClassifier.cs → src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs
View file

@ -15,19 +15,20 @@ using BotSharp.Plugin.RoutingSpeeder.Providers.Models;
using Microsoft.Extensions.DependencyInjection;
using System.Linq;
using Tensorflow.Keras;
using BotSharp.Abstraction.Knowledges.Settings;
namespace BotSharp.Plugin.RoutingSpeeder.Providers;
public class DialogueClassifier
public class IntentClassifier
{
private readonly IServiceProvider _services;
Model _model;
public Model model => _model;
private bool _isModelReady;
public bool isModelReady => _isModelReady;
private classifierSetting _settings;
private ClassifierSetting _settings;
public DialogueClassifier(IServiceProvider services, classifierSetting settings)
public IntentClassifier(IServiceProvider services, ClassifierSetting settings)
{
_services = services;
_settings = settings;
@ -51,7 +52,7 @@ public class DialogueClassifier
keras.layers.InputLayer((300), name: "Input"),
keras.layers.Dense(256, activation:"relu"),
keras.layers.Dense(256, activation:"relu"),
keras.layers.Dense(_settings.labelMappingDict.Count, activation: keras.activations.Softmax)
keras.layers.Dense(_settings.LabelMappingDict.Count, activation: keras.activations.Softmax)
};
_model = keras.Sequential(layers);
@ -64,10 +65,6 @@ public class DialogueClassifier
private void Fit(NDArray x, NDArray y, TrainingParams trainingParams)
{
// release more memory
var vector = _services.GetRequiredService<ITextEmbedding>();
// vector.UnloadModel();
_model.compile(optimizer: keras.optimizers.Adam(trainingParams.LearningRate),
loss: keras.losses.SparseCategoricalCrossentropy(),
metrics: new[] { "accuracy" }
@ -101,7 +98,7 @@ public class DialogueClassifier
public string LoadWeights()
{
var weightsFile = Path.Combine(_settings.MODEL_DIR, $"wo-dialogue-classifier.h5");
var weightsFile = Path.Combine(_settings.MODEL_DIR, $"intent-classifier.h5");
if (File.Exists(weightsFile))
{
_model.load_weights(weightsFile);
@ -116,7 +113,7 @@ public class DialogueClassifier
public (NDArray x, NDArray y) Vectorize(List<DialoguePredictionModel> items)
{
var x = np.zeros((items.Count, 300), dtype: np.float32);
var x = np.zeros((items.Count, vector.Dimension), dtype: np.float32);
var y = np.zeros((items.Count, 1), dtype: np.float32);
var vector = _services.GetRequiredService<ITextEmbedding>();
@ -124,13 +121,23 @@ public class DialogueClassifier
for (int i = 0; i < items.Count; i++)
{
x[i] = vector.GetVector(TextClean(items[i].text));
if (_settings.labelMappingDict.ContainsKey(items[i].label))
if (_settings.LabelMappingDict.ContainsKey(items[i].label))
{
y[i] = _settings.labelMappingDict[items[i].label];
y[i] = _settings.LabelMappingDict[items[i].label];
}
}
return (x, y);
}
public float[] GetTextEmbedding(string text)
{
var knowledgeSettings = _services.GetRequiredService<KnowledgeBaseSettings>();
var embedding = _services.GetServices<ITextEmbedding>()
.FirstOrDefault(x => x.GetType().FullName.EndsWith(knowledgeSettings.TextEmbedding));
return embedding.GetVector(text);
}
public string TextClean(string text)
{
// Remove punctuation

2
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Settings/trainingParams.cs → src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/Models/TrainingParams.cs
View file

@ -2,7 +2,7 @@ using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Plugin.RoutingSpeeder.Settings;
namespace BotSharp.Plugin.RoutingSpeeder.Providers.Models;
public class TrainingParams
{

70
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/fastTextEmbeddingProvider.cs
View file

@ -1,70 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime;
using System.Text;
using System.Text.RegularExpressions;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Plugin.RoutingSpeeder.Settings;
using FastText.NetWrapper;
namespace BotSharp.Plugin.RoutingSpeeder.Providers;
public class fastTextEmbeddingProvider : ITextEmbedding
{
private FastTextWrapper _fastText;
private readonly fastTextSetting _settings;
public int Dimension
{
get
{
if (!_fastText.IsModelReady())
{
_fastText.LoadModel(_settings.ModelPath);
}
return _fastText.GetModelDimension();
}
}
public fastTextEmbeddingProvider(fastTextSetting settings)
{
_settings = settings;
}
public float[] GetVector(string text)
{
LoadModel();
return _fastText.GetSentenceVector(text);
}
public List<float[]> GetVectors(List<string> texts)
{
LoadModel();
var vectors = new List<float[]>();
for (int i = 0; i < texts.Count; i++)
{
vectors.Add(GetVector(texts[i]));
}
return vectors;
}
private void LoadModel()
{
if (_fastText == null)
{
if (!File.Exists(_settings.ModelPath))
{
throw new FileNotFoundException($"Can't load pre-trained word vectors from {_settings.ModelPath}.\n Try to download from https://fasttext.cc/docs/en/english-vectors.html.");
}
_fastText = new FastTextWrapper();
if (!_fastText.IsModelReady())
{
_fastText.LoadModel(_settings.ModelPath);
}
}
}
}

11
src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs
View file

@ -7,24 +7,25 @@ using Microsoft.Extensions.DependencyInjection;
using System;
using System.Linq;
using System.Threading.Tasks;
using FastText.NetWrapper;
using BotSharp.Plugin.RoutingSpeeder.Settings;
using BotSharp.Abstraction.Templating;
using BotSharp.Plugin.RoutingSpeeder.Providers;
namespace BotSharp.Plugin.RoutingSpeeder;
public class RoutingConversationHook: ConversationHookBase
{
private readonly IServiceProvider _services;
private routerSpeedSettings _settings;
public RoutingConversationHook(IServiceProvider service, routerSpeedSettings settings)
private RouterSpeederSettings _settings;
public RoutingConversationHook(IServiceProvider service, RouterSpeederSettings settings)
{
_services = service;
_settings = settings;
}
public override async Task BeforeCompletion(RoleDialogModel message)
{
var embedding = _services.GetServices<ITextEmbedding>()
.FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding));
var intentClassifier = _services.GetRequiredService<IntentClassifier>();
var vector = intentClassifier.GetTextEmbedding(message.Content);
// Utilize local discriminative model to predict intent
message.IntentName = "greeting";

10
src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingSpeederPlugin.cs
View file

@ -12,11 +12,13 @@ public class RoutingSpeederPlugin : IBotSharpPlugin
{
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
var settings = new routerSpeedSettings();
config.Bind("routerSpeed", settings);
var settings = new RouterSpeederSettings();
config.Bind("RouterSpeeder", settings);
services.AddSingleton(x => settings);
services.AddSingleton(x => settings.fastText);
services.AddSingleton<ClassifierSetting>();
services.AddScoped<IConversationHook, RoutingConversationHook>();
services.AddSingleton<ITextEmbedding, fastTextEmbeddingProvider>();
services.AddSingleton<IntentClassifier>();
}
}

15
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Settings/classifierSetting.cs
View file

@ -4,18 +4,15 @@ using System.Text;
namespace BotSharp.Plugin.RoutingSpeeder.Settings;
public class classifierSetting
public class ClassifierSetting
{
public Dictionary<string, float> labelMappingDict { get; set; } = new Dictionary<string, float>()
public Dictionary<string, float> LabelMappingDict { get; set; } = new Dictionary<string, float>()
{
{"goodbye", 0f},
{"greeting", 1f},
{"other", 2f},
{"wo-followup", 3f},
{"wo-identifer", 4f},
{"wo-scheduler", 5}
{"other", 2f}
};
public string RAW_DATA_DIR { get; set; } = "C:\\new_wenbocao\\one_brain\\WebStarter\\data\\raw_data";
public string MODEL_DIR { get; set; } = "C:\\new_wenbocao\\one_brain\\WebStarter\\data\\models";
public string RAW_DATA_DIR { get; set; } = "";
public string MODEL_DIR { get; set; } = "";
}

7
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Settings/fastTextSetting.cs
View file

@ -1,7 +0,0 @@
namespace BotSharp.Plugin.RoutingSpeeder.Settings;
public class fastTextSetting
{
public string ModelPath { get; set; }
}

4
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Settings/routerSpeedSettings.cs
View file

@ -4,8 +4,6 @@ using System.Text;
namespace BotSharp.Plugin.RoutingSpeeder.Settings;
public class routerSpeedSettings
public class RouterSpeederSettings
{
public fastTextSetting fastText { get; set; }
public string TextEmbedding { get; set; }
}

13
src/WebStarter/appsettings.json
View file

@ -47,11 +47,14 @@
}
},
"MetaAi": {
"fastText": {
"ModelPath": "crawl-300d-2M-subword.bin"
}
},
"MetaAi": {
"fastText": {
"ModelPath": "crawl-300d-2M-subword.bin"
}
},
"RoutingSpeeder": {
},
"MetaMessenger": {
"Endpoint": "https://graph.facebook.com",