SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add DialoguePrediction

Browse source
This commit is contained in:
Wenbo Cao 2023-08-31 09:52:09 -05:00
parent 20a7676442
commit 0d017f1e86
10 changed files with 309 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/Plugins/BotSharp.Plugin.RoutingSpeeder/BotSharp.Plugin.RoutingSpeeder.csproj
View file

@ -11,4 +11,10 @@
<ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="FastText.NetWrapper" Version="1.3.0" />
<PackageReference Include="TensorFlow.Keras" Version="0.11.2" />
<PackageReference Include="TensorFlow.NET" Version="0.110.2" />
</ItemGroup>
</Project>

144
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/DialogueClassifier.cs Normal file
View file

@ -0,0 +1,144 @@
using System;
using System.IO;
using System.Text;
using System.Collections.Generic;
using Tensorflow;
using static Tensorflow.KerasApi;
using Tensorflow.Keras.Engine;
using Tensorflow.NumPy;
using static Tensorflow.Binding;
using Tensorflow.Keras.Callbacks;
using System.Text.RegularExpressions;
using BotSharp.Plugin.RoutingSpeeder.Settings;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Plugin.RoutingSpeeder.Providers.Models;
using Microsoft.Extensions.DependencyInjection;
using System.Linq;
using Tensorflow.Keras;
namespace BotSharp.Plugin.RoutingSpeeder.Providers;
public class DialogueClassifier
{
private readonly IServiceProvider _services;
Model _model;
public Model model => _model;
private bool _isModelReady;
public bool isModelReady => _isModelReady;
private classifierSetting _settings;
public DialogueClassifier(IServiceProvider services, classifierSetting settings)
{
_services = services;
_settings = settings;
}
private void Reset()
{
keras.backend.clear_session();
_isModelReady = false;
}
private void Build()
{
if (_isModelReady)
{
return;
}
var layers = new List<ILayer>
{
keras.layers.InputLayer((300), name: "Input"),
keras.layers.Dense(256, activation:"relu"),
keras.layers.Dense(256, activation:"relu"),
keras.layers.Dense(_settings.labelMappingDict.Count, activation: keras.activations.Softmax)
};
_model = keras.Sequential(layers);
#if DEBUG
Console.WriteLine();
_model.summary();
#endif
_isModelReady = true;
}
private void Fit(NDArray x, NDArray y, TrainingParams trainingParams)
{
// release more memory
var vector = _services.GetRequiredService<ITextEmbedding>();
// vector.UnloadModel();
_model.compile(optimizer: keras.optimizers.Adam(trainingParams.LearningRate),
loss: keras.losses.SparseCategoricalCrossentropy(),
metrics: new[] { "accuracy" }
);
CallbackParams callback_parameters = new CallbackParams
{
Model = _model,
Epochs = trainingParams.Epochs,
Verbose = 1,
Steps = 10
};
ICallback earlyStop = new EarlyStopping(callback_parameters, "accuracy");
var callbacks = new List<ICallback>() { earlyStop };
var weights = LoadWeights();
_model.fit(x, y,
batch_size: trainingParams.BatchSize,
epochs: trainingParams.Epochs,
callbacks: callbacks,
// validation_split: 0.1f,
shuffle: true);
_model.save_weights(weights);
_isModelReady = true;
}
public string LoadWeights()
{
var weightsFile = Path.Combine(_settings.MODEL_DIR, $"wo-dialogue-classifier.h5");
if (File.Exists(weightsFile))
{
_model.load_weights(weightsFile);
Console.WriteLine($"Successfully load the weights!");
}
else
{
Console.WriteLine("No available weights.");
}
return weightsFile;
}
public (NDArray x, NDArray y) Vectorize(List<DialoguePredictionModel> items)
{
var x = np.zeros((items.Count, 300), dtype: np.float32);
var y = np.zeros((items.Count, 1), dtype: np.float32);
var vector = _services.GetRequiredService<ITextEmbedding>();
for (int i = 0; i < items.Count; i++)
{
x[i] = vector.GetVector(TextClean(items[i].text));
if (_settings.labelMappingDict.ContainsKey(items[i].label))
{
y[i] = _settings.labelMappingDict[items[i].label];
}
}
return (x, y);
}
public string TextClean(string text)
{
// Remove punctuation
// Remove digits
// To lowercase
var processedText = Regex.Replace(text, "[AB0-9]", " ");
processedText = string.Join("", processedText.Select(c => char.IsPunctuation(c) ? ' ' : c).ToList());
processedText = processedText.Replace(" ", " ").ToLower();
return processedText;
}
}

13
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/Models/DialoguePredictionModel.cs Normal file
View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Plugin.RoutingSpeeder.Providers.Models;
public class DialoguePredictionModel
{
public int Id { get; set; }
public string text { get; set; }
public string? label { get; set; }
public string? prediction { get; set; }
}

70
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/fastTextEmbeddingProvider.cs Normal file
View file

@ -0,0 +1,70 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime;
using System.Text;
using System.Text.RegularExpressions;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Plugin.RoutingSpeeder.Settings;
using FastText.NetWrapper;
namespace BotSharp.Plugin.RoutingSpeeder.Providers;
public class fastTextEmbeddingProvider : ITextEmbedding
{
private FastTextWrapper _fastText;
private readonly fastTextSetting _settings;
public int Dimension
{
get
{
if (!_fastText.IsModelReady())
{
_fastText.LoadModel(_settings.ModelPath);
}
return _fastText.GetModelDimension();
}
}
public fastTextEmbeddingProvider(fastTextSetting settings)
{
_settings = settings;
}
public float[] GetVector(string text)
{
LoadModel();
return _fastText.GetSentenceVector(text);
}
public List<float[]> GetVectors(List<string> texts)
{
LoadModel();
var vectors = new List<float[]>();
for (int i = 0; i < texts.Count; i++)
{
vectors.Add(GetVector(texts[i]));
}
return vectors;
}
private void LoadModel()
{
if (_fastText == null)
{
if (!File.Exists(_settings.ModelPath))
{
throw new FileNotFoundException($"Can't load pre-trained word vectors from {_settings.ModelPath}.\n Try to download from https://fasttext.cc/docs/en/english-vectors.html.");
}
_fastText = new FastTextWrapper();
if (!_fastText.IsModelReady())
{
_fastText.LoadModel(_settings.ModelPath);
}
}
}
}

16
src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingConversationHook.cs
View file

@ -1,13 +1,29 @@
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Conversations.Models;
using BotSharp.Abstraction.MLTasks;
using Microsoft.Extensions.DependencyInjection;
using System;
using System.Linq;
using System.Threading.Tasks;
using FastText.NetWrapper;
using BotSharp.Plugin.RoutingSpeeder.Settings;
namespace BotSharp.Plugin.RoutingSpeeder;
public class RoutingConversationHook: ConversationHookBase
{
private readonly IServiceProvider _services;
private routerSpeedSettings _settings;
public RoutingConversationHook(IServiceProvider service, routerSpeedSettings settings)
{
_services = service;
_settings = settings;
}
public override async Task BeforeCompletion(RoleDialogModel message)
{
var embedding = _services.GetServices<ITextEmbedding>()
.FirstOrDefault(x => x.GetType().FullName.EndsWith(_settings.TextEmbedding));
// Utilize local discriminative model to predict intent
message.Content = "response content";
message.StopCompletion = true;

8
src/Plugins/BotSharp.Plugin.RoutingSpeeder/RoutingSpeederPlugin.cs
View file

@ -1,5 +1,8 @@
using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Plugins;
using BotSharp.Plugin.RoutingSpeeder.Settings;
using BotSharp.Plugin.RoutingSpeeder.Providers;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
@ -9,6 +12,11 @@ public class RoutingSpeederPlugin : IBotSharpPlugin
{
public void RegisterDI(IServiceCollection services, IConfiguration config)
{
var settings = new routerSpeedSettings();
config.Bind("routerSpeed", settings);
services.AddSingleton(x => settings);
services.AddSingleton(x => settings.fastText);
services.AddScoped<IConversationHook, RoutingConversationHook>();
services.AddSingleton<ITextEmbedding, fastTextEmbeddingProvider>();
}
}

21
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Settings/classifierSetting.cs Normal file
View file

@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Plugin.RoutingSpeeder.Settings;
public class classifierSetting
{
public Dictionary<string, float> labelMappingDict { get; set; } = new Dictionary<string, float>()
{
{"goodbye", 0f},
{"greeting", 1f},
{"other", 2f},
{"wo-followup", 3f},
{"wo-identifer", 4f},
{"wo-scheduler", 5}
};
public string RAW_DATA_DIR { get; set; } = "C:\\new_wenbocao\\one_brain\\WebStarter\\data\\raw_data";
public string MODEL_DIR { get; set; } = "C:\\new_wenbocao\\one_brain\\WebStarter\\data\\models";
}

7
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Settings/fastTextSetting.cs Normal file
View file

@ -0,0 +1,7 @@
namespace BotSharp.Plugin.RoutingSpeeder.Settings;
public class fastTextSetting
{
public string ModelPath { get; set; }
}

11
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Settings/routerSpeedSettings.cs Normal file
View file

@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Plugin.RoutingSpeeder.Settings;
public class routerSpeedSettings
{
public fastTextSetting fastText { get; set; }
public string TextEmbedding { get; set; }
}

13
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Settings/trainingParams.cs Normal file
View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BotSharp.Plugin.RoutingSpeeder.Settings;
public class TrainingParams
{
public int ClientId { get; set; }
public int Epochs { get; set; } = 10;
public int BatchSize { get; set; } = 16;
public float LearningRate { get; set; } = 1.0e-4f;
}