From ea8ea433530bc3731bdb9def2aec959911f17fdf Mon Sep 17 00:00:00 2001 From: Wenbo Cao <104199@smsassist.com> Date: Fri, 1 Sep 2023 16:02:53 -0500 Subject: [PATCH] Fix the previous issues --- ...troller.cs => RoutingSpeederController.cs} | 8 ++-- .../Providers/IntentClassifier.cs | 43 +++++++++++++------ .../Providers/Models/TrainingParams.cs | 2 +- 3 files changed, 35 insertions(+), 18 deletions(-) rename src/Plugins/BotSharp.Plugin.RoutingSpeeder/Controllers/{TrainIntentClassifierController.cs => RoutingSpeederController.cs} (76%) diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Controllers/TrainIntentClassifierController.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Controllers/RoutingSpeederController.cs similarity index 76% rename from src/Plugins/BotSharp.Plugin.RoutingSpeeder/Controllers/TrainIntentClassifierController.cs rename to src/Plugins/BotSharp.Plugin.RoutingSpeeder/Controllers/RoutingSpeederController.cs index b4f5f18e..0be1ed92 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Controllers/TrainIntentClassifierController.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Controllers/RoutingSpeederController.cs @@ -12,19 +12,19 @@ using Microsoft.Extensions.DependencyInjection; namespace BotSharp.Plugin.RoutingSpeeder.Controllers; [AllowAnonymous] -public class TrainIntentClassifierController : ControllerBase +public class RoutingSpeederController : ControllerBase { private readonly IServiceProvider _service; - public TrainIntentClassifierController(IServiceProvider service) + public RoutingSpeederController(IServiceProvider service) { _service = service; } - [HttpPost("/intent/classifier/training")] + [HttpPost("/routingspeeder/classifier/train")] public IActionResult TrainIntentClassifier(TrainingParams trainingParams) { var intentClassifier = _service.GetRequiredService(); - intentClassifier.InitClassifer(trainingParams.Reference); + intentClassifier.InitClassifer(trainingParams.Inference); intentClassifier.Train(trainingParams); return Ok(intentClassifier.Labels); } diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs index 5da02130..440bebf0 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs @@ -11,35 +11,46 @@ using Tensorflow.Keras.Callbacks; using System.Text.RegularExpressions; using BotSharp.Plugin.RoutingSpeeder.Settings; using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.Knowledges.Settings; using BotSharp.Plugin.RoutingSpeeder.Providers.Models; using Microsoft.Extensions.DependencyInjection; using System.Linq; using Tensorflow.Keras; -using BotSharp.Abstraction.Knowledges.Settings; using System.Numerics; using Newtonsoft.Json; using Tensorflow.Keras.Layers; using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Knowledges; namespace BotSharp.Plugin.RoutingSpeeder.Providers; public class IntentClassifier { private readonly IServiceProvider _services; + private KnowledgeBaseSettings _knowledgeBaseSettings; Model _model; public Model model => _model; private bool _isModelReady; public bool isModelReady => _isModelReady; private ClassifierSetting _settings; - private string[] _labels => GetLabels(); + private string[] _labels; - public string[] Labels => _labels; + public string[] Labels => GetLabels(); - public IntentClassifier(IServiceProvider services, ClassifierSetting settings) + private int _numLabels + { + get + { + return Labels.Length; + } + } + + public IntentClassifier(IServiceProvider services, ClassifierSetting settings, KnowledgeBaseSettings knowledgeBaseSettings) { _services = services; _settings = settings; + _knowledgeBaseSettings = knowledgeBaseSettings; } private void Reset() @@ -55,14 +66,15 @@ public class IntentClassifier return; } - var vector = _services.GetRequiredService(); + var vector = _services.GetServices() + .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding)); var layers = new List { keras.layers.InputLayer((vector.Dimension), name: "Input"), keras.layers.Dense(256, activation:"relu"), keras.layers.Dense(256, activation:"relu"), - keras.layers.Dense(_labels.Length, activation: keras.activations.Softmax) + keras.layers.Dense(_numLabels, activation: keras.activations.Softmax) }; _model = keras.Sequential(layers); @@ -92,7 +104,7 @@ public class IntentClassifier var callbacks = new List() { earlyStop }; - var weights = LoadWeights(trainingParams.Reference); + var weights = LoadWeights(trainingParams.Inference); _model.fit(x, y, batch_size: trainingParams.BatchSize, @@ -159,12 +171,12 @@ public class IntentClassifier { var texts = File.ReadAllLines(filePath, Encoding.UTF8).Select(x => TextClean(x)).ToList(); vectorList.AddRange(vector.GetVectors(texts)); - string fileName = Path.GetFileNameWithoutExtension(filePath).Replace("intent.", ""); + string fileName = Path.GetFileNameWithoutExtension(filePath); labelList.AddRange(Enumerable.Repeat(fileName, texts.Count).ToList()); } // Write label into local file - var uniqueLabelList = labelList.Distinct().Select(x => x.Replace("intent.", "")).OrderBy(x => x).ToArray(); + var uniqueLabelList = labelList.Distinct().OrderBy(x => x).ToArray(); File.WriteAllLines(saveLabelDirectory, uniqueLabelList); var x = np.zeros((vectorList.Count, vector.Dimension), dtype: np.float32); @@ -188,10 +200,15 @@ public class IntentClassifier public string[] GetLabels() { - var agentService = _services.CreateScope().ServiceProvider.GetRequiredService(); - string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, _settings.LABEL_FILE_NAME); - var labelText = File.ReadAllLines(rootDirectory); - return labelText.Select(x => x.Replace("intent.", "")).OrderBy(x => x).ToArray(); + if (_labels == null) + { + var agentService = _services.CreateScope().ServiceProvider.GetRequiredService(); + string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, _settings.LABEL_FILE_NAME); + var labelText = File.ReadAllLines(rootDirectory); + _labels = labelText.OrderBy(x => x).ToArray(); + } + + return _labels; } public string TextClean(string text) diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/Models/TrainingParams.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/Models/TrainingParams.cs index 7a4a8e67..4cd9829c 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/Models/TrainingParams.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/Models/TrainingParams.cs @@ -10,5 +10,5 @@ public class TrainingParams public int Epochs { get; set; } = 10; public int BatchSize { get; set; } = 16; public float LearningRate { get; set; } = 1.0e-4f; - public bool Reference { get; set; } = false; + public bool Inference { get; set; } = false; }