diff --git a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs index 440bebf0..6e685905 100644 --- a/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs +++ b/src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs @@ -2,7 +2,6 @@ using System; using System.IO; using System.Text; using System.Collections.Generic; -using Tensorflow; using static Tensorflow.KerasApi; using Tensorflow.Keras.Engine; using Tensorflow.NumPy; @@ -16,11 +15,7 @@ using BotSharp.Plugin.RoutingSpeeder.Providers.Models; using Microsoft.Extensions.DependencyInjection; using System.Linq; using Tensorflow.Keras; -using System.Numerics; -using Newtonsoft.Json; -using Tensorflow.Keras.Layers; using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Knowledges; namespace BotSharp.Plugin.RoutingSpeeder.Providers; @@ -33,11 +28,8 @@ public class IntentClassifier private bool _isModelReady; public bool isModelReady => _isModelReady; private ClassifierSetting _settings; - private string[] _labels; - public string[] Labels => GetLabels(); - private int _numLabels { get @@ -67,7 +59,7 @@ public class IntentClassifier } var vector = _services.GetServices() - .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding)); + .FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding)); var layers = new List { @@ -89,10 +81,9 @@ public class IntentClassifier { _model.compile(optimizer: keras.optimizers.Adam(trainingParams.LearningRate), loss: keras.losses.SparseCategoricalCrossentropy(), - metrics: new[] { "accuracy" } - ); + metrics: new[] { "accuracy" }); - CallbackParams callback_parameters = new CallbackParams + var callback_parameters = new CallbackParams { Model = _model, Epochs = trainingParams.Epochs, @@ -100,9 +91,12 @@ public class IntentClassifier Steps = 10 }; - ICallback earlyStop = new EarlyStopping(callback_parameters, "accuracy"); + var earlyStop = new EarlyStopping(callback_parameters, "accuracy"); - var callbacks = new List() { earlyStop }; + var callbacks = new List() + { + earlyStop + }; var weights = LoadWeights(trainingParams.Inference); @@ -110,7 +104,6 @@ public class IntentClassifier batch_size: trainingParams.BatchSize, epochs: trainingParams.Epochs, callbacks: callbacks, - // validation_split: 0.1f, shuffle: true); _model.save_weights(weights); @@ -120,7 +113,9 @@ public class IntentClassifier public string LoadWeights(bool inference = true) { - var agentService = _services.CreateScope().ServiceProvider.GetRequiredService(); + var agentService = _services.CreateScope() + .ServiceProvider + .GetRequiredService(); var weightsFile = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, $"intent-classifier.h5"); @@ -129,13 +124,13 @@ public class IntentClassifier _model.load_weights(weightsFile); _isModelReady = true; Console.WriteLine($"Successfully load the weights!"); - } else { var logInfo = inference ? "No available weights." : "Will implement model training process and write trained weights into local"; Console.WriteLine(logInfo); } + return weightsFile; } @@ -152,24 +147,33 @@ public class IntentClassifier public (NDArray, NDArray) PrepareLoadData() { - var agentService = _services.CreateScope().ServiceProvider.GetRequiredService(); - string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.RAW_DATA_DIR); - string saveLabelDirectory = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, _settings.LABEL_FILE_NAME); + var agentService = _services.CreateScope() + .ServiceProvider + .GetRequiredService(); + string rootDirectory = Path.Combine( + agentService.GetDataDir(), + _settings.RAW_DATA_DIR); + string saveLabelDirectory = Path.Combine( + agentService.GetDataDir(), + _settings.MODEL_DIR, + _settings.LABEL_FILE_NAME); if (!Directory.Exists(rootDirectory)) { throw new Exception($"No training data found! Please put training data in this path: {rootDirectory}"); } + // Do embedding and store results var vector = _services.GetRequiredService(); - var vectorList = new List(); - var labelList = new List(); foreach (var filePath in GetFiles()) { - var texts = File.ReadAllLines(filePath, Encoding.UTF8).Select(x => TextClean(x)).ToList(); + var texts = File.ReadAllLines(filePath, Encoding.UTF8) + .Select(x => TextClean(x)) + .ToList(); + vectorList.AddRange(vector.GetVectors(texts)); string fileName = Path.GetFileNameWithoutExtension(filePath); labelList.AddRange(Enumerable.Repeat(fileName, texts.Count).ToList()); @@ -185,25 +189,39 @@ public class IntentClassifier for (int i = 0; i < vectorList.Count; i++) { x[i] = vectorList[i]; - // y[i] = (float)uniqueLabelList.IndexOf(labelList[i]); y[i] = (float)Array.IndexOf(uniqueLabelList, labelList[i]); } + return (x, y); } public string[] GetFiles(string prefix = "intent") { - var agentService = _services.CreateScope().ServiceProvider.GetRequiredService(); + var agentService = _services.CreateScope() + .ServiceProvider + .GetRequiredService(); string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.RAW_DATA_DIR); - return Directory.GetFiles(rootDirectory).Where(x => Path.GetFileNameWithoutExtension(x).StartsWith(prefix)).OrderBy(x => x).ToArray(); + + return Directory.GetFiles(rootDirectory) + .Where(x => Path.GetFileNameWithoutExtension(x) + .StartsWith(prefix)) + .OrderBy(x => x) + .ToArray(); } public string[] GetLabels() { if (_labels == null) { - var agentService = _services.CreateScope().ServiceProvider.GetRequiredService(); - string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, _settings.LABEL_FILE_NAME); + var agentService = _services.CreateScope() + .ServiceProvider + .GetRequiredService(); + string rootDirectory = Path.Combine( + agentService.GetDataDir(), + _settings.MODEL_DIR, + _settings.LABEL_FILE_NAME + ); + var labelText = File.ReadAllLines(rootDirectory); _labels = labelText.OrderBy(x => x).ToArray(); } @@ -217,9 +235,11 @@ public class IntentClassifier // Remove digits // To lowercase var processedText = Regex.Replace(text, "[AB0-9]", " "); - processedText = string.Join("", processedText.Select(c => char.IsPunctuation(c) ? ' ' : c).ToList()); - processedText = processedText.Replace(" ", " ").ToLower(); - return processedText; + var replacedTextList = processedText.Select(c => char.IsPunctuation(c) ? ' ' : c).ToList(); + + return string.Join("", replacedTextList) + .Replace(" ", " ") + .ToLower(); } public string Predict(NDArray vector, float confidenceScore = 0.9f) @@ -229,8 +249,8 @@ public class IntentClassifier InitClassifer(); } + // Generate and post-process prediction var prob = _model.predict(vector).numpy(); - var probLabel = tf.arg_max(prob, -1).numpy().ToArray(); prob = np.squeeze(prob, axis: 0); @@ -239,9 +259,9 @@ public class IntentClassifier return string.Empty; } - var prediction = _labels[probLabel[0]]; + var labelIndex = probLabel[0]; - return prediction; + return _labels[labelIndex]; } public void InitClassifer(bool inference = true) {