SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #132 from evan-cao-wb/re-format-the-code

Browse source 
re format the code
This commit is contained in:
Haiping 2023-09-04 21:01:50 -05:00 committed by GitHub
parent a2bae7424f f7f9e96f0e
commit a238ae5406
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 54 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs
View file

@ -2,7 +2,6 @@ using System;
using System.IO;
using System.Text;
using System.Collections.Generic;
using Tensorflow;
using static Tensorflow.KerasApi;
using Tensorflow.Keras.Engine;
using Tensorflow.NumPy;
@ -16,11 +15,7 @@ using BotSharp.Plugin.RoutingSpeeder.Providers.Models;
using Microsoft.Extensions.DependencyInjection;
using System.Linq;
using Tensorflow.Keras;
using System.Numerics;
using Newtonsoft.Json;
using Tensorflow.Keras.Layers;
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Knowledges;
namespace BotSharp.Plugin.RoutingSpeeder.Providers;
@ -33,11 +28,8 @@ public class IntentClassifier
private bool _isModelReady;
public bool isModelReady => _isModelReady;
private ClassifierSetting _settings;
private string[] _labels;
public string[] Labels => GetLabels();
private int _numLabels
{
get
@ -67,7 +59,7 @@ public class IntentClassifier
}
var vector = _services.GetServices<ITextEmbedding>()
.FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding));
.FirstOrDefault(x => x.GetType().FullName.EndsWith(_knowledgeBaseSettings.TextEmbedding));
var layers = new List<ILayer>
{
@ -89,10 +81,9 @@ public class IntentClassifier
{
_model.compile(optimizer: keras.optimizers.Adam(trainingParams.LearningRate),
loss: keras.losses.SparseCategoricalCrossentropy(),
metrics: new[] { "accuracy" }
);
metrics: new[] { "accuracy" });
CallbackParams callback_parameters = new CallbackParams
var callback_parameters = new CallbackParams
{
Model = _model,
Epochs = trainingParams.Epochs,
@ -100,9 +91,12 @@ public class IntentClassifier
Steps = 10
};
ICallback earlyStop = new EarlyStopping(callback_parameters, "accuracy");
var earlyStop = new EarlyStopping(callback_parameters, "accuracy");
var callbacks = new List<ICallback>() { earlyStop };
var callbacks = new List<ICallback>()
{
earlyStop
};
var weights = LoadWeights(trainingParams.Inference);
@ -110,7 +104,6 @@ public class IntentClassifier
batch_size: trainingParams.BatchSize,
epochs: trainingParams.Epochs,
callbacks: callbacks,
// validation_split: 0.1f,
shuffle: true);
_model.save_weights(weights);
@ -120,7 +113,9 @@ public class IntentClassifier
public string LoadWeights(bool inference = true)
{
var agentService = _services.CreateScope().ServiceProvider.GetRequiredService<IAgentService>();
var agentService = _services.CreateScope()
.ServiceProvider
.GetRequiredService<IAgentService>();
var weightsFile = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, $"intent-classifier.h5");
@ -129,13 +124,13 @@ public class IntentClassifier
_model.load_weights(weightsFile);
_isModelReady = true;
Console.WriteLine($"Successfully load the weights!");
}
else
{
var logInfo = inference ? "No available weights." : "Will implement model training process and write trained weights into local";
Console.WriteLine(logInfo);
}
return weightsFile;
}
@ -152,24 +147,33 @@ public class IntentClassifier
public (NDArray, NDArray) PrepareLoadData()
{
var agentService = _services.CreateScope().ServiceProvider.GetRequiredService<IAgentService>();
string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.RAW_DATA_DIR);
string saveLabelDirectory = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, _settings.LABEL_FILE_NAME);
var agentService = _services.CreateScope()
.ServiceProvider
.GetRequiredService<IAgentService>();
string rootDirectory = Path.Combine(
agentService.GetDataDir(),
_settings.RAW_DATA_DIR);
string saveLabelDirectory = Path.Combine(
agentService.GetDataDir(),
_settings.MODEL_DIR,
_settings.LABEL_FILE_NAME);
if (!Directory.Exists(rootDirectory))
{
throw new Exception($"No training data found! Please put training data in this path: {rootDirectory}");
}
// Do embedding and store results
var vector = _services.GetRequiredService<ITextEmbedding>();
var vectorList = new List<float[]>();
var labelList = new List<string>();
foreach (var filePath in GetFiles())
{
var texts = File.ReadAllLines(filePath, Encoding.UTF8).Select(x => TextClean(x)).ToList();
var texts = File.ReadAllLines(filePath, Encoding.UTF8)
.Select(x => TextClean(x))
.ToList();
vectorList.AddRange(vector.GetVectors(texts));
string fileName = Path.GetFileNameWithoutExtension(filePath);
labelList.AddRange(Enumerable.Repeat(fileName, texts.Count).ToList());
@ -185,25 +189,39 @@ public class IntentClassifier
for (int i = 0; i < vectorList.Count; i++)
{
x[i] = vectorList[i];
// y[i] = (float)uniqueLabelList.IndexOf(labelList[i]);
y[i] = (float)Array.IndexOf(uniqueLabelList, labelList[i]);
}
return (x, y);
}
public string[] GetFiles(string prefix = "intent")
{
var agentService = _services.CreateScope().ServiceProvider.GetRequiredService<IAgentService>();
var agentService = _services.CreateScope()
.ServiceProvider
.GetRequiredService<IAgentService>();
string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.RAW_DATA_DIR);
return Directory.GetFiles(rootDirectory).Where(x => Path.GetFileNameWithoutExtension(x).StartsWith(prefix)).OrderBy(x => x).ToArray();
return Directory.GetFiles(rootDirectory)
.Where(x => Path.GetFileNameWithoutExtension(x)
.StartsWith(prefix))
.OrderBy(x => x)
.ToArray();
}
public string[] GetLabels()
{
if (_labels == null)
{
var agentService = _services.CreateScope().ServiceProvider.GetRequiredService<IAgentService>();
string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.MODEL_DIR, _settings.LABEL_FILE_NAME);
var agentService = _services.CreateScope()
.ServiceProvider
.GetRequiredService<IAgentService>();
string rootDirectory = Path.Combine(
agentService.GetDataDir(),
_settings.MODEL_DIR,
_settings.LABEL_FILE_NAME
);
var labelText = File.ReadAllLines(rootDirectory);
_labels = labelText.OrderBy(x => x).ToArray();
}
@ -217,9 +235,11 @@ public class IntentClassifier
// Remove digits
// To lowercase
var processedText = Regex.Replace(text, "[AB0-9]", " ");
processedText = string.Join("", processedText.Select(c => char.IsPunctuation(c) ? ' ' : c).ToList());
processedText = processedText.Replace(" ", " ").ToLower();
return processedText;
var replacedTextList = processedText.Select(c => char.IsPunctuation(c) ? ' ' : c).ToList();
return string.Join("", replacedTextList)
.Replace(" ", " ")
.ToLower();
}
public string Predict(NDArray vector, float confidenceScore = 0.9f)
@ -229,8 +249,8 @@ public class IntentClassifier
InitClassifer();
}
// Generate and post-process prediction
var prob = _model.predict(vector).numpy();
var probLabel = tf.arg_max(prob, -1).numpy().ToArray<long>();
prob = np.squeeze(prob, axis: 0);
@ -239,9 +259,9 @@ public class IntentClassifier
return string.Empty;
}
var prediction = _labels[probLabel[0]];
var labelIndex = probLabel[0];
return prediction;
return _labels[labelIndex];
}
public void InitClassifer(bool inference = true)
{