SemianiakaVY/BotSharp
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #134 from evan-cao-wb/master

Browse source 
fix-local-file-loading-issue
This commit is contained in:
Haiping 2023-09-07 07:11:34 -05:00 committed by GitHub
parent 9f74b04fd7 27bcaad52f
commit 1e89b0162f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 25 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
src/Plugins/BotSharp.Plugin.RoutingSpeeder/Providers/IntentClassifier.cs
View file

@ -7,7 +7,6 @@ using Tensorflow.Keras.Engine;
using Tensorflow.NumPy;
using static Tensorflow.Binding;
using Tensorflow.Keras.Callbacks;
using System.Text.RegularExpressions;
using BotSharp.Plugin.RoutingSpeeder.Settings;
using BotSharp.Abstraction.MLTasks;
using BotSharp.Abstraction.Knowledges.Settings;
@ -151,11 +150,11 @@ public class IntentClassifier
.ServiceProvider
.GetRequiredService<IAgentService>();
string rootDirectory = Path.Combine(
agentService.GetDataDir(),
agentService.GetDataDir(),
_settings.RAW_DATA_DIR);
string saveLabelDirectory = Path.Combine(
agentService.GetDataDir(),
_settings.MODEL_DIR,
agentService.GetDataDir(),
_settings.MODEL_DIR,
_settings.LABEL_FILE_NAME);
if (!Directory.Exists(rootDirectory))
@ -170,18 +169,15 @@ public class IntentClassifier
foreach (var filePath in GetFiles())
{
var texts = File.ReadAllLines(filePath, Encoding.UTF8)
.Select(x => TextClean(x))
.ToList();
var texts = File.ReadAllLines(filePath, Encoding.UTF8).ToList();
vectorList.AddRange(vector.GetVectors(texts));
string fileName = Path.GetFileNameWithoutExtension(filePath);
labelList.AddRange(Enumerable.Repeat(fileName, texts.Count).ToList());
}
// Write label into local file
// Sort label to keep the same order
var uniqueLabelList = labelList.Distinct().OrderBy(x => x).ToArray();
File.WriteAllLines(saveLabelDirectory, uniqueLabelList);
var x = np.zeros((vectorList.Count, vector.Dimension), dtype: np.float32);
var y = np.zeros((vectorList.Count, 1), dtype: np.float32);
@ -195,13 +191,20 @@ public class IntentClassifier
return (x, y);
}
public string[] GetFiles(string prefix = "intent")
public string[] GetFiles(string prefix = "")
{
var agentService = _services.CreateScope()
.ServiceProvider
.GetRequiredService<IAgentService>();
string rootDirectory = Path.Combine(agentService.GetDataDir(), _settings.RAW_DATA_DIR);
if (string.IsNullOrEmpty(prefix))
{
return Directory.GetFiles(rootDirectory)
.OrderBy(x => Path.GetFileName(x).Split(".")[^2])
.ToArray();
}
return Directory.GetFiles(rootDirectory)
.Where(x => Path.GetFileNameWithoutExtension(x)
.StartsWith(prefix))
@ -216,32 +219,24 @@ public class IntentClassifier
var agentService = _services.CreateScope()
.ServiceProvider
.GetRequiredService<IAgentService>();
string rootDirectory = Path.Combine(
agentService.GetDataDir(),
string[] labels = GetFiles()
.Select(x => Path.GetFileName(x).Split(".")[^2])
.ToArray();
string writePath = Path.Combine(
agentService.GetDataDir(),
_settings.MODEL_DIR,
_settings.LABEL_FILE_NAME
);
_settings.LABEL_FILE_NAME);
var labelText = File.ReadAllLines(rootDirectory);
_labels = labelText.OrderBy(x => x).ToArray();
_labels = labels.OrderBy(x => x).ToArray();
// Write labels into the local txt file
File.WriteAllLines(writePath, _labels);
}
return _labels;
}
public string TextClean(string text)
{
// Remove punctuation
// Remove digits
// To lowercase
var processedText = Regex.Replace(text, "[AB0-9]", " ");
var replacedTextList = processedText.Select(c => char.IsPunctuation(c) ? ' ' : c).ToList();
return string.Join("", replacedTextList)
.Replace(" ", " ")
.ToLower();
}
public string Predict(NDArray vector, float confidenceScore = 0.9f)
{
if (!_isModelReady)
@ -260,7 +255,6 @@ public class IntentClassifier
}
var labelIndex = probLabel[0];
return _labels[labelIndex];
}
public void InitClassifer(bool inference = true)