43 lines
1.4 KiB
C#
43 lines
1.4 KiB
C#
using BotSharp.NLP.Corpus;
|
|
using BotSharp.NLP.Tokenize;
|
|
using BotSharp.NLP.Txt2Vec;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text;
|
|
|
|
namespace BotSharp.NLP.UnitTest.Vector
|
|
{
|
|
[TestClass]
|
|
public class OneHotEncodingTest : TestEssential
|
|
{
|
|
[TestMethod]
|
|
public void OneHotTest()
|
|
{
|
|
var reader = new FasttextDataReader();
|
|
var sentences = reader.Read(new ReaderOptions
|
|
{
|
|
DataDir = Path.Combine(Configuration.GetValue<String>("MachineLearning:dataDir"), "Text Classification", "cooking.stackexchange"),
|
|
FileName = "cooking.stackexchange.txt"
|
|
});
|
|
|
|
var tokenizer = new TokenizerFactory(new TokenizationOptions { }, SupportedLanguage.English);
|
|
tokenizer.GetTokenizer<TreebankTokenizer>();
|
|
|
|
var newSentences = tokenizer.Tokenize(sentences.Select(x => x.Text).ToList());
|
|
for (int i = 0; i < newSentences.Count; i++)
|
|
{
|
|
newSentences[i].Label = sentences[i].Label;
|
|
}
|
|
sentences = newSentences.ToList();
|
|
|
|
var encoder = new OneHotEncoder();
|
|
encoder.Sentences = sentences;
|
|
encoder.EncodeAll();
|
|
}
|
|
}
|
|
}
|