40 lines
1.1 KiB
C#
40 lines
1.1 KiB
C#
|
using System;
|
|||
|
|
using System.Collections.Generic;
|
|||
|
|
using System.IO;
|
|||
|
|
using System.Linq;
|
|||
|
|
using System.Text;
|
|||
|
|
using System.Text.RegularExpressions;
|
|||
|
|
|
|||
|
|
namespace BotSharp.NLP.Corpus
|
|||
|
|
{
|
|||
|
|
/// <summary>
|
|||
|
|
/// Fasttext labeled data reader
|
|||
|
|
/// </summary>
|
|||
|
|
public class FasttextDataReader
|
|||
|
|
{
|
|||
|
|
public List<Sentence> Read(ReaderOptions options)
|
|||
|
|
{
|
|||
|
|
var sentences = new List<Sentence>();
|
|||
|
|
using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName)))
|
|||
|
|
{
|
|||
|
|
while (!reader.EndOfStream)
|
|||
|
|
{
|
|||
|
|
string line = reader.ReadLine();
|
|||
|
|
if (!String.IsNullOrEmpty(line))
|
|||
|
|
{
|
|||
|
|
var ms = Regex.Matches(line, @"__label__\w+\s").Cast<Match>().ToList();
|
|||
|
|
|
|||
|
|
sentences.Add(new Sentence
|
|||
|
|
{
|
|||
|
|
// Label = lable,
|
|||
|
|
Text = line
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return sentences;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|