28 lines
834 B
C#
28 lines
834 B
C#
|
using System;
|
|||
|
|
using System.Collections.Generic;
|
|||
|
|
using System.Text;
|
|||
|
|
|
|||
|
|
namespace BotSharp.NLP.Tokenize
|
|||
|
|
{
|
|||
|
|
/// <summary>
|
|||
|
|
/// BotSharp Tokenizer Factory
|
|||
|
|
/// Tokenizers divide strings into lists of substrings.
|
|||
|
|
/// The particular tokenizer requires implement interface
|
|||
|
|
/// models to be installed.BotSharp.NLP also provides a simpler, regular-expression based tokenizer, which splits text on whitespace and punctuation.
|
|||
|
|
/// </summary>
|
|||
|
|
public class TokenizerFactory<ITokenize> where ITokenize : ITokenizer, new()
|
|||
|
|
{
|
|||
|
|
private ITokenize _tokenizer;
|
|||
|
|
|
|||
|
|
public TokenizerFactory()
|
|||
|
|
{
|
|||
|
|
_tokenizer = new ITokenize();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public Token[] Tokenize(string text, TokenizationOptions options)
|
|||
|
|
{
|
|||
|
|
return _tokenizer.Tokenize(text, options);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|