27 lines
809 B
C#
27 lines
809 B
C#
|
using System;
|
|||
|
|
using System.Collections.Generic;
|
|||
|
|
using System.Text;
|
|||
|
|
|
|||
|
|
namespace BotSharp.NLP.Tokenize
|
|||
|
|
{
|
|||
|
|
/// <summary>
|
|||
|
|
/// A tokenizer is a component used for dividing text intotokens.
|
|||
|
|
/// A tokenizer is language specific and takes into account the peculiarities of the language, e.g. don’t in English is tokenized as two tokens.
|
|||
|
|
/// </summary>
|
|||
|
|
public interface ITokenizer
|
|||
|
|
{
|
|||
|
|
/// <summary>
|
|||
|
|
/// Language
|
|||
|
|
/// </summary>
|
|||
|
|
SupportedLanguage Lang { get; set; }
|
|||
|
|
|
|||
|
|
/// <summary>
|
|||
|
|
/// Tokenize
|
|||
|
|
/// </summary>
|
|||
|
|
/// <param name="text">input</param>
|
|||
|
|
/// <param name="options">Options such as: regex expression</param>
|
|||
|
|
/// <returns></returns>
|
|||
|
|
Token[] Tokenize(string text, TokenizationOptions options);
|
|||
|
|
}
|
|||
|
|
}
|