27 lines
788 B
C#
27 lines
788 B
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Text;
|
|
|
|
namespace BotSharp.NLP.Tokenize
|
|
{
|
|
public class TokenizationOptions
|
|
{
|
|
/// <summary>
|
|
/// Regex pattern
|
|
/// </summary>
|
|
public string Pattern { get; set; }
|
|
|
|
/// <summary>
|
|
/// True if this tokenizer's pattern should be used to find separators between tokens;
|
|
/// False if this tokenizer's pattern should be used to find the tokens themselves.
|
|
/// </summary>
|
|
public bool IsGap { get; set; }
|
|
|
|
/// <summary>
|
|
/// True if any empty tokens generated by the tokenizer should be discarded.
|
|
/// Empty tokens can only be generated if `IsGap == True`
|
|
/// </summary>
|
|
public bool IgnoreEmpty { get; set; }
|
|
}
|
|
}
|