37 lines
1.1 KiB
C#
37 lines
1.1 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Text;
|
|
|
|
namespace BotSharp.NLP.Tokenize
|
|
{
|
|
public class TokenizationOptions
|
|
{
|
|
/// <summary>
|
|
/// Regex pattern
|
|
/// </summary>
|
|
public string Pattern { get; set; }
|
|
|
|
/// <summary>
|
|
/// True if this tokenizer's pattern should be used to find separators between tokens;
|
|
/// False if this tokenizer's pattern should be used to find the tokens themselves.
|
|
/// </summary>
|
|
public bool IsGap { get; set; }
|
|
|
|
/// <summary>
|
|
/// True if any empty tokens generated by the tokenizer should be discarded.
|
|
/// Empty tokens can only be generated if `IsGap == True`
|
|
/// </summary>
|
|
public bool IgnoreEmpty { get; set; }
|
|
|
|
/// <summary>
|
|
/// Split "isn't" into "is", "n't"
|
|
/// </summary>
|
|
public List<string> SpecialWords { get; set; }
|
|
|
|
/// <summary>
|
|
/// Convert bracket-like characters to avoid confusion with parse brackets.
|
|
/// </summary>
|
|
public bool ConvertParentheses { get; set; }
|
|
}
|
|
}
|