BotSharp/BotSharp.NLP/Tokenize/TokenizationOptions.cs

using System;
using System.Collections.Generic;
using System.Text;

namespace BotSharp.NLP.Tokenize
{
    public class TokenizationOptions
    {
        /// <summary>
        /// Regex pattern
        /// </summary>
        public string Pattern { get; set; }

        /// <summary>
        /// True if this tokenizer's pattern should be used to find separators between tokens; 
        /// False if this tokenizer's pattern should be used to find the tokens themselves.
        /// </summary>
        public bool IsGap { get; set; }

        /// <summary>
        /// True if any empty tokens generated by the tokenizer should be discarded.
        /// Empty tokens can only be generated if `IsGap == True`
        /// </summary>
        public bool IgnoreEmpty { get; set; }
    }
}